diff --git a/.github/actions/setup-kind-cluster/action.yaml b/.github/actions/setup-kind-cluster/action.yaml index 90d3115d3e..398f51f818 100644 --- a/.github/actions/setup-kind-cluster/action.yaml +++ b/.github/actions/setup-kind-cluster/action.yaml @@ -144,10 +144,14 @@ runs: KIND_IMAGE_PULL_USER: ${{ inputs['kind-image-pull-user'] }} NMP_E2E_REGISTRY: ${{ inputs['image-registry'] }} NMP_E2E_TAG: ${{ inputs['image-tag'] }} + # CPU generic HTTP smoke image; tag must match e2e/test_models_deployments_plugin.py. + GENERIC_HTTP_IMAGE: docker.io/library/python + GENERIC_HTTP_TAG: 3.12-alpine run: | e2e/k8s/scripts/prepull_kind_images.sh \ "${NMP_E2E_REGISTRY}/nmp-api:${NMP_E2E_TAG}" \ - "${NMP_E2E_REGISTRY}/nmp-cpu-tasks:${NMP_E2E_TAG}" + "${NMP_E2E_REGISTRY}/nmp-cpu-tasks:${NMP_E2E_TAG}" \ + "${GENERIC_HTTP_IMAGE}:${GENERIC_HTTP_TAG}" - name: Install NeMo Platform if: ${{ inputs['install-nemo-platform'] == 'true' }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3bb03b3ecc..4b0e54e2fb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -447,6 +447,7 @@ jobs: export NMP_BASE_URL="${NMP_E2E_CLUSTER_URL}" uv run --frozen pytest \ e2e/test_jobs.py::test_job_using_secret_environment_variable \ + e2e/test_models_deployments_plugin.py::test_generic_model_deployment_lifecycle \ -v \ --run-e2e \ --no-cov \ diff --git a/contrib/auth/authentik/helm/values.yaml b/contrib/auth/authentik/helm/values.yaml index d7778e5b27..4de9af9ac7 100644 --- a/contrib/auth/authentik/helm/values.yaml +++ b/contrib/auth/authentik/helm/values.yaml @@ -255,10 +255,8 @@ nemo-platform: models: controller: backends: - none: + deployments_plugin: enabled: true - nim_operator: - enabled: false integration: nemoPlatform: diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx index 5926a619d3..f602736ddb 100644 --- a/docs/set-up/config-reference.mdx +++ b/docs/set-up/config-reference.mdx @@ -504,149 +504,6 @@ models: interval_seconds: 5 # Dict of custom backend configurations for the Models Controller backends: - docker: - # Default NIM image when none is specified (multi-LLM image) | default: 'nvcr.io/nim/nvidia/llm-nim' - default_nimservice_image: nvcr.io/nim/nvidia/llm-nim - # Default NIM image tag when none is specified | default: '1.13.1' - default_nimservice_image_tag: 1.13.1 - # Default vLLM image when none is specified for engine='vllm' | default: 'vllm/vllm-openai' - default_vllm_image: vllm/vllm-openai - # Default vLLM image tag when none is specified for engine='vllm' | default: 'v0.22.1' - default_vllm_image_tag: v0.22.1 - # NIM guided decoding backend | default: 'outlines' - nim_guided_decoding_backend: outlines - # PEFT/LoRA source URL for models service | default: '' - peft_source: '' - # PEFT/LoRA refresh interval in seconds | default: 30 - peft_refresh_interval: 30 - # Secret name for Files service authentication | default: 'files-hf-token' - files_auth_secret: files-hf-token - # Docker client timeout in seconds for long-running operations (default: 10 minutes) | default: 600 - docker_timeout: 600 - # Networking mode for NIM containers: 'local' (port forwarding to localhost for local dev), 'dond' (container names on shared network for quickstart), 'dind' (port forwarding to docker service for DinD setups). | default: 'local' | values: 'local' | 'dond' | 'dind' - models_docker_networking_mode: local - # Docker network name for 'dond' mode. NIMs will join this network to communicate with the NeMo Platform container. Required when MODELS_DOCKER_NETWORKING_MODE='dond'. Quickstart sets this automatically via MODELS_DOCKER_NETWORK env var. | default: '' - models_docker_network: '' - # Container name for 'dond' mode. Used to replace localhost in URLs passed to NIMs so they can reach services via the Docker network. Quickstart sets this automatically via MODELS_DOCKER_CONTAINER_NAME env var. | default: '' - models_docker_container_name: '' - # Hostname for port forwarding in 'dind' mode. Typically 'localhost' or the DinD service name. Used to construct URLs like http://{hostname}:{port}. | default: 'localhost' - models_docker_host_service_name: localhost - # Start of port range for port forwarding (inclusive). Defaults to start of IANA dynamic/ephemeral port range. | default: 49152 - models_docker_port_range_start: 49152 - # End of port range for port forwarding (inclusive). Defaults to 500-port range within IANA dynamic/ephemeral range. | default: 49652 - models_docker_port_range_end: 49652 - # Image used to pull model weights. Its entrypoint is overridden to the Hugging Face CLI ('hf download ...'), so any image with the 'hf' CLI on PATH works. Defaults to the platform's nmp-api image (registry/tag from platform config); override to use a different puller image. - huggingface_model_puller: my-registry/nmp-api:local - # Extra environment variables for the model puller container. Values override the reconciler defaults (HF_ENDPOINT, HF_TOKEN). E.g. set HF_HUB_ENABLE_HF_TRANSFER='0' to disable the hf_transfer download path. - huggingface_model_puller_env: {} - # Timeout in seconds for the model puller container to complete (default: 30 minutes) | default: 1800 - model_puller_timeout: 1800 - # Per-file HTTP download timeout in seconds for the model puller (HF_HUB_DOWNLOAD_TIMEOUT). Increase if large model files fail with IncompleteRead/ChunkedEncodingError (default: 2 hours). | default: 7200 - model_puller_download_timeout: 7200 - # Max concurrent file downloads in the model puller (hf download --max-workers). Default 1 (sequential) reduces IncompleteRead/ChunkedEncodingError on slow or flaky links; increase for speed. | default: 1 - model_puller_max_workers: 1 - # Number of retries when the puller fails with a transient error (IncompleteRead, ChunkedEncodingError, connection broken). Same volume is reused so partial downloads can be completed. | default: 3 - model_puller_retries: 3 - # Additional labels copied onto model resources managed by the models controller. - model_labels: {} - # BusyBox image repository used for helper containers (permissions/find/chown). | default: 'busybox' - busybox_image: busybox - # BusyBox image tag used for helper containers. | default: 'latest' - busybox_image_tag: latest - # Image name (without registry/tag) used for the LoRA adapters sidecar container. Registry and tag are taken from NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG. The sidecar is invoked via the lora_sidecar_command. | default: 'nmp-api' - lora_sidecar_image_name: nmp-api - # Command passed to the LoRA sidecar container. Default uses the nmp-platform-runner entrypoint present in nmp-api. - lora_sidecar_command: - - --sidecars - - adapters - - --port - - '60830' - # Optional entrypoint override for the LoRA sidecar container. Leave empty to use the image's default entrypoint (correct for nmp-api). | default: '' - lora_sidecar_entrypoint: '' - # Maximum time (in seconds) a deployment may stay in PENDING before being transitioned to ERROR. Default: 7200 (2 hours). | default: 7200 - pending_timeout_seconds: 7200 - # Maximum number of container restarts before a PENDING deployment is transitioned to ERROR (crash loop detection). Default: 5. | default: 5 - max_restart_count: 5 - # Optional fixed shared memory size (/dev/shm) for multi-GPU NIM containers. If set, overrides the per-GPU calculation. Leave empty to use shm_size_per_gpu x GPU count. Override via MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE env. Format: e.g. '2g', '4g'. | default: '' - nim_multi_gpu_shm_size: '' - # Shared memory size (/dev/shm) per GPU in megabytes for multi-GPU NIM containers. Total shm = this value x GPU count (e.g. 1024 x 2 GPUs = 2048m). Override via MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU env (integer string). | default: 1024 - nim_multi_gpu_shm_size_per_gpu: 1024 - # Whether this backend is enabled | default: False - enabled: false - nim_operator: - # Default storage class for PVCs. If not set, the cluster's default StorageClass is used. - default_storage_class: - # Default PVC size for model storage (used if not specified in deployment config) | default: '200Gi' - default_pvc_size: 200Gi - # LoRA/PEFT source endpoint (only used when lora_enabled is true) | default: 'http://nemo-entity-store:8000' - peft_source: http://nemo-entity-store:8000 - # PEFT refresh interval in seconds (only used when lora_enabled is true) | default: 30 - peft_refresh_interval: 30 - # Image name (without registry/tag) for the LoRA adapters sidecar container. Registry and tag are taken from the platform config (NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG). Override to 'nmp-customizer-tasks' for local dev when that image is already available but nmp-api is not. | default: 'nmp-api' - lora_sidecar_image_name: nmp-api - # Kubernetes container command (entrypoint) for the LoRA sidecar. Default uses the nmp-platform-runner entrypoint present in nmp-api. When using nmp-customizer-tasks set to ['python'] and set lora_sidecar_args to ['-m', 'nmp.core.models.sidecars.adapters.main']. - lora_sidecar_command: - - nemo - - services - - run - - --sidecars - - adapters - # Kubernetes container args for the LoRA sidecar (appended after lora_sidecar_command). Leave empty for nmp-api. Set to ['-m', 'nmp.core.models.sidecars.adapters.main'] when using nmp-customizer-tasks. | default: [] - lora_sidecar_args: [] - # Default user ID for NIM containers (security context) - default_user_id: - # Default group ID for NIM containers (security context) - default_group_id: - # Default user ID for vLLM puller + server pods (security context). Defaults to 2000 to match the upstream vLLM image's 'vllm' user, which has an /etc/passwd entry (avoids torch getpwuid crashes from an unknown uid). | default: 2000 - default_vllm_user_id: 2000 - # Default group ID / fsGroup for vLLM puller + server pods. Defaults to 0 (root group) to match the upstream vLLM image and keep weights readable across the puller and server pods. | default: 0 - default_vllm_group_id: 0 - # Kubernetes secret name for Files service authentication (HF_TOKEN) | default: 'nemo-models-files-token' - files_auth_secret: nemo-models-files-token - # The name of the image pull secret for the modelPuller image | default: 'nvcrimagepullsecret' - huggingface_model_puller_image_pull_secret: nvcrimagepullsecret - # BusyBox image repository used by plugin init containers. Fully qualified (docker.io/library/...) so it resolves on container runtimes that enforce fully-qualified image names (short names like 'busybox' fail there). | default: 'docker.io/library/busybox' - busybox_image: docker.io/library/busybox - # BusyBox image tag used by plugin init containers. | default: 'latest' - busybox_image_tag: latest - # NGC API key secret name for pulling NIM images | default: 'ngc-api' - auth_secret: ngc-api - # Default NIMService image repository (used if not specified in deployment config) | default: 'nvcr.io/nim/nvidia/llm-nim' - default_nimservice_image: nvcr.io/nim/nvidia/llm-nim - # Default NIMService image tag (used if not specified in deployment config) | default: '1.13.1' - default_nimservice_image_tag: 1.13.1 - # Default vLLM server image repository (used if not specified in deployment config) | default: 'vllm/vllm-openai' - default_vllm_image: vllm/vllm-openai - # Default vLLM server image tag (used if not specified in deployment config) | default: 'v0.22.1' - default_vllm_image_tag: v0.22.1 - # Default guided decoding backend for NIM (e.g., 'outlines', 'auto', 'lm-format-enforcer') | default: 'outlines' - nim_guided_decoding_backend: outlines - # Kubernetes namespace for NIM deployments (defaults to controller's namespace if not set) - namespace: - # ServiceAccount name for directly-emitted vLLM Deployment pods and the weight-puller Job. If not set, the namespace default ServiceAccount is used. - service_account_name: - # Shared memory (/dev/shm) size limit for vLLM Deployment pods (e.g. '8Gi'). If not set, the emptyDir uses the node default size. - default_shared_memory_size_limit: - # Default Kubernetes resource requirements for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: {'requests': {'cpu': '2', 'memory': '8Gi'}, 'limits': {'memory': '16Gi'}} - default_resources: - # Default Kubernetes tolerations for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: [{'key': 'nvidia.com/gpu', 'operator': 'Exists', 'effect': 'NoSchedule'}] - default_tolerations: - # Default Kubernetes node selector for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: {'node-type': 'gpu-node', 'zone': 'us-west1-a'} - default_node_selector: - # Additional labels copied onto model resources managed by the models controller. - model_labels: - # Default Kubernetes labels applied to NIMService and NIMCache resources and their child resources (e.g. pods). Merged with controller-managed labels; controller labels take precedence on conflict. Example: {'team': 'ml-platform', 'environment': 'prod'} - default_labels: - # Default Kubernetes annotations applied to NIMService and NIMCache resources and their child resources (e.g. pods, PVCs). Merged with controller-managed annotations; controller annotations take precedence on conflict. Example: {'prometheus.io/scrape': 'true'} - default_annotations: - # Default grace period in seconds for NIM startup. Can be overridden per-deployment via k8s_nim_operator_config. Determines how long Kubernetes will wait for the NIM to become ready before restarting it. If not set, defaults to 600 seconds (10 minutes). Example: 600 (10 minutes) - default_startup_probe_grace_period_seconds: - # Maximum time in seconds a deployment may stay in PENDING before being transitioned to ERROR. Default: 7200 (2 hours). | default: 7200 - pending_timeout_seconds: 7200 - # Maximum number of pod container restarts before a PENDING deployment is transitioned to ERROR (crash loop detection). Default: 5. | default: 5 - max_restart_count: 5 - # Whether this backend is enabled | default: False - enabled: false deployments_plugin: default_executor: docker_executor: @@ -662,10 +519,10 @@ models: default_nimservice_image: nvcr.io/nim/meta/llama-3.1-8b-instruct # default: '1.8.5' default_nimservice_image_tag: 1.8.5 - # default: 'vllm/vllm-openai' - default_vllm_image: vllm/vllm-openai - # default: 'v0.8.5' - default_vllm_image_tag: v0.8.5 + # Default vLLM image repository. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/vllm/vllm-openai' + default_vllm_image: docker.io/vllm/vllm-openai + # default: 'v0.22.1' + default_vllm_image_tag: v0.22.1 # default: 1000 default_user_id: 1000 # default: 2000 @@ -689,15 +546,10 @@ models: busybox_image: docker.io/library/busybox # BusyBox image tag for LoRA cache init containers. | default: 'latest' busybox_image_tag: latest - # default: 5.0 - delete_wait_seconds: 5.0 - # default: 0.5 - delete_poll_seconds: 0.5 + # Maximum seconds a deployment may stay DELETING before ERROR. 0 disables timeout escalation. | default: 60 + deleting_timeout_seconds: 60 # default: False enabled: false - none: - # Whether this backend is enabled | default: False - enabled: false # Time-to-live in seconds for DELETED deployments before they are permanently removed from the database | default: 30 model_deployment_garbage_collection_ttl_seconds: 30 # Time-to-live in seconds for ERROR deployments before backend resources are garbage collected | default: 10800 diff --git a/docs/set-up/helm/persistent-volumes.mdx b/docs/set-up/helm/persistent-volumes.mdx index 8f304c2469..068b7a14e7 100644 --- a/docs/set-up/helm/persistent-volumes.mdx +++ b/docs/set-up/helm/persistent-volumes.mdx @@ -31,11 +31,11 @@ As an alternative to PVC-based file storage, you can configure the Files service ```yaml core: - storage: - storageClass: "nfs" - accessModes: - - ReadWriteMany - size: 200Gi + storage: + storageClass: "nfs" + accessModes: + - ReadWriteMany + size: 200Gi ``` If `storageClass` is empty (default), the cluster’s default StorageClass is used. @@ -46,8 +46,8 @@ To use a pre-created PersistentVolume instead of having the chart create a PVC, ```yaml core: - storage: - existingPersistentVolumeName: "my-existing-pv-name" + storage: + existingPersistentVolumeName: "my-existing-pv-name" ``` When set, the chart does not create a new PVC; pods mount the named volume. @@ -58,12 +58,11 @@ For NIM deployments launched via the NeMo Platform, you can set the default Stor ```yaml platformConfig: - models: - controller: - backends: - nim_operator: - config: - default_storage_class: "nfs" + models: + controller: + backends: + deployments_plugin: + default_storage_class: "nfs" ``` Replace `"nfs"` with your StorageClass name (e.g. `oci-nfs`, `gp3`). For NIM scaling and multi-node deployments, use a ReadWriteMany-capable StorageClass. diff --git a/e2e/k8s/scripts/prepull_kind_images.sh b/e2e/k8s/scripts/prepull_kind_images.sh index 22d5490497..dc8b6cecbf 100755 --- a/e2e/k8s/scripts/prepull_kind_images.sh +++ b/e2e/k8s/scripts/prepull_kind_images.sh @@ -37,6 +37,11 @@ pull_image_on_node() { return 1 fi auth_args=(--user "\$oauthtoken:${NGC_API_KEY}") + elif [[ "${image}" == docker.io/* || "${image}" == registry-1.docker.io/* ]]; then + # Public Docker Hub images are pulled anonymously. Passing the GHCR + # KIND_IMAGE_PULL_* credentials here makes auth.docker.io reject the + # request with a 401, so leave auth_args empty. + auth_args=() elif [ -n "${KIND_IMAGE_PULL_USER:-}" ] && [ -n "${KIND_IMAGE_PULL_TOKEN:-}" ]; then auth_args=(--user "${KIND_IMAGE_PULL_USER}:${KIND_IMAGE_PULL_TOKEN}") fi diff --git a/e2e/k8s/values/default.yaml b/e2e/k8s/values/default.yaml index c267ded693..8a63cf1493 100644 --- a/e2e/k8s/values/default.yaml +++ b/e2e/k8s/values/default.yaml @@ -1,9 +1,5 @@ # Default E2E values for running tests in a Kubernetes cluster, for ephemeral testing -# Always disable nim operator for E2E tests -k8s-nim-operator: - enabled: false - postgresql: image: repository: docker.io/library/postgres diff --git a/e2e/k8s/values/kind.yaml b/e2e/k8s/values/kind.yaml index cc6880fc6e..734c77515a 100644 --- a/e2e/k8s/values/kind.yaml +++ b/e2e/k8s/values/kind.yaml @@ -1,10 +1,6 @@ # kind values for CPU-only GitHub Actions Kubernetes E2E runs. -k8s-nim-operator: - enabled: false - rbac: - k8sNimOperatorEnabled: false volcanoEnabled: false postgresql: @@ -31,8 +27,10 @@ platformConfig: models: controller: backends: - nim_operator: - enabled: false + deployments_plugin: + enabled: true + k8s_executor: kind-k8s + default_executor: kind-k8s # Wire a nemo-deployments k8s executor so deployment_mode=k8s agent # deployments (test_nemo_agents_k8s.py) run as real Deployment+Service # workloads. default_namespace is omitted so the executor uses its own pod diff --git a/e2e/k8s/values/minikube-auth-portforward.yaml b/e2e/k8s/values/minikube-auth-portforward.yaml index 0a262c289c..a84228dbc0 100644 --- a/e2e/k8s/values/minikube-auth-portforward.yaml +++ b/e2e/k8s/values/minikube-auth-portforward.yaml @@ -8,9 +8,6 @@ # is blocked and the ingress addon cannot bootstrap. The chart still keeps split # pods on the API service URL while the API pod itself loops back to localhost. -k8s-nim-operator: - enabled: false - postgresql: persistence: storageClass: standard @@ -38,5 +35,14 @@ platformConfig: models: controller: backends: - nim_operator: - enabled: false + deployments_plugin: + enabled: true + k8s_executor: local-k8s + default_executor: local-k8s + deployments: + executors: + - name: local-k8s + backend: k8s + config: + default_namespace: default + default_executor: local-k8s diff --git a/e2e/k8s/values/minikube-auth.yaml b/e2e/k8s/values/minikube-auth.yaml index a3f7eab3aa..713fc3a23e 100644 --- a/e2e/k8s/values/minikube-auth.yaml +++ b/e2e/k8s/values/minikube-auth.yaml @@ -10,9 +10,6 @@ # while the API pod itself loops back to localhost # - keeps ingress enabled for browser/curl access through minikube ingress -k8s-nim-operator: - enabled: false - postgresql: persistence: storageClass: standard @@ -44,5 +41,14 @@ platformConfig: models: controller: backends: - nim_operator: - enabled: false + deployments_plugin: + enabled: true + k8s_executor: local-k8s + default_executor: local-k8s + deployments: + executors: + - name: local-k8s + backend: k8s + config: + default_namespace: default + default_executor: local-k8s diff --git a/e2e/k8s/values/minikube.yaml b/e2e/k8s/values/minikube.yaml index 2570a9b716..0f48a4f718 100644 --- a/e2e/k8s/values/minikube.yaml +++ b/e2e/k8s/values/minikube.yaml @@ -3,13 +3,6 @@ # # Usage: helm upgrade -i nemo-platform k8s/helm -f e2e/k8s/values/minikube.yaml -# Enable NIM operator for local GPU testing -k8s-nim-operator: - enabled: true - operator: - admissionController: - enabled: false - postgresql: image: repository: docker.io/library/postgres diff --git a/e2e/test_models_deployments_plugin.py b/e2e/test_models_deployments_plugin.py new file mode 100644 index 0000000000..dbe9405a75 --- /dev/null +++ b/e2e/test_models_deployments_plugin.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kind smoke e2e for models → deployments_plugin → plugin k8s backend. + +Runs against a real cluster when ``NMP_BASE_URL`` / ``NMP_E2E_CLUSTER_URL`` is set +(the ``kind-cpu-smoke`` CI job). Uses a CPU-only generic container image with no +NGC credentials. +""" + +from __future__ import annotations + +import time +import uuid + +import pytest +from nemo_platform import NeMoPlatform, NotFoundError + +# Kind smoke generic deployment image (python -m http.server). Keep in sync with +# .github/actions/setup-kind-cluster/action.yaml GENERIC_HTTP_* prepull vars. +GENERIC_HTTP_IMAGE = "docker.io/library/python" +GENERIC_HTTP_TAG = "3.12-alpine" + +pytestmark = [ + pytest.mark.container_only, + pytest.mark.timeout(900), +] + + +def _unique_name(prefix: str) -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +def _deployment_diagnostic(sdk: NeMoPlatform, *, workspace: str, name: str, prefix: str) -> str: + try: + deployment = sdk.inference.deployments.retrieve(name, workspace=workspace) + except NotFoundError: + return f"{prefix}\nDeployment {name!r} not found." + return ( + f"{prefix}\n" + f"status={deployment.status!r}\n" + f"status_message={deployment.status_message!r}\n" + f"model_provider_id={deployment.model_provider_id!r}" + ) + + +def _wait_for_deployment_ready( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + timeout_seconds: float = 600, +) -> None: + deadline = time.monotonic() + timeout_seconds + last_status: str | None = None + last_message: str | None = None + + while time.monotonic() < deadline: + deployment = sdk.inference.deployments.retrieve(name, workspace=workspace) + last_status = deployment.status + last_message = deployment.status_message + if deployment.status == "READY": + assert deployment.model_provider_id is not None, _deployment_diagnostic( + sdk, + workspace=workspace, + name=name, + prefix="Deployment reached READY without model_provider_id", + ) + return + if deployment.status == "ERROR": + pytest.fail( + _deployment_diagnostic( + sdk, + workspace=workspace, + name=name, + prefix=f"Deployment {name!r} entered ERROR", + ) + ) + time.sleep(2) + + pytest.fail( + f"Deployment {name!r} did not reach READY within {timeout_seconds}s; " + f"last status={last_status!r}, status_message={last_message!r}" + ) + + +def _wait_for_deployment_deleted( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + timeout_seconds: float = 300, +) -> None: + deadline = time.monotonic() + timeout_seconds + last_status: str | None = None + + while time.monotonic() < deadline: + try: + deployment = sdk.inference.deployments.retrieve(name, workspace=workspace) + last_status = deployment.status + if deployment.status == "ERROR": + pytest.fail( + _deployment_diagnostic( + sdk, + workspace=workspace, + name=name, + prefix=f"Deployment {name!r} entered ERROR while waiting for deletion", + ) + ) + except NotFoundError: + return + time.sleep(2) + + pytest.fail(f"Deployment {name!r} was not deleted within {timeout_seconds}s; last status={last_status!r}") + + +def test_generic_model_deployment_lifecycle(sdk: NeMoPlatform, workspace: str) -> None: + """Create → READY → delete a generic CPU deployment on the plugin k8s backend.""" + config_name = _unique_name("kind-generic-cfg") + deployment_name = _unique_name("kind-generic-dep") + + sdk.inference.deployment_configs.create( + workspace=workspace, + name=config_name, + engine="generic", + model_spec={}, + executor_config={ + "gpu": 0, + "image_name": GENERIC_HTTP_IMAGE, + "image_tag": GENERIC_HTTP_TAG, + "additional_args": ["python3", "-m", "http.server", "8000"], + "health_check_path": "/", + }, + ) + sdk.inference.deployments.create( + workspace=workspace, + name=deployment_name, + config=config_name, + ) + + try: + _wait_for_deployment_ready(sdk, workspace=workspace, name=deployment_name) + finally: + try: + sdk.inference.deployments.delete(deployment_name, workspace=workspace) + except NotFoundError: + pass + + _wait_for_deployment_deleted(sdk, workspace=workspace, name=deployment_name) diff --git a/k8s/helm/Chart.yaml b/k8s/helm/Chart.yaml index e4f919bef0..c1900168d2 100644 --- a/k8s/helm/Chart.yaml +++ b/k8s/helm/Chart.yaml @@ -10,9 +10,3 @@ version: 0.1.0 # appVersion is the version of the application deployed in the chart. appVersion: "0.2.0" home: https://nvidia.com -dependencies: - ## NMP dependencies - - name: k8s-nim-operator - condition: k8s-nim-operator.enabled - repository: https://helm.ngc.nvidia.com/nvidia/ - version: "~3.1.0" diff --git a/k8s/helm/README.md b/k8s/helm/README.md index e4cd7e650f..bb00415eec 100644 --- a/k8s/helm/README.md +++ b/k8s/helm/README.md @@ -237,8 +237,6 @@ secrets will not decrypt with a new key. | ingress.enabled | bool | `false` | Specifies whether to enable the ingress. | | ingress.hosts[0] | object | `{"name":"","paths":[{"path":"/","pathType":"Exact","port":"{{ include \"nemo-platform.ingressBackendPort\" . }}","service":"{{ include \"nemo-platform.ingressBackendService\" . }}"},{"path":"/apis","pathType":"Prefix","port":"{{ include \"nemo-platform.ingressBackendPort\" . }}","service":"{{ include \"nemo-platform.ingressBackendService\" . }}"},{"path":"/studio","pathType":"Prefix","port":"{{ include \"nemo-platform.ingressBackendPort\" . }}","service":"{{ include \"nemo-platform.ingressBackendService\" . }}"},{"path":"/cluster-info","pathType":"Exact","port":"{{ include \"nemo-platform.ingressBackendPort\" . }}","service":"{{ include \"nemo-platform.ingressBackendService\" . }}"},{"path":"/status","pathType":"Exact","port":"{{ include \"nemo-platform.ingressBackendPort\" . }}","service":"{{ include \"nemo-platform.ingressBackendService\" . }}"}]}` | Hostname used by ingress. If blank, use path-only routing. | | ingress.tls | list | `[]` | TLS configurations. | -| k8s-nim-operator.enabled | bool | `true` | Specifies whether to enable the default NIM Operator installation. To learn more, see [Install NIM Operator](https://docs.nvidia.com/nim-operator/latest/install.html). If you are using an existing NIM Operator installation, set this to false. | -| k8s-nim-operator.nfd.nodeFeatureRules.deviceID | bool | `false` | Specifies whether to enable device ID feature rules. | | multinodeNetworking | object | `{"aws":{"efaDevicesPerGPU":1,"enabled":false},"azure":{"enabled":false,"rdmaDeviceName":"hca_shared_devices_a","rdmaDevicesPerGPU":1},"gcp":{"enabled":false},"oci":{"enabled":false,"rdmaDevicesPerGPU":8}}` | Multi-node networking configuration for distributed GPU training. These settings control Kyverno policies that inject cloud-specific networking and NCCL configurations. Requirements: - Kyverno policy engine must be installed in your cluster (required for multi-node networking) - Kyverno is NOT included as a subchart dependency and must be installed separately To install Kyverno: helm install kyverno kyverno/kyverno --namespace kyverno --create-namespace --version 3.2.0 Documentation: https://kyverno.io/docs/installation/ Helm chart: https://kyverno.github.io/kyverno/ Note: Only enable ONE cloud provider per cluster deployment. | | multinodeNetworking.aws | object | `{"efaDevicesPerGPU":1,"enabled":false}` | AWS-specific configuration for EFA device injection | | multinodeNetworking.aws.efaDevicesPerGPU | int | `1` | Number of EFA devices to request per GPU (typically 1 or 4) | @@ -270,7 +268,7 @@ secrets will not decrypt with a new key. | openshiftRoute.service | string | `"{{ include \"nemo-platform.ingressBackendService\" . }}"` | Service name to route to. Defaults to Envoy when auth+envoy enabled, otherwise API (tpl-evaluated). | | openshiftRoute.targetPort | string | `"{{ include \"nemo-platform.ingressBackendPort\" . }}"` | Target port on the service. Defaults to Envoy or API port depending on auth (tpl-evaluated). | | openshiftRoute.tls | object | `{}` | Optional TLS configuration (termination, certificate, key, etc.). See OpenShift Route spec. | -| platformConfig | object | `{}` | Platform-wide configuration settings Set configuration here to apply custom, structured configuration across all services. Applied after the base platform config is evaluated for templates. Enables adding / overriding YAML-based elements in the evaluated platform config. It is usually recommended to use this config section instead of `basePlatformConfig` unless you need to use templating features. For example, you can set the NIM default StorageClass via models.controller.backends.k8s-nim-operator.config.default_storage_class. For full configuration reference, see the NeMo Platform's config reference: https://docs.nvidia.com/nemo-platform | +| platformConfig | object | `{}` | Platform-wide configuration settings Set configuration here to apply custom, structured configuration across all services. Applied after the base platform config is evaluated for templates. Enables adding / overriding YAML-based elements in the evaluated platform config. It is usually recommended to use this config section instead of `basePlatformConfig` unless you need to use templating features. For example, you can set the NIM default StorageClass via models.controller.backends.deployments_plugin.default_storage_class. For full configuration reference, see https://docs.nvidia.com/nemo-platform | | platformSeedJob | object | This object has the following default values for the platform seed Job configuration. | Platform seed Job (Helm hook: runs after install/upgrade) Runs the platform-seed task (guardrails configs, evaluator system entities, data designer filesets). Uses post-install,post-upgrade hooks so it runs on fresh installs and can be re-triggered on no-op upgrade. | | platformSeedJob.activeDeadlineSeconds | int | `600` | Maximum time in seconds the Job can run. | | platformSeedJob.affinity | object | `{}` | Affinity for the platform seeding Job pod. | @@ -303,8 +301,7 @@ secrets will not decrypt with a new key. | postgresql.serviceAccount.create | bool | `true` | Specifies whether a service account should be created for the PostgreSQL pod. | | postgresql.serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated from the release fullname. | | postgresql.tolerations | list | `[]` | Tolerations for the PostgreSQL pod. | -| rbac | object | `{"k8sNimOperatorEnabled":true,"volcanoEnabled":true}` | RBAC configuration settings for optional dependencies | -| rbac.k8sNimOperatorEnabled | bool | `true` | Specifies whether to enable the core Controller to have RBAC permissions to k8s-nim-operator's NIMService for scheduling NIMs. | +| rbac | object | `{"volcanoEnabled":true}` | RBAC configuration settings for optional dependencies | | rbac.volcanoEnabled | bool | `true` | Specifies whether to enable the core Controller to have RBAC permissions to Volcano for scheduling distributed jobs. | | secrets | object | `{"defaultEncryptionKey":{"existingSecret":{"key":"NMP_SECRETS_DEFAULT_ENCRYPTION_KEY","name":""},"generated":{"activeDeadlineSeconds":120,"affinity":{},"backoffLimit":3,"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"docker.io/library/python","tag":"3.12-slim"},"nodeSelector":{},"podSecurityContext":{},"resources":{},"securityContext":{},"serviceAccount":{"annotations":{},"create":true,"name":""},"tolerations":[],"ttlSecondsAfterFinished":300},"value":""}}` | Secrets service configuration. | | secrets.defaultEncryptionKey.existingSecret | object | `{"key":"NMP_SECRETS_DEFAULT_ENCRYPTION_KEY","name":""}` | Existing Kubernetes Secret containing the key for encrypting platform secrets. If name is set, the chart does not create or generate the default api-env Secret. | diff --git a/k8s/helm/templates/core/controller-role.yaml b/k8s/helm/templates/core/controller-role.yaml index 712eb2fb96..5c7d1a374c 100644 --- a/k8s/helm/templates/core/controller-role.yaml +++ b/k8s/helm/templates/core/controller-role.yaml @@ -43,15 +43,6 @@ rules: resources: ["queues", "queues/status", "podgroups"] verbs: ["create", "get", "list", "watch", "update", "patch", "bind", "updateStatus", "delete"] {{- end }} -{{- if .Values.rbac.k8sNimOperatorEnabled }} -# NIM Operator -- apiGroups: ["apps.nvidia.com"] - resources: ["nimservices", "nimservices/status"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -- apiGroups: ["apps.nvidia.com"] - resources: ["nimcaches", "nimcaches/status"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml index 2e7deceb0c..2780e50b6d 100644 --- a/k8s/helm/values.yaml +++ b/k8s/helm/values.yaml @@ -73,8 +73,6 @@ imagePullSecrets: [] rbac: # -- Specifies whether to enable the core Controller to have RBAC permissions to Volcano for scheduling distributed jobs. volcanoEnabled: true - # -- Specifies whether to enable the core Controller to have RBAC permissions to k8s-nim-operator's NIMService for scheduling NIMs. - k8sNimOperatorEnabled: true # -- Multi-node networking configuration for distributed GPU training. # These settings control Kyverno policies that inject cloud-specific networking and NCCL configurations. @@ -167,16 +165,6 @@ ncclTest: cpu: "8" memory: 16Gi -## Optional dependencies configuration. For production deployments, it is recommended to use existing installations of these dependencies. -k8s-nim-operator: - # -- Specifies whether to enable the default NIM Operator installation. To learn more, see [Install NIM Operator](https://docs.nvidia.com/nim-operator/latest/install.html). - # If you are using an existing NIM Operator installation, set this to false. - enabled: true - nfd: - nodeFeatureRules: - # -- Specifies whether to enable device ID feature rules. - deviceID: false - # -- Local PostgreSQL configuration for the NeMo Platform. # @default -- This object has the following default values for the PostgreSQL configuration. postgresql: @@ -258,9 +246,8 @@ externalDatabase: # Set configuration here to apply custom, structured configuration across all services. # Applied after the base platform config is evaluated for templates. Enables adding / overriding YAML-based elements in the evaluated platform config. # It is usually recommended to use this config section instead of `basePlatformConfig` unless you need to use templating features. -# For example, you can set the NIM default StorageClass via models.controller.backends.k8s-nim-operator.config.default_storage_class. -# For full configuration reference, see the NeMo Platform's config reference: -# https://docs.nvidia.com/nemo-platform +# For example, you can set the NIM default StorageClass via models.controller.backends.deployments_plugin.default_storage_class. +# For full configuration reference, see https://docs.nvidia.com/nemo-platform platformConfig: {} # -- Base platform configuration settings @@ -344,9 +331,18 @@ basePlatformConfig: | models: controller: backends: - nim_operator: + deployments_plugin: enabled: true - files_auth_secret: {{ include "nemo-platform.modelsFilesAuthSecretName" . | quote }} + k8s_executor: local-k8s + default_executor: local-k8s + + deployments: + executors: + - name: local-k8s + backend: k8s + config: + default_namespace: {{ .Release.Namespace | quote }} + default_executor: local-k8s # -- inference_gateway is the configuration specific to inference request routing inference_gateway: {} diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 175812eac9..f91c0ade04 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -9172,9 +9172,12 @@ components: Ignored by non-NIM engines. override_config: title: Override Config - description: Raw NIMService spec configuration that takes precedence over - generated config (NIM engine on k8s). Allows advanced configuration options - directly. Ignored by non-NIM engines. + description: 'Partial NIMService Spec fragments deep-merged after generated + defaults and k8s_nim_operator_config (NIM engine on k8s only). Supported + keys: image, command, args, resources, env, readinessProbe, livenessProbe, + startupProbe, nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored + by non-NIM engines and docker runtime.' additionalProperties: true type: object type: object diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 175812eac9..f91c0ade04 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -9172,9 +9172,12 @@ components: Ignored by non-NIM engines. override_config: title: Override Config - description: Raw NIMService spec configuration that takes precedence over - generated config (NIM engine on k8s). Allows advanced configuration options - directly. Ignored by non-NIM engines. + description: 'Partial NIMService Spec fragments deep-merged after generated + defaults and k8s_nim_operator_config (NIM engine on k8s only). Supported + keys: image, command, args, resources, env, readinessProbe, livenessProbe, + startupProbe, nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored + by non-NIM engines and docker runtime.' additionalProperties: true type: object type: object diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 175812eac9..f91c0ade04 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -9172,9 +9172,12 @@ components: Ignored by non-NIM engines. override_config: title: Override Config - description: Raw NIMService spec configuration that takes precedence over - generated config (NIM engine on k8s). Allows advanced configuration options - directly. Ignored by non-NIM engines. + description: 'Partial NIMService Spec fragments deep-merged after generated + defaults and k8s_nim_operator_config (NIM engine on k8s only). Supported + keys: image, command, args, resources, env, readinessProbe, livenessProbe, + startupProbe, nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored + by non-NIM engines and docker runtime.' additionalProperties: true type: object type: object diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index e83a956d60..d490c52581 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -211,11 +211,9 @@ models-service = [ "pydantic-settings>=2.8.1", "sqlmodel>=0.0.22", "pyyaml>=6.0.2", - "kubernetes>=31.0.0", "urllib3>=2.7.0", "nmp-common", "nemo-platform-plugin", - "docker>=7.1.0", "tenacity>=8.5.0", ] diff --git a/packages/nmp_platform/config/local.yaml b/packages/nmp_platform/config/local.yaml index 9dfd8f13f0..fa2dbc301d 100644 --- a/packages/nmp_platform/config/local.yaml +++ b/packages/nmp_platform/config/local.yaml @@ -122,13 +122,25 @@ evaluator: safe_synthesizer: {} +deployments: + executors: + - name: local-docker + backend: docker + config: + pull_images: false + port_range_start: 9000 + port_range_end: 9100 + default_executor: local-docker + models: controller: interval_seconds: 5 model_deployment_garbage_collection_ttl_seconds: 30 backends: - docker: + deployments_plugin: enabled: true + docker_executor: local-docker + default_executor: local-docker # LoRA adapter-download sidecar for deployments with lora_enabled. The # docker backend has no separate args field (unlike k8s) and keeps the # image ENTRYPOINT — nmp-customizer-tasks is `/opt/venv/bin/python`, so diff --git a/packages/nmp_platform_runner/src/nmp/platform_runner/config/local.yaml b/packages/nmp_platform_runner/src/nmp/platform_runner/config/local.yaml index 637e52fa85..2636247775 100644 --- a/packages/nmp_platform_runner/src/nmp/platform_runner/config/local.yaml +++ b/packages/nmp_platform_runner/src/nmp/platform_runner/config/local.yaml @@ -61,8 +61,10 @@ models: interval_seconds: 5 model_deployment_garbage_collection_ttl_seconds: 30 backends: - docker: + deployments_plugin: enabled: true + docker_executor: local-docker + default_executor: local-docker # LoRA adapter-download sidecar for deployments with lora_enabled. The # docker backend has no separate args field (unlike k8s) and keeps the # image ENTRYPOINT — nmp-customizer-tasks is `/opt/venv/bin/python`, so @@ -70,6 +72,16 @@ models: lora_sidecar_image_name: nmp-customizer-tasks lora_sidecar_command: ["-m", "nmp.core.models.sidecars.adapters.main"] +deployments: + executors: + - name: local-docker + backend: docker + config: + pull_images: false + port_range_start: 9000 + port_range_end: 9100 + default_executor: local-docker + inference_gateway: {} secrets: diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py index 559442790b..91389e8cc6 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py @@ -120,6 +120,7 @@ async def create_deployment( labels=labels, backend_config=backend_config, config=config, + executor_image_pull_secrets=self._executor_config.image_pull_secrets, ) return await job_ops.create_job( @@ -131,6 +132,7 @@ async def create_deployment( labels=labels, backend_config=backend_config, config=config, + executor_image_pull_secrets=self._executor_config.image_pull_secrets, ) async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate: diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py index 01b5886d19..d3a43be341 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/compiler.py @@ -32,6 +32,7 @@ VolumeMount, ) from nemo_deployments_plugin.types import RestartPolicy +from nemo_platform_plugin.config import ImagePullSecret, get_platform_config CONFIG_FILES_VOLUME = "config-files" NATIVE_SIDECAR_RESTART_POLICY: RestartPolicy = "Always" @@ -344,6 +345,19 @@ def _build_config_file_volume(configmap_name: str, config_files: list[ConfigFile ) +def build_pod_image_pull_secrets( + executor_image_pull_secrets: list[ImagePullSecret] | None = None, +) -> list[Any]: + """Merge platform and executor image pull secrets for pod specs.""" + k8s = k8s_client_module() + merged_names: dict[str, None] = {} + for secret in get_platform_config().image_pull_secrets: + merged_names[secret.name] = None + for secret in executor_image_pull_secrets or []: + merged_names[secret.name] = None + return [k8s.client.V1LocalObjectReference(name=name) for name in merged_names] + + def compile_workload( *, config: DeploymentConfig, @@ -352,6 +366,7 @@ def compile_workload( labels: dict[str, str], k8s_config: K8sDeploymentConfig | None, pod_restart_policy: RestartPolicy, + executor_image_pull_secrets: list[ImagePullSecret] | None = None, ) -> CompiledWorkload: """Compile pod spec kwargs and optional ConfigMap for a Job or Deployment.""" validate_workload_config(config) @@ -388,6 +403,10 @@ def compile_workload( if volumes: pod_spec_kwargs["volumes"] = volumes + image_pull_secrets = build_pod_image_pull_secrets(executor_image_pull_secrets) + if image_pull_secrets: + pod_spec_kwargs["image_pull_secrets"] = image_pull_secrets + if k8s_config is not None: tolerations = build_tolerations(k8s_config.tolerations) if tolerations: diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py index 8e91834fbc..b51c1c46fc 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py @@ -8,6 +8,7 @@ import os import re +from nemo_platform_plugin.config import ImagePullSecret from pydantic import BaseModel, Field, field_validator _DNS_LABEL_PATTERN = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") @@ -45,6 +46,10 @@ class K8sExecutorConfig(BaseModel): ge=1, description="Kubernetes API client timeout in seconds.", ) + image_pull_secrets: list[ImagePullSecret] = Field( + default_factory=list, + description="Image pull secrets merged with platform image_pull_secrets on every pod.", + ) @field_validator("default_namespace") @classmethod diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py index db72bfa2d8..0f5cb1dd1b 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/deployments.py @@ -77,6 +77,7 @@ def build_deployment_body( workspace: str, deployment_name: str, k8s_config: K8sDeploymentConfig | None, + executor_image_pull_secrets: list | None = None, ) -> BuiltDeployment: """Build an ``apps/v1.Deployment`` for create and its compiled workload.""" k8s = k8s_client_module() @@ -89,6 +90,7 @@ def build_deployment_body( labels=labels, k8s_config=k8s_config, pod_restart_policy="Always", + executor_image_pull_secrets=executor_image_pull_secrets, ) deployment = k8s.client.V1Deployment( api_version="apps/v1", @@ -224,6 +226,7 @@ async def create_deployment( labels: dict[str, str], backend_config: dict[str, Any], config: DeploymentConfig, + executor_image_pull_secrets: list | None = None, ) -> BackendStatusUpdate: resource_name = k8s_deployment_resource_name(workspace, name) try: @@ -245,6 +248,7 @@ async def create_deployment( workspace=workspace, deployment_name=name, k8s_config=k8s_config, + executor_image_pull_secrets=executor_image_pull_secrets, ) deployment_body = built.deployment compiled = built.compiled diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py index df254b10ae..f03e4e87a5 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/jobs.py @@ -113,6 +113,7 @@ def build_job_body( workspace: str, deployment_name: str, k8s_config: K8sDeploymentConfig | None, + executor_image_pull_secrets: list | None = None, ) -> BuiltJob: """Build a ``batch/v1.Job`` for create.""" k8s = k8s_client_module() @@ -123,6 +124,7 @@ def build_job_body( labels=labels, k8s_config=k8s_config, pod_restart_policy=config.restart_policy, + executor_image_pull_secrets=executor_image_pull_secrets, ) job = k8s.client.V1Job( api_version="batch/v1", @@ -183,6 +185,7 @@ async def create_job( labels: dict[str, str], backend_config: dict[str, Any], config: DeploymentConfig, + executor_image_pull_secrets: list | None = None, ) -> BackendStatusUpdate: job_name = k8s_deployment_resource_name(workspace, name) try: @@ -204,6 +207,7 @@ async def create_job( workspace=workspace, deployment_name=name, k8s_config=k8s_config, + executor_image_pull_secrets=executor_image_pull_secrets, ) body = built.job compiled = built.compiled diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/config.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/config.py index 59550aa893..b789ac5d60 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/config.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/config.py @@ -46,6 +46,11 @@ class ControllerConfig(BaseModel): ge=0, description="Max seconds in STARTING before FAILED (0 disables).", ) + deleting_timeout_seconds: int = Field( + default=300, + ge=0, + description="Max seconds in DELETING before FAILED (0 disables).", + ) @model_validator(mode="after") def _validate_backoff(self) -> ControllerConfig: diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/deployment_reconciler.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/deployment_reconciler.py index 3c5ded485a..cb59eced05 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/deployment_reconciler.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/deployment_reconciler.py @@ -20,6 +20,16 @@ logger = logging.getLogger(__name__) +_DELETE_COMPLETE_STATUSES = frozenset({"SUCCEEDED", "DELETED"}) + + +def _is_terminal_deleting_timeout_failure(deployment: Deployment) -> bool: + """Return True when delete reconciliation already recorded a deleting timeout.""" + if deployment.status != "FAILED": + return False + details = deployment.error_details + return isinstance(details, dict) and details.get("reason") == "deleting_timeout" + def deployment_id(deployment: Deployment) -> str: return f"{deployment.workspace}/{deployment.name}" @@ -180,6 +190,13 @@ async def _reconcile_create( async def _reconcile_delete(self, deployment: Deployment) -> None: dep_id = deployment_id(deployment) self._drift_cache.remove(dep_id) + if _is_terminal_deleting_timeout_failure(deployment): + return + timeout_update = self._check_deleting_timeout(deployment) + if timeout_update is not None: + await self._update_deployment_status(deployment, timeout_update) + return + backend = self._try_resolve_backend(deployment) if deployment.status != "DELETING": await self._update_deployment_status( @@ -189,10 +206,18 @@ async def _reconcile_delete(self, deployment: Deployment) -> None: if backend is not None: try: - await backend.delete_deployment(deployment.workspace, deployment.name) + status_update = await backend.delete_deployment(deployment.workspace, deployment.name) except Exception: logger.warning("Backend delete failed for %s — will retry", dep_id, exc_info=True) return + if status_update.status not in _DELETE_COMPLETE_STATUSES: + logger.debug( + "Backend delete not complete for %s: %s — %s", + dep_id, + status_update.status, + status_update.status_message, + ) + return else: logger.warning("No executor for delete of %s — removing entity only", dep_id) @@ -351,6 +376,30 @@ def _check_starting_timeout(self, deployment: Deployment) -> BackendStatusUpdate }, ) + def _check_deleting_timeout(self, deployment: Deployment) -> BackendStatusUpdate | None: + timeout = self._controller_config.deleting_timeout_seconds + if timeout <= 0: + return None + deleting_at = _deleting_timestamp(deployment) + if deleting_at is None: + return None + elapsed = (datetime.now(timezone.utc) - deleting_at).total_seconds() + if elapsed < timeout: + return None + elapsed_int = int(elapsed) + return BackendStatusUpdate( + status="FAILED", + status_message=( + f"Deployment stuck in DELETING for {elapsed_int}s (timeout: {timeout}s). " + "Backend resources were not removed." + ), + error_details={ + "reason": "deleting_timeout", + "elapsed_seconds": elapsed_int, + "timeout_seconds": timeout, + }, + ) + async def _update_deployment_status_pending(self, deployment: Deployment, message: str) -> None: if deployment.status == "PENDING" and deployment.status_message == message: return @@ -405,6 +454,13 @@ def _starting_timestamp(deployment: Deployment) -> datetime | None: return None +def _deleting_timestamp(deployment: Deployment) -> datetime | None: + for event in reversed(deployment.status_history): + if event.status == "DELETING" and event.timestamp: + return datetime.fromisoformat(event.timestamp) + return None + + def _prerequisite_failed( result: PrerequisiteResult, deployments_by_name: dict[tuple[str, str], Deployment], diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/volume_mounts.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/volume_mounts.py index e7ffe2b479..bdc5e45b20 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/volume_mounts.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/reconciler/volume_mounts.py @@ -30,7 +30,11 @@ def volume_mounts_ready( workspace: str, volumes_by_name: dict[tuple[str, str], Volume], ) -> VolumeMountResult: - """Return whether all mounted volumes exist and are BOUND.""" + """Return whether all mounted volumes exist and are usable for pod create. + + ``PENDING`` volumes are allowed so Jobs can mount WaitForFirstConsumer PVCs + and become the first consumer that triggers binding. ``FAILED`` volumes block. + """ for mount_name in sorted(collect_volume_mount_names(config)): volume = volumes_by_name.get((workspace, mount_name)) if volume is None: @@ -45,10 +49,10 @@ def volume_mounts_ready( reason=f"Volume '{mount_name}' failed", blocking_volume=mount_name, ) - if volume.status != "BOUND": + if volume.status not in {"PENDING", "BOUND"}: return VolumeMountResult( ready=False, - reason=f"Waiting for volume '{mount_name}' to reach BOUND (currently {volume.status})", + reason=f"Waiting for volume '{mount_name}' to become ready (currently {volume.status})", blocking_volume=mount_name, ) return VolumeMountResult(ready=True) diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py index a58d8efc5b..402db73cb1 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_compiler.py @@ -3,6 +3,9 @@ from __future__ import annotations +from types import SimpleNamespace +from unittest.mock import patch + import pytest from backends.k8s.k8s_helpers import sample_always_config, sample_config from kubernetes.client import ApiClient @@ -22,6 +25,7 @@ ContainerPort, K8sDeploymentConfig, ) +from nemo_platform_plugin.config import ImagePullSecret def _serialized(obj: object) -> dict: @@ -148,6 +152,28 @@ def test_compile_applies_k8s_deployment_config() -> None: assert security_context.run_as_user == 1000 +def test_compile_workload_emits_image_pull_secrets() -> None: + config = sample_config(restart_policy="Never") + platform_secret = ImagePullSecret(name="platform-secret") + executor_secret = ImagePullSecret(name="executor-secret") + with patch( + "nemo_deployments_plugin.backends.k8s.compiler.get_platform_config", + return_value=SimpleNamespace(image_pull_secrets=[platform_secret]), + ): + compiled = compile_workload( + config=config, + workspace="default", + deployment_name="task", + labels={"managed-by": "nemo-deployments"}, + k8s_config=None, + pod_restart_policy="Never", + executor_image_pull_secrets=[executor_secret], + ) + pod_spec = _serialized(compiled.pod_spec_kwargs) + secret_names = {secret["name"] for secret in pod_spec["image_pull_secrets"]} + assert secret_names == {"platform-secret", "executor-secret"} + + def test_compile_config_files_emit_configmap_and_mounts() -> None: config = sample_always_config().model_copy( update={"config_files": [ConfigFile(path="/etc/app/config.yaml", content="key: value")]} diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_rbac_manifest.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_rbac_manifest.py index c0f3add6d0..fc34283f53 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_rbac_manifest.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_rbac_manifest.py @@ -3,12 +3,9 @@ """Assert the deploy chart's controller Role grants the k8s backend's required RBAC. -The chart under ``k8s/helm`` declares an unconditional ``k8s-nim-operator`` chart -dependency, so ``helm template`` cannot render without that subchart present in -``charts/`` (fetched from an NGC repo) — a network dependency this suite must not -require. Assertions instead parse the static YAML rule entries directly out of the -Go-template source, which is safe because the base RBAC rules (unlike the -Volcano/NIM-Operator blocks) are plain YAML with no Helm expressions inside them. +Assertions parse the static YAML rule entries directly out of the Go-template +source, which is safe because the base RBAC rules (unlike the Volcano block) +are plain YAML with no Helm expressions inside them. """ from __future__ import annotations diff --git a/plugins/nemo-deployments/tests/unit/reconciler/test_deployment_reconciler.py b/plugins/nemo-deployments/tests/unit/reconciler/test_deployment_reconciler.py index eaddeb2c76..4288e611d6 100644 --- a/plugins/nemo-deployments/tests/unit/reconciler/test_deployment_reconciler.py +++ b/plugins/nemo-deployments/tests/unit/reconciler/test_deployment_reconciler.py @@ -19,6 +19,7 @@ NOW = datetime(2026, 6, 30, 12, 0, 0, tzinfo=timezone.utc) STARTING_TIMEOUT_SECONDS = 60 +DELETING_TIMEOUT_SECONDS = 60 @pytest.fixture @@ -36,6 +37,7 @@ def _starting_timeout_reconciler( mock_backend: MockDeploymentBackend, *, starting_timeout_seconds: int = STARTING_TIMEOUT_SECONDS, + deleting_timeout_seconds: int = DELETING_TIMEOUT_SECONDS, ) -> DeploymentReconciler: return DeploymentReconciler( mock_entities, @@ -45,6 +47,7 @@ def _starting_timeout_reconciler( drift_recovery_initial_delay_seconds=1, drift_recovery_max_delay_seconds=10, starting_timeout_seconds=starting_timeout_seconds, + deleting_timeout_seconds=deleting_timeout_seconds, ), ) @@ -346,19 +349,126 @@ async def test_delete_proceeds_when_config_missing( mock_entities.delete.assert_awaited_once() +def _deployment_stuck_deleting(*, elapsed_seconds: int) -> Deployment: + deleting_at = NOW - timedelta(seconds=elapsed_seconds) + dep = make_deployment() + dep.desired_state = "STOPPED" + dep.status = "DELETING" + dep.status_history = [ + StatusEvent(status="DELETING", message="Stopping deployment", timestamp=deleting_at.isoformat()), + ] + return dep + + +async def _reconcile_stuck_deleting( + reconciler: DeploymentReconciler, + mock_backend: MockDeploymentBackend, + dep: Deployment, +) -> None: + cfg = make_deployment_config() + reconciler.set_config_cache({("default", "cfg1"): cfg}) + await reconciler.reconcile_one(dep, deployments_by_name={}, volumes_by_name=NO_VOLUMES) + + @pytest.mark.asyncio -async def test_delete_retains_deleting_when_backend_delete_fails( +async def test_delete_retains_deleting_when_backend_returns_failed( deployment_reconciler: DeploymentReconciler, mock_backend: MockDeploymentBackend, mock_entities: AsyncMock, ) -> None: dep = make_deployment() dep.desired_state = "STOPPED" + mock_backend.delete_status = BackendStatusUpdate(status="FAILED", status_message="delete failed") + + await deployment_reconciler.reconcile_one(dep, deployments_by_name={}, volumes_by_name=NO_VOLUMES) - async def failing_delete(workspace: str, name: str) -> BackendStatusUpdate: - raise RuntimeError("delete failed") + assert dep.status == "DELETING" + mock_entities.delete.assert_not_awaited() - mock_backend.delete_deployment = failing_delete # type: ignore[method-assign] + +@pytest.mark.asyncio +async def test_delete_timeout_fails_at_boundary( + patch_reconciler_now, + mock_entities: AsyncMock, + mock_backend: MockDeploymentBackend, +) -> None: + reconciler = _starting_timeout_reconciler(mock_entities, mock_backend) + dep = _deployment_stuck_deleting(elapsed_seconds=DELETING_TIMEOUT_SECONDS) + + await _reconcile_stuck_deleting(reconciler, mock_backend, dep) + + assert dep.status == "FAILED" + assert dep.error_details is not None + assert dep.error_details["reason"] == "deleting_timeout" + mock_entities.delete.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_delete_timeout_failure_is_not_re_evaluated( + patch_reconciler_now, + mock_entities: AsyncMock, + mock_backend: MockDeploymentBackend, +) -> None: + reconciler = _starting_timeout_reconciler(mock_entities, mock_backend) + dep = _deployment_stuck_deleting(elapsed_seconds=DELETING_TIMEOUT_SECONDS + 120) + dep.status = "FAILED" + dep.error_details = { + "reason": "deleting_timeout", + "elapsed_seconds": DELETING_TIMEOUT_SECONDS + 120, + "timeout_seconds": DELETING_TIMEOUT_SECONDS, + } + + await _reconcile_stuck_deleting(reconciler, mock_backend, dep) + await _reconcile_stuck_deleting(reconciler, mock_backend, dep) + + mock_entities.update.assert_not_awaited() + mock_entities.delete.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_delete_timeout_before_boundary_stays_deleting( + patch_reconciler_now, + mock_entities: AsyncMock, + mock_backend: MockDeploymentBackend, +) -> None: + reconciler = _starting_timeout_reconciler(mock_entities, mock_backend) + dep = _deployment_stuck_deleting(elapsed_seconds=DELETING_TIMEOUT_SECONDS - 1) + mock_backend.delete_status = BackendStatusUpdate(status="DELETING", status_message="still terminating") + + await _reconcile_stuck_deleting(reconciler, mock_backend, dep) + + assert dep.status == "DELETING" + mock_entities.delete.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_delete_timeout_disabled_when_zero( + patch_reconciler_now, + mock_entities: AsyncMock, + mock_backend: MockDeploymentBackend, +) -> None: + reconciler = _starting_timeout_reconciler( + mock_entities, + mock_backend, + deleting_timeout_seconds=0, + ) + dep = _deployment_stuck_deleting(elapsed_seconds=DELETING_TIMEOUT_SECONDS + 60) + + await _reconcile_stuck_deleting(reconciler, mock_backend, dep) + + assert dep.status == "DELETING" + mock_entities.delete.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_delete_retains_deleting_when_backend_delete_fails( + deployment_reconciler: DeploymentReconciler, + mock_backend: MockDeploymentBackend, + mock_entities: AsyncMock, +) -> None: + dep = make_deployment() + dep.desired_state = "STOPPED" + mock_backend.delete_deployment = AsyncMock(side_effect=RuntimeError("delete failed")) await deployment_reconciler.reconcile_one(dep, deployments_by_name={}, volumes_by_name=NO_VOLUMES) @@ -591,6 +701,28 @@ async def test_volume_mount_gating( volumes_by_name={("default", "data"): vol}, ) + assert dep.status == "STARTING" + assert len(mock_backend.create_calls) == 1 + + +@pytest.mark.asyncio +async def test_volume_mount_gating_waits_for_missing_volume( + deployment_reconciler: DeploymentReconciler, + mock_backend: MockDeploymentBackend, +) -> None: + from nemo_deployments_plugin.entities import VolumeMount + + dep = make_deployment() + cfg = make_deployment_config() + cfg.volume_mounts = [VolumeMount(name="data", mountPath="/data")] + deployment_reconciler.set_config_cache({("default", "cfg1"): cfg}) + + await deployment_reconciler.reconcile_one( + dep, + deployments_by_name={}, + volumes_by_name={}, + ) + assert dep.status == "PENDING" assert "volume" in dep.status_message.lower() assert mock_backend.create_calls == [] @@ -633,11 +765,11 @@ async def test_drift_recovery_create_failure_stays_lost_and_backoffs( deployment_reconciler.set_config_cache({("default", "cfg1"): cfg}) mock_backend.read_status_result = BackendStatusUpdate(status="LOST", status_message="missing") - async def failing_create(**kwargs: object) -> BackendStatusUpdate: + async def _failing_create(**kwargs: object) -> BackendStatusUpdate: mock_backend.create_calls.append(kwargs) raise RuntimeError("create failed") - mock_backend.create_deployment = failing_create # type: ignore[method-assign] + mock_backend.create_deployment = AsyncMock(side_effect=_failing_create) await deployment_reconciler.reconcile_one(dep, deployments_by_name={}, volumes_by_name=NO_VOLUMES) assert dep.status == "LOST" diff --git a/plugins/nemo-deployments/tests/unit/reconciler/test_volume_mounts.py b/plugins/nemo-deployments/tests/unit/reconciler/test_volume_mounts.py index e016dec67a..89dc3a3284 100644 --- a/plugins/nemo-deployments/tests/unit/reconciler/test_volume_mounts.py +++ b/plugins/nemo-deployments/tests/unit/reconciler/test_volume_mounts.py @@ -30,10 +30,19 @@ def test_volume_mounts_ready_when_all_bound() -> None: assert result.ready is True -def test_volume_mounts_wait_when_pending() -> None: +def test_volume_mounts_ready_when_pending() -> None: cfg = make_deployment_config() cfg.volume_mounts = [VolumeMount(name="data", mountPath="/data")] vol = make_volume("data") result = volume_mounts_ready(cfg, "default", {("default", "data"): vol}) + assert result.ready is True + + +def test_volume_mounts_failed_blocks() -> None: + cfg = make_deployment_config() + cfg.volume_mounts = [VolumeMount(name="data", mountPath="/data")] + vol = make_volume("data") + vol.status = "FAILED" + result = volume_mounts_ready(cfg, "default", {("default", "data"): vol}) assert result.ready is False - assert "BOUND" in result.reason + assert "failed" in result.reason.lower() diff --git a/plugins/nemo-deployments/tests/unit/test_config.py b/plugins/nemo-deployments/tests/unit/test_config.py index 08bb7001e5..1c9c5255c9 100644 --- a/plugins/nemo-deployments/tests/unit/test_config.py +++ b/plugins/nemo-deployments/tests/unit/test_config.py @@ -11,6 +11,7 @@ def test_controller_config_defaults() -> None: assert cfg.controller.drift_recovery_max_attempts == 5 assert cfg.controller.orphan_cleanup_interval_seconds == 30 assert cfg.controller.starting_timeout_seconds == 3600 + assert cfg.controller.deleting_timeout_seconds == 300 def test_controller_config_custom_orphan_interval() -> None: diff --git a/pytest.ini b/pytest.ini index f6ed98dc22..5131d6eced 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,6 +14,7 @@ pythonpath = plugins/nemo-deployments/tests/integration plugins/nemo-safe-synthesizer/src services/core/jobs/tests/controllers + services/core/models/tests/unit # Test discovery paths - packages and stable services # Option A: Centralized Testing - All tests runnable from root diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 175812eac9..f91c0ade04 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -9172,9 +9172,12 @@ components: Ignored by non-NIM engines. override_config: title: Override Config - description: Raw NIMService spec configuration that takes precedence over - generated config (NIM engine on k8s). Allows advanced configuration options - directly. Ignored by non-NIM engines. + description: 'Partial NIMService Spec fragments deep-merged after generated + defaults and k8s_nim_operator_config (NIM engine on k8s only). Supported + keys: image, command, args, resources, env, readinessProbe, livenessProbe, + startupProbe, nodeSelector, tolerations, userID, groupID, labels, initContainers, + sidecarContainers. Unsupported keys are rejected at compile time. Ignored + by non-NIM engines and docker runtime.' additionalProperties: true type: object type: object diff --git a/services/core/inference-gateway/tests/integration/conftest.py b/services/core/inference-gateway/tests/integration/conftest.py index 7c0008c21c..baeabe10a3 100644 --- a/services/core/inference-gateway/tests/integration/conftest.py +++ b/services/core/inference-gateway/tests/integration/conftest.py @@ -9,13 +9,16 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from nemo_deployments_plugin.config import ControllerConfig, DeploymentsConfig, ExecutorConfigEntry +from nemo_deployments_plugin.controller import DeploymentsController from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nmp.common.config import Runtime from nmp.core.inference_gateway.api.dependencies import global_model_cache from nmp.core.inference_gateway.api.model_cache import ModelCache, model_provider_getter_from_sdk, refresh_model_cache from nmp.core.inference_gateway.config import InferenceGatewayConfig from nmp.core.inference_gateway.service import InferenceGatewayService from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend -from nmp.core.models.controllers.backends.docker import DockerServiceBackend +from nmp.core.models.controllers.backends.deployments_plugin.backend import DeploymentsPluginServiceBackend from nmp.core.models.controllers.backends.registry import BackendRegistry from nmp.core.models.controllers.models_controller import ModelsController from nmp.core.models.service import ModelsService @@ -264,6 +267,15 @@ def docker_backend_config( } +@pytest.fixture +def deployments_plugin_backend_config() -> dict[str, Any]: + return { + "docker_executor": "local-docker", + "default_executor": "local-docker", + "deleting_timeout_seconds": 300, + } + + @pytest.fixture def controller_with_docker_and_igw( test_clients: ClientContext, @@ -273,6 +285,8 @@ def controller_with_docker_and_igw( docker_backend_config, docker_test_context, models_controller_container_cleanup, + deployments_plugin_backend_config, + worker_id: str, ) -> Generator[ tuple[ModelsController, ModelCache, NeMoPlatform, str, DockerTestContext, AsyncNeMoPlatform], None, None ]: @@ -297,43 +311,80 @@ def patched_get_qualified_image(name: str, tag=None, registry=None): return mock_sidecar_image return real_get_qualified_image(name, tag=tag, registry=registry) - # Create Docker backend - docker_backend = DockerServiceBackend( + start_port, end_port = get_worker_port_range(worker_id) + deployments_config = DeploymentsConfig( + executors=[ + ExecutorConfigEntry( + name="local-docker", + backend="docker", + config={ + "pull_images": False, + "port_range_start": start_port, + "port_range_end": end_port, + }, + ) + ], + default_executor="local-docker", + controller=ControllerConfig(interval_seconds=1, orphan_cleanup_interval_seconds=0), + ) + + plugin_backend = DeploymentsPluginServiceBackend( nmp_sdk=test_clients.async_sdk, - config=docker_backend_config, + config=deployments_plugin_backend_config, + huggingface_model_puller="alpine:3.20", ) - backend_registry = BackendRegistry(registry={"docker": docker_backend}) + plugin_backend.init() + backend_registry = BackendRegistry(registry={"deployments_plugin": plugin_backend}) + deployments_controller = DeploymentsController() mock_platform_config = MagicMock() mock_platform_config.models_url = "http://testserver" + mock_platform_config.base_url = "http://testserver" + mock_platform_config.runtime = Runtime.DOCKER mock_platform_config.get_service_url.return_value = "http://testserver" + mock_platform_config.service_discovery = {"files": "http://testserver"} + with ( patch("nmp.core.models.config.get_platform_config", return_value=mock_platform_config), patch("nmp.core.models.controllers.main.get_platform_config", return_value=mock_platform_config), + patch( + "nmp.core.models.controllers.backends.deployments_plugin.resolve.get_platform_config", + return_value=mock_platform_config, + ), patch("nmp.core.models.controllers.models_controller.get_async_platform_sdk") as mock_sdk_factory, + patch("nemo_platform_plugin.sdk_provider.get_async_platform_sdk") as mock_sdk, + patch("nemo_deployments_plugin.config.DeploymentsConfig.get", return_value=deployments_config), patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.get_qualified_image", + "nemo_platform_plugin.jobs.image.get_qualified_image", side_effect=patched_get_qualified_image, ), ): mock_sdk_factory.return_value = test_clients.async_sdk + mock_sdk.return_value = test_clients.async_sdk controller = ModelsController( backend_registry=backend_registry, stop_signal=None, ) - - # Mock the provider reconciler to avoid event loop conflicts - # when it tries to call through the IGW proxy for autodiscovery controller._provider_reconciler.reconcile_model_providers = AsyncMock(return_value=None) + controller._loop.run_until_complete(deployments_controller.on_startup()) + + original_step = controller.step + + def step_with_deployments_plugin() -> None: + original_step() + controller._loop.run_until_complete(deployments_controller.reconcile()) + + controller.step = step_with_deployments_plugin - # Access IGW model cache model_cache = global_model_cache() yield controller, model_cache, test_clients.sdk, mock_nim_image, docker_test_context, test_clients.async_sdk - # Clean up controller resources (event loop, backend registry, etc.) - controller.shutdown() + try: + controller._loop.run_until_complete(deployments_controller.on_shutdown()) + finally: + controller.shutdown() # ============================================================================= diff --git a/services/core/inference-gateway/tests/integration/test_inference.py b/services/core/inference-gateway/tests/integration/test_inference.py index c14dc4ddae..a9ea4bf54f 100644 --- a/services/core/inference-gateway/tests/integration/test_inference.py +++ b/services/core/inference-gateway/tests/integration/test_inference.py @@ -17,6 +17,8 @@ import pytest from docker.errors import NotFound +from nemo_deployments_plugin.backends.labels import container_name as plugin_container_name +from nemo_deployments_plugin.backends.labels import docker_volume_name from nemo_platform import ConflictError, NeMoPlatform, NotFoundError from nemo_platform.types.inference.model_deployment import ModelDeployment from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig @@ -24,7 +26,7 @@ from nemo_platform.types.inference.virtual_model import VirtualModel as SDKVirtualModel from nmp.core.inference_gateway.api.dependencies import global_virtual_model_cache from nmp.core.inference_gateway.api.model_cache import ModelCache, ModelProviderInfo -from nmp.core.models.app.utils import get_docker_container_name, get_docker_volume_name +from nmp.core.models.controllers.backends.deployments_plugin.naming import entity_names from nmp.core.models.controllers.models_controller import ModelsController from tenacity import retry, stop_after_delay, wait_fixed @@ -69,6 +71,37 @@ def _poll(): return _poll() +def _wait_for_deployment_deleted( + controller: ModelsController, + sdk: NeMoPlatform, + deployment_name: str, + max_wait: float = 30, + poll_interval: float = 0.1, +) -> None: + """Step the controller until a deployment reaches DELETED (or is gone). + + The deployments_plugin delete path is non-blocking, so a single + ``controller.step()`` is not enough to drive a deployment to DELETED: the + reconciler must run several times (DELETING -> backend teardown -> DELETED). + Deleting the ModelDeploymentConfig before its deployment is DELETED fails + with a 409 Conflict. + """ + + @retry(stop=stop_after_delay(max_wait), wait=wait_fixed(poll_interval), reraise=True) + def _poll() -> None: + controller.step() + try: + deployment = sdk.inference.deployments.retrieve( + deployment_name, + workspace=DEFAULT_WORKSPACE, + ) + except NotFoundError: + return + assert deployment.status == "DELETED", f"Deployment not DELETED: {deployment.status}" + + _poll() + + def _create_deployment_with_config( sdk: NeMoPlatform, config_name: str, @@ -246,11 +279,13 @@ def test_igw_routes_to_deployed_mock_nim( test_uuid = uuid.uuid4().hex[:8] config_name = f"test-igw-e2e-{test_uuid}" deployment_name = f"test-igw-e2e-{test_uuid}" - container_name = get_docker_container_name(DEFAULT_WORKSPACE, deployment_name) + names = entity_names(deployment_name) + container_name = plugin_container_name(DEFAULT_WORKSPACE, names.server) # Register for cleanup ctx.register_container(container_name) - ctx.register_volume(get_docker_volume_name(DEFAULT_WORKSPACE, deployment_name)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.volume)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.scratch)) # === Phase 1: Create deployment config and deployment === config, deployment = _create_deployment_with_config(sdk, config_name, deployment_name, mock_nim_image) @@ -358,9 +393,10 @@ def get_running_container(): # === Phase 8: Cleanup === sdk.inference.deployments.delete(deployment_name, workspace=DEFAULT_WORKSPACE) - controller.step() # Process deletion + # Drive the non-blocking delete to completion before removing the config. + _wait_for_deployment_deleted(controller, sdk, deployment_name) - # Poll for container to be removed or stopped (DinD may be slow) + # Container should be gone once the deployment is DELETED (DinD may lag). @retry(stop=stop_after_delay(15), wait=wait_fixed(0.1), reraise=True) def wait_for_container_deleted(): try: @@ -411,10 +447,12 @@ def test_igw_cache_removes_deleted_deployment_provider( test_uuid = uuid.uuid4().hex[:8] config_name = f"test-igw-delete-{test_uuid}" deployment_name = f"test-igw-delete-{test_uuid}" - container_name = get_docker_container_name(DEFAULT_WORKSPACE, deployment_name) + names = entity_names(deployment_name) + container_name = plugin_container_name(DEFAULT_WORKSPACE, names.server) ctx.register_container(container_name) - ctx.register_volume(get_docker_volume_name(DEFAULT_WORKSPACE, deployment_name)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.volume)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.scratch)) # Create deployment config, deployment = _create_deployment_with_config(sdk, config_name, deployment_name, mock_nim_image) @@ -431,7 +469,7 @@ def test_igw_cache_removes_deleted_deployment_provider( # Delete deployment sdk.inference.deployments.delete(deployment_name, workspace=DEFAULT_WORKSPACE) - controller.step() + _wait_for_deployment_deleted(controller, sdk, deployment_name) # Manually remove from cache (simulating cache refresh) cache_key = (DEFAULT_WORKSPACE, deployment_name) diff --git a/services/core/inference-gateway/tests/integration/test_middleware_pipeline.py b/services/core/inference-gateway/tests/integration/test_middleware_pipeline.py index e3c10aae65..5ced30d534 100644 --- a/services/core/inference-gateway/tests/integration/test_middleware_pipeline.py +++ b/services/core/inference-gateway/tests/integration/test_middleware_pipeline.py @@ -31,6 +31,9 @@ from typing import Any import pytest +from nemo_deployments_plugin.backends.labels import container_name as plugin_container_name +from nemo_deployments_plugin.backends.labels import docker_volume_name +from nemo_platform import NeMoPlatform, NotFoundError from nemo_platform.types.inference.virtual_model import VirtualModel as SDKVirtualModel from nemo_platform_plugin.inference_middleware import ( ImmediateResponse, @@ -49,11 +52,44 @@ ) from nmp.core.inference_gateway.api.model_cache import ModelProviderInfo from nmp.core.inference_gateway.api.virtual_model_cache import VirtualModelCache -from nmp.core.models.app.utils import get_docker_container_name, get_docker_volume_name +from nmp.core.models.controllers.backends.deployments_plugin.naming import entity_names +from nmp.core.models.controllers.models_controller import ModelsController from tenacity import retry, stop_after_delay, wait_fixed DEFAULT_WORKSPACE = "default" + +def _wait_for_deployment_deleted( + controller: ModelsController, + sdk: NeMoPlatform, + deployment_name: str, + max_wait: float = 30, + poll_interval: float = 0.1, +) -> None: + """Step the controller until a deployment reaches DELETED (or is gone). + + The deployments_plugin delete path is non-blocking, so a single + ``controller.step()`` is not enough to drive a deployment to DELETED: the + reconciler must run several times (DELETING -> backend teardown -> DELETED). + Deleting the ModelDeploymentConfig before its deployment is DELETED fails + with a 409 Conflict. + """ + + @retry(stop=stop_after_delay(max_wait), wait=wait_fixed(poll_interval), reraise=True) + def _poll() -> None: + controller.step() + try: + deployment = sdk.inference.deployments.retrieve( + deployment_name, + workspace=DEFAULT_WORKSPACE, + ) + except NotFoundError: + return + assert deployment.status == "DELETED", f"Deployment not DELETED: {deployment.status}" + + _poll() + + # Sentinel stamped on the response by response middleware so tests can assert # the pipeline ran end-to-end. RESPONSE_MIDDLEWARE_MARKER = "middleware-response-applied" @@ -394,10 +430,12 @@ def test_middleware_request_and_response_mutation_through_backend( vm_name = f"test-mw-alias-{test_uuid}" router_key = f"test-router-{test_uuid}" marker_key = f"test-marker-{test_uuid}" - container_name = get_docker_container_name(DEFAULT_WORKSPACE, deployment_name) + names = entity_names(deployment_name) + container_name = plugin_container_name(DEFAULT_WORKSPACE, names.server) ctx.register_container(container_name) - ctx.register_volume(get_docker_volume_name(DEFAULT_WORKSPACE, deployment_name)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.volume)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.scratch)) # ---- Phase 1: Deploy mock NIM ---------------------------------------- image_name, image_tag = mock_nim_image.rsplit(":", 1) @@ -486,7 +524,7 @@ def _wait_ready(): # ---- Phase 5: Cleanup -------------------------------------------- sdk.inference.deployments.delete(deployment_name, workspace=DEFAULT_WORKSPACE) - controller.step() + _wait_for_deployment_deleted(controller, sdk, deployment_name) sdk.inference.deployment_configs.delete(config_name, workspace=DEFAULT_WORKSPACE) @@ -516,10 +554,12 @@ def test_model_endpoint_request_and_response_mutation_through_backend( vm_name = f"test-mep-alias-{test_uuid}" router_key = f"test-mep-router-{test_uuid}" marker_key = f"test-mep-marker-{test_uuid}" - container_name = get_docker_container_name(DEFAULT_WORKSPACE, deployment_name) + names = entity_names(deployment_name) + container_name = plugin_container_name(DEFAULT_WORKSPACE, names.server) ctx.register_container(container_name) - ctx.register_volume(get_docker_volume_name(DEFAULT_WORKSPACE, deployment_name)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.volume)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.scratch)) # ---- Phase 1: Deploy mock NIM ---------------------------------------- image_name, image_tag = mock_nim_image.rsplit(":", 1) @@ -595,7 +635,7 @@ def _wait_ready(): finally: _cleanup(registry, vm_cache, DEFAULT_WORKSPACE, vm_name, [router_key, marker_key]) sdk.inference.deployments.delete(deployment_name, workspace=DEFAULT_WORKSPACE) - controller.step() + _wait_for_deployment_deleted(controller, sdk, deployment_name) sdk.inference.deployment_configs.delete(config_name, workspace=DEFAULT_WORKSPACE) @@ -635,10 +675,12 @@ def test_model_endpoint_non_model_body_mutation_regression( vm_name = f"test-mep-reg-alias-{test_uuid}" router_key = f"test-mep-reg-router-{test_uuid}" echo_key = f"test-mep-reg-echo-{test_uuid}" - container_name = get_docker_container_name(DEFAULT_WORKSPACE, deployment_name) + names = entity_names(deployment_name) + container_name = plugin_container_name(DEFAULT_WORKSPACE, names.server) ctx.register_container(container_name) - ctx.register_volume(get_docker_volume_name(DEFAULT_WORKSPACE, deployment_name)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.volume)) + ctx.register_volume(docker_volume_name(DEFAULT_WORKSPACE, names.scratch)) # ---- Phase 1: Deploy mock NIM ---------------------------------------- image_name, image_tag = mock_nim_image.rsplit(":", 1) @@ -736,5 +778,5 @@ async def process_response(self, ctx, response, cfg) -> InferenceResponse: finally: _cleanup(registry, vm_cache, DEFAULT_WORKSPACE, vm_name, [router_key, echo_key]) sdk.inference.deployments.delete(deployment_name, workspace=DEFAULT_WORKSPACE) - controller.step() + _wait_for_deployment_deleted(controller, sdk, deployment_name) sdk.inference.deployment_configs.delete(config_name, workspace=DEFAULT_WORKSPACE) diff --git a/services/core/models/config/local.yaml b/services/core/models/config/local.yaml index 730d28ec58..f512ebd390 100644 --- a/services/core/models/config/local.yaml +++ b/services/core/models/config/local.yaml @@ -4,7 +4,9 @@ models: interval_seconds: 5 # Faster iteration for development model_deployment_garbage_collection_ttl_seconds: 30 backends: - docker: + deployments_plugin: enabled: true + docker_executor: local-docker + default_executor: local-docker inference_gateway: {} diff --git a/services/core/models/pyproject.toml b/services/core/models/pyproject.toml index 4b5cfe6cd7..6c5f5a9c44 100644 --- a/services/core/models/pyproject.toml +++ b/services/core/models/pyproject.toml @@ -20,12 +20,10 @@ dependencies = [ "pydantic-settings>=2.8.1", "sqlmodel>=0.0.22", "pyyaml>=6.0.2", - "kubernetes>=31.0.0", "urllib3>=2.7.0", "nmp-common", "nemo-platform-plugin", - "docker>=7.1.0", - "tenacity>=8.5.0", # Retry logic for Docker backend operations + "tenacity>=8.5.0", ] version = "0.0.0" @@ -49,7 +47,6 @@ dev = [ "aiosqlite>=0.20.0", "ruff>=0.11.8", "ty>=0.0.1a14", - "types-docker>=7.1.0", "pytest-xdist>=3.8.0", "datamodel-code-generator>=0.35.0", "openai>=1.0.0", # For e2e testing of sdk.models helper methods diff --git a/services/core/models/scripts/generate_nim_types.py b/services/core/models/scripts/generate_nim_types.py deleted file mode 100644 index 958d818089..0000000000 --- a/services/core/models/scripts/generate_nim_types.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Generate Pydantic models from k8s-nim-operator CRD definitions.""" - -import argparse -import json -import subprocess -import sys -from pathlib import Path - -import yaml - -# Configuration: CRDs to process -# Each tuple is (crd_filename, class_name) -# see: https://github.com/NVIDIA/k8s-nim-operator/tree/main/config/crd/bases -CRDS_TO_PROCESS = [ - ("apps.nvidia.com_nimservices.yaml", "NIMService"), - ("apps.nvidia.com_nimcaches.yaml", "NIMCache"), -] - - -def extract_openapi_schema(crd_path: Path) -> dict: - """Extract OpenAPI v3 schema from CRD YAML.""" - crd = yaml.safe_load(crd_path.read_text()) - versions = crd["spec"]["versions"] - if not versions: - raise ValueError(f"No versions in {crd_path.name}") - return versions[0]["schema"]["openAPIV3Schema"] - - -def fix_kubernetes_int_or_string_types(content: str) -> str: - """Fix Kubernetes IntOrString types that have pattern constraints on Union[int, str]. - - The datamodel-code-generator doesn't handle x-kubernetes-int-or-string properly, - generating types like Union[int, str] with a pattern constraint that only applies - to strings. This causes Pydantic validation errors when integers are passed. - - We fix this by removing the pattern constraint from RootModel types. - """ - import re - - # Pattern to match RootModel classes with Field(..., pattern=..., ...) - # We need to match the complete Field() call including description and pattern - # Example: - # class Limits(RootModel[int]): - # root: int = Field( - # ..., - # description="...", - # pattern='^...$', - # ) - pattern = re.compile( - r"class\s+(\w+)\(RootModel\[(int|str)\]\):\n" - r"\s+root:\s+(int|str)\s+=\s+Field\(\n" - r"(?:\s+\.\.\..*?\n)?" # Ellipsis line (optional) - r"(?:\s+description=.*?\n)?" # Description line (optional) - r"\s+pattern=.*?\n" # Pattern line - r"\s+\)", # Closing paren - re.MULTILINE, - ) - - def replace_root_model(match): - class_name = match.group(1) - type_name = match.group(2) - # Return simplified version without Field and pattern - return f"class {class_name}(RootModel[{type_name}]):\n root: {type_name}" - - return pattern.sub(replace_root_model, content) - - -def generate_pydantic_models(schema: dict, output_file: Path, class_name: str, version: str = "unknown"): - """Generate Pydantic models using datamodel-code-generator.""" - temp_schema = output_file.parent / "temp_schema.json" - - openapi_doc = { - "openapi": "3.0.0", - "info": {"title": f"{class_name} Schema", "version": "1.0.0"}, - "components": {"schemas": {class_name: schema}}, - } - - temp_schema.write_text(json.dumps(openapi_doc, indent=2)) - - try: - cmd = [ - "uv", - "run", - "datamodel-codegen", - "--input", - str(temp_schema), - "--output", - str(output_file), - "--input-file-type", - "openapi", - "--output-model-type", - "pydantic_v2.BaseModel", - "--field-constraints", - "--use-standard-collections", - "--use-schema-description", - "--use-title-as-name", - "--target-python-version", - "3.11", - ] - - subprocess.run(cmd, check=True, capture_output=True, text=True) - - # Read generated content - content = output_file.read_text() - - # Fix Kubernetes IntOrString types - content = fix_kubernetes_int_or_string_types(content) - - # Add version header to generated file - header = f"# Generated from k8s-nim-operator {version}\n# Source: https://github.com/NVIDIA/k8s-nim-operator\n" - output_file.write_text(header + content) - - print(f"Generated: {output_file.name}") - finally: - temp_schema.unlink(missing_ok=True) - - -def main(): - parser = argparse.ArgumentParser(description="Generate Pydantic models from k8s-nim-operator CRDs") - parser.add_argument("--version", default="unknown", help="k8s-nim-operator version tag") - args = parser.parse_args() - version = args.version - - script_dir = Path(__file__).parent - models_dir = script_dir.parent - operator_dir = models_dir / "k8s-nim-operator" - crd_dir = operator_dir / "config" / "crd" / "bases" - output_dir = models_dir / "src" / "models" / "nim_operator_types" - - output_dir.mkdir(parents=True, exist_ok=True) - - print(f"Generating Pydantic models from k8s-nim-operator {version}") - print("-" * 60) - - for crd_filename, class_name in CRDS_TO_PROCESS: - crd_path = crd_dir / crd_filename - output_file = output_dir / f"{class_name.lower()}.py" - - try: - schema = extract_openapi_schema(crd_path) - generate_pydantic_models(schema, output_file, class_name, version) - except Exception as e: - print(f"Error processing {class_name}: {e}", file=sys.stderr) - sys.exit(1) - - # Generate __init__.py - imports = "\n".join( - f"from nmp.core.models.controllers.backends.k8s_nim_operator.types.{cls.lower()} import {cls}" - for _, cls in CRDS_TO_PROCESS - ) - exports = [cls for _, cls in CRDS_TO_PROCESS] - - init_content = f'''# Generated from k8s-nim-operator {version} -# Source: https://github.com/NVIDIA/k8s-nim-operator -"""Generated Pydantic models from k8s-nim-operator CRDs.""" - -{imports} - -__all__ = {exports} -''' - - init_file = output_dir / "__init__.py" - init_file.write_text(init_content) - print(f"Generated: {init_file.name}") - print("-" * 60) - print("Done!") - - -if __name__ == "__main__": - main() diff --git a/services/core/models/scripts/update-types.sh b/services/core/models/scripts/update-types.sh deleted file mode 100755 index f8a693a340..0000000000 --- a/services/core/models/scripts/update-types.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# -# Update NIM Operator Types -# -# This script clones the k8s-nim-operator repository, generates Pydantic types -# from the CRD definitions, and cleans up the cloned repository. -# -# Usage: -# ./scripts/update-types.sh # List available versions -# ./scripts/update-types.sh --version v2.0.2 # Generate types from specific version - -set -e # Exit on error - -# Get the directory where this script is located -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Get the models service directory (parent of scripts/) -MODELS_DIR="$(dirname "$SCRIPT_DIR")" - -OPERATOR_DIR="${MODELS_DIR}/k8s-nim-operator" -OPERATOR_REPO="https://github.com/NVIDIA/k8s-nim-operator.git" - -VERSION="" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case $1 in - --version) - VERSION="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" - echo "Usage: $0 [--version ]" - exit 1 - ;; - esac -done - -echo "================================================================================" -echo "NIM Operator Type Generator" -echo "================================================================================" -echo "" - -# Step 1: Clone or update the k8s-nim-operator repository -if [ -d "$OPERATOR_DIR" ]; then - echo "📁 Repository already exists at: $OPERATOR_DIR" - echo " Fetching latest tags..." - cd "$OPERATOR_DIR" - git fetch --tags --quiet - cd "$MODELS_DIR" -else - echo "📥 Cloning k8s-nim-operator repository..." - git clone --quiet "$OPERATOR_REPO" "$OPERATOR_DIR" - echo " ✓ Cloned to: $OPERATOR_DIR" -fi -echo "" - -# Step 2: If no version specified, list available tags -if [ -z "$VERSION" ]; then - echo "📋 Available versions (tags):" - echo "" - cd "$OPERATOR_DIR" - git tag --list --sort=-version:refname | head -20 - echo "" - echo "================================================================================" - echo "To generate types for a specific version, run:" - echo " $0 --version " - echo "" - echo "Example:" - echo " $0 --version v3.0.0" - echo "================================================================================" - echo "" - echo "Repository left at: $OPERATOR_DIR" - echo "To remove it manually: rm -rf $OPERATOR_DIR" - exit 0 -fi - -# Step 3: Checkout the specified version -echo "🔖 Checking out version: $VERSION" -cd "$OPERATOR_DIR" -if ! git checkout --quiet "$VERSION" 2>/dev/null; then - echo " ✗ Error: Version '$VERSION' not found" - echo " Available tags:" - git tag --list | head -20 - exit 1 -fi -echo " ✓ Checked out: $VERSION" -echo "" - -# Step 4: Generate the Pydantic types -echo "🔨 Generating Pydantic types from CRDs..." -cd "$MODELS_DIR" -uv run python scripts/generate_nim_types.py --version "$VERSION" -echo "" - -# Step 5: Clean up the cloned repository -echo "🧹 Cleaning up..." -rm -rf "$OPERATOR_DIR" -echo " ✓ Removed: $OPERATOR_DIR" -echo "" - -echo "================================================================================" -echo "✓ Successfully updated NIM Operator types from version $VERSION!" -echo "================================================================================" -echo "" -echo "Updated files:" -echo " - src/models/nim_operator_types/nimservice.py" -echo " - src/models/nim_operator_types/nimcache.py" -echo " - src/models/nim_operator_types/__init__.py" -echo "" - diff --git a/services/core/models/src/nmp/core/models/config.py b/services/core/models/src/nmp/core/models/config.py index 63bc96c3d2..f4b3319280 100644 --- a/services/core/models/src/nmp/core/models/config.py +++ b/services/core/models/src/nmp/core/models/config.py @@ -6,13 +6,8 @@ from nemo_platform_plugin.jobs.image import get_qualified_image from nmp.common.config import Runtime, create_service_config_class, get_platform_config, get_service_config -from nmp.core.models.controllers.backends.registry import ( - BackendConfig, - DeploymentsPluginBackendConfigModel, - DockerBackendConfigModel, - K8sNimOperatorBackendConfigModel, - NoneBackendConfigModel, -) +from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginBackendConfigModel +from nmp.core.models.controllers.backends.registry import BackendConfig from pydantic import BaseModel, Field, field_validator logger = logging.getLogger(__name__) @@ -164,15 +159,11 @@ class ParallelismConfig(BaseModel): # Backend and controller configuration # ----------------------------------------------------------------------------- -BackendName = Literal["docker", "nim_operator", "deployments_plugin", "none"] +BackendName = Literal["deployments_plugin"] # Map backend names to their config model classes. -# ``none`` is the no-op substrate used when platform.runtime is ``none``. BACKEND_CONFIG_MODELS: dict[str, type[BackendConfig]] = { - "docker": DockerBackendConfigModel, - "nim_operator": K8sNimOperatorBackendConfigModel, "deployments_plugin": DeploymentsPluginBackendConfigModel, - "none": NoneBackendConfigModel, } @@ -191,11 +182,19 @@ def get_default_backends_for_runtime(runtime: Runtime) -> dict[BackendName, Back # Default backend for each runtime is enabled so that a minimal platform config # (e.g. only platform.runtime: "docker") works without requiring models.controller.backends if runtime == Runtime.DOCKER: - backends["docker"] = DockerBackendConfigModel(enabled=True) + backends["deployments_plugin"] = DeploymentsPluginBackendConfigModel( + enabled=True, + docker_executor="local-docker", + default_executor="local-docker", + ) elif runtime == Runtime.KUBERNETES: - backends["nim_operator"] = K8sNimOperatorBackendConfigModel(enabled=True) + backends["deployments_plugin"] = DeploymentsPluginBackendConfigModel( + enabled=True, + k8s_executor="local-k8s", + default_executor="local-k8s", + ) elif runtime == Runtime.NONE: - backends["none"] = NoneBackendConfigModel(enabled=True) + backends["deployments_plugin"] = DeploymentsPluginBackendConfigModel(enabled=True) if not backends: logger.warning(f"No default backends defined for runtime type: {runtime}") @@ -272,37 +271,31 @@ def merge_backends( else: merged_backends[backend_name] = custom_config - # If the runtime-default backend is "none", the platform's preferred - # runtime (docker/kubernetes) was auto-demoted because it wasn't - # available (see NemoPlatformConfig.validate_runtime). The "none" backend - # is the only viable option in that state, so we must guarantee it is - # the single enabled backend regardless of what the user's config says: - # - Force-enable merged["none"], even if the user explicitly disabled - # it — otherwise we can end up with zero enabled backends and crash - # the registry with "No backends are enabled". - # - Force-disable any other still-enabled custom backends — otherwise - # the registry crashes with "Multiple backends are enabled". - none_default = default_backends.get("none") - if none_default is not None and none_default.enabled: - current_none = merged_backends.get("none", none_default) - if not current_none.enabled: - logger.warning( - "Backend 'none' was disabled in config but the platform runtime is " - "not available; force-enabling 'none' since it is the only viable " - "backend in this state." - ) - current_none = current_none.model_copy(update={"enabled": True}) - merged_backends["none"] = current_none - - for name in list(merged_backends): - cfg = merged_backends[name] - if name != "none" and cfg.enabled: + # When platform.runtime is NONE (docker/k8s unavailable), the default + # deployments_plugin backend is the only viable option. Force-enable it and + # disable any other enabled custom backends so the registry sees exactly one. + if get_platform_config().runtime == Runtime.NONE: + fallback_default = default_backends.get("deployments_plugin") + if fallback_default is not None and fallback_default.enabled: + current = merged_backends.get("deployments_plugin", fallback_default) + if not current.enabled: logger.warning( - "Backend '%s' was enabled in config but the platform runtime " - "is not available; disabling it and using 'none' backend instead.", - name, + "Backend 'deployments_plugin' was disabled in config but the platform runtime is " + "not available; force-enabling 'deployments_plugin' since it is the only viable " + "backend in this state." ) - merged_backends[name] = cfg.model_copy(update={"enabled": False}) + current = current.model_copy(update={"enabled": True}) + merged_backends["deployments_plugin"] = current + + for name in list(merged_backends): + cfg = merged_backends[name] + if name != "deployments_plugin" and cfg.enabled: + logger.warning( + "Backend '%s' was enabled in config but the platform runtime " + "is not available; disabling it and using 'deployments_plugin' instead.", + name, + ) + merged_backends[name] = cfg.model_copy(update={"enabled": False}) return merged_backends @@ -372,9 +365,7 @@ def validate_backends(cls, v: Any) -> dict[str, BackendConfig]: result: dict[str, BackendConfig] = {} for backend_name, config_data in v.items(): # If already a BackendConfig instance, use it directly - if isinstance( - config_data, (DockerBackendConfigModel, K8sNimOperatorBackendConfigModel, NoneBackendConfigModel) - ): + if isinstance(config_data, DeploymentsPluginBackendConfigModel): result[backend_name] = config_data continue diff --git a/services/core/models/src/nmp/core/models/controllers/backends/__init__.py b/services/core/models/src/nmp/core/models/controllers/backends/__init__.py index a0e2cee985..c683de3a8f 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/__init__.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/__init__.py @@ -4,6 +4,4 @@ """Backends package for Models Controller service.""" from .backends import ServiceBackend as ServiceBackend -from .docker import DockerServiceBackend as DockerServiceBackend -from .k8s_nim_operator import K8sNimOperatorServiceBackend as K8sNimOperatorServiceBackend from .registry import BackendRegistry as BackendRegistry diff --git a/services/core/models/src/nmp/core/models/controllers/backends/backends.py b/services/core/models/src/nmp/core/models/controllers/backends/backends.py index 23662a8bef..44d4dc5fe3 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/backends.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/backends.py @@ -116,7 +116,13 @@ async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStat ... @abstractmethod - async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: + async def delete_model_deployment( + self, + workspace: str, + name: str, + *, + deleting_elapsed_seconds: float | None = None, + ) -> DeploymentStatusUpdate: """Delete a model deployment by workspace and name (model deployment ID). Used for both regular reconciliation (controller passes deployment.workspace/name) @@ -125,6 +131,8 @@ async def delete_model_deployment(self, workspace: str, name: str) -> Deployment Args: workspace: Deployment workspace name: Deployment name + deleting_elapsed_seconds: Seconds the deployment has been in DELETING; used by + backends that escalate stuck teardown to ERROR. Returns: DeploymentStatusUpdate with the current status after deletion attempt diff --git a/services/core/models/src/nmp/core/models/controllers/backends/common.py b/services/core/models/src/nmp/core/models/controllers/backends/common.py index 97b4affb43..fd9a903604 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/common.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/common.py @@ -89,18 +89,26 @@ def deployment_config_view(config: Optional[_DeploymentConfigLike]) -> Deploymen ) +def _elapsed_seconds_since(timestamp: datetime | None) -> float: + if timestamp is None: + return 0.0 + if timestamp.tzinfo is None: + timestamp = timestamp.replace(tzinfo=timezone.utc) + return (datetime.now(timezone.utc) - timestamp).total_seconds() + + def deployment_elapsed_seconds(deployment: ModelDeployment) -> float: """Seconds since the deployment entity was created. Uses the entity-store ``created_at`` timestamp so the value survives controller restarts. """ - created_at = deployment.created_at - if created_at is None: - return 0.0 - if created_at.tzinfo is None: - created_at = created_at.replace(tzinfo=timezone.utc) - return (datetime.now(timezone.utc) - created_at).total_seconds() + return _elapsed_seconds_since(deployment.created_at) + + +def deleting_elapsed_seconds(deployment: ModelDeployment) -> float: + """Seconds since the deployment last changed state (for DELETING timeout).""" + return _elapsed_seconds_since(deployment.updated_at or deployment.created_at) def format_duration(seconds: float) -> str: diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/backend.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/backend.py index 0f75cef113..2e83db03b1 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/backend.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/backend.py @@ -3,9 +3,7 @@ """Models ServiceBackend backed by nemo-deployments plugin entities.""" -import asyncio import logging -import time from typing import Any from nemo_deployments_plugin.entities import Deployment, DeploymentConfig, Prerequisite, Volume @@ -22,7 +20,11 @@ from nmp.core.models.controllers.backends.deployments_plugin.executor import executor_for_runtime from nmp.core.models.controllers.backends.deployments_plugin.naming import entity_names from nmp.core.models.controllers.backends.deployments_plugin.resolve import resolve_plugin_deployment -from nmp.core.models.controllers.backends.deployments_plugin.status import aggregate_status, apply_pending_timeout +from nmp.core.models.controllers.backends.deployments_plugin.status import ( + aggregate_status, + apply_deleting_timeout, + apply_pending_timeout, +) from nmp.core.models.controllers.backends.engine import ENGINE_GENERIC, config_engine from nmp.core.models.controllers.context import ModelContext @@ -163,6 +165,11 @@ async def _rollback_create(self, ctx: ModelContext) -> None: ) async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: + """Project plugin entity health into models deployment status. + + Aggregates Volume, puller, and server Deployment entities, then applies + ``pending_timeout_seconds`` when the deployment remains PENDING too long. + """ if ctx.model_deployment is None: return DeploymentStatusUpdate(status="UNKNOWN", status_message="Model deployment unavailable.") names = entity_names(ctx.model_deployment.name) @@ -182,14 +189,26 @@ async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUp del ctx return DeploymentStatusUpdate(status="ERROR", status_message="Update via recreate not yet supported.") - async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Stop deployments, wait for each to disappear, then remove their configs.""" + async def delete_model_deployment( + self, + workspace: str, + name: str, + *, + deleting_elapsed_seconds: float | None = None, + ) -> DeploymentStatusUpdate: + """Stop deployments, then remove configs and volumes once substrate is gone.""" names = entity_names(name) for deployment_name, config_name in ((names.server, names.server), (names.puller, names.puller)): - if not await self._delete_deployment_and_config(workspace, deployment_name, config_name): - return DeploymentStatusUpdate( + if not await self._complete_deployment_delete(workspace, deployment_name, config_name): + result = DeploymentStatusUpdate( status="DELETING", status_message="Waiting for plugin deployment teardown." ) + return apply_deleting_timeout( + result, + elapsed_seconds=deleting_elapsed_seconds or 0.0, + timeout_seconds=self._cfg.deleting_timeout_seconds, + deployment_name=name, + ) for volume_name in (names.scratch, names.volume): try: await self._entity_client().delete(Volume, name=volume_name, workspace=workspace) @@ -197,14 +216,23 @@ async def delete_model_deployment(self, workspace: str, name: str) -> Deployment pass return DeploymentStatusUpdate(status="DELETED", status_message="Deleted deployments-plugin entities.") - async def _delete_deployment_and_config(self, workspace: str, deployment_name: str, config_name: str) -> bool: + async def _complete_deployment_delete(self, workspace: str, deployment_name: str, config_name: str) -> bool: + """Initiate plugin deployment stop and return True once config can be removed.""" deployment = await self._get_optional(Deployment, workspace, deployment_name) if deployment is not None: - if deployment.status != "DELETING": + if deployment.status == "FAILED": + # Plugin reconciler gave up on substrate teardown; remove the stale + # entity so models delete can finish config/volume cleanup. + try: + await self._entity_client().delete(Deployment, name=deployment_name, workspace=workspace) + except NemoEntityNotFoundError: + pass + elif deployment.status != "DELETING" or deployment.desired_state != "STOPPED": deployment.status = "DELETING" deployment.desired_state = "STOPPED" await self._entity_client().update(deployment) - if not await self._wait_for_deployment_gone(workspace, deployment_name): + return False + else: return False try: await self._entity_client().delete(DeploymentConfig, name=config_name, workspace=workspace) @@ -212,14 +240,6 @@ async def _delete_deployment_and_config(self, workspace: str, deployment_name: s pass return True - async def _wait_for_deployment_gone(self, workspace: str, name: str) -> bool: - deadline = time.monotonic() + self._cfg.delete_wait_seconds - while time.monotonic() < deadline: - if await self._get_optional(Deployment, workspace, name) is None: - return True - await asyncio.sleep(self._cfg.delete_poll_seconds) - return False - async def _get_optional(self, entity_type: type[Any], workspace: str, name: str) -> Any | None: try: return await self._entity_client().get(entity_type, name=name, workspace=workspace) @@ -227,8 +247,11 @@ async def _get_optional(self, entity_type: type[Any], workspace: str, name: str) return None async def list_managed_deployment_names(self) -> list[str]: - # Labels live on immutable DeploymentConfig entities; deployments-plugin - # does not currently mirror them onto Deployment. + """List workspace/name IDs for model deployments managed by this backend. + + Discovers server-role DeploymentConfig entities stamped with models-controller + ownership labels (deployments-plugin does not mirror labels onto Deployment). + """ result = await self._entity_client().list(DeploymentConfig, workspace="-") names = { f"{config.labels[_DEPLOYMENT_WORKSPACE_LABEL]}/{config.labels[_DEPLOYMENT_NAME_LABEL]}" diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py index 549f2ac01d..4ddecb15fc 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py @@ -1,13 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Compile ModelDeployments into deployments-plugin entity specifications.""" +"""Compile ModelDeployments into deployments-plugin entity specifications. + +Parity ports from ``k8s_nim_operator/nimservice_compiler.py`` — please review +mapping of ``k8s_nim_operator_config`` → plugin ``K8sDeploymentConfig``. +""" from dataclasses import dataclass from nemo_deployments_plugin.entities import ( Container, ContainerPort, + DeploymentBackendConfig, DeploymentConfig, EnvVar, HTTPGetAction, @@ -17,12 +22,22 @@ VolumeBackendConfig, VolumeMount, ) +from nemo_platform_plugin.config import get_platform_config from nemo_platform_plugin.jobs.image import get_qualified_image from nmp.common.config import Runtime from nmp.core.models.app import ModelWeightsType from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginConfig from nmp.core.models.controllers.backends.deployments_plugin.naming import EntityNames, entity_names +from nmp.core.models.controllers.backends.deployments_plugin.nim_compiler import ( + apply_container_resources, + apply_k8s_nim_operator_container_overrides, + apply_nim_override_config, + build_k8s_deployment_backend_config, + compile_nim_server_env, + tool_call_plugin_init_containers, + tool_call_plugin_install_path, +) from nmp.core.models.controllers.backends.deployments_plugin.resolve import ResolvedPluginDeployment from nmp.core.models.controllers.backends.engine import ( ENGINE_GENERIC, @@ -38,6 +53,7 @@ ) from nmp.core.models.controllers.backends.vllm_compiler import ( MODEL_STORE_PATH, + VLLM_SERVER_PORT, compile_vllm_args, compile_vllm_env_vars, resolve_vllm_image, @@ -92,6 +108,20 @@ def _env(values: dict[str, str]) -> list[EnvVar]: return [EnvVar(name=name, value=value) for name, value in values.items()] +def _apply_gpu_resources(container: Container, gpu: int) -> None: + """Map executor_config.gpu to nvidia.com/gpu requests and limits.""" + if gpu < 1: + return + quantity = str(gpu) + apply_container_resources( + container, + { + "requests": {"nvidia.com/gpu": quantity}, + "limits": {"nvidia.com/gpu": quantity}, + }, + ) + + def _lora_sidecar( resolved: ResolvedPluginDeployment, *, @@ -100,17 +130,26 @@ def _lora_sidecar( names: EntityNames, weighted: bool, ) -> Container: - """Build the adapters sidecar with the same env contract as existing backends.""" + """Build the adapters sidecar with the same env contract as existing backends. + + ``NMP_BASE_URL`` must point at the platform API (not the sidecar's own listen + address). ``nemo services run --sidecars adapters`` binds localhost:8080 inside + the sidecar, so an unset base URL makes the SDK call itself and 404. + """ entity_workspace = resolved.model_entity.workspace if resolved.model_entity else resolved.deployment.workspace entity_name = resolved.model_entity.name if resolved.model_entity else resolved.deployment.name + platform = get_platform_config() sidecar_env = { "NIM_PEFT_SOURCE": _LORA_MOUNT, "NIM_PEFT_REFRESH_INTERVAL": str(config.peft_refresh_interval), "NMP_MODEL_ENTITY_WORKSPACE": entity_workspace, "NMP_MODEL_ENTITY_NAME": entity_name, + "NMP_BASE_URL": platform.base_url, } if engine == ENGINE_VLLM: sidecar_env["VLLM_LORA_BASE_MODEL_OVERRIDE"] = MODEL_STORE_PATH + # Native sidecar / pod-local network: talk to the sibling vLLM server. + sidecar_env["VLLM_ENDPOINT"] = f"http://127.0.0.1:{VLLM_SERVER_PORT}" mounts = [VolumeMount(name=names.scratch, mountPath=_SCRATCH_MOUNT)] if weighted: mounts.append(VolumeMount(name=names.volume, mountPath=_WEIGHTS_MOUNT, readOnly=True)) @@ -128,13 +167,24 @@ def _lora_sidecar( def compile_model_deployment( resolved: ResolvedPluginDeployment, config: DeploymentsPluginConfig ) -> CompiledModelDeployment: - """Compile volume, puller, and always-on serving config specifications.""" + """Compile a model deployment into deployments-plugin entity specifications. + + Chooses the engine compiler path (nim / vllm / generic), optionally emits a + weighted puller chain, and maps ``k8s_nim_operator_config`` onto plugin k8s + backend settings when the platform runtime is Kubernetes. + """ engine = config_engine(resolved.config) if engine not in {ENGINE_NIM, ENGINE_VLLM, ENGINE_GENERIC}: raise ValueError(f"Unsupported engine {engine!r}.") names = entity_names(resolved.deployment.name) weighted = _weighted(resolved, engine) lora_enabled = resolved.view.lora_enabled and engine != ENGINE_GENERIC + k8s_backend = ( + build_k8s_deployment_backend_config(engine, resolved.view, config) + if resolved.runtime == Runtime.KUBERNETES + else None + ) + backend_config = k8s_backend if k8s_backend is not None and k8s_backend.k8s is not None else None volume = None scratch_volume = None puller_config = None @@ -145,7 +195,7 @@ def compile_model_deployment( size=resolved.view.disk_size or config.default_pvc_size, backendConfig=VolumeBackendConfig(k8s=K8sVolumeConfig(storageClass=config.default_storage_class)), ) - puller_env = {"HF_ENDPOINT": resolved.files_hf_url} + puller_env = {"HF_ENDPOINT": resolved.files_hf_url, "HF_TOKEN": "service:models"} puller_args = ["download", f"{resolved.model_namespace}/{resolved.model_name}", "--local-dir", _WEIGHTS_MOUNT] if resolved.model_revision: puller_args.extend(["--revision", resolved.model_revision]) @@ -157,6 +207,7 @@ def compile_model_deployment( env=_env(puller_env), volumeMounts=[VolumeMount(name=names.volume, mountPath=_WEIGHTS_MOUNT)], ) + _apply_gpu_resources(puller, resolved.view.gpu) puller_config = DeploymentConfig( name=names.puller, workspace=resolved.deployment.workspace, @@ -164,8 +215,19 @@ def compile_model_deployment( labels=_labels(resolved, engine, "puller"), restartPolicy="OnFailure", backoffLimit=config.max_restart_count, + backendConfig=backend_config or DeploymentBackendConfig(), ) + tool_call_inits = tool_call_plugin_init_containers( + resolved, + config, + names_volume=names.volume, + names_scratch=names.scratch, + weighted=weighted, + ) + needs_scratch = lora_enabled or tool_call_inits is not None + tool_call_plugin_path = tool_call_plugin_install_path(weighted=weighted) if tool_call_inits is not None else None + if engine == ENGINE_VLLM: image_name, image_tag = resolve_vllm_image( resolved.view, config.default_vllm_image, config.default_vllm_image_tag @@ -178,12 +240,12 @@ def compile_model_deployment( resolved.view.image_tag or config.default_nimservice_image_tag, ) args = list(resolved.view.additional_args or []) - env = dict(resolved.view.additional_envs or {}) - env.update({"NIM_MODEL_NAME": _WEIGHTS_MOUNT, "NIM_MODEL_PATH": _WEIGHTS_MOUNT}) - if resolved.model_name: - env["NIM_SERVED_MODEL_NAME"] = ( - f"{resolved.model_namespace}/{resolved.model_name}" if resolved.model_namespace else resolved.model_name - ) + env = compile_nim_server_env( + resolved, + config, + weighted=weighted, + tool_call_plugin_path=tool_call_plugin_path, + ) if lora_enabled: env["NIM_PEFT_SOURCE"] = _LORA_MOUNT env["NIM_PEFT_REFRESH_INTERVAL"] = str(config.peft_refresh_interval) @@ -195,9 +257,11 @@ def compile_model_deployment( mounts: list[VolumeMount] = [] if weighted: mounts.append(VolumeMount(name=names.volume, mountPath=_WEIGHTS_MOUNT, readOnly=True)) - init_containers: list[Container] = [] + init_containers: list[Container] = list(tool_call_inits or []) server_config_containers: list[Container] - if lora_enabled: + readiness_probe = Probe(httpGet=HTTPGetAction(path=resolve_health_path(engine, resolved.view), port=8000)) + vllm_command = ["vllm", "serve"] if engine == ENGINE_VLLM else None + if needs_scratch: scratch_volume = Volume( name=names.scratch, workspace=resolved.deployment.workspace, @@ -205,6 +269,7 @@ def compile_model_deployment( backendConfig=VolumeBackendConfig(k8s=K8sVolumeConfig(storageClass=config.default_storage_class)), ) mounts.append(VolumeMount(name=names.scratch, mountPath=_SCRATCH_MOUNT)) + if lora_enabled: # Ensure the LoRA cache dir exists before the server/sidecar start. init_containers.append( Container( @@ -218,12 +283,16 @@ def compile_model_deployment( server = Container( name="server", image=_image(image_name, image_tag), + command=vllm_command or [], args=args, env=_env(env), ports=[ContainerPort(name="http", containerPort=8000)], volumeMounts=mounts, - readinessProbe=Probe(httpGet=HTTPGetAction(path=resolve_health_path(engine, resolved.view), port=8000)), + readinessProbe=readiness_probe, ) + _apply_gpu_resources(server, resolved.view.gpu) + if engine == ENGINE_NIM: + apply_k8s_nim_operator_container_overrides(server, readiness_probe, resolved.view) if resolved.runtime == Runtime.DOCKER: # Docker v1 is single-container today; emit a second container so the # shape matches the locked design for when the plugin docker backend @@ -235,17 +304,20 @@ def compile_model_deployment( server_config_containers = [server] init_containers.append(lora) else: - server_config_containers = [ - Container( - name="server", - image=_image(image_name, image_tag), - args=args, - env=_env(env), - ports=[ContainerPort(name="http", containerPort=8000)], - volumeMounts=mounts, - readinessProbe=Probe(httpGet=HTTPGetAction(path=resolve_health_path(engine, resolved.view), port=8000)), - ) - ] + server = Container( + name="server", + image=_image(image_name, image_tag), + command=vllm_command or [], + args=args, + env=_env(env), + ports=[ContainerPort(name="http", containerPort=8000)], + volumeMounts=mounts, + readinessProbe=readiness_probe, + ) + _apply_gpu_resources(server, resolved.view.gpu) + if engine == ENGINE_NIM: + apply_k8s_nim_operator_container_overrides(server, readiness_probe, resolved.view) + server_config_containers = [server] server_config = DeploymentConfig( name=names.server, @@ -254,7 +326,16 @@ def compile_model_deployment( initContainers=init_containers, labels=_labels(resolved, engine, "server"), restartPolicy="Always", + backendConfig=backend_config or DeploymentBackendConfig(), ) + if engine == ENGINE_NIM: + apply_nim_override_config( + server, + server_config, + resolved.view, + engine=engine, + runtime=resolved.runtime, + ) return CompiledModelDeployment( names=names, volume=volume, diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py index 7d03911285..19dca500c4 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/config.py @@ -20,8 +20,11 @@ class DeploymentsPluginConfig(BaseModel): default_pvc_size: str = "200Gi" default_nimservice_image: str = "nvcr.io/nim/meta/llama-3.1-8b-instruct" default_nimservice_image_tag: str = "1.8.5" - default_vllm_image: str = "vllm/vllm-openai" - default_vllm_image_tag: str = "v0.8.5" + default_vllm_image: str = Field( + default="docker.io/vllm/vllm-openai", + description="Default vLLM image repository. Fully qualified so it resolves on runtimes that block docker.io short names.", + ) + default_vllm_image_tag: str = "v0.22.1" default_user_id: int | None = 1000 default_group_id: int | None = 2000 default_vllm_user_id: int | None = 2000 @@ -41,8 +44,11 @@ class DeploymentsPluginConfig(BaseModel): default="latest", description="BusyBox image tag for LoRA cache init containers.", ) - delete_wait_seconds: float = Field(default=5.0, gt=0) - delete_poll_seconds: float = Field(default=0.5, gt=0) + deleting_timeout_seconds: int = Field( + default=60, + ge=0, + description=("Maximum seconds a deployment may stay DELETING before ERROR. 0 disables timeout escalation."), + ) class DeploymentsPluginBackendConfigModel(DeploymentsPluginConfig): diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/nim_compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/nim_compiler.py new file mode 100644 index 0000000000..b1d9e13177 --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/nim_compiler.py @@ -0,0 +1,608 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""NIM-specific compilation helpers for the deployments-plugin models backend. + +Parity ports from ``k8s_nim_operator/nimservice_compiler.py`` — please review +mapping of ``k8s_nim_operator_config`` → plugin ``K8sDeploymentConfig``. +""" + +from __future__ import annotations + +import math +from typing import Any + +from nemo_deployments_plugin.entities import ( + Affinity, + Container, + DeploymentBackendConfig, + DeploymentConfig, + EnvVar, + ExecAction, + HTTPGetAction, + K8sDeploymentConfig, + PodSecurityContext, + Probe, + ResourceRequirements, + Toleration, + VolumeMount, +) +from nemo_platform.types.inference.k8s_nim_operator_config import K8sNIMOperatorConfig +from nemo_platform.types.models.model_entity import ModelEntity +from nmp.common.config import Runtime +from nmp.core.models.app import is_multi_llm_image, parse_model_name_revision +from nmp.core.models.controllers.backends.common import DeploymentConfigView +from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginConfig +from nmp.core.models.controllers.backends.deployments_plugin.resolve import ResolvedPluginDeployment +from nmp.core.models.controllers.backends.engine import ENGINE_GENERIC, ENGINE_NIM, ENGINE_VLLM + +_WEIGHTS_MOUNT = "/model-store" +_SCRATCH_MOUNT = "/scratch" +_TOOL_CALL_PLUGIN_PATH = "/model-store/plugin/plugin.py" +_TOOL_CALL_PLUGIN_SCRATCH_PATH = "/scratch/plugin/plugin.py" +_TOOL_CALL_PLUGIN_SCRATCH_DIR = "/scratch/plugin" +_TOOL_CALL_PLUGIN_FINALIZE_SCRIPT = """set -euo pipefail +py_files="$(find "{scratch_dir}" -type f -name '*.py' || true)" +count="$(printf '%s\\n' "$py_files" | sed '/^$/d' | wc -l | tr -d ' ')" +if [ "$count" -eq 0 ]; then + echo "tool_call_plugin fileset contains no .py files" + exit 1 +fi +if [ "$count" -ne 1 ]; then + echo "tool_call_plugin fileset must contain exactly one .py file, found $count" + printf '%s\\n' "$py_files" + exit 1 +fi +plugin_file="$(printf '%s\\n' "$py_files" | sed '/^$/d' | sed -n '1p')" +if [ "$plugin_file" != "{plugin_path}" ]; then + mv "$plugin_file" "{plugin_path}" +fi +""" + +_SUPPORTED_NIM_OVERRIDE_CONFIG_KEYS = frozenset( + { + "image", + "command", + "args", + "resources", + "env", + "readinessProbe", + "livenessProbe", + "startupProbe", + "nodeSelector", + "tolerations", + "userID", + "groupID", + "labels", + "initContainers", + "sidecarContainers", + } +) + + +def _plugin_fileset(view: DeploymentConfigView, model_entity: ModelEntity | None) -> str | None: + if view.tool_call_config and view.tool_call_config.tool_call_plugin: + return view.tool_call_config.tool_call_plugin + if ( + model_entity + and model_entity.spec + and model_entity.spec.tool_call_config + and model_entity.spec.tool_call_config.tool_call_plugin + ): + return model_entity.spec.tool_call_config.tool_call_plugin + return None + + +def _puller_image_parts(image: str) -> tuple[str, str] | None: + last_slash_idx = image.rfind("/") + last_colon_idx = image.rfind(":") + if last_colon_idx <= last_slash_idx: + return None + return image[:last_colon_idx], image[last_colon_idx + 1 :] + + +def tool_call_plugin_install_path(*, weighted: bool) -> str: + """Return the in-container path where the tool-call plugin is installed.""" + return _TOOL_CALL_PLUGIN_PATH if weighted else _TOOL_CALL_PLUGIN_SCRATCH_PATH + + +def tool_call_plugin_init_containers( + resolved: ResolvedPluginDeployment, + config: DeploymentsPluginConfig, + *, + names_volume: str, + names_scratch: str, + weighted: bool, +) -> list[Container] | None: + """Build init containers that fetch and install a tool-call plugin fileset.""" + plugin_fileset = _plugin_fileset(resolved.view, resolved.model_entity) + if plugin_fileset is None: + return None + if not resolved.huggingface_model_puller: + return None + puller_parts = _puller_image_parts(resolved.huggingface_model_puller) + if puller_parts is None: + return None + puller_repo, puller_tag = puller_parts + scratch_mount = VolumeMount(name=names_scratch, mountPath=_SCRATCH_MOUNT) + weights_mount = VolumeMount(name=names_volume, mountPath=_WEIGHTS_MOUNT) + plugin_path = tool_call_plugin_install_path(weighted=weighted) + busybox = _image(config.busybox_image, config.busybox_image_tag) + prepare_mounts = [scratch_mount] + finalize_mounts = [scratch_mount] + if weighted: + prepare_mounts.append(weights_mount) + finalize_mounts.append(weights_mount) + prepare_script = ( + "set -e; mkdir -p /model-store/plugin /scratch/plugin; " + f"rm -f {_TOOL_CALL_PLUGIN_PATH}; rm -rf /scratch/plugin/*" + ) + else: + prepare_script = f"set -e; mkdir -p /scratch/plugin; rm -f {plugin_path}; rm -rf /scratch/plugin/*" + return [ + Container( + name="tool-call-plugin-prepare", + image=busybox, + command=["sh", "-c", prepare_script], + volumeMounts=prepare_mounts, + ), + Container( + name="tool-call-plugin-pull", + image=f"{puller_repo}:{puller_tag}", + command=["download", plugin_fileset, "--local-dir", _TOOL_CALL_PLUGIN_SCRATCH_DIR], + env=[ + EnvVar(name="HF_ENDPOINT", value=resolved.files_hf_url), + EnvVar(name="HF_TOKEN", value="service:models"), + ], + volumeMounts=[scratch_mount], + ), + Container( + name="tool-call-plugin-finalize", + image=busybox, + command=[ + "sh", + "-c", + _TOOL_CALL_PLUGIN_FINALIZE_SCRIPT.format( + scratch_dir=_TOOL_CALL_PLUGIN_SCRATCH_DIR, + plugin_path=plugin_path, + ), + ], + volumeMounts=finalize_mounts, + ), + ] + + +def compile_nim_server_env( + resolved: ResolvedPluginDeployment, + config: DeploymentsPluginConfig, + *, + weighted: bool, + tool_call_plugin_path: str | None, +) -> dict[str, str]: + """Compile NIM server environment variables.""" + view = resolved.view + env: dict[str, str] = dict(view.additional_envs or {}) + + model_fqdn: str | None = None + if view.model_name: + parsed_namespace, parsed_name, parsed_revision = parse_model_name_revision( + model_namespace=view.model_namespace, + model_name=view.model_name, + model_revision=view.model_revision, + ) + if parsed_namespace and parsed_name: + model_fqdn = f"{parsed_namespace}/{parsed_name}" + elif parsed_name: + model_fqdn = parsed_name + if model_fqdn and parsed_revision: + model_fqdn += f"@{parsed_revision}" + + if model_fqdn: + env["NIM_SERVED_MODEL_NAME"] = model_fqdn + + if weighted: + env["NIM_MODEL_NAME"] = _WEIGHTS_MOUNT + env["NIM_MODEL_PATH"] = _WEIGHTS_MOUNT + effective_image = view.image_name or config.default_nimservice_image + if not is_multi_llm_image(effective_image): + env["NIM_FT_MODEL"] = _WEIGHTS_MOUNT + env["NIM_CUSTOM_MODEL"] = _WEIGHTS_MOUNT + elif resolved.model_name: + served = ( + f"{resolved.model_namespace}/{resolved.model_name}" if resolved.model_namespace else resolved.model_name + ) + env.setdefault("NIM_SERVED_MODEL_NAME", served) + + model_entity = resolved.model_entity + if model_entity: + env["NMP_MODEL_ENTITY_WORKSPACE"] = model_entity.workspace + env["NMP_MODEL_ENTITY_NAME"] = model_entity.name + if model_entity.trust_remote_code: + env["NIM_FORCE_TRUST_REMOTE_CODE"] = "1" + env["NIM_TRUST_CUSTOM_CODE"] = "1" + if model_entity.spec: + if model_entity.spec.chat_template: + env["NIM_CHAT_TEMPLATE"] = model_entity.spec.chat_template + tool_cfg = model_entity.spec.tool_call_config + if tool_cfg: + if tool_cfg.tool_call_parser: + env["NIM_TOOL_CALL_PARSER"] = tool_cfg.tool_call_parser + if tool_call_plugin_path: + env["NIM_TOOL_PARSER_PLUGIN"] = tool_call_plugin_path + if tool_cfg.auto_tool_choice is not None: + env["NIM_ENABLE_AUTO_TOOL_CHOICE"] = "1" if tool_cfg.auto_tool_choice else "0" + + if view.chat_template: + env["NIM_CHAT_TEMPLATE"] = view.chat_template + + deploy_tool_cfg = view.tool_call_config + if deploy_tool_cfg: + if deploy_tool_cfg.tool_call_parser: + env["NIM_TOOL_CALL_PARSER"] = deploy_tool_cfg.tool_call_parser + if deploy_tool_cfg.tool_call_plugin and tool_call_plugin_path: + env["NIM_TOOL_PARSER_PLUGIN"] = tool_call_plugin_path + if deploy_tool_cfg.auto_tool_choice is not None: + env["NIM_ENABLE_AUTO_TOOL_CHOICE"] = "1" if deploy_tool_cfg.auto_tool_choice else "0" + + return env + + +def _image(name: str, tag: str) -> str: + return name if "@" in name or name.endswith(f":{tag}") else f"{name}:{tag}" + + +def _k8s_config_dict(k8s_config: K8sNIMOperatorConfig | dict[str, Any] | Any) -> dict[str, Any]: + if hasattr(k8s_config, "model_dump"): + return k8s_config.model_dump(exclude_none=True) + if isinstance(k8s_config, dict): + return {key: value for key, value in k8s_config.items() if value is not None} + return {} + + +def _tolerations_from_config(raw: list[dict[str, Any]]) -> list[Toleration]: + tolerations: list[Toleration] = [] + for item in raw: + if isinstance(item, dict): + tolerations.append(Toleration(**{key: value for key, value in item.items() if value is not None})) + return tolerations + + +def _affinity_from_node_selector(node_selector: dict[str, str]) -> Affinity: + return Affinity( + node_affinity={ + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": key, "operator": "In", "values": [value]} for key, value in node_selector.items() + ] + } + ] + } + } + ) + + +def k8s_backend_config_from_nim_operator(view: DeploymentConfigView) -> K8sDeploymentConfig | None: + """Map per-deployment ``k8s_nim_operator_config`` onto plugin ``K8sDeploymentConfig``.""" + if view.k8s_nim_operator_config is None: + return None + config_dict = _k8s_config_dict(view.k8s_nim_operator_config) + if not config_dict: + return None + + k8s_kwargs: dict[str, Any] = {} + tolerations = config_dict.get("tolerations") + if isinstance(tolerations, list): + parsed = _tolerations_from_config(tolerations) + if parsed: + k8s_kwargs["tolerations"] = parsed + node_selector = config_dict.get("node_selector") + if isinstance(node_selector, dict) and node_selector: + k8s_kwargs["affinity"] = _affinity_from_node_selector(node_selector) + if not k8s_kwargs: + return None + return K8sDeploymentConfig(**k8s_kwargs) + + +def pod_security_context_for_engine( + engine: str, + view: DeploymentConfigView, + config: DeploymentsPluginConfig, +) -> PodSecurityContext | None: + """Map backend default uid/gid onto pod securityContext for k8s workloads.""" + if engine == ENGINE_VLLM: + user_id = view.run_as_user if view.run_as_user is not None else config.default_vllm_user_id + group_id = view.run_as_group if view.run_as_group is not None else config.default_vllm_group_id + elif engine == ENGINE_GENERIC: + # Generic containers use the image's own user unless run_as_* is set explicitly. + user_id = view.run_as_user + group_id = view.run_as_group + else: + user_id = view.run_as_user if view.run_as_user is not None else config.default_user_id + group_id = view.run_as_group if view.run_as_group is not None else config.default_group_id + if user_id is None and group_id is None: + return None + return PodSecurityContext(run_as_user=user_id, run_as_group=group_id, fs_group=group_id) + + +def build_k8s_deployment_backend_config( + engine: str, + view: DeploymentConfigView, + config: DeploymentsPluginConfig, +) -> DeploymentBackendConfig: + """Merge operator overrides and engine security defaults into backend_config.k8s.""" + if engine == ENGINE_NIM: + k8s = k8s_backend_config_from_nim_operator(view) or K8sDeploymentConfig() + else: + k8s = K8sDeploymentConfig() + security_context = pod_security_context_for_engine(engine, view, config) + if security_context is not None: + k8s.security_context = security_context + if any( + ( + k8s.tolerations, + k8s.affinity, + k8s.security_context, + k8s.namespace, + k8s.service_account, + ) + ): + return DeploymentBackendConfig(k8s=k8s) + return DeploymentBackendConfig() + + +def startup_probe_failure_threshold(view: DeploymentConfigView, *, period_seconds: int = 10) -> int | None: + """Derive readiness failure threshold from ``startup_probe_grace_seconds``.""" + if view.k8s_nim_operator_config is None: + return None + config_dict = _k8s_config_dict(view.k8s_nim_operator_config) + grace_seconds = config_dict.get("startup_probe_grace_seconds") + if grace_seconds is None: + return None + return max(1, math.ceil(int(grace_seconds) / period_seconds)) + + +def apply_container_resources(container: Container, resources: dict[str, Any]) -> None: + """Apply k8s resource requirements to a plugin container.""" + requests = resources.get("requests") if isinstance(resources.get("requests"), dict) else {} + limits = resources.get("limits") if isinstance(resources.get("limits"), dict) else {} + if not requests and not limits: + return + existing = container.resources + merged_requests = dict(existing.requests) if existing and existing.requests else {} + merged_limits = dict(existing.limits) if existing and existing.limits else {} + merged_requests.update({str(key): str(value) for key, value in requests.items()}) + merged_limits.update({str(key): str(value) for key, value in limits.items()}) + container.resources = ResourceRequirements( + requests=merged_requests, + limits=merged_limits, + ) + + +def apply_k8s_nim_operator_container_overrides( + container: Container, + probe: Probe, + view: DeploymentConfigView, +) -> None: + """Apply per-deployment operator config fields that target the server container.""" + if view.k8s_nim_operator_config is None: + return + config_dict = _k8s_config_dict(view.k8s_nim_operator_config) + resources = config_dict.get("resources") + if isinstance(resources, dict): + apply_container_resources(container, resources) + failure_threshold = startup_probe_failure_threshold(view, period_seconds=probe.period_seconds) + if failure_threshold is not None: + probe.failure_threshold = failure_threshold + + +def _resource_string(value: Any) -> str: + if isinstance(value, dict) and "root" in value: + return str(value["root"]) + return str(value) + + +def _normalize_resources(resources: dict[str, Any]) -> dict[str, dict[str, str]]: + normalized: dict[str, dict[str, str]] = {} + for section in ("requests", "limits"): + raw = resources.get(section) + if isinstance(raw, dict): + normalized[section] = {str(key): _resource_string(val) for key, val in raw.items()} + return normalized + + +def _merge_env_override(container: Container, env_list: list[Any]) -> None: + by_name = {item.name: index for index, item in enumerate(container.env)} + for item in env_list: + if not isinstance(item, dict): + continue + name = item.get("name") + if not name: + continue + value = item.get("value") + if value is None: + continue + env_var = EnvVar(name=str(name), value=str(value)) + if name in by_name: + container.env[by_name[name]] = env_var + else: + container.env.append(env_var) + by_name[name] = len(container.env) - 1 + + +def _probe_from_k8s_dict(data: dict[str, Any]) -> Probe | None: + if not isinstance(data, dict): + return None + kwargs: dict[str, Any] = {} + for src, dest in ( + ("initialDelaySeconds", "initial_delay_seconds"), + ("periodSeconds", "period_seconds"), + ("timeoutSeconds", "timeout_seconds"), + ("failureThreshold", "failure_threshold"), + ): + if src in data and data[src] is not None: + kwargs[dest] = int(data[src]) + http_get = data.get("httpGet") + if isinstance(http_get, dict): + kwargs["http_get"] = HTTPGetAction( + path=http_get.get("path", "/"), + port=http_get.get("port", 8000), + scheme=http_get.get("scheme", "HTTP"), + ) + exec_action = data.get("exec") + if isinstance(exec_action, dict) and exec_action.get("command"): + kwargs["exec_action"] = ExecAction(command=[str(item) for item in exec_action["command"]]) + if not kwargs: + return None + return Probe(**kwargs) + + +def _probe_from_nim_service_probe(probe_wrapper: dict[str, Any]) -> Probe | None: + if probe_wrapper.get("enabled") is False: + return None + inner = probe_wrapper.get("probe") + if isinstance(inner, dict): + return _probe_from_k8s_dict(inner) + return _probe_from_k8s_dict(probe_wrapper) + + +def _container_from_spec_entry(spec_container: dict[str, Any], *, native_sidecar: bool) -> Container | None: + name = spec_container.get("name") + image_info = spec_container.get("image") + if not name or not isinstance(image_info, dict): + return None + repository = image_info.get("repository") + if not repository: + return None + tag = image_info.get("tag") or "latest" + env: list[EnvVar] = [] + for item in spec_container.get("env") or []: + if isinstance(item, dict) and item.get("name") and item.get("value") is not None: + env.append(EnvVar(name=str(item["name"]), value=str(item["value"]))) + return Container( + name=str(name), + image=_image(str(repository), str(tag)), + command=[str(item) for item in spec_container.get("command") or []], + args=[str(item) for item in spec_container.get("args") or []], + env=env, + restartPolicy="Always" if native_sidecar else None, + ) + + +def _ensure_k8s_backend(server_config: DeploymentConfig) -> K8sDeploymentConfig: + backend = server_config.backend_config + if backend.k8s is None: + backend.k8s = K8sDeploymentConfig() + return backend.k8s + + +def _validate_nim_override_config_keys(override: dict[str, Any]) -> None: + """Reject override_config keys that the deployments_plugin NIM compiler does not apply.""" + unsupported = sorted(set(override) - _SUPPORTED_NIM_OVERRIDE_CONFIG_KEYS) + if unsupported: + supported = ", ".join(sorted(_SUPPORTED_NIM_OVERRIDE_CONFIG_KEYS)) + raise ValueError( + f"override_config contains unsupported keys: {', '.join(unsupported)}. Supported keys: {supported}." + ) + + +def apply_nim_override_config( + container: Container, + server_config: DeploymentConfig, + view: DeploymentConfigView, + *, + engine: str, + runtime: Runtime, +) -> None: + """Apply ``override_config`` NIMService Spec fragments onto plugin entities. + + Precedence: generated defaults < ``k8s_nim_operator_config`` < ``override_config``. + Only honored for NIM engine deployments on the Kubernetes runtime. + """ + if engine != ENGINE_NIM or runtime != Runtime.KUBERNETES: + return + override = view.override_config + if not override: + return + + _validate_nim_override_config_keys(override) + + image_info = override.get("image") + if isinstance(image_info, dict): + repository = image_info.get("repository") + if repository: + container.image = _image(str(repository), str(image_info.get("tag") or "latest")) + + if "command" in override: + container.command = [str(item) for item in override["command"]] + if "args" in override: + container.args = [str(item) for item in override["args"]] + + resources = override.get("resources") + if isinstance(resources, dict): + apply_container_resources(container, _normalize_resources(resources)) + + env_list = override.get("env") + if isinstance(env_list, list): + _merge_env_override(container, env_list) + + readiness = override.get("readinessProbe") + if isinstance(readiness, dict): + container.readiness_probe = _probe_from_nim_service_probe(readiness) + + liveness = override.get("livenessProbe") + if isinstance(liveness, dict): + container.liveness_probe = _probe_from_nim_service_probe(liveness) + + startup = override.get("startupProbe") + if isinstance(startup, dict): + probe = container.readiness_probe or Probe(httpGet=HTTPGetAction(path="/v1/health/ready", port=8000)) + startup_probe = _probe_from_nim_service_probe(startup) + if startup_probe is not None and startup_probe.failure_threshold is not None: + probe.failure_threshold = startup_probe.failure_threshold + container.readiness_probe = probe + + k8s = _ensure_k8s_backend(server_config) + node_selector = override.get("nodeSelector") + if isinstance(node_selector, dict) and node_selector: + k8s.affinity = _affinity_from_node_selector({str(k): str(v) for k, v in node_selector.items()}) + + tolerations = override.get("tolerations") + if isinstance(tolerations, list): + parsed = _tolerations_from_config([item for item in tolerations if isinstance(item, dict)]) + if parsed: + k8s.tolerations = parsed + + user_id = override.get("userID") + group_id = override.get("groupID") + if user_id is not None or group_id is not None: + security_context = k8s.security_context or PodSecurityContext() + if user_id is not None: + security_context.run_as_user = int(user_id) + if group_id is not None: + security_context.run_as_group = int(group_id) + security_context.fs_group = int(group_id) + k8s.security_context = security_context + + labels = override.get("labels") + if isinstance(labels, dict): + server_config.labels.update({str(key): str(value) for key, value in labels.items()}) + + init_containers = override.get("initContainers") + if isinstance(init_containers, list): + for entry in init_containers: + if isinstance(entry, dict): + parsed = _container_from_spec_entry(entry, native_sidecar=False) + if parsed is not None: + server_config.init_containers.append(parsed) + + sidecar_containers = override.get("sidecarContainers") + if isinstance(sidecar_containers, list): + for entry in sidecar_containers: + if isinstance(entry, dict): + parsed = _container_from_spec_entry(entry, native_sidecar=True) + if parsed is not None: + server_config.init_containers.append(parsed) + + server_config.backend_config = DeploymentBackendConfig(k8s=k8s) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/status.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/status.py index 3335267a7d..4a3fbce966 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/status.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/status.py @@ -144,3 +144,43 @@ def apply_pending_timeout( timeout_seconds=timeout_seconds, substrate=substrate, ) + + +def build_deleting_timeout_error( + *, + deployment_name: str, + elapsed_seconds: float, + timeout_seconds: int, +) -> DeploymentStatusUpdate: + """Build ERROR status when delete exceeds ``deleting_timeout_seconds``.""" + status_msg = ( + f"Deployment '{deployment_name}' timed out after {format_duration(elapsed_seconds)} waiting for " + f"deployments-plugin substrate teardown (timeout: {format_duration(timeout_seconds)})." + ) + return DeploymentStatusUpdate( + status="ERROR", + status_message=status_msg, + error_details={ + "reason": "deleting_timeout", + "elapsed_seconds": int(elapsed_seconds), + "timeout_seconds": timeout_seconds, + "deployment_name": deployment_name, + }, + ) + + +def apply_deleting_timeout( + result: DeploymentStatusUpdate, + *, + elapsed_seconds: float, + timeout_seconds: int, + deployment_name: str, +) -> DeploymentStatusUpdate: + """Escalate a DELETING delete result to ERROR once teardown ages out.""" + if result.status != "DELETING" or timeout_seconds <= 0 or elapsed_seconds < timeout_seconds: + return result + return build_deleting_timeout_error( + deployment_name=deployment_name, + elapsed_seconds=elapsed_seconds, + timeout_seconds=timeout_seconds, + ) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/docker/__init__.py b/services/core/models/src/nmp/core/models/controllers/backends/docker/__init__.py deleted file mode 100644 index 5ac9939d05..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/docker/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Docker backend for Models Controller service.""" - -from .backend import DockerServiceBackend as DockerServiceBackend -from .config import DockerBackendConfig as DockerBackendConfig diff --git a/services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py b/services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py deleted file mode 100644 index 860980ee64..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py +++ /dev/null @@ -1,742 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Docker backend implementation for Models Controller service. - -Implements the :class:`ServiceBackend` CRUD interface for managing model -deployments as Docker containers. Heavy creation-pipeline logic (image -pulling, model downloading, container creation) is delegated to -:class:`DockerDeploymentCreationReconciler`. -""" - -import asyncio -import os -from logging import getLogger -from typing import Any - -import httpx -from docker.errors import APIError, NotFound -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform_plugin.client.adapter import client_from_platform -from nemo_platform_plugin.client.errors import NotFoundError -from nemo_platform_plugin.secrets.client import AsyncSecretsClient -from nmp.common.config import get_platform_config -from nmp.common.docker.gpu_pool import DockerGPUPool -from nmp.common.resources import SharedResourceManager -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend -from nmp.core.models.controllers.backends.common import ( - LOG_MAX_CHARS, - LOG_TAIL_LINES, - deployment_elapsed_seconds, - format_duration, -) -from nmp.core.models.controllers.backends.docker.config import DockerBackendConfig -from nmp.core.models.controllers.backends.docker.creation_reconciler import ( - NGC_IMAGE_REGISTRY, - NGC_IMAGE_REGISTRY_USER_NAME, - DockerDeploymentCreationReconciler, -) -from nmp.core.models.controllers.context import ModelContext -from requests.exceptions import ConnectionError as RequestsConnectionError -from requests.exceptions import ReadTimeout -from urllib3.exceptions import ReadTimeoutError as Urllib3ReadTimeoutError - -import docker - -logger = getLogger(__name__) - - -def _matches_labels(container_labels: dict[str, str], expected_labels: dict[str, str]) -> bool: - """Return True when all expected Docker labels are present.""" - return all(container_labels.get(key) == value for key, value in expected_labels.items()) - - -class DockerServiceBackend(ServiceBackend): - """Docker-based backend for managing model deployments. - - Schedules models as Docker containers on the local Docker daemon. - Creation pipeline orchestration is delegated to - :class:`DockerDeploymentCreationReconciler`; this class owns the - ``ServiceBackend`` CRUD interface, container health monitoring, - and deployment lifecycle management. - """ - - def init(self) -> None: - """Initialize Docker backend.""" - logger.info("Initializing Docker service backend") - - self._backend_config = DockerBackendConfig(**self._config) - logger.debug(f"Backend config: {self._backend_config.model_dump()}") - - logger.info( - f"Port forwarding enabled: will allocate ports from " - f"{self._backend_config.models_docker_port_range_start} " - f"to {self._backend_config.models_docker_port_range_end}" - ) - - resource_manager = SharedResourceManager.get_instance() - self._gpu_pool: DockerGPUPool | None = resource_manager.get_gpu_pool() - - if self._gpu_pool is None: - logger.warning( - "No GPU pool available - no GPUs were detected on this system. " - "GPU model deployments will fail until GPUs are available." - ) - - try: - timeout = self._backend_config.docker_timeout - - env_timeout = os.getenv("DOCKER_TIMEOUT") - if env_timeout: - logger.warning( - f"DOCKER_TIMEOUT env var is set to {env_timeout}s, which may override " - f"the configured timeout of {timeout}s. Consider unsetting DOCKER_TIMEOUT." - ) - - self._client = docker.from_env(timeout=timeout) - self._client.api.timeout = timeout - - docker_host = os.getenv("DOCKER_HOST", "unix:///var/run/docker.sock") - logger.info(f"Connected to Docker daemon at {docker_host} with {timeout}s timeout") - except Exception as e: - logger.error(f"Failed to initialize Docker client: {e}") - raise - - self._reconciler = DockerDeploymentCreationReconciler( - client=self._client, - backend_config=self._backend_config, - nmp_sdk=self._nmp_sdk, - gpu_pool=self._gpu_pool, - ) - - def shutdown(self) -> None: - """Shutdown Docker backend and release resources.""" - logger.info("Shutting down Docker service backend") - # Cancel all in-flight image-pull / puller tasks so they don't block - # the event loop from draining after the controller receives SIGINT/SIGTERM. - self._reconciler.shutdown() - if hasattr(self, "_client") and self._client is not None: - try: - self._client.close() - logger.debug("Docker client closed") - except Exception as e: - logger.warning(f"Error closing Docker client: {e}") - - # ------------------------------------------------------------------ - # Convenience accessors - # ------------------------------------------------------------------ - - def _get_deployment_key(self, deployment: ModelDeployment) -> str: - """Shorthand for ``_reconciler.get_deployment_key``.""" - return self._reconciler.get_deployment_key(deployment.workspace, deployment.name) - - # ------------------------------------------------------------------ - # NGC authentication - # ------------------------------------------------------------------ - - async def _resolve_ngc_api_key(self) -> str | None: - """Resolve NGC API key from secrets service (or platform env fallback).""" - secret_ref = get_platform_config().ngc_api_key_secret.strip() - if not secret_ref: - fallback = os.environ.get(get_platform_config().ngc_api_key_env_var) - if fallback: - logger.debug("No platform.ngc_api_key_secret configured; using env fallback for NGC API key") - return fallback or None - parts = secret_ref.split("/") - if len(parts) != 2 or not parts[0] or not parts[1]: - logger.warning( - "platform.ngc_api_key_secret must be 'workspace/name'; got %r, skipping NGC key resolution", - secret_ref, - ) - return os.environ.get(get_platform_config().ngc_api_key_env_var) or None - workspace, name = parts[0], parts[1] - try: - secrets = client_from_platform(self._nmp_sdk, AsyncSecretsClient) - response = (await secrets.access_secret(name=name, workspace=workspace)).data() - if response.value: - logger.debug("Resolved NGC API key from secret %s/%s", workspace, name) - return response.value - logger.warning("Secret %s/%s has no data", workspace, name) - except NotFoundError: - logger.info( - "NGC API key secret %s/%s not found; falling back to env %s", - workspace, - name, - get_platform_config().ngc_api_key_env_var, - ) - return os.environ.get(get_platform_config().ngc_api_key_env_var) or None - except Exception as e: - logger.warning("Failed to resolve NGC API key from secret %s/%s: %s", workspace, name, e) - return os.environ.get(get_platform_config().ngc_api_key_env_var) or None - return None - - async def _ensure_ngc_login(self, ngc_api_key: str | None) -> None: - """Log in to the NGC Docker registry if an API key is provided.""" - if not ngc_api_key: - return - try: - logger.info("Authenticating to NGC registry: %s", NGC_IMAGE_REGISTRY) - await asyncio.to_thread( - self._client.login, - username=NGC_IMAGE_REGISTRY_USER_NAME, - password=ngc_api_key, - registry=NGC_IMAGE_REGISTRY, - ) - logger.info("Successfully authenticated to NGC registry") - except Exception as e: - logger.warning("Failed to authenticate to NGC registry: %s", e) - - # ================================================================== - # ServiceBackend CRUD interface - # ================================================================== - - async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Create a new model deployment as a Docker container. - - Resolves NGC credentials and delegates the multi-stage creation - pipeline to :class:`DockerDeploymentCreationReconciler`. - """ - deployment = ctx.model_deployment - config = ctx.model_deployment_config - model_entity = ctx.model_entity - resolved_ngc_key = await self._resolve_ngc_api_key() - await self._ensure_ngc_login(resolved_ngc_key) - - return await self._reconciler.register_deployment( - deployment, - config, - model_entity, - resolved_ngc_key, - ) - - async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Update a model deployment by recreating the container.""" - deployment = ctx.model_deployment - logger.info(f"Updating Docker deployment: {deployment.workspace}/{deployment.name}") - delete_result = await self.delete_model_deployment(deployment.workspace, deployment.name) - if delete_result.status == "ERROR": - return delete_result - return await self.create_model_deployment(ctx) - - async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Get the status of a Docker model deployment. - - While the deployment is still progressing through the creation - pipeline this delegates to the reconciler's ``advance`` method. - """ - deployment = ctx.model_deployment - if self._reconciler.is_deploying(deployment.workspace, deployment.name): - deployment_key = self._reconciler.get_deployment_key(deployment.workspace, deployment.name) - return await self._reconciler.advance(deployment_key) - - container_name = self._reconciler.get_container_name(deployment.workspace, deployment.name) - - try: - container = await asyncio.to_thread(self._reconciler.get_container, container_name) - await asyncio.to_thread(self._reconciler.reload_container, container) - - state = container.status - container_id = container.id[:12] - - host_port = None - ports = container.ports - if ports and "8000/tcp" in ports: - bindings = ports["8000/tcp"] - if bindings and len(bindings) > 0: - host_port = bindings[0].get("HostPort") - - host_url = self._reconciler.get_host_url(container_name, host_port) - - logger.debug("Container status check", extra={"container": container_name, "state": state}) - - if state == "running": - started_at = container.attrs.get("State", {}).get("StartedAt", "unknown") - health_path = self._reconciler.get_health_path_from_container(container) - is_healthy, health_failure_reason = await self._probe_nim_health( - host_url, container_id=container_id, health_path=health_path - ) - - if is_healthy: - return DeploymentStatusUpdate( - status="READY", - status_message=( - f"Container is running and ready for inference (ID: {container_id}, started: {started_at})" - ), - host_url=host_url, - ) - else: - elapsed = deployment_elapsed_seconds(deployment) - timeout = self._backend_config.pending_timeout_seconds - restart_count = container.attrs.get("RestartCount", 0) - max_restarts = self._backend_config.max_restart_count - - error_update = await self._check_crash_loop( - container_name, container_id, restart_count, max_restarts - ) or await self._check_pending_timeout(container_name, container_id, elapsed, timeout) - if error_update: - return error_update - - # Use a stable message (no elapsed/timeout) so we don't create a new history entry every poll - status_msg = ( - f"Container is running but the inference engine is still initializing " - f"(ID: {container_id}, started: {started_at}, " - f"health_url: {host_url}{health_path}" - ) - if health_failure_reason: - status_msg += f", probe_result: {health_failure_reason}" - if restart_count > 0: - status_msg += f", restarts: {restart_count}" - status_msg += ")" - return DeploymentStatusUpdate( - status="PENDING", - status_message=status_msg, - host_url=host_url, - ) - - elif state in ["created", "restarting"]: - elapsed = deployment_elapsed_seconds(deployment) - timeout = self._backend_config.pending_timeout_seconds - restart_count = container.attrs.get("RestartCount", 0) - max_restarts = self._backend_config.max_restart_count - - error_update = await self._check_crash_loop( - container_name, - container_id, - restart_count, - max_restarts, - container_state=state, - ) or await self._check_pending_timeout( - container_name, - container_id, - elapsed, - timeout, - container_state=state, - ) - if error_update: - return error_update - - # Use a stable message (no elapsed/timeout) so we don't create a new history entry every poll - status_msg = f"Container is starting up (state: {state}, ID: {container_id}" - if state == "restarting": - status_msg += f", restart count: {restart_count}" - status_msg += ")" - return DeploymentStatusUpdate( - status="PENDING", - status_message=status_msg, - host_url=host_url, - ) - - elif state in ["exited", "dead"]: - dk = self._reconciler.get_deployment_key(deployment.workspace, deployment.name) - if self._gpu_pool is not None: - released_gpus = self._gpu_pool.release_gpu(dk) - if released_gpus: - logger.info( - "Released GPUs from terminated deployment", - extra={ - "gpu_ids": released_gpus, - "deployment_key": dk, - "container_state": state, - }, - ) - - exit_code = container.attrs.get("State", {}).get("ExitCode", "unknown") - error_msg = f"Container exited with code {exit_code}" - error_stack = "" - - try: - raw_logs = await asyncio.to_thread(container.logs, tail=LOG_TAIL_LINES) - logs = raw_logs.decode("utf-8", errors="ignore") - error_stack = logs - if len(error_stack) > LOG_MAX_CHARS: - error_stack = error_stack[-LOG_MAX_CHARS:] - - last_lines = "\n".join([line for line in logs.split("\n")[-5:] if line.strip()]) - if last_lines: - error_msg += f"\n\nLast log lines:\n{last_lines}" - except Exception as e: - logger.warning( - "Failed to retrieve container logs", - extra={"container_name": container_name, "error": str(e)}, - ) - - return DeploymentStatusUpdate( - status="ERROR", - status_message=error_msg, - error_details={ - "exit_code": exit_code, - "container_state": state, - "container_id": container_id, - "error_stack": error_stack if error_stack else None, - }, - host_url=None, - ) - - elif state == "removing": - return DeploymentStatusUpdate( - status="DELETING", - status_message=f"Container is being removed (ID: {container_id})", - host_url=None, - ) - else: - return DeploymentStatusUpdate( - status="UNKNOWN", - status_message=f"Container in unexpected state: {state} (ID: {container_id})", - host_url=host_url if state != "paused" else None, - ) - - except NotFound: - dk = self._reconciler.get_deployment_key(deployment.workspace, deployment.name) - if self._gpu_pool is not None: - released_gpus = self._gpu_pool.release_gpu(dk) - if released_gpus: - logger.info( - "Released GPUs from lost deployment", - extra={"gpu_ids": released_gpus, "deployment_key": dk}, - ) - - logger.warning( - "Container not found for deployment", - extra={ - "container_name": container_name, - "deployment": f"{deployment.workspace}/{deployment.name}", - }, - ) - return DeploymentStatusUpdate( - status="LOST", - status_message=( - f"Container not found - may have been manually deleted or never created. " - f"Expected container name: {container_name}" - ), - error_details={"expected_container_name": container_name}, - host_url=None, - ) - except (APIError, ReadTimeout, Urllib3ReadTimeoutError, RequestsConnectionError) as e: - logger.error( - "Docker API error checking container status", - extra={"container_name": container_name, "error": str(e)}, - ) - return DeploymentStatusUpdate( - status="UNKNOWN", - status_message=f"Docker API error while checking container status: {e}", - error_details={"error": str(e), "container_name": container_name}, - host_url=None, - ) - except Exception as e: - logger.error( - "Unexpected error checking container status", - extra={"container_name": container_name, "error": str(e)}, - exc_info=True, - ) - return DeploymentStatusUpdate( - status="UNKNOWN", - status_message=f"Failed to get container status: {e}", - error_details={"error": str(e), "container_name": container_name}, - host_url=None, - ) - - async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete a Docker model deployment by workspace and name.""" - return await self._delete_by_model_deployment_id(workspace, name) - - async def list_managed_deployment_names(self) -> list[str]: - """List deployment names the backend currently manages via Docker labels.""" - try: - containers = await asyncio.to_thread( - self._reconciler.list_containers, - all=True, - filters={"label": f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}"}, - ignore_removed=True, - ) - except Exception as e: - logger.warning(f"Failed to list managed containers for orphan reconciliation: {e}") - return [] - - seen: set[str] = set() - owner_labels = self._backend_config.model_labels - for container in containers: - labels = container.labels or {} - if owner_labels and not _matches_labels(labels, owner_labels): - continue - ws = labels.get("nmp.nvidia.com/deployment-workspace") - n = labels.get("nmp.nvidia.com/deployment-name") - if ws and n: - seen.add(f"{ws}/{n}") - return sorted(seen) - - # ================================================================== - # Deletion - # ================================================================== - - async def _delete_by_model_deployment_id(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete a Docker deployment by workspace/name.""" - container_name = self._reconciler.get_container_name(workspace, name) - puller_container_name = self._reconciler.get_puller_container_name(workspace, name) - volume_name = self._reconciler.get_volume_name(workspace, name) - deployment_key = self._reconciler.get_deployment_key(workspace, name) - - # Cancel any in-progress creation task and clean up the reconciler's state. - await self._reconciler.cleanup(deployment_key) - - logger.info( - "Deleting Docker model deployment", - extra={"workspace": workspace, "deployment_name": name, "container": container_name}, - ) - - container_names = [ - (puller_container_name, False), - (f"{container_name}-sidecar", False), - (container_name, True), - ] - - for c_name, fail_on_exception in container_names: - try: - container = await asyncio.to_thread(self._reconciler.get_container, c_name) - container_id = container.id[:12] - - logger.info(f"Stopping container {c_name} (ID: {container_id})") - try: - await asyncio.to_thread(self._reconciler.stop_container, container, timeout=30) - except NotFound: - logger.debug(f"Container {c_name} already stopped/removed") - except (APIError, ReadTimeout, Urllib3ReadTimeoutError, RequestsConnectionError) as e: - logger.warning(f"Failed to stop container {c_name} after retries: {e}") - except Exception as e: - logger.warning(f"Unexpected error stopping container {c_name}: {e}") - - logger.info(f"Removing container {c_name}") - try: - await asyncio.to_thread(self._reconciler.remove_container, container, force=True) - logger.info(f"Successfully removed container {c_name}") - except NotFound: - logger.debug(f"Container {c_name} already removed") - except (APIError, ReadTimeout, Urllib3ReadTimeoutError, RequestsConnectionError) as e: - if "removal" in str(e).lower() or "already" in str(e).lower(): - logger.debug(f"Container {c_name} removal already in progress") - else: - logger.warning(f"Failed to remove container {c_name} after retries: {e}") - except Exception as e: - logger.warning(f"Unexpected error removing container {c_name}: {e}") - - except NotFound: - logger.info(f"Container {c_name} not found, may have been already deleted") - except (APIError, ReadTimeout, Urllib3ReadTimeoutError, RequestsConnectionError) as e: - logger.error(f"Docker API error deleting container {c_name}: {e}", exc_info=True) - if fail_on_exception: - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to delete container due to Docker API error: {e}", - error_details={"error": str(e), "container_name": c_name}, - host_url=None, - ) - except Exception as e: - logger.error(f"Unexpected error deleting container {c_name}: {e}", exc_info=True) - if fail_on_exception: - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to delete container: {e}", - error_details={"error": str(e), "container_name": c_name}, - host_url=None, - ) - - # Remove volumes - volume_names = [volume_name, f"{volume_name}-scratch"] - for v_name in volume_names: - try: - volume = await asyncio.to_thread(self._client.volumes.get, v_name) - logger.info(f"Removing volume {v_name}") - await asyncio.to_thread(volume.remove, force=True) - logger.info(f"Successfully removed volume {v_name}") - except NotFound: - logger.info(f"Volume {v_name} not found, already deleted") - except Exception as e: - logger.warning(f"Error removing volume {v_name}: {e}") - - if self._gpu_pool is not None: - released_gpus = self._gpu_pool.release_gpu(deployment_key) - if released_gpus: - logger.debug(f"Released GPU(s) {released_gpus} from deployment {deployment_key}") - - return DeploymentStatusUpdate( - status="DELETED", - status_message=f"Successfully deleted container {container_name} and cleaned up volume {volume_name}", - host_url=None, - ) - - # ================================================================== - # Container health probing - # ================================================================== - - async def _fetch_container_error_logs(self, container_name: str) -> str: - """Fetch recent container logs for error reporting, truncated to LOG_MAX_CHARS.""" - try: - container = await asyncio.to_thread(self._reconciler.get_container, container_name) - raw_logs = await asyncio.to_thread(container.logs, tail=LOG_TAIL_LINES) - logs = raw_logs.decode("utf-8", errors="ignore") - if len(logs) > LOG_MAX_CHARS: - logs = logs[-LOG_MAX_CHARS:] - return logs - except Exception as e: - logger.warning( - "Failed to retrieve container logs for error report", - extra={"container_name": container_name, "error": str(e)}, - ) - return "" - - async def _build_pending_timeout_error( - self, - container_name: str, - elapsed: float, - ) -> tuple[str, str]: - """Build user-facing error message and error_stack for a PENDING timeout.""" - error_stack = await self._fetch_container_error_logs(container_name) - status_msg = ( - f"Deployment timed out after {format_duration(elapsed)} waiting for NIM " - f"to pass health checks (timeout: {format_duration(self._backend_config.pending_timeout_seconds)}).\n\n" - f"Inspect the NIM container logs by running this command where quickstart is installed:\n" - f" docker logs {container_name}" - ) - return status_msg, error_stack - - async def _build_crash_loop_error( - self, - container_name: str, - restart_count: int, - ) -> tuple[str, str]: - """Build user-facing error message and error_stack for a crash loop.""" - error_stack = await self._fetch_container_error_logs(container_name) - status_msg = ( - f"Deployment entered crash loop after {restart_count} container restarts " - f"(max: {self._backend_config.max_restart_count}).\n\n" - f"Inspect the NIM container logs by running this command where quickstart is installed:\n" - f" docker logs {container_name}" - ) - return status_msg, error_stack - - async def _check_crash_loop( - self, - container_name: str, - container_id: str, - restart_count: int, - max_restarts: int, - container_state: str | None = None, - ) -> DeploymentStatusUpdate | None: - """Return an ERROR status if the container has exceeded the restart limit.""" - if restart_count < max_restarts: - return None - logger.warning( - "Container entered crash loop", - extra={ - "container_name": container_name, - "restart_count": restart_count, - "max_restarts": max_restarts, - }, - ) - status_msg, error_stack = await self._build_crash_loop_error(container_name, restart_count) - error_details: dict[str, Any] = { - "reason": "crash_loop", - "restart_count": restart_count, - "max_restart_count": max_restarts, - "container_name": container_name, - "container_id": container_id, - "error_stack": error_stack if error_stack else None, - } - if container_state is not None: - error_details["container_state"] = container_state - return DeploymentStatusUpdate( - status="ERROR", - status_message=status_msg, - error_details=error_details, - host_url=None, - ) - - async def _check_pending_timeout( - self, - container_name: str, - container_id: str, - elapsed: float, - timeout: int, - container_state: str | None = None, - ) -> DeploymentStatusUpdate | None: - """Return an ERROR status if the pending timeout has been exceeded.""" - if elapsed < timeout: - return None - logger.warning( - "Deployment PENDING timeout exceeded", - extra={ - "container_name": container_name, - "elapsed": format_duration(elapsed), - "timeout": format_duration(timeout), - }, - ) - status_msg, error_stack = await self._build_pending_timeout_error(container_name, elapsed) - error_details: dict[str, Any] = { - "reason": "pending_timeout", - "elapsed_seconds": int(elapsed), - "timeout_seconds": timeout, - "container_name": container_name, - "container_id": container_id, - "error_stack": error_stack if error_stack else None, - } - if container_state is not None: - error_details["container_state"] = container_state - return DeploymentStatusUpdate( - status="ERROR", - status_message=status_msg, - error_details=error_details, - host_url=None, - ) - - async def _probe_nim_health( - self, - host_url: str, - timeout: float = 5.0, - container_id: str | None = None, - health_path: str = "/v1/health/ready", - ) -> tuple[bool, str]: - """Probe the inference engine's health endpoint to check readiness. - - NIM exposes /v1/health/ready and vLLM exposes /health; both return 200 when - the model is fully loaded and ready to serve. The path is engine-specific and - provided by the caller. - - Args: - host_url: The base URL of the container (e.g., http://localhost:8500) - timeout: Request timeout in seconds - container_id: Optional container ID for logging context - health_path: Engine-specific readiness path appended to host_url - - Returns: - Tuple of (is_healthy, failure_reason). failure_reason is empty string if healthy. - """ - health_url = f"{host_url}{health_path}" - container_ctx = f" (container: {container_id})" if container_id else "" - try: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.get(health_url) - if response.status_code == 200: - logger.debug(f"NIM health check passed: {health_url}{container_ctx}") - return True, "" - else: - reason = f"HTTP {response.status_code}" - try: - body = response.text[:200] if response.text else "" - if body: - reason += f" - {body}" - except Exception: - pass - logger.info(f"NIM health check returned {response.status_code}: {health_url}{container_ctx}") - return False, reason - except httpx.TimeoutException: - reason = f"timeout after {timeout}s" - logger.info(f"NIM health check timed out ({timeout}s): {health_url}{container_ctx}") - return False, reason - except httpx.ConnectError as e: - reason = f"connection error: {e}" - logger.info(f"NIM health check connection error: {health_url}{container_ctx} - {e}") - return False, reason - except Exception as e: - reason = f"{type(e).__name__}: {e}" - logger.info(f"NIM health check failed: {health_url}{container_ctx} - {reason}") - return False, reason diff --git a/services/core/models/src/nmp/core/models/controllers/backends/docker/config.py b/services/core/models/src/nmp/core/models/controllers/backends/docker/config.py deleted file mode 100644 index 46c86d684a..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/docker/config.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Configuration model for docker backend.""" - -import os -from enum import Enum -from typing import Literal - -from nemo_platform_plugin.jobs.image import get_qualified_image -from pydantic import BaseModel, Field - -MODELS_DOCKER_NETWORKING_MODE = os.getenv("MODELS_DOCKER_NETWORKING_MODE", "local") -MODELS_DOCKER_NETWORK = os.getenv("MODELS_DOCKER_NETWORK", "") -MODELS_DOCKER_CONTAINER_NAME = os.getenv("MODELS_DOCKER_CONTAINER_NAME", "") -MODELS_DOCKER_HOST_SERVICE_NAME = os.getenv("MODELS_DOCKER_HOST_SERVICE_NAME", "localhost") -MODELS_DOCKER_PORT_RANGE_START = os.getenv("MODELS_DOCKER_PORT_RANGE_START", "49152") -MODELS_DOCKER_PORT_RANGE_END = os.getenv("MODELS_DOCKER_PORT_RANGE_END", "49652") -MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE = os.getenv("MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE", "") -MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU = os.getenv("MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU", "") - - -class DockerNetworkingMode(str, Enum): - """Networking mode for the Docker backend. - - Determines how NeMo Platform communicates with NIM containers. - """ - - LOCAL = "local" - """Local development: NeMo Platform runs on host, NIMs get ports mapped to localhost.""" - - DOND = "dond" - """Docker on Docker: NeMo Platform runs in a container with Docker socket mounted. - NIMs are sibling containers and communicate via a shared Docker network.""" - - DIND = "dind" - """Docker in Docker: NeMo Platform runs in a container with a DinD sidecar. - NIMs run inside DinD and communicate via port forwarding through the DinD service.""" - - -class DockerBackendConfig(BaseModel): - """ - Configuration for the Docker backend. - """ - - default_nimservice_image: str = Field( - default="nvcr.io/nim/nvidia/llm-nim", - description="Default NIM image when none is specified (multi-LLM image)", - ) - - default_nimservice_image_tag: str = Field( - default="1.13.1", - description="Default NIM image tag when none is specified", - ) - - default_vllm_image: str = Field( - default="vllm/vllm-openai", - description="Default vLLM image when none is specified for engine='vllm'", - ) - - default_vllm_image_tag: str = Field( - default="v0.22.1", - description="Default vLLM image tag when none is specified for engine='vllm'", - ) - - nim_guided_decoding_backend: str = Field( - default="outlines", - description="NIM guided decoding backend", - ) - - peft_source: str = Field( - default="", - description="PEFT/LoRA source URL for models service", - ) - - peft_refresh_interval: int = Field( - default=30, - description="PEFT/LoRA refresh interval in seconds", - ) - - files_auth_secret: str = Field( - default="files-hf-token", - description="Secret name for Files service authentication", - ) - - docker_timeout: int = Field( - default=600, - description="Docker client timeout in seconds for long-running operations (default: 10 minutes)", - ) - - # ========================================================================== - # Networking configuration - # ========================================================================== - - models_docker_networking_mode: Literal["local", "dond", "dind"] = Field( - default=MODELS_DOCKER_NETWORKING_MODE, - description="Networking mode for NIM containers: " - "'local' (port forwarding to localhost for local dev), " - "'dond' (container names on shared network for quickstart), " - "'dind' (port forwarding to docker service for DinD setups).", - ) - - models_docker_network: str = Field( - default=MODELS_DOCKER_NETWORK, - description="Docker network name for 'dond' mode. NIMs will join this network to communicate " - "with the NeMo Platform container. Required when MODELS_DOCKER_NETWORKING_MODE='dond'. " - "Quickstart sets this automatically via MODELS_DOCKER_NETWORK env var.", - ) - - models_docker_container_name: str = Field( - default=MODELS_DOCKER_CONTAINER_NAME, - description="Container name for 'dond' mode. Used to replace localhost in URLs passed to NIMs " - "so they can reach services via the Docker network. " - "Quickstart sets this automatically via MODELS_DOCKER_CONTAINER_NAME env var.", - ) - - models_docker_host_service_name: str = Field( - default=MODELS_DOCKER_HOST_SERVICE_NAME, - description="Hostname for port forwarding in 'dind' mode. Typically 'localhost' or the " - "DinD service name. Used to construct URLs like http://{hostname}:{port}.", - ) - - models_docker_port_range_start: int = Field( - default=int(MODELS_DOCKER_PORT_RANGE_START), - description="Start of port range for port forwarding (inclusive). " - "Defaults to start of IANA dynamic/ephemeral port range.", - ) - - models_docker_port_range_end: int = Field( - default=int(MODELS_DOCKER_PORT_RANGE_END), - description="End of port range for port forwarding (inclusive). " - "Defaults to 500-port range within IANA dynamic/ephemeral range.", - ) - - huggingface_model_puller: str = Field( - default_factory=lambda: get_qualified_image("nmp-api"), - description="Image used to pull model weights. Its entrypoint is overridden to the " - "Hugging Face CLI ('hf download ...'), so any image with the 'hf' CLI on PATH works. " - "Defaults to the platform's nmp-api image (registry/tag from platform config); override " - "to use a different puller image.", - ) - - huggingface_model_puller_env: dict[str, str] = Field( - default_factory=dict, - description="Extra environment variables for the model puller container. Values override the " - "reconciler defaults (HF_ENDPOINT, HF_TOKEN). E.g. set HF_HUB_ENABLE_HF_TRANSFER='0' to " - "disable the hf_transfer download path.", - ) - - model_puller_timeout: int = Field( - default=1800, - description="Timeout in seconds for the model puller container to complete (default: 30 minutes)", - ) - - model_puller_download_timeout: int = Field( - default=7200, - description="Per-file HTTP download timeout in seconds for the model puller (HF_HUB_DOWNLOAD_TIMEOUT). " - "Increase if large model files fail with IncompleteRead/ChunkedEncodingError (default: 2 hours).", - ) - - model_puller_max_workers: int = Field( - default=1, - ge=1, - le=16, - description="Max concurrent file downloads in the model puller (hf download --max-workers). " - "Default 1 (sequential) reduces IncompleteRead/ChunkedEncodingError on slow or flaky links; increase for speed.", - ) - - model_puller_retries: int = Field( - default=3, - ge=1, - le=10, - description="Number of retries when the puller fails with a transient error (IncompleteRead, " - "ChunkedEncodingError, connection broken). Same volume is reused so partial downloads can be completed.", - ) - - model_labels: dict[str, str] = Field( - default_factory=dict, - description=("Additional labels copied onto model resources managed by the models controller. "), - ) - - busybox_image: str = Field( - default="busybox", - description="BusyBox image repository used for helper containers (permissions/find/chown).", - ) - - busybox_image_tag: str = Field( - default="latest", - description="BusyBox image tag used for helper containers.", - ) - - lora_sidecar_image_name: str = Field( - default="nmp-api", - description=( - "Image name (without registry/tag) used for the LoRA adapters sidecar container. " - "Registry and tag are taken from NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG. " - "The sidecar is invoked via the lora_sidecar_command." - ), - ) - - lora_sidecar_command: list[str] = Field( - default=["--sidecars", "adapters", "--port", "60830"], - description=( - "Command passed to the LoRA sidecar container. " - "Default uses the nmp-platform-runner entrypoint present in nmp-api. " - ), - ) - - lora_sidecar_entrypoint: str = Field( - default="", - description=( - "Optional entrypoint override for the LoRA sidecar container. " - "Leave empty to use the image's default entrypoint (correct for nmp-api). " - ), - ) - - # ========================================================================== - # PENDING timeout and crash loop detection - # ========================================================================== - - pending_timeout_seconds: int = Field( - default=7200, - ge=60, - description="Maximum time (in seconds) a deployment may stay in PENDING before being " - "transitioned to ERROR. Default: 7200 (2 hours).", - ) - - max_restart_count: int = Field( - default=5, - ge=1, - description="Maximum number of container restarts before a PENDING deployment is " - "transitioned to ERROR (crash loop detection). Default: 5.", - ) - - nim_multi_gpu_shm_size: str = Field( - default="", - description="Optional fixed shared memory size (/dev/shm) for multi-GPU NIM containers. " - "If set, overrides the per-GPU calculation. Leave empty to use shm_size_per_gpu x GPU count. " - "Override via MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE env. Format: e.g. '2g', '4g'.", - ) - - nim_multi_gpu_shm_size_per_gpu: int = Field( - default=1024, - ge=1, - description="Shared memory size (/dev/shm) per GPU in megabytes for multi-GPU NIM containers. " - "Total shm = this value x GPU count (e.g. 1024 x 2 GPUs = 2048m). " - "Override via MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU env (integer string).", - ) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py b/services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py deleted file mode 100644 index 8d92c431c5..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py +++ /dev/null @@ -1,1583 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Docker deployment creation reconciler for the Models Controller service. - -Manages the multi-stage, non-blocking pipeline that creates a model deployment -as a Docker container: image pull → (optional) model puller → container creation. - -The DockerServiceBackend delegates to this class for all creation-related Docker -orchestration, keeping the backend focused on the ServiceBackend CRUD interface, -container health monitoring, and lifecycle management. -""" - -import asyncio -import logging -import os -import socket -from dataclasses import dataclass, field -from enum import Enum -from logging import getLogger -from typing import Any, Dict, Optional -from urllib.parse import urljoin - -from docker.errors import APIError, ImageNotFound, NotFound -from docker.models.containers import Container -from docker.models.volumes import Volume -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig -from nemo_platform.types.models.model_entity import ModelEntity -from nemo_platform_plugin.client.adapter import client_from_platform -from nemo_platform_plugin.jobs.image import get_qualified_image -from nemo_platform_plugin.secrets.client import AsyncSecretsClient -from nmp.common.config import get_auth_config, get_platform_config -from nmp.common.config.base import LOOPBACK_ADDRESSES -from nmp.common.docker.gpu_pool import DockerGPUPool, GPUAllocationError -from nmp.common.sdk_factory import get_sdk_on_behalf_of -from nmp.core.models.app import ModelWeightsType, get_model_weights_type, is_multi_llm_image, parse_model_name_revision -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.app.utils import ( - get_docker_container_name, - get_docker_plugin_puller_container_name, - get_docker_puller_container_name, - get_docker_volume_name, -) -from nmp.core.models.controllers.backends import generic_compiler, vllm_compiler -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.common import deployment_config_view -from nmp.core.models.controllers.backends.docker.config import ( - MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE, - MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU, - DockerBackendConfig, -) -from nmp.core.models.controllers.backends.engine import ( - ENGINE_GENERIC, - ENGINE_HEALTH_PATHS, - ENGINE_LABEL, - ENGINE_NIM, - ENGINE_VLLM, - HEALTH_PATH_LABEL, -) -from nmp.core.models.controllers.backends.engine import ( - config_engine as _config_engine, -) -from nmp.core.models.controllers.backends.engine import ( - resolve_health_path as _resolve_health_path, -) -from requests.exceptions import ConnectionError as RequestsConnectionError -from requests.exceptions import ReadTimeout -from tenacity import before_sleep_log, retry, stop_after_attempt, wait_exponential -from urllib3.exceptions import ReadTimeoutError as Urllib3ReadTimeoutError - -import docker - -logger = getLogger(__name__) - -# --------------------------------------------------------------------------- -# Retry configuration for Docker operations -# --------------------------------------------------------------------------- - -DOCKER_RETRY_ATTEMPTS = int(os.getenv("MODELS_DOCKER_RETRY_ATTEMPTS", "7")) -DOCKER_RETRY_WAIT_MIN = 2 -DOCKER_RETRY_WAIT_MAX = 10 - -HUGGINGFACE_HUB_URL = "https://huggingface.co" - -NGC_IMAGE_REGISTRY = os.getenv("NGC_IMAGE_REGISTRY", "nvcr.io") -NGC_IMAGE_REGISTRY_USER_NAME = os.getenv("NGC_IMAGE_REGISTRY_USER_NAME", "$oauthtoken") - - -def _should_retry_docker_error(exception: BaseException) -> bool: - """Determine if a Docker exception should be retried.""" - if isinstance(exception, (NotFound, ImageNotFound)): - return False - if isinstance(exception, APIError) and exception.status_code == 409: - return False - return isinstance( - exception, (ReadTimeout, Urllib3ReadTimeoutError, RequestsConnectionError, TimeoutError, APIError) - ) - - -docker_retry = retry( - stop=stop_after_attempt(DOCKER_RETRY_ATTEMPTS), - wait=wait_exponential(multiplier=1, min=DOCKER_RETRY_WAIT_MIN, max=DOCKER_RETRY_WAIT_MAX), - retry=_should_retry_docker_error, - before_sleep=before_sleep_log(logger, logging.WARNING), - reraise=True, -) - - -def _compute_multi_gpu_shm_size(fixed_total: str, per_gpu_mb: int, gpu_count: int) -> str: - """Compute shm_size for multi-GPU deployment. - - Returns fixed_total if non-empty (e.g. '4g'), else per_gpu_mb * gpu_count as '{n}m'. - """ - if fixed_total: - return fixed_total - return f"{per_gpu_mb * gpu_count}m" - - -# --------------------------------------------------------------------------- -# Creation pipeline data types -# --------------------------------------------------------------------------- - - -class CreationStage(str, Enum): - """Stages of the non-blocking deployment creation pipeline.""" - - PULLING_NIM_IMAGE = "pulling_nim_image" - PULLING_PULLER_IMAGE = "pulling_puller_image" - RUNNING_PULLER = "running_puller" - CREATING_CONTAINER = "creating_container" - - -@dataclass -class CreationState: - """In-memory state for a deployment progressing through the creation pipeline.""" - - stage: CreationStage - task: asyncio.Task | None = None - deployment: Any = None - config: Any = None - model_entity: Any = None - model_weights_type: Any = None - volume_name: str = "" - scratch_volume_name: str = "" - nim_image: str = "" - ngc_api_key: str | None = None - is_multi_llm: bool = False - puller_container_name: str = "" - tool_call_plugin_path: str | None = None - plugin_fileset: str | None = None - error_details: dict[str, Any] = field(default_factory=dict) - - -# --------------------------------------------------------------------------- -# DockerDeploymentCreationReconciler -# --------------------------------------------------------------------------- - - -class DockerDeploymentCreationReconciler: - """Manages the multi-stage deployment creation pipeline and Docker operations. - - Handles image pulling, model weight downloading (via the HuggingFace puller - container), GPU/port allocation, and final NIM container creation. - - Owns the in-progress creation state for all deployments; the backend delegates - to :meth:`register_deployment`, :meth:`is_deploying`, :meth:`advance`, - :meth:`cleanup`, and :meth:`shutdown`. - """ - - def __init__( - self, - client: docker.DockerClient, - backend_config: DockerBackendConfig, - nmp_sdk: Any, - gpu_pool: DockerGPUPool | None, - ) -> None: - self._client = client - self._backend_config = backend_config - self._nmp_sdk = nmp_sdk - self._gpu_pool = gpu_pool - # In-progress creation state: maps deployment key -> CreationState. - self._creation_states: dict[str, CreationState] = {} - - # ====================================================================== - # Helpers - # ====================================================================== - - def _get_busybox_image(self) -> str: - """Return the configured BusyBox image reference.""" - return f"{self._backend_config.busybox_image}:{self._backend_config.busybox_image_tag}" - - # ====================================================================== - # Docker SDK wrappers (retry-decorated) - # ====================================================================== - - @docker_retry - def get_container(self, container_name: str) -> Container: - return self._client.containers.get(container_name) - - @docker_retry - def reload_container(self, container: Container) -> None: - container.reload() - - @docker_retry - def stop_container(self, container: Container, timeout: int = 30) -> None: - container.stop(timeout=timeout) - - @docker_retry - def remove_container(self, container: Container, force: bool = True) -> None: - container.remove(force=force) - - @docker_retry - def list_containers(self, **kwargs: Any) -> list[Container]: - return self._client.containers.list(**kwargs) - - @docker_retry - def create_volume(self, volume_name: str) -> Volume: - return self._client.volumes.create(volume_name) - - @docker_retry - def pull_image(self, image_name: str, image_tag: str | None = None) -> None: - self._client.images.pull(image_name, tag=image_tag) - - @docker_retry - def run_container(self, **kwargs: Any) -> Container: - return self._client.containers.run(**kwargs) - - @docker_retry - def create_and_start_container(self, create_args: Dict[str, Any]) -> Container: - container = self._client.containers.create(**create_args) - container.start() - return container - - def pull_image_if_not_local(self, image: str) -> None: - """Pull an image only when it is not already available locally.""" - logger.info(f"Checking for image {image}...") - try: - self._client.images.get(image) - logger.info(f"Image {image} found locally") - except ImageNotFound: - logger.info(f"Image {image} not found locally, pulling from registry...") - self.pull_image(image) - logger.info(f"Successfully pulled image {image}") - - # ====================================================================== - # Name / key generators - # ====================================================================== - - def get_container_name(self, workspace: str, name: str) -> str: - """Primary NIM container name; capped at 55 chars to leave room for ``-sidecar``.""" - return get_docker_container_name(workspace, name) - - def get_volume_name(self, workspace: str, name: str) -> str: - """Model cache volume name (hashed ``workspace/name`` identity).""" - return get_docker_volume_name(workspace, name) - - def get_puller_container_name(self, workspace: str, name: str) -> str: - """SFT/model puller container name (hashed ``workspace/name`` identity).""" - return get_docker_puller_container_name(workspace, name) - - def get_plugin_puller_container_name(self, workspace: str, name: str) -> str: - """Tool-call plugin fileset puller container name.""" - return get_docker_plugin_puller_container_name(workspace, name) - - def get_deployment_key(self, workspace: str, name: str) -> str: - return f"{workspace}/{name}" - - def get_health_path_from_container(self, container: Container) -> str: - """Resolve the readiness probe path from the container's labels. - - Prefers the explicit health-path label (which already accounts for a - user-supplied ``executor_config.health_check_path`` resolved at create - time). Falls back to the engine label's standard endpoint, then the NIM - path for older containers that predate these labels. - """ - try: - labels = container.labels or {} - except Exception: - labels = {} - explicit_path = labels.get(HEALTH_PATH_LABEL) - if explicit_path: - return explicit_path - engine = str(labels.get(ENGINE_LABEL, ENGINE_NIM)).lower() - return ENGINE_HEALTH_PATHS.get(engine, ENGINE_HEALTH_PATHS[ENGINE_NIM]) - - def _managed_container_labels( - self, - deployment: ModelDeployment, - extra_labels: dict[str, str] | None = None, - ) -> dict[str, str]: - """Build labels for Docker resources managed by the models controller.""" - return { - **self._backend_config.model_labels, - "nmp.nvidia.com/deployment-workspace": deployment.workspace, - "nmp.nvidia.com/deployment-name": deployment.name, - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - **(extra_labels or {}), - } - - # ====================================================================== - # Network / URL helpers - # ====================================================================== - - def get_host_url(self, container_name: str, host_port: int | None) -> str: - match self._backend_config.models_docker_networking_mode: - case "dond": - return f"http://{container_name}:8000" - case "dind": - return f"http://{self._backend_config.models_docker_host_service_name}:{host_port}" - case _: - return f"http://localhost:{host_port}" - - def _should_attach_network(self) -> bool: - return self._backend_config.models_docker_networking_mode == "dond" - - def _assign_network(self, run_kwargs: dict) -> None: - if self._backend_config.models_docker_networking_mode == "local": - run_kwargs["network_mode"] = "host" - logger.info("Container will use host network (local mode)") - elif self._should_attach_network() and self._backend_config.models_docker_network: - run_kwargs["network"] = self._backend_config.models_docker_network - logger.info(f"Container will join network: {self._backend_config.models_docker_network}") - - def _get_hf_compatible_files_url(self) -> str: - files_url = get_platform_config().get_service_url("files") - if ( - self._backend_config.models_docker_networking_mode == "dond" - and self._backend_config.models_docker_container_name - ): - for loopback in LOOPBACK_ADDRESSES: - if loopback in files_url: - files_url = files_url.replace(loopback, self._backend_config.models_docker_container_name) - logger.info( - f"DOND mode: replaced {loopback} with {self._backend_config.models_docker_container_name} in files_url" - ) - break - return urljoin(files_url, "/apis/files/v2/hf") - - # ====================================================================== - # Port allocation - # ====================================================================== - - def _is_remote_docker_host(self) -> bool: - docker_host = os.environ.get("DOCKER_HOST", "") - return docker_host.startswith("tcp://") - - def _is_port_free(self, port: int) -> bool: - if self._is_remote_docker_host(): - logger.debug(f"Remote Docker host detected, skipping local port check for {port}") - return True - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind(("0.0.0.0", port)) # noqa: S104 # nosec B104 - return True - except OSError: - logger.debug(f"Port {port} is not free (system process may be using it)") - return False - - async def find_available_port(self) -> Optional[int]: - try: - all_containers = await asyncio.to_thread( - self.list_containers, - all=True, - filters={"label": f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}"}, - ignore_removed=True, - ) - except Exception as e: - logger.error(f"Failed to list containers: {e}") - return None - - used_ports: set[int] = set() - for container in all_containers: - try: - ports = container.ports - if ports: - for port_bindings in ports.values(): - if port_bindings: - for binding in port_bindings: - if binding and "HostPort" in binding: - used_ports.add(int(binding["HostPort"])) - except Exception as e: - logger.warning(f"Failed to get ports for container {container.name}: {e}") - - for port in range( - self._backend_config.models_docker_port_range_start, - self._backend_config.models_docker_port_range_end + 1, - ): - if port not in used_ports and self._is_port_free(port): - logger.debug(f"Found available port: {port}") - return port - - logger.error( - f"No available ports in range " - f"{self._backend_config.models_docker_port_range_start}-{self._backend_config.models_docker_port_range_end}" - ) - return None - - # ====================================================================== - # Model helpers - # ====================================================================== - - def _extract_model_repo_from_artifact(self, model_entity: ModelEntity) -> str: - if not model_entity.fileset: - raise ValueError("Model entity fileset is required for Files service weights") - files_url_str = str(model_entity.fileset) - model_repo = files_url_str.removeprefix("hf://").removeprefix("fileset://") - return model_repo - - def _get_model_repo_from_entity( - self, - model_entity: ModelEntity | None, - model_weights_type: ModelWeightsType, - nim_config: Any = None, - ) -> str: - if model_weights_type == ModelWeightsType.FILES_SERVICE: - if model_entity and model_entity.fileset: - return self._extract_model_repo_from_artifact(model_entity) - if not nim_config: - raise ValueError("nim_config is required for FILES_SERVICE without model entity artifact") - model_workspace, model_name, _ = parse_model_name_revision( - model_namespace=nim_config.model_namespace, - model_name=nim_config.model_name, - model_revision=nim_config.model_revision, - ) - if not model_workspace or not model_name: - raise ValueError( - f"Cannot determine fileset path: missing workspace or name " - f"(workspace={model_workspace}, name={model_name})" - ) - return f"{model_workspace}/{model_name}" - elif model_weights_type == ModelWeightsType.HUGGINGFACE: - if not nim_config: - raise ValueError("nim_config is required for HUGGINGFACE model type") - model_workspace, model_name, _ = parse_model_name_revision( - model_namespace=nim_config.model_namespace, - model_name=nim_config.model_name, - model_revision=nim_config.model_revision, - ) - if not model_workspace or not model_name: - raise ValueError( - f"Cannot determine HF model repo: missing model workspace or name " - f"(workspace={model_workspace}, name={model_name})" - ) - return f"{model_workspace}/{model_name}" - else: - raise ValueError( - f"Model weights type: {model_weights_type} is not supported. " - f"Supported types: {ModelWeightsType.FILES_SERVICE}, {ModelWeightsType.HUGGINGFACE}" - ) - - # ====================================================================== - # Creation pipeline – public entry points - # ====================================================================== - - async def register_deployment( - self, - deployment: ModelDeployment, - config: ModelDeploymentConfig, - model_entity: Optional[ModelEntity], - ngc_api_key: str | None, - ) -> DeploymentStatusUpdate: - """Kick off the creation pipeline for a new deployment. - - Performs quick validation and setup, then starts the first long-running - phase (NIM image pull) as a background task. Stores the ``CreationState`` - internally and returns an initial PENDING status (or ERROR on setup failure). - """ - container_name = self.get_container_name(deployment.workspace, deployment.name) - deployment_key = self.get_deployment_key(deployment.workspace, deployment.name) - logger.info( - f"Creating Docker deployment: {deployment.workspace}/{deployment.name} (container: {container_name})" - ) - - # Release any stale GPU allocations from a previous deployment attempt. - if self._gpu_pool is not None: - released_gpus = self._gpu_pool.release_gpu(deployment_key) - if released_gpus: - logger.info(f"Released stale GPU allocation {released_gpus} for {deployment_key} before recreation") - - model_weights_type = get_model_weights_type( - model_deployment=deployment, - model_deployment_config=config, - model_entity=model_entity, - ) - weights_from_files = model_weights_type == ModelWeightsType.FILES_SERVICE - if weights_from_files: - logger.info( - f"Pulling weights from Files service for {deployment.workspace}/{deployment.name}, " - "will run model puller first" - ) - - # Check if container already exists - try: - existing_container = await asyncio.to_thread(self.get_container, container_name) - logger.warning(f"Container {container_name} already exists, removing it first") - try: - await asyncio.to_thread(existing_container.stop, timeout=10) - except Exception: - pass - await asyncio.to_thread(existing_container.remove, force=True) - except NotFound: - pass - except Exception as e: - logger.error(f"Error checking for existing container {container_name}: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to check for existing container: {e}", - error_details={"error": str(e)}, - host_url=None, - ) - - engine = _config_engine(config) - view = deployment_config_view(config) - if engine == ENGINE_VLLM: - image_name, image_tag = vllm_compiler.resolve_vllm_image( - view, - self._backend_config.default_vllm_image, - self._backend_config.default_vllm_image_tag, - ) - elif engine == ENGINE_GENERIC: - # Generic containers have no platform-default image; image_name is - # required (enforced at the API layer and again in the compiler). - image_name, image_tag = generic_compiler.resolve_generic_image(view) - else: - image_name = view.image_name or self._backend_config.default_nimservice_image - image_tag = view.image_tag or self._backend_config.default_nimservice_image_tag - full_image = f"{image_name}:{image_tag}" - logger.info(f"Using image: {full_image} (engine={engine})") - - # Create volumes for model cache + scratch. A generic container that pulls - # no weights runs raw (no platform volumes mounted -- see container create - # below), so skip provisioning them; every other case mounts them. - volume_name = self.get_volume_name(deployment.workspace, deployment.name) - scratch_volume_name = volume_name + "-scratch" - provision_volumes = engine != ENGINE_GENERIC or weights_from_files - if provision_volumes: - try: - await asyncio.to_thread(self.create_volume, volume_name) - logger.info(f"Created volume: {volume_name}") - except Exception as e: - logger.warning(f"Failed to create volume {volume_name} (may already exist): {e}") - - try: - await asyncio.to_thread(self.create_volume, scratch_volume_name) - logger.info(f"Created volume: {scratch_volume_name}") - except Exception as e: - logger.warning(f"Failed to create volume {scratch_volume_name} (may already exist): {e}") - - # Multi-LLM detection only applies to NIM images; vLLM/generic are never multi-LLM. - is_multi_llm = False - if engine == ENGINE_NIM: - effective_image = view.image_name or self._backend_config.default_nimservice_image - is_multi_llm = is_multi_llm_image(effective_image) - logger.debug(f"Detected multi-LLM image: {is_multi_llm} (effective_image={effective_image})") - - plugin_fileset: str | None = None - if view.tool_call_config and view.tool_call_config.tool_call_plugin: - plugin_fileset = view.tool_call_config.tool_call_plugin - elif ( - model_entity - and model_entity.spec - and model_entity.spec.tool_call_config - and model_entity.spec.tool_call_config.tool_call_plugin - ): - plugin_fileset = model_entity.spec.tool_call_config.tool_call_plugin - - # Start NIM image pull in background - logger.info(f"Starting background image pull for {full_image}") - pull_task = asyncio.ensure_future(asyncio.to_thread(self.pull_image_if_not_local, full_image)) - - state = CreationState( - stage=CreationStage.PULLING_NIM_IMAGE, - task=pull_task, - deployment=deployment, - config=config, - model_entity=model_entity, - model_weights_type=model_weights_type, - volume_name=volume_name, - scratch_volume_name=scratch_volume_name, - nim_image=full_image, - ngc_api_key=ngc_api_key, - is_multi_llm=is_multi_llm, - plugin_fileset=plugin_fileset, - ) - - self._creation_states[deployment_key] = state - - return DeploymentStatusUpdate( - status="PENDING", - status_message=( - f"Pulling container image {full_image}. This may take several minutes for large NIM images." - ), - ) - - def is_deploying(self, workspace: str, name: str) -> bool: - """Return True while the deployment is still progressing through the creation pipeline.""" - return self.get_deployment_key(workspace, name) in self._creation_states - - async def advance(self, deployment_key: str) -> DeploymentStatusUpdate: - """Advance the creation pipeline one step and return the current status. - - The full lifecycle is readable here: - PULLING_NIM_IMAGE → (needs puller?) PULLING_PULLER_IMAGE → RUNNING_PULLER - (no puller?) CREATING_CONTAINER - - RUNNING_PULLER and CREATING_CONTAINER are complex enough to warrant their - own helpers; the transitions into and out of those stages remain visible below. - """ - state = self._creation_states.get(deployment_key) - if state is None: - return DeploymentStatusUpdate( - status="ERROR", - status_message="Internal error: creation state lost", - error_details={"error": "creation_state_missing", "deployment_key": deployment_key}, - ) - try: - match state.stage: - # ── Stage 1: pull NIM container image ─────────────────────────── - case CreationStage.PULLING_NIM_IMAGE: - if not self._is_task_complete(state): - return DeploymentStatusUpdate( - status="PENDING", - status_message=( - f"Pulling container image {state.nim_image}. " - "This may take several minutes for large NIM images." - ), - ) - - error = self._get_task_error(state) - if error is not None: - error_str = str(error) - logger.error("Image pull failed for %s: %s", deployment_key, error_str) - self._creation_states.pop(deployment_key, None) - if isinstance(error, ImageNotFound): - return DeploymentStatusUpdate( - status="ERROR", - status_message=( - f"Image not found in registry: {state.nim_image}. " - "Ensure the image exists and platform.ngc_api_key_secret " - "(or NGC_API_KEY env) is set correctly." - ), - error_details={"error": error_str, "image": state.nim_image}, - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to pull image {state.nim_image}: {error_str}", - error_details={"error": error_str, "image": state.nim_image, "stage": "image_pull"}, - ) - - logger.info("NIM image pull complete for %s", deployment_key) - - if self._needs_puller(state): - # ── Transition → PULLING_PULLER_IMAGE ─────────────────── - puller_image = self._backend_config.huggingface_model_puller - logger.info("Starting background puller image pull for %s", deployment_key) - state.task = asyncio.ensure_future( - asyncio.to_thread(self.pull_image_if_not_local, puller_image) - ) - state.stage = CreationStage.PULLING_PULLER_IMAGE - return DeploymentStatusUpdate( - status="PENDING", - status_message=f"NIM image ready. Pulling model puller image {puller_image}...", - ) - - # ── Transition → CREATING_CONTAINER ───────────────────────── - state.stage = CreationStage.CREATING_CONTAINER - state.task = None - status, complete = await self._advance_creating_container(deployment_key, state) - if complete: - self._creation_states.pop(deployment_key, None) - return status - - # ── Stage 2 (optional): pull model-puller image ────────────────── - case CreationStage.PULLING_PULLER_IMAGE: - if not self._is_task_complete(state): - return DeploymentStatusUpdate( - status="PENDING", - status_message="Pulling model puller image...", - ) - - error = self._get_task_error(state) - if error is not None: - error_str = str(error) - logger.error("Puller image pull failed for %s: %s", deployment_key, error_str) - self._creation_states.pop(deployment_key, None) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to pull model puller image: {error_str}", - error_details={"error": error_str, "stage": "puller_image_pull"}, - ) - - logger.info("Puller image ready for %s, starting model puller container", deployment_key) - try: - puller_container = await self._start_model_puller_container(state) - except Exception as e: - logger.error("Failed to start model puller for %s: %s", deployment_key, e) - self._creation_states.pop(deployment_key, None) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to start model puller: {e}", - error_details={"error": str(e), "stage": "model_puller"}, - ) - - # ── Transition → RUNNING_PULLER ────────────────────────────── - state.puller_container_name = puller_container.name - state.stage = CreationStage.RUNNING_PULLER - state.task = None - return DeploymentStatusUpdate( - status="PENDING", - status_message="Downloading model weights. This may take several minutes.", - ) - - # ── Stage 3 (optional): run model-puller container ─────────────── - case CreationStage.RUNNING_PULLER: - status, complete = await self._advance_running_puller(deployment_key, state) - if complete: - self._creation_states.pop(deployment_key, None) - return status - - # ── Stage 4: create and start NIM container ────────────────────── - case CreationStage.CREATING_CONTAINER: - status, complete = await self._advance_creating_container(deployment_key, state) - if complete: - self._creation_states.pop(deployment_key, None) - return status - - except Exception as e: - logger.error("Unexpected error advancing creation for %s: %s", deployment_key, e, exc_info=True) - self._creation_states.pop(deployment_key, None) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Unexpected error during deployment creation: {e}", - error_details={"error": str(e), "stage": state.stage.value}, - ) - - self._creation_states.pop(deployment_key, None) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Internal error: unknown creation stage {state.stage}", - error_details={"stage": state.stage.value}, - ) - - def shutdown(self) -> None: - """Cancel all in-flight creation tasks on service shutdown.""" - for key, state in list(self._creation_states.items()): - if state.task is not None and not state.task.done(): - state.task.cancel() - logger.info("Cancelled in-flight creation task for %s on shutdown", key) - self._creation_states.clear() - - async def cleanup(self, deployment_key: str) -> None: - """Cancel any in-flight background task and clean up resources for a deployment.""" - state = self._creation_states.pop(deployment_key, None) - if state is None: - return - if state.task is not None and not state.task.done(): - state.task.cancel() - logger.info("Cancelled in-flight creation task for %s (stage: %s)", deployment_key, state.stage.value) - if state.puller_container_name: - try: - puller = await asyncio.to_thread(self.get_container, state.puller_container_name) - try: - await asyncio.to_thread(puller.stop, timeout=10) - except Exception: - pass - await asyncio.to_thread(puller.remove, force=True) - logger.info("Cleaned up puller container %s for %s", state.puller_container_name, deployment_key) - except NotFound: - pass - except Exception as e: - logger.warning("Error cleaning up puller container for %s: %s", deployment_key, e) - - # ====================================================================== - # Stage status checking - # ====================================================================== - - def _is_task_complete(self, state: CreationState) -> bool: - """Return True when the current async task has finished (or is absent).""" - return state.task is None or state.task.done() - - def _get_task_error(self, state: CreationState) -> Exception | None: - """Return the exception from the completed task, or None on success/cancel.""" - if state.task is None: - return None - if state.task.cancelled(): - return asyncio.CancelledError() - return state.task.exception() - - # ====================================================================== - # Stage machine - # ====================================================================== - - def _needs_puller(self, state: CreationState) -> bool: - """Return True when a model-puller container is required before NIM can start.""" - return state.model_weights_type == ModelWeightsType.FILES_SERVICE or state.is_multi_llm - - # ------------------------------------------------------------------ - # RUNNING_PULLER - # ------------------------------------------------------------------ - - async def _advance_running_puller( - self, deployment_key: str, state: CreationState - ) -> tuple[DeploymentStatusUpdate, bool]: - # Check puller container status - try: - puller = await asyncio.to_thread(self.get_container, state.puller_container_name) - await asyncio.to_thread(self.reload_container, puller) - except NotFound: - logger.error("Puller container %s not found for %s", state.puller_container_name, deployment_key) - return DeploymentStatusUpdate( - status="ERROR", - status_message="Model puller container disappeared unexpectedly", - error_details={"stage": "model_puller", "puller_container": state.puller_container_name}, - ), True - except Exception as e: - logger.warning("Error checking puller container for %s: %s", deployment_key, e) - return DeploymentStatusUpdate( - status="PENDING", - status_message="Downloading model weights (checking puller status...)", - ), False - - container_status = puller.status - - # Still running — keep waiting - if container_status == "running": - return DeploymentStatusUpdate( - status="PENDING", - status_message="Downloading model weights. This may take several minutes.", - ), False - - # Finished — check exit code - if container_status in ("exited", "dead"): - exit_code = puller.attrs.get("State", {}).get("ExitCode", -1) - if exit_code == 0: - logger.info("Model puller completed successfully for %s", deployment_key) - try: - await asyncio.to_thread( - self._client.containers.run, - image=self._get_busybox_image(), - command=["sh", "-c", "chown -R 1000:1000 /model-store"], - volumes={state.volume_name: {"bind": "/model-store", "mode": "rw"}}, - remove=True, - ) - except Exception as e: - logger.warning("Post-puller chown failed (continuing): %s", e) - try: - await asyncio.to_thread(puller.remove, force=True) - except Exception as e: - logger.warning("Failed to remove puller container: %s", e) - - # Handle plugin puller if needed - if state.plugin_fileset: - logger.info("Running plugin puller for %s", deployment_key) - plugin_path, plugin_error = await self._run_plugin_puller( - state.deployment, - state.plugin_fileset, - state.volume_name, - target_subdir="tool_call_plugin", - ) - if plugin_error: - logger.error("Plugin puller failed for %s: %s", deployment_key, plugin_error) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to download tool_call_plugin fileset: {plugin_error}", - error_details={"error": plugin_error, "stage": "plugin_puller"}, - ), True - state.tool_call_plugin_path = plugin_path - - state.stage = CreationStage.CREATING_CONTAINER - state.task = None - return await self._advance_creating_container(deployment_key, state) - else: - logs = "" - try: - raw_logs = await asyncio.to_thread(puller.logs, tail=50) - logs = raw_logs.decode("utf-8", errors="ignore") - logger.error("Puller logs:\n%s", logs) - except Exception: - logs = "Unable to retrieve logs" - error_msg = ( - f"Model puller failed with exit code {exit_code}. " - f"Last logs: {logs[-500:] if len(logs) > 500 else logs}" - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to download model weights: {error_msg}", - error_details={"error": error_msg, "stage": "model_puller"}, - ), True - - # Still starting up or unknown state - return DeploymentStatusUpdate( - status="PENDING", - status_message=f"Model puller container is {container_status}...", - ), False - - # ------------------------------------------------------------------ - # CREATING_CONTAINER (final stage) - # ------------------------------------------------------------------ - - async def _advance_creating_container( - self, deployment_key: str, state: CreationState - ) -> tuple[DeploymentStatusUpdate, bool]: - deployment = state.deployment - config = state.config - engine = _config_engine(config) - view = deployment_config_view(config) - nim_config = view - container_name = self.get_container_name(deployment.workspace, deployment.name) - full_image = state.nim_image - - # Run plugin puller if needed and not yet done - if state.plugin_fileset and not state.tool_call_plugin_path: - logger.info("Running plugin puller for %s", deployment_key) - plugin_path, plugin_error = await self._run_plugin_puller( - deployment, - state.plugin_fileset, - state.volume_name, - target_subdir="tool_call_plugin", - ) - if plugin_error: - logger.error("Plugin puller failed for %s: %s", deployment_key, plugin_error) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to download tool_call_plugin fileset: {plugin_error}", - error_details={"error": plugin_error, "stage": "plugin_puller"}, - ), True - state.tool_call_plugin_path = plugin_path - - # Compile engine-specific environment variables (and serve args for - # arg-configured engines: vLLM and generic). NIM is configured purely - # via env, so it leaves the container command unset. - serve_args: list[str] | None = None - if engine == ENGINE_GENERIC: - # Generic: run the image with the user's raw env + args verbatim. The - # platform synthesizes nothing (no served-model-name, no LoRA, etc.). - env_vars = generic_compiler.compile_generic_env_vars(view) - generic_args = generic_compiler.compile_generic_args(view) - # Only override the image's command when the user supplied args. - serve_args = generic_args or None - elif engine == ENGINE_VLLM: - env_vars = vllm_compiler.compile_vllm_env_vars(view) - serve_args = vllm_compiler.compile_vllm_args(view, state.model_entity) - if view.lora_enabled: - # vLLM's lora_filesystem_resolver validates that - # VLLM_LORA_RESOLVER_CACHE_DIR exists at startup, before the adapter - # sidecar has a chance to create it. Pre-create the directory in the - # shared scratch volume so the vLLM container doesn't crash-loop while - # waiting for the first adapter to land. - lora_subdir = vllm_compiler.VLLM_LORA_CACHE_DIR.removeprefix("/scratch/") - try: - await asyncio.to_thread( - self._client.containers.run, - image=self._get_busybox_image(), - command=["sh", "-c", f"mkdir -p /scratch/{lora_subdir} && chmod -R 777 /scratch/{lora_subdir}"], - volumes={state.scratch_volume_name: {"bind": "/scratch", "mode": "rw"}}, - remove=True, - ) - logger.info("Pre-created LoRA cache dir %s in scratch volume", vllm_compiler.VLLM_LORA_CACHE_DIR) - except Exception as e: - logger.warning("Failed to pre-create LoRA cache dir (continuing anyway): %s", e) - else: - env_vars = await self._compile_env_vars( - deployment, - config, - state.model_entity, - model_weights_type=state.model_weights_type, - is_multi_llm=state.is_multi_llm, - ngc_api_key=state.ngc_api_key, - tool_call_plugin_path=state.tool_call_plugin_path, - ) - - # GPU allocation - device_requests: list = [] - allocated_gpu_ids: list[int] = [] - container: Container | None = None - sidecar_container: Container | None = None - - async def cleanup_and_error(status_message: str, error_details: dict) -> tuple[DeploymentStatusUpdate, bool]: - if allocated_gpu_ids and self._gpu_pool is not None: - self._gpu_pool.release_gpu(deployment_key) - logger.info(f"Released GPU(s) {allocated_gpu_ids} after container creation failure") - for ctr in [sidecar_container, container]: - if ctr: - try: - await asyncio.to_thread(self.stop_container, ctr, timeout=30) - await asyncio.to_thread(self.remove_container, ctr, force=True) - except NotFound: - pass - except (APIError, ReadTimeout, Urllib3ReadTimeoutError, RequestsConnectionError) as e: - logger.warning(f"Failed to stop container {ctr} after retries: {e}") - except Exception as e: - logger.warning(f"Unexpected error stopping container {ctr}: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=status_message, - error_details=error_details, - host_url=None, - ), True - - if nim_config.gpu > 0: - if self._gpu_pool is None: - logger.error(f"Cannot deploy {deployment_key}: no GPUs detected on this system") - return await cleanup_and_error( - status_message="No GPUs available on this system. GPU model deployments require NVIDIA GPUs.", - error_details={"error": "No GPUs detected", "stage": "gpu_allocation"}, - ) - try: - allocated_gpu_ids = self._gpu_pool.allocate_gpu(deployment_key, num_requested=nim_config.gpu) - device_requests = [ - docker.types.DeviceRequest( - driver="nvidia", - device_ids=[str(gpu_id) for gpu_id in allocated_gpu_ids], - capabilities=[["gpu"]], - ) - ] - logger.info("Allocated GPU(s) %s for deployment %s", allocated_gpu_ids, deployment_key) - except GPUAllocationError as e: - logger.error(f"Failed to allocate GPU(s) for deployment {deployment_key}: {e}") - return await cleanup_and_error( - status_message=f"Failed to allocate GPU resources: {e}", - error_details={"error": str(e), "stage": "gpu_allocation"}, - ) - - # Port allocation - host_port = await self.find_available_port() - if host_port is None: - logger.error(f"Failed to allocate port for deployment {deployment_key}") - if allocated_gpu_ids and self._gpu_pool is not None: - self._gpu_pool.release_gpu(deployment_key) - logger.info("Released GPU(s) %s after port allocation failure", allocated_gpu_ids) - return await cleanup_and_error( - status_message=( - f"Failed to allocate host port for deployment. No ports available in range " - f"{self._backend_config.models_docker_port_range_start}" - f"-{self._backend_config.models_docker_port_range_end}." - ), - error_details={"error": "Port allocation failed"}, - ) - - ports = {"8000/tcp": host_port} - logger.info("Allocated port %s for deployment %s", host_port, deployment_key) - - try: - logger.info("Creating container %s with image %s...", container_name, full_image) - - # Platform volumes (/model-store, /scratch) hold pulled weights + scratch - # space. NIM/vLLM always mount them. A generic container runs the user's - # image as-is, so only mount them when the platform actually pulls weights - # for it (a fileset-backed model deployment); otherwise the mounts would - # shadow the image's own contents at those paths. - mount_platform_volumes = engine != ENGINE_GENERIC or self._needs_puller(state) - volumes: dict[str, Any] = {} - if mount_platform_volumes: - volumes = { - state.volume_name: {"bind": "/model-store", "mode": "rw"}, - state.scratch_volume_name: {"bind": "/scratch", "mode": "rw"}, - } - - create_args: dict[str, Any] = { - "image": full_image, - "name": container_name, - "environment": env_vars, - "detach": True, - "device_requests": device_requests, - "volumes": volumes, - "labels": self._managed_container_labels( - deployment, - { - ENGINE_LABEL: engine, - HEALTH_PATH_LABEL: _resolve_health_path(engine, view), - }, - ), - "restart_policy": {"Name": "unless-stopped"}, - } - - # Serve args are passed as the container command (appended to the - # image's entrypoint). vLLM uses its compiled `vllm serve` args; - # generic uses the user's raw additional_args. NIM is configured - # purely via env and leaves the command unset. - if serve_args is not None: - create_args["command"] = serve_args - - if nim_config.gpu > 1: - fixed = MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE or self._backend_config.nim_multi_gpu_shm_size - per_gpu_mb = ( - int(MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU) - if MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU - else self._backend_config.nim_multi_gpu_shm_size_per_gpu - ) - shm_size = _compute_multi_gpu_shm_size(fixed, per_gpu_mb, nim_config.gpu) - create_args["shm_size"] = shm_size - logger.info("Using shm_size=%s for multi-GPU deployment (%s GPU(s))", shm_size, nim_config.gpu) - - create_args["ports"] = ports - - if self._should_attach_network(): - network = self._backend_config.models_docker_network - if network: - create_args["network"] = network - logger.info("NIM container will join network: %s", network) - else: - logger.warning( - "MODELS_DOCKER_NETWORKING_MODE='dond' but MODELS_DOCKER_NETWORK is not set. " - "NIM container may not be reachable." - ) - - container = await asyncio.to_thread(self.create_and_start_container, create_args) - - container_id = container.id[:12] - logger.info("Container %s started successfully (ID: %s)", container_name, container_id) - - # The generic engine has no LoRA semantics (no engine compiler to - # wire the adapter sidecar against), so never attach the sidecar for - # it even if lora_enabled was set. - if view.lora_enabled and engine != ENGINE_GENERIC: - cfg = get_platform_config() - image = get_qualified_image(self._backend_config.lora_sidecar_image_name) - sidecar_envs = cfg.to_shared_envvars() - sidecar_envs.update(env_vars) - # The adapters sidecar is engine-agnostic: it downloads enabled LoRA - # adapter filesets for the base model entity into NIM_PEFT_SOURCE. NIM - # already sets NIM_PEFT_SOURCE + the model-entity env in its env_vars; - # vLLM watches VLLM_LORA_RESOLVER_CACHE_DIR instead, so point the - # sidecar's NIM_PEFT_SOURCE at that same directory and supply the - # model-entity identity the sidecar needs to resolve adapters. - if engine == ENGINE_VLLM: - sidecar_envs["NIM_PEFT_SOURCE"] = vllm_compiler.VLLM_LORA_CACHE_DIR - sidecar_envs["NIM_PEFT_REFRESH_INTERVAL"] = str(self._backend_config.peft_refresh_interval) - # vLLM's filesystem resolver only loads an adapter whose - # base_model_name_or_path equals vLLM's --model value (the local - # model path). Tell the sidecar to rewrite each adapter to match. - sidecar_envs["VLLM_LORA_BASE_MODEL_OVERRIDE"] = vllm_compiler.MODEL_STORE_PATH - # Endpoint the sidecar uses to eagerly (un)load adapters via vLLM's - # runtime LoRA API. Without this the filesystem resolver only loads an - # adapter on the first request that names it, so it never appears in - # /v1/models and model-provider discovery never surfaces it. Reuse the - # same host URL the controller uses to reach the deployment, which is - # reachable from the sidecar across the supported networking modes. - sidecar_envs["VLLM_ENDPOINT"] = self.get_host_url(container_name, host_port) - if state.model_entity is not None: - sidecar_envs["NMP_MODEL_ENTITY_WORKSPACE"] = state.model_entity.workspace - sidecar_envs["NMP_MODEL_ENTITY_NAME"] = state.model_entity.name - sidecar_args: dict[str, Any] = { - "image": image, - "name": f"{container_name}-sidecar", - "environment": sidecar_envs, - "command": self._backend_config.lora_sidecar_command, - "detach": True, - "volumes": { - state.volume_name: {"bind": "/model-store", "mode": "rw"}, - state.scratch_volume_name: {"bind": "/scratch", "mode": "rw"}, - }, - "labels": self._managed_container_labels(deployment), - "restart_policy": {"Name": "unless-stopped"}, - "healthcheck": {"test": ["NONE"]}, - "ports": {}, - } - if self._backend_config.lora_sidecar_entrypoint: - sidecar_args["entrypoint"] = self._backend_config.lora_sidecar_entrypoint - - self._assign_network(sidecar_args) - - await asyncio.to_thread(self.pull_image_if_not_local, image) - sidecar_container = await asyncio.to_thread(self.create_and_start_container, sidecar_args) - sidecar_container_id = sidecar_container.id[:12] - logger.info("Container %s-sidecar started successfully (ID: %s)", container_name, sidecar_container_id) - - host_url = self.get_host_url(container_name, host_port) - logger.info( - "Successfully created container %s (ID: %s) for %s/%s. Host URL: %s", - container_name, - container_id, - deployment.workspace, - deployment.name, - host_url, - ) - - return DeploymentStatusUpdate( - status="PENDING", - status_message=( - f"Container created and starting with image {full_image} (ID: {container_id}). " - "The inference engine is initializing, this may take several minutes." - ), - host_url=host_url, - ), True # pipeline complete - - except APIError as e: - logger.error(f"Docker API error creating container {container_name}: {e}") - return await cleanup_and_error( - status_message=f"Docker API error: {e}", - error_details={"error": str(e)}, - ) - - # ====================================================================== - # Model puller container - # ====================================================================== - - async def _start_model_puller_container(self, state: CreationState) -> Container: - """Start the model puller container (detached, no wait).""" - deployment = state.deployment - config = state.config - model_entity = state.model_entity - model_weights_type = state.model_weights_type - volume_name = state.volume_name - - puller_container_name = self.get_puller_container_name(deployment.workspace, deployment.name) - puller_image = self._backend_config.huggingface_model_puller - nim_config = deployment_config_view(config) - - model_repo = self._get_model_repo_from_entity(model_entity, model_weights_type, nim_config) - - # Build environment vars based on weights type - if model_weights_type == ModelWeightsType.FILES_SERVICE: - logger.info("Configuring model puller for Files service model source") - files_url = self._get_hf_compatible_files_url() - env_vars = {"HF_ENDPOINT": files_url, "HF_TOKEN": "service:models"} - model_revision = None - elif model_weights_type == ModelWeightsType.HUGGINGFACE: - logger.info("Configuring model puller for Hugging Face model source") - _, _, model_revision = parse_model_name_revision( - model_namespace=nim_config.model_namespace, - model_name=nim_config.model_name, - model_revision=nim_config.model_revision, - ) - hf_token = None - if deployment.hf_token_secret_name: - try: - sdk = self._nmp_sdk - if deployment.auth_context: - sdk = get_sdk_on_behalf_of(self._nmp_sdk, deployment.auth_context.principal_id) - elif get_auth_config().enabled: - logger.warning( - "Deployment %s/%s has no auth_context; accessing secret as service principal", - deployment.workspace, - deployment.name, - ) - secrets = client_from_platform(sdk, AsyncSecretsClient) - response = ( - await secrets.access_secret( - name=deployment.hf_token_secret_name, workspace=deployment.workspace - ) - ).data() - hf_token = response.value - logger.info("Retrieved HF token from secrets service") - except Exception as e: - logger.warning("Failed to retrieve HF token from secrets service: %s", e) - env_vars = {"HF_ENDPOINT": HUGGINGFACE_HUB_URL} - if hf_token: - env_vars["HF_TOKEN"] = hf_token - else: - logger.warning("No HF token found, might not be able to pull model from Hugging Face Hub") - else: - raise ValueError(f"Unsupported model weights type for puller: {model_weights_type}") - - # Operator-provided overrides win over the defaults set above (e.g. HF_ENDPOINT/HF_TOKEN). - if self._backend_config.huggingface_model_puller_env: - env_vars = {**env_vars, **self._backend_config.huggingface_model_puller_env} - - logger.info( - "Running model puller for %s (container: %s, image: %s)", - model_repo, - puller_container_name, - puller_image, - ) - - # Clean up any existing puller container - try: - existing_puller = await asyncio.to_thread(self.get_container, puller_container_name) - logger.warning("Puller container %s already exists, removing it", puller_container_name) - try: - await asyncio.to_thread(existing_puller.stop, timeout=10) - except Exception: - pass - await asyncio.to_thread(existing_puller.remove, force=True) - except NotFound: - pass - except Exception as e: - logger.warning("Error cleaning up existing puller container: %s", e) - - # Build download command - command = ["download", model_repo, "--local-dir", "/model-store"] - if model_revision: - command.extend(["--revision", model_revision]) - - # Fix volume permissions - try: - logger.info("Setting volume permissions for %s...", volume_name) - await asyncio.to_thread( - self._client.containers.run, - image=self._get_busybox_image(), - command=["sh", "-c", "chown -R 1000:1000 /model-store && chmod -R 755 /model-store"], - volumes={volume_name: {"bind": "/model-store", "mode": "rw"}}, - remove=True, - ) - logger.info("Volume permissions set successfully") - except Exception as e: - logger.warning("Failed to set volume permissions (continuing anyway): %s", e) - - run_kwargs: dict = { - "image": puller_image, - "name": puller_container_name, - # The puller image is the platform nmp-api image, whose entrypoint is - # `nemo services run`. Override it to the Hugging Face CLI so `command` - # (["download", , "--local-dir", "/model-store", ...]) runs as - # `hf download ...`. - "entrypoint": ["hf"], - "command": command, - "environment": env_vars, - "user": "1000:1000", - "volumes": {volume_name: {"bind": "/model-store", "mode": "rw"}}, - "labels": self._managed_container_labels( - deployment, - {"nmp.nvidia.com/container-type": "model-puller"}, - ), - "detach": True, - "remove": False, - } - - if self._backend_config.models_docker_networking_mode == "local": - run_kwargs["network_mode"] = "host" - logger.info("Puller container will use host network (local mode)") - elif self._should_attach_network() and self._backend_config.models_docker_network: - run_kwargs["network"] = self._backend_config.models_docker_network - logger.info("Puller container will join network: %s", self._backend_config.models_docker_network) - - puller_container = await asyncio.to_thread(self.run_container, **run_kwargs) - logger.info("Puller container %s started (ID: %s)", puller_container_name, puller_container.id[:12]) - return puller_container - - # ====================================================================== - # Plugin puller - # ====================================================================== - - async def _run_plugin_puller( - self, - deployment: ModelDeployment, - fileset_ref: str, - volume_name: str, - target_subdir: str = "tool_call_plugin", - ) -> tuple[str | None, str | None]: - """Pull a plugin fileset and discover the Python file inside it. - - Returns ``(container_path_to_py_file, None)`` on success, - or ``(None, error_message)`` on failure. - """ - container_name = self.get_plugin_puller_container_name(deployment.workspace, deployment.name) - puller_image = self._backend_config.huggingface_model_puller - target_path = f"/model-store/{target_subdir}" - - logger.info( - f"Pulling plugin fileset '{fileset_ref}' into {target_path} for {deployment.workspace}/{deployment.name}" - ) - - files_url = self._get_hf_compatible_files_url() - env_vars = {"HF_ENDPOINT": files_url, "HF_TOKEN": "service:models"} - command = ["download", fileset_ref, "--local-dir", target_path] - - # Clean up any existing plugin puller container - try: - existing = await asyncio.to_thread(self.get_container, container_name) - logger.warning(f"Plugin puller container {container_name} already exists, removing") - try: - await asyncio.to_thread(existing.stop, timeout=10) - except Exception: - logger.warning(f"Failed to stop existing plugin puller container {container_name}") - await asyncio.to_thread(existing.remove, force=True) - except NotFound: - pass - except Exception as e: - logger.warning(f"Error cleaning up existing plugin puller container: {e}") - - try: - await asyncio.to_thread(self.pull_image_if_not_local, puller_image) - except Exception as e: - return None, f"Failed to pull plugin puller image {puller_image}: {e}" - - # Create the target subdirectory and fix permissions - try: - await asyncio.to_thread( - self._client.containers.run, - image=self._get_busybox_image(), - command=["sh", "-c", f"mkdir -p {target_path} && chown -R 1000:1000 {target_path}"], - volumes={volume_name: {"bind": "/model-store", "mode": "rw"}}, - remove=True, - ) - except Exception as e: - logger.warning(f"Failed to create plugin directory (continuing): {e}") - - # Run the puller container - try: - run_kwargs: dict = { - "image": puller_image, - "name": container_name, - # The puller image is the platform nmp-api image, whose entrypoint - # is `nemo services run`. Override it to the Hugging Face CLI so - # `command` (["download", , "--local-dir", ...]) runs as - # `hf download ...`. - "entrypoint": ["hf"], - "command": command, - "environment": env_vars, - "user": "1000:1000", - "volumes": {volume_name: {"bind": "/model-store", "mode": "rw"}}, - "labels": self._managed_container_labels( - deployment, - {"nmp.nvidia.com/container-type": "plugin-puller"}, - ), - "detach": True, - "remove": False, - } - - self._assign_network(run_kwargs) - - puller_container = await asyncio.to_thread(self.run_container, **run_kwargs) - logger.info(f"Plugin puller container {container_name} started (ID: {puller_container.id[:12]})") - - timeout = self._backend_config.model_puller_timeout - try: - result = await asyncio.to_thread(puller_container.wait, timeout=timeout) - exit_code = result.get("StatusCode", -1) - - if exit_code == 0: - logger.info(f"Plugin puller completed successfully for fileset '{fileset_ref}'") - try: - await asyncio.to_thread( - self._client.containers.run, - image=self._get_busybox_image(), - command=["sh", "-c", f"chown -R 1000:1000 {target_path}"], - volumes={volume_name: {"bind": "/model-store", "mode": "rw"}}, - remove=True, - ) - except Exception as e: - logger.warning(f"Post-puller chown failed for plugin (continuing): {e}") - try: - await asyncio.to_thread(puller_container.remove, force=True) - except Exception as e: - logger.warning(f"Failed to remove plugin puller container: {e}") - - # Discover the .py file in the pulled fileset - try: - find_result = await asyncio.to_thread( - self._client.containers.run, - image=self._get_busybox_image(), - command=["find", target_path, "-name", "*.py", "-type", "f"], - volumes={volume_name: {"bind": "/model-store", "mode": "rw"}}, - remove=True, - ) - py_files = [f.strip() for f in find_result.decode("utf-8").strip().split("\n") if f.strip()] - except Exception as e: - return None, f"Failed to discover Python files in plugin fileset: {e}" - - if len(py_files) == 0: - return None, ( - f"tool_call_plugin fileset '{fileset_ref}' contains no .py files. " - "The fileset must contain exactly one Python file." - ) - if len(py_files) > 1: - return None, ( - f"tool_call_plugin fileset '{fileset_ref}' contains {len(py_files)} .py files: " - f"{py_files}. The fileset must contain exactly one Python file." - ) - - plugin_path = py_files[0] - logger.info(f"Discovered tool_call_plugin Python file: {plugin_path}") - return plugin_path, None - else: - try: - raw_logs = await asyncio.to_thread(puller_container.logs, tail=50) - logs = raw_logs.decode("utf-8", errors="ignore") - logger.error(f"Plugin puller logs:\n{logs}") - except Exception: - logs = "Unable to retrieve logs" - return ( - None, - f"Plugin puller failed with exit code {exit_code}. " - f"Last logs: {logs[-500:] if len(logs) > 500 else logs}", - ) - - except Exception as wait_error: - logger.error(f"Error waiting for plugin puller container: {wait_error}") - try: - await asyncio.to_thread(puller_container.stop, timeout=10) - except Exception: - logger.warning("Failed to stop plugin puller container", exc_info=True) - return None, f"Plugin puller timed out or failed: {wait_error}" - - except APIError as e: - return None, f"Docker API error running plugin puller: {e}" - except Exception as e: - return None, f"Failed to run plugin puller: {e}" - - # ====================================================================== - # Environment variable compilation - # ====================================================================== - - async def _compile_env_vars( - self, - deployment: ModelDeployment, - config: ModelDeploymentConfig, - model_entity: Optional[ModelEntity] = None, - model_weights_type: Optional[ModelWeightsType] = None, - is_multi_llm: bool = False, - ngc_api_key: str | None = None, - tool_call_plugin_path: str | None = None, - ) -> Dict[str, str]: - """Compile environment variables for the NIM container.""" - nim_config = deployment_config_view(config) - env_vars: Dict[str, str] = { - "NIM_GUIDED_DECODING_BACKEND": self._backend_config.nim_guided_decoding_backend, - } - - model_fqdn: str | None = None - if nim_config.model_name: - if nim_config.model_namespace: - model_fqdn = f"{nim_config.model_namespace}/{nim_config.model_name}" - else: - model_fqdn = nim_config.model_name - - if model_fqdn: - env_vars["NIM_SERVED_MODEL_NAME"] = model_fqdn - - weights_from_files = model_weights_type is not None and model_weights_type == ModelWeightsType.FILES_SERVICE - puller_ran = weights_from_files or is_multi_llm - if puller_ran: - env_vars["NIM_MODEL_NAME"] = "/model-store" - env_vars["NIM_MODEL_PATH"] = "/model-store" - - if puller_ran and not is_multi_llm: - logger.info("Adding fine-tuned model environment variables for pre-downloaded weights") - env_vars["NIM_FT_MODEL"] = "/model-store" - env_vars["NIM_CUSTOM_MODEL"] = "/model-store" - - if nim_config.lora_enabled: - env_vars["NIM_PEFT_SOURCE"] = "/scratch/loras" - env_vars["NIM_PEFT_REFRESH_INTERVAL"] = str(self._backend_config.peft_refresh_interval) - - if ngc_api_key: - env_vars["NGC_API_KEY"] = ngc_api_key - logger.info("Passing NGC_API_KEY to container for model downloads") - - if nim_config.additional_envs: - env_vars.update(nim_config.additional_envs) - - if model_entity: - env_vars["NMP_MODEL_ENTITY_WORKSPACE"] = model_entity.workspace - env_vars["NMP_MODEL_ENTITY_NAME"] = model_entity.name - - if model_entity.trust_remote_code: - env_vars["NIM_FORCE_TRUST_REMOTE_CODE"] = "1" - if model_entity.spec: - if model_entity.spec.chat_template: - env_vars["NIM_CHAT_TEMPLATE"] = model_entity.spec.chat_template - - if model_entity.spec.tool_call_config: - tool_cfg = model_entity.spec.tool_call_config - if tool_cfg.tool_call_parser: - env_vars["NIM_TOOL_CALL_PARSER"] = tool_cfg.tool_call_parser - if tool_call_plugin_path: - env_vars["NIM_TOOL_PARSER_PLUGIN"] = tool_call_plugin_path - if tool_cfg.auto_tool_choice is not None: - env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] = "1" if tool_cfg.auto_tool_choice else "0" - - # Deployment-level overrides (highest priority) - if nim_config.chat_template: - env_vars["NIM_CHAT_TEMPLATE"] = nim_config.chat_template - - if nim_config.tool_call_config: - deploy_cfg = nim_config.tool_call_config - if deploy_cfg.tool_call_parser: - env_vars["NIM_TOOL_CALL_PARSER"] = deploy_cfg.tool_call_parser - if deploy_cfg.tool_call_plugin: - if tool_call_plugin_path: - env_vars["NIM_TOOL_PARSER_PLUGIN"] = tool_call_plugin_path - else: - logger.warning( - "Deployment tool_call_config.tool_call_plugin is set but no plugin .py file " - "was discovered. Ensure the fileset was pulled successfully." - ) - if deploy_cfg.auto_tool_choice is not None: - env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] = "1" if deploy_cfg.auto_tool_choice else "0" - - return env_vars diff --git a/services/core/models/src/nmp/core/models/controllers/backends/generic_compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/generic_compiler.py index 2df6a85245..5e571034d5 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/generic_compiler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/generic_compiler.py @@ -45,9 +45,12 @@ def compile_generic_args(view: DeploymentConfigView) -> list[str]: """Return the container arg vector for a generic container. The platform synthesizes nothing for the generic engine: the user's - ``additional_args`` are the entire arg vector (appended to the image's own - entrypoint). Returns an empty list when none are supplied, in which case the - image's default command/args are used unchanged. + ``additional_args`` become the container arg vector. On Kubernetes this + replaces the image CMD (ENTRYPOINT is unchanged); on Docker it replaces the + run command. When the image has no ENTRYPOINT, include the executable as + argv[0] (e.g. ``["python3", "-m", "http.server", "8000"]`` for + ``python:alpine``). Returns an empty list when none are supplied, in which + case the image's default command/args are used unchanged. """ return list(view.additional_args or []) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/__init__.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/__init__.py deleted file mode 100644 index 584f74a778..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Kubernetes NIM Operator backend for Models Controller service.""" - -from .backend import K8sNimOperatorServiceBackend as K8sNimOperatorServiceBackend -from .config import K8sNimOperatorConfig as K8sNimOperatorConfig diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py deleted file mode 100644 index 5d813aa24f..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py +++ /dev/null @@ -1,391 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Kubernetes NIM Operator backend implementation for Models Controller service. - -This ``ServiceBackend`` owns the ``nemo_platform`` SDK, determines the state of -the *API object* (ModelDeployment / ModelDeploymentConfig), resolves every input -a reconciler needs (weight source, resource names, Files endpoint) into a -:class:`ResolvedDeployment`, selects the correct reconciler by engine, and -delegates. It holds NO Kubernetes-reconciliation logic itself -- that lives in -the two reconcilers under :mod:`.reconcilers`: - -* :class:`NimOperatorReconciler` -- emits ``NIMService`` / ``NIMCache`` CRs. -* :class:`K8sReconciler` -- emits native Kubernetes objects directly (vLLM). -""" - -import os -from logging import getLogger -from typing import Optional -from urllib.parse import urljoin - -from kubernetes import client as k8s_client -from kubernetes import config as k8s_config -from kubernetes.dynamic import DynamicClient -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.models.model_entity import ModelEntity -from nmp.common.config import get_platform_config -from nmp.core.models.app import ( - get_deployment_resource_name, - get_model_weights_type, - get_nimcache_resource_name, - parse_model_name_revision, -) -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend -from nmp.core.models.controllers.backends.common import ( - DeploymentConfigView, - deployment_config_view, - deployment_elapsed_seconds, -) -from nmp.core.models.controllers.backends.engine import ENGINE_GENERIC, ENGINE_NIM, ENGINE_VLLM, config_engine -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.base import Reconciler, ResolvedDeployment -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.k8s import K8sReconciler -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator import ( - NimOperatorReconciler, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector -from nmp.core.models.controllers.context import ModelContext - -logger = getLogger(__name__) - - -class K8sNimOperatorServiceBackend(ServiceBackend): - """Kubernetes backend for managing model deployments. - - Resolves API-object state and delegates reconciliation to the engine-specific - reconciler (NIM operator CRs vs. native Kubernetes objects). - """ - - def __init__(self, nmp_sdk, config, huggingface_model_puller: str): - self._k8s_client: k8s_client.ApiClient | None = None - self._dynamic_client: DynamicClient | None = None - self._k8s_namespace: str | None = None - self._backend_config: K8sNimOperatorConfig | None = None - self._huggingface_model_puller = huggingface_model_puller - self._status_projector: StatusProjector | None = None - self._resource_deleter: ResourceDeleter | None = None - self._nim_reconciler: NimOperatorReconciler | None = None - self._k8s_reconciler: K8sReconciler | None = None - super().__init__(nmp_sdk, config) - - def init(self) -> None: - """Initialize Kubernetes backend and build the engine reconcilers.""" - logger.info("Initializing Kubernetes NIM Operator service backend") - - self._backend_config = K8sNimOperatorConfig(**self._config) - logger.debug(f"Backend config: {self._backend_config.model_dump()}") - - try: - # Try in-cluster config first (for running inside k8s) - k8s_config.load_incluster_config() - logger.info("Loaded in-cluster Kubernetes configuration") - except k8s_config.ConfigException: - # Fall back to kubeconfig (for local development) - k8s_config.load_kube_config() - logger.info("Loaded kubeconfig configuration") - - self._k8s_client = k8s_client.ApiClient() - self._dynamic_client = DynamicClient(self._k8s_client) - - self._k8s_namespace = self._get_current_namespace() - logger.info(f"Models controller will deploy models to namespace: {self._k8s_namespace}") - - # Shared collaborators composed into both reconcilers (and used directly - # by the PENDING-timeout policy below). - self._status_projector = StatusProjector( - k8s_client_=self._k8s_client, - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - ) - self._resource_deleter = ResourceDeleter(k8s_namespace=self._k8s_namespace) - - self._nim_reconciler = NimOperatorReconciler( - dynamic_client=self._dynamic_client, - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - huggingface_model_puller=self._huggingface_model_puller, - status=self._status_projector, - deleter=self._resource_deleter, - ) - self._k8s_reconciler = K8sReconciler( - k8s_client_=self._k8s_client, - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - huggingface_model_puller=self._huggingface_model_puller, - status=self._status_projector, - deleter=self._resource_deleter, - ) - - def shutdown(self) -> None: - """Shutdown Kubernetes backend and release resources.""" - logger.info("Shutting down Kubernetes NIM Operator service backend") - if self._k8s_client is not None: - try: - self._k8s_client.close() - logger.debug("Kubernetes API client closed") - except Exception as e: - logger.warning(f"Error closing Kubernetes API client: {e}") - - def _get_current_namespace(self) -> str: - """Get the Kubernetes namespace where the controller is running.""" - if self._backend_config and self._backend_config.namespace: - return self._backend_config.namespace - - # Try to read from the service account namespace file (in-cluster) - namespace_file = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" - if os.path.exists(namespace_file): - with open(namespace_file, "r") as f: - return f.read().strip() - - logger.warning("Could not determine k8s namespace, using 'default'") - return "default" - - # ------------------------------------------------------------------ - # Name + weight-source resolution (API-object work owned by the backend) - # ------------------------------------------------------------------ - - def _get_resource_name(self, deployment: ModelDeployment) -> str: - """Generate the k8s resource name for NIMService/PVC resources (63-char limit).""" - return get_deployment_resource_name(deployment.workspace, deployment.name) - - def _get_nimcache_resource_name(self, deployment: ModelDeployment) -> str: - """Generate the k8s resource name for NIMCache resources (59-char limit). - - NIMCache names are capped at 59 characters instead of 63 because - k8s-nim-operator appends '-job' (4 chars) when creating its internal - batch Job, and the resulting name must not exceed the 63-char K8s - label limit. - """ - return get_nimcache_resource_name(deployment.workspace, deployment.name) - - def _resolve_model_source( - self, - model_entity: Optional[ModelEntity], - nim_config: DeploymentConfigView, - ) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Derive the model namespace/name for NIMCache from the model entity's fileset. - - The HF-compatible Files API resolves models by *fileset* name, not by - model-entity name. When a model entity carries a fileset reference - (e.g. ``hf://workspace/fileset`` or ``fileset://workspace/fileset``), - the NIMCache source must use that fileset path so the model puller can - actually find the files. Falls back to ``nim_config`` fields when no - fileset is available - """ - model_namespace, model_name, model_revision = parse_model_name_revision( - model_namespace=nim_config.model_namespace, - model_name=nim_config.model_name, - model_revision=nim_config.model_revision, - ) - - if model_entity and model_entity.fileset: - fileset_path = str(model_entity.fileset).removeprefix("hf://").removeprefix("fileset://") - parts = fileset_path.split("/", 1) - if len(parts) == 2: - logger.info(f"Resolved model source from entity fileset: namespace={parts[0]}, name={parts[1]}") - return parts[0], parts[1], model_revision - logger.warning( - f"model_entity.fileset '{model_entity.fileset}' does not contain namespace/name, falling back to nim_config" - ) - - return model_namespace, model_name, model_revision - - def _remote_files_hf_url(self) -> str: - """Cluster-routable Files HF endpoint for the puller Job. - - ``_get_files_hf_url`` resolves via the platform config's local-service - routing, which returns ``localhost`` when the Files service runs in this - same process. The puller is a *separate pod* and cannot reach localhost, so - we resolve the Files URL from ``service_discovery``/``base_url`` directly - (the cluster-routable address) and append the HF-compatible path. - """ - platform_config = get_platform_config() - files_url = platform_config.service_discovery.get("files") or platform_config.base_url - return urljoin(files_url.rstrip("/") + "/", "apis/files/v2/hf") - - def _resolve(self, ctx: ModelContext) -> ResolvedDeployment: - """Resolve everything a reconciler needs from the API object + SDK state.""" - deployment = ctx.model_deployment - config = ctx.model_deployment_config - model_entity = ctx.model_entity - view = deployment_config_view(config) - model_namespace, model_name, model_revision = self._resolve_model_source(model_entity, view) - weights_type = get_model_weights_type( - model_deployment=deployment, - model_deployment_config=config, - model_entity=model_entity, - ) - return ResolvedDeployment( - deployment=deployment, - config=config, - model_entity=model_entity, - view=view, - resource_name=self._get_resource_name(deployment), - nimcache_resource_name=self._get_nimcache_resource_name(deployment), - weights_type=weights_type, - model_namespace=model_namespace, - model_name=model_name, - model_revision=model_revision, - files_hf_url=self._remote_files_hf_url(), - huggingface_model_puller=self._huggingface_model_puller, - ) - - def _select_reconciler(self, engine: str) -> Optional[Reconciler]: - """Select the reconciler for an engine. - - The direct-emission :class:`K8sReconciler` handles ``vllm`` and - ``generic``; ``nim`` uses the NIM-operator reconciler. Any other value is - unsupported and yields ``None``, which the callers turn into the - "unsupported engine" rejection (see :meth:`_unsupported_engine`). - """ - if engine in (ENGINE_VLLM, ENGINE_GENERIC): - return self._k8s_reconciler - if engine == ENGINE_NIM: - return self._nim_reconciler - return None - - @staticmethod - def _unsupported_engine(engine: str) -> DeploymentStatusUpdate: - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"The '{engine}' engine is not supported on the k8s backend.", - error_details={"error": "unsupported_engine", "engine": engine}, - host_url=None, - ) - - # ------------------------------------------------------------------ - # ServiceBackend interface (resolve + select + delegate) - # ------------------------------------------------------------------ - - async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Create a new model deployment (dispatches on the config's engine).""" - engine = config_engine(ctx.model_deployment_config) - reconciler = self._select_reconciler(engine) - if reconciler is None: - return self._unsupported_engine(engine) - resolved = self._resolve(ctx) - return await reconciler.create(resolved) - - async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Update an existing model deployment (dispatches on the config's engine).""" - engine = config_engine(ctx.model_deployment_config) - reconciler = self._select_reconciler(engine) - if reconciler is None: - return self._unsupported_engine(engine) - resolved = self._resolve(ctx) - return await reconciler.update(resolved) - - async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Get the current status of a model deployment. - - The engine is taken from the config (same selection as create/update), so a - config is required. When ``ctx.model_deployment_config`` is ``None`` (e.g. - the controller failed to fetch it this cycle) the backend cannot determine - the deployment's state and returns ``UNKNOWN``; the controller retries on - the next poll (which normally has a config) and escalates to ERROR after - its retry budget. - - In addition to the reconciler's status, this method enforces the PENDING - timeout policy: if the deployment has been alive longer than - ``pending_timeout_seconds`` and is still PENDING, transition to ERROR with - diagnostic information. (Crash-loop detection is handled inside the - reconciler's pod drill-down.) - """ - deployment = ctx.model_deployment - config = ctx.model_deployment_config - logger.debug( - f"Checking deployment status: {deployment.workspace}/{deployment.name} " - f"(version: {deployment.entity_version})" - ) - - if config is None: - logger.warning( - f"No config available for {deployment.workspace}/{deployment.name}; cannot determine status this cycle" - ) - return DeploymentStatusUpdate( - status="UNKNOWN", - status_message="Deployment config unavailable; will retry.", - host_url=None, - ) - - try: - resource_name = self._get_resource_name(deployment) - - engine = config_engine(config) - reconciler = self._select_reconciler(engine) - if reconciler is None: - return self._unsupported_engine(engine) - # A reconciler MAY advance creation in get_status; it needs the - # resolved config to compile the serving spec. - resolved = self._resolve(ctx) - result = await reconciler.get_status(resolved) - - if result.status == "PENDING": - elapsed = deployment_elapsed_seconds(deployment) - - if elapsed >= self._backend_config.pending_timeout_seconds: - pod_name = self._status_projector.find_pod_name(resource_name) - return self._status_projector.build_pending_timeout_error(resource_name, elapsed, pod_name) - - # Use a stable message (no elapsed/timeout) so we don't create a new history entry every poll - - return result - except Exception as e: - logger.error(f"Failed to get status for {deployment.workspace}/{deployment.name}: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message="Unable to determine deployment status due to a service backend error", - host_url=None, - ) - - async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete a model deployment by workspace and name (model deployment ID). - - Delete has only workspace/name (no config/engine -- it is also called for - orphan reconciliation), so BOTH reconcilers are asked to delete the - resources they own (NIMService/NIMCache CRs and the directly-emitted vLLM - objects). Each delete is independent and 404-tolerant; one reconciler's - failure never aborts the other. Real (non-404) failures are aggregated and - surfaced as ERROR so we never report DELETED while cluster resources may - remain. - """ - logger.info(f"Deleting model deployment: {workspace}/{name}") - return await self._delete_resources_by_model_deployment_id(workspace, name) - - async def _delete_resources_by_model_deployment_id(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Aggregate both reconcilers' deletes into a single status update.""" - errors: list[str] = [] - for result in ( - await self._nim_reconciler.delete(workspace, name), - await self._k8s_reconciler.delete(workspace, name), - ): - if result.status == "ERROR" and result.error_details: - errors.extend(result.error_details.get("errors", [])) - - if errors: - summary = "; ".join(errors) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to fully delete deployment {workspace}/{name}: {summary}", - error_details={"errors": errors}, - host_url=None, - ) - return DeploymentStatusUpdate( - status="DELETED", - status_message="Deployment deletion initiated successfully", - host_url=None, - ) - - async def list_managed_deployment_names(self) -> list[str]: - """List deployment names (workspace/name) the backend manages. - - Unions the operator path (NIMServices) and the directly-emitted vLLM path - (raw Deployments), both labelled by the same managed-by + workspace/name - labels, for orphan reconciliation. - """ - seen: set[str] = set() - seen.update(await self._nim_reconciler.list_managed_deployment_names()) - seen.update(await self._k8s_reconciler.list_managed_deployment_names()) - return sorted(seen) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py deleted file mode 100644 index 75e8a3e938..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py +++ /dev/null @@ -1,245 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Configuration model for k8s-nim-operator backend.""" - -from typing import Any, Dict, List, Optional - -from pydantic import BaseModel, Field - - -class K8sNimOperatorConfig(BaseModel): - """ - Configuration for the Kubernetes NIM Operator backend. - - These settings control how NIMService CRDs are generated and deployed. - """ - - # Storage configuration - default_storage_class: Optional[str] = Field( - default=None, - description="Default storage class for PVCs. If not set, the cluster's default StorageClass is used.", - ) - default_pvc_size: str = Field( - default="200Gi", - description="Default PVC size for model storage (used if not specified in deployment config)", - ) - - # PEFT/LoRA configuration - peft_source: str = Field( - default="http://nemo-entity-store:8000", - description="LoRA/PEFT source endpoint (only used when lora_enabled is true)", - ) - peft_refresh_interval: int = Field( - default=30, - ge=1, - description="PEFT refresh interval in seconds (only used when lora_enabled is true)", - ) - lora_sidecar_image_name: str = Field( - default="nmp-api", - description=( - "Image name (without registry/tag) for the LoRA adapters sidecar container. " - "Registry and tag are taken from the platform config (NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG). " - "Override to 'nmp-customizer-tasks' for local dev when that image is already available " - "but nmp-api is not." - ), - ) - lora_sidecar_command: list[str] = Field( - default=["nemo", "services", "run", "--sidecars", "adapters"], - description=( - "Kubernetes container command (entrypoint) for the LoRA sidecar. " - "Default uses the nmp-platform-runner entrypoint present in nmp-api. " - "When using nmp-customizer-tasks set to ['python'] and set lora_sidecar_args to " - "['-m', 'nmp.core.models.sidecars.adapters.main']." - ), - ) - lora_sidecar_args: list[str] = Field( - default=[], - description=( - "Kubernetes container args for the LoRA sidecar (appended after lora_sidecar_command). " - "Leave empty for nmp-api. " - "Set to ['-m', 'nmp.core.models.sidecars.adapters.main'] when using nmp-customizer-tasks." - ), - ) - - # Security context for the NIM (operator) path: applied to the NIMService / - # NIMCache CRs. NOTE: securityContext uid/gid is engine-specific -- NIM images - # expect different values (operator default 1000/2000) than the vLLM image - # (2000/0, see default_vllm_*). When NIM is migrated onto the raw-object - # compilers (vllm_k8s_compiler), keep passing THESE fields for the NIM path -- - # do not reuse default_vllm_user_id/_group_id. See the FUTURE note in - # vllm_k8s_compiler.py. - default_user_id: Optional[int] = Field( - default=None, - description="Default user ID for NIM containers (security context)", - ) - default_group_id: Optional[int] = Field( - default=None, - description="Default group ID for NIM containers (security context)", - ) - - # Security context for the directly-emitted vLLM path (puller Job + server - # Deployment). Defaults match the user the upstream vllm/vllm-openai image - # ships ("vllm", uid 2000, gid 0): a non-root uid that HAS an /etc/passwd - # entry, so torch/inductor's getpass.getuser() (pwd.getpwuid) does not crash. - # gid 0 (root group) is the image's group and is the standard - # arbitrary-uid-friendly group. The puller writes weights under this uid/gid - # so the server can read them. - default_vllm_user_id: Optional[int] = Field( - default=2000, - description="Default user ID for vLLM puller + server pods (security context). " - "Defaults to 2000 to match the upstream vLLM image's 'vllm' user, which has an " - "/etc/passwd entry (avoids torch getpwuid crashes from an unknown uid).", - ) - default_vllm_group_id: Optional[int] = Field( - default=0, - description="Default group ID / fsGroup for vLLM puller + server pods. Defaults to 0 " - "(root group) to match the upstream vLLM image and keep weights readable across the " - "puller and server pods.", - ) - - # Files service configuration - files_auth_secret: str = Field( - default="nemo-models-files-token", - description="Kubernetes secret name for Files service authentication (HF_TOKEN)", - ) - huggingface_model_puller_image_pull_secret: str = Field( - default="nvcrimagepullsecret", - description="The name of the image pull secret for the modelPuller image", - ) - - busybox_image: str = Field( - default="docker.io/library/busybox", - description="BusyBox image repository used by plugin init containers. " - "Fully qualified (docker.io/library/...) so it resolves on container runtimes " - "that enforce fully-qualified image names (short names like 'busybox' fail there).", - ) - - busybox_image_tag: str = Field( - default="latest", - description="BusyBox image tag used by plugin init containers.", - ) - - # Auth configuration - auth_secret: str = Field( - default="ngc-api", - description="NGC API key secret name for pulling NIM images", - ) - - # NIMService image configuration - default_nimservice_image: str = Field( - default="nvcr.io/nim/nvidia/llm-nim", - description="Default NIMService image repository (used if not specified in deployment config)", - ) - default_nimservice_image_tag: str = Field( - default="1.13.1", - description="Default NIMService image tag (used if not specified in deployment config)", - ) - - # vLLM image configuration (vLLM engine on k8s; raw-object emission path) - default_vllm_image: str = Field( - default="vllm/vllm-openai", - description="Default vLLM server image repository (used if not specified in deployment config)", - ) - default_vllm_image_tag: str = Field( - default="v0.22.1", - description="Default vLLM server image tag (used if not specified in deployment config)", - ) - - # NIM runtime configuration - nim_guided_decoding_backend: str = Field( - default="outlines", - description="Default guided decoding backend for NIM (e.g., 'outlines', 'auto', 'lm-format-enforcer')", - ) - - # Kubernetes namespace (optional override) - namespace: Optional[str] = Field( - default=None, - description="Kubernetes namespace for NIM deployments (defaults to controller's namespace if not set)", - ) - - # ServiceAccount for directly-emitted workloads (vLLM Deployment pods + weight - # puller Job). A single shared models ServiceAccount is used; the platform Helm - # chart is responsible for creating it and granting any required RBAC/SCC. - # If not set, pods run under the namespace's default ServiceAccount. - service_account_name: Optional[str] = Field( - default=None, - description="ServiceAccount name for directly-emitted vLLM Deployment pods and the weight-puller Job. " - "If not set, the namespace default ServiceAccount is used.", - ) - - # Shared memory (/dev/shm) for directly-emitted vLLM Deployment pods. vLLM uses - # /dev/shm for tensor-parallel NCCL communication. If not set, the dshm emptyDir - # is mounted with no explicit size limit (uses the node default). - default_shared_memory_size_limit: Optional[str] = Field( - default=None, - description="Shared memory (/dev/shm) size limit for vLLM Deployment pods (e.g. '8Gi'). " - "If not set, the emptyDir uses the node default size.", - ) - - # Default Kubernetes configuration for all NIM deployments - default_resources: Optional[Dict[str, Any]] = Field( - default=None, - description="Default Kubernetes resource requirements for all NIM deployments. " - "Can be overridden per-deployment via k8s_nim_operator_config. " - "Example: {'requests': {'cpu': '2', 'memory': '8Gi'}, 'limits': {'memory': '16Gi'}}", - examples=[{"requests": {"cpu": "2", "memory": "8Gi"}, "limits": {"memory": "16Gi"}}], - ) - default_tolerations: Optional[List[Dict[str, Any]]] = Field( - default=None, - description="Default Kubernetes tolerations for all NIM deployments. " - "Can be overridden per-deployment via k8s_nim_operator_config. " - "Example: [{'key': 'nvidia.com/gpu', 'operator': 'Exists', 'effect': 'NoSchedule'}]", - examples=[[{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}]], - ) - default_node_selector: Optional[Dict[str, str]] = Field( - default=None, - description="Default Kubernetes node selector for all NIM deployments. " - "Can be overridden per-deployment via k8s_nim_operator_config. " - "Example: {'node-type': 'gpu-node', 'zone': 'us-west1-a'}", - examples=[{"node-type": "gpu-node", "zone": "us-west1-a"}], - ) - model_labels: Optional[Dict[str, str]] = Field( - default=None, - description="Additional labels copied onto model resources managed by the models controller.", - examples=[{"example.com/test-label": "A", "example.com/test-suite": "B"}], - ) - default_labels: Optional[Dict[str, str]] = Field( - default=None, - description="Default Kubernetes labels applied to NIMService and NIMCache resources and their child resources (e.g. pods). " - "Merged with controller-managed labels; controller labels take precedence on conflict. " - "Example: {'team': 'ml-platform', 'environment': 'prod'}", - examples=[{"team": "ml-platform", "environment": "prod"}], - ) - default_annotations: Optional[Dict[str, str]] = Field( - default=None, - description="Default Kubernetes annotations applied to NIMService and NIMCache resources and their child resources (e.g. pods, PVCs). " - "Merged with controller-managed annotations; controller annotations take precedence on conflict. " - "Example: {'prometheus.io/scrape': 'true'}", - examples=[{"prometheus.io/scrape": "true"}], - ) - default_startup_probe_grace_period_seconds: Optional[int] = Field( - default=None, - description="Default grace period in seconds for NIM startup. " - "Can be overridden per-deployment via k8s_nim_operator_config. " - "Determines how long Kubernetes will wait for the NIM to become ready before restarting it. " - "If not set, defaults to 600 seconds (10 minutes). " - "Example: 600 (10 minutes)", - examples=[600], - gt=0, - ) - - # PENDING timeout and crash loop detection - pending_timeout_seconds: int = Field( - default=7200, - ge=60, - description="Maximum time in seconds a deployment may stay in PENDING before being " - "transitioned to ERROR. Default: 7200 (2 hours).", - ) - - max_restart_count: int = Field( - default=5, - ge=1, - description="Maximum number of pod container restarts before a PENDING deployment is " - "transitioned to ERROR (crash loop detection). Default: 5.", - ) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/nimservice_compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/nimservice_compiler.py deleted file mode 100644 index 55d9f09fa3..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/nimservice_compiler.py +++ /dev/null @@ -1,723 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NIMService compiler for transforming ModelDeploymentConfig into NIMService CRD.""" - -from logging import getLogger -from typing import Any, Optional -from urllib.parse import urljoin - -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig -from nemo_platform.types.models.model_entity import ModelEntity -from nmp.common.config import get_platform_config -from nmp.core.models.app import is_multi_llm_image, parse_model_name_revision -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.app.utils import _get_k8s_safe_name, get_docker_plugin_puller_container_name -from nmp.core.models.controllers.backends.common import DeploymentConfigView, deployment_config_view -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.types.nimcache import ( - Hf, - NIMCache, - Pvc, - Source, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.types.nimcache import ( - Resources as NIMCacheResources, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.types.nimcache import ( - Spec as NIMCacheSpec, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.types.nimcache import ( - Storage as NIMCacheStorage, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.types.nimcache import ( - Toleration as NIMCacheToleration, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.types.nimservice import ( - ContainerSpec, - EnvItem, - Expose, - HttpGet2, - Image, - Limits1, - NIMService, - Probe2, - Requests1, - Resources, - SecretKeyRef, - Spec, - StartupProbe, - Storage, - ValueFrom, -) - -logger = getLogger(__name__) -TOOL_CALL_PLUGIN_PATH = "/model-store/plugin/plugin.py" -TOOL_CALL_PLUGIN_SCRATCH_DIR = "/scratch/plugin" -TOOL_CALL_PLUGIN_FINALIZE_SCRIPT_TEMPLATE = """set -euo pipefail -py_files="$(find "{scratch_dir}" -type f -name '*.py' || true)" -count="$(printf '%s\n' "$py_files" | sed '/^$/d' | wc -l | tr -d ' ')" -if [ "$count" -eq 0 ]; then - echo "tool_call_plugin fileset contains no .py files" - exit 1 -fi -if [ "$count" -ne 1 ]; then - echo "tool_call_plugin fileset must contain exactly one .py file, found $count" - printf '%s\n' "$py_files" - exit 1 -fi -plugin_file="$(printf '%s\n' "$py_files" | sed '/^$/d' | sed -n '1p')" -mv "$plugin_file" "{plugin_path}" -""" - - -def _get_files_hf_url() -> str: - """Get Files service HF-compatible API URL.""" - return urljoin(get_platform_config().get_service_url("files"), "apis/files/v2/hf") - - -def _nimcache_default_resources(backend_config: K8sNimOperatorConfig) -> Optional[NIMCacheResources]: - """Map backend default_resources (K8s requests/limits) to NIMCache Resources (cpu, memory).""" - raw = backend_config.default_resources - if not raw or not isinstance(raw, dict): - return None - requests = raw.get("requests") or {} - limits = raw.get("limits") or {} - if not isinstance(requests, dict): - requests = {} - if not isinstance(limits, dict): - limits = {} - cpu = requests.get("cpu") or limits.get("cpu") - memory = requests.get("memory") or limits.get("memory") - if cpu is None and memory is None: - return None - return NIMCacheResources(cpu=cpu, memory=memory) - - -def _nimcache_default_tolerations(backend_config: K8sNimOperatorConfig) -> Optional[list[NIMCacheToleration]]: - """Convert backend default_tolerations to NIMCache Toleration list.""" - raw = backend_config.default_tolerations - if not raw or not isinstance(raw, list): - return None - out = [] - for item in raw: - if not isinstance(item, dict): - continue - out.append(NIMCacheToleration(**{k: v for k, v in item.items() if v is not None})) - return out if out else None - - -def compile_nimcache( - backend_config: K8sNimOperatorConfig, - k8s_namespace: str, - resource_name: str, - model_namespace: str, - model_name: str, - pvc_size: str, - huggingface_model_puller: str, - model_revision: Optional[str] = None, -) -> NIMCache: - """Generate a NIMCache CR for models whose weights are served via the Files service HF-compatible API. - - Used for Files service weights (e.g. fileset-backed or SFT full weights in Files). - models, e.g. multi-LLM pulling through Files). The NIMCache job pulls from the Files endpoint - and populates the cache so the NIM pod uses pre-filled storage. - - Args: - backend_config: Backend configuration - k8s_namespace: Kubernetes namespace - resource_name: Name for the NIMCache resource (already K8s-safe) - model_namespace: Model namespace in Entity Store - model_name: Model name in Entity Store - pvc_size: PVC size for storage - model_revision: Optional model revision - - Returns: - NIMCache CR object - """ - logger.info(f"Generating NIMCache for Files service model: {model_namespace}/{model_name}") - - files_full_url = _get_files_hf_url() - - cr_labels = _merge_model_labels( - backend_config, - { - "app.kubernetes.io/name": resource_name, - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - }, - ) - cr_annotations = _merge_default_annotations(backend_config.default_annotations) - nimcache_metadata: dict[str, Any] = { - "name": resource_name, - "namespace": k8s_namespace, - "labels": cr_labels, - } - if cr_annotations: - nimcache_metadata["annotations"] = cr_annotations - - nimcache = NIMCache( - apiVersion="apps.nvidia.com/v1alpha1", - kind="NIMCache", - metadata=nimcache_metadata, - spec=NIMCacheSpec( - storage=NIMCacheStorage( - pvc=Pvc( - create=True, - name=resource_name, - size=pvc_size, - storageClass=backend_config.default_storage_class or None, - volumeAccessMode="ReadWriteOnce", - annotations=cr_annotations, - ) - ), - source=Source( - # Using Hf (HuggingFace) source type for Files service HF-compatible API - hf=Hf( - endpoint=files_full_url, - namespace=model_namespace, - authSecret=backend_config.files_auth_secret, - modelPuller=huggingface_model_puller, - pullSecret=backend_config.huggingface_model_puller_image_pull_secret, - modelName=model_name, - revision=model_revision, - ) - ), - userID=backend_config.default_user_id, - groupID=backend_config.default_group_id, - resources=_nimcache_default_resources(backend_config), - tolerations=_nimcache_default_tolerations(backend_config), - nodeSelector=backend_config.default_node_selector, - ), - ) - - logger.info(f"Generated NIMCache {k8s_namespace}/{resource_name} for model {model_namespace}/{model_name}") - return nimcache - - -def _generate_tool_plugin_container( - deployment: ModelDeployment, - config: ModelDeploymentConfig, - model_entity: ModelEntity | None, - backend_config: K8sNimOperatorConfig, - huggingface_model_puller: str | None, -) -> list[ContainerSpec] | None: - nim_config = deployment_config_view(config) - plugin_fileset: str | None = None - if nim_config.tool_call_config and nim_config.tool_call_config.tool_call_plugin: - plugin_fileset = nim_config.tool_call_config.tool_call_plugin - elif ( - model_entity - and model_entity.spec - and model_entity.spec.tool_call_config - and model_entity.spec.tool_call_config.tool_call_plugin - ): - plugin_fileset = model_entity.spec.tool_call_config.tool_call_plugin - - if plugin_fileset: - logger.info(f"Pulling tool_call_plugin fileset '{plugin_fileset}' for {deployment.workspace}/{deployment.name}") - container_name = get_docker_plugin_puller_container_name(deployment.workspace, deployment.name) - if not huggingface_model_puller: - logger.warning( - "tool_call_plugin is configured but huggingface_model_puller image is unavailable; " - "skipping plugin init containers" - ) - return None - - files_url = _get_files_hf_url() - - # Require explicit tag. The tag separator ':' must be after the last '/' - # so registry ports like registry:5000/... are not treated as a tag. - hmp = huggingface_model_puller - last_slash_idx = hmp.rfind("/") - last_colon_idx = hmp.rfind(":") - if last_colon_idx <= last_slash_idx: - logger.warning(f"huggingface_model_puller image {huggingface_model_puller} does not have a tag") - return None - - puller_repo = hmp[:last_colon_idx] - puller_tag = hmp[last_colon_idx + 1 :] - - return [ - ContainerSpec( - name=_get_k8s_safe_name( - container_name, max_length=63, suffix="-prepare", name_type="label", include_hash=False - ), - image=Image( - repository=backend_config.busybox_image, - tag=backend_config.busybox_image_tag, - pullPolicy="IfNotPresent", - ), - command=[ - "sh", - "-c", - "set -e; mkdir -p /model-store/plugin /scratch/plugin; " - f"rm -f {TOOL_CALL_PLUGIN_PATH}; rm -rf /scratch/plugin/*", - ], - ), - ContainerSpec( - name=_get_k8s_safe_name( - container_name, max_length=63, suffix="-pull", name_type="label", include_hash=False - ), - image=Image( - repository=puller_repo, - tag=puller_tag, - pullPolicy="IfNotPresent", - pullSecrets=( - [backend_config.huggingface_model_puller_image_pull_secret] - if backend_config.huggingface_model_puller_image_pull_secret - else None - ), - ), - command=["download", plugin_fileset, "--local-dir", TOOL_CALL_PLUGIN_SCRATCH_DIR], - env=[ - EnvItem(name="HF_ENDPOINT", value=files_url), - EnvItem(name="HF_TOKEN", value="service:models"), - ], - ), - ContainerSpec( - name=_get_k8s_safe_name( - container_name, max_length=63, suffix="-finalize", name_type="label", include_hash=False - ), - image=Image( - repository=backend_config.busybox_image, - tag=backend_config.busybox_image_tag, - pullPolicy="IfNotPresent", - ), - command=[ - "sh", - "-c", - TOOL_CALL_PLUGIN_FINALIZE_SCRIPT_TEMPLATE.format( - scratch_dir=TOOL_CALL_PLUGIN_SCRATCH_DIR, - plugin_path=TOOL_CALL_PLUGIN_PATH, - ), - ], - ), - ] - - return None - - -def compile_nimservice( - deployment: ModelDeployment, - config: ModelDeploymentConfig, - backend_config: K8sNimOperatorConfig, - k8s_namespace: str, - resource_name: str, - nimcache_name: str | None = None, - model_entity: ModelEntity | None = None, - huggingface_model_puller: str | None = None, -) -> NIMService: - """Compile a NIMService CRD from a ModelDeployment and its configuration. - - Args: - deployment: The ModelDeployment object - config: The ModelDeploymentConfig - backend_config: Backend configuration - k8s_namespace: Kubernetes namespace - resource_name: Name for the NIMService resource - nimcache_name: Optional NIMCache name when pulling weights from Files service (weights in /model-store). - model_entity: Optional ModelEntity for propagating entity-level settings (e.g. trust_remote_code). - - Returns: - NIMService CR object - """ - logger.info( - f"Compiling NIMService for deployment {deployment.workspace}/{deployment.name} " - f"with config {config.workspace}/{config.name}@{config.entity_version}" - ) - - nim_config = deployment_config_view(config) - platform_config = get_platform_config() - image_pull_secrets = [secret.name for secret in platform_config.image_pull_secrets] - pvc_size = nim_config.disk_size if nim_config.disk_size else backend_config.default_pvc_size - - # Determine startup_probe_grace_seconds with precedence: backend default < per-deployment config - startup_grace_seconds = backend_config.default_startup_probe_grace_period_seconds - if nim_config.k8s_nim_operator_config: - if hasattr(nim_config.k8s_nim_operator_config, "model_dump"): - config_dict = nim_config.k8s_nim_operator_config.model_dump(exclude_none=True) - if "startup_probe_grace_seconds" in config_dict: - startup_grace_seconds = config_dict.get("startup_probe_grace_seconds") - elif isinstance(nim_config.k8s_nim_operator_config, dict): - if "startup_probe_grace_seconds" in nim_config.k8s_nim_operator_config: - startup_grace_seconds = nim_config.k8s_nim_operator_config.get("startup_probe_grace_seconds") - - plugin_containers = _generate_tool_plugin_container( - deployment=deployment, - config=config, - model_entity=model_entity, - backend_config=backend_config, - huggingface_model_puller=huggingface_model_puller, - ) - plugin_path = TOOL_CALL_PLUGIN_PATH if plugin_containers is not None else None - - env_vars = _compile_env_vars( - backend_config, nim_config, nimcache_name, model_entity, tool_call_plugin_path=plugin_path - ) - sidecar_env_vars = env_vars + [EnvItem(name=k, value=v) for k, v in platform_config.to_shared_envvars().items()] - # Operator uses authSecret for NGC_API_KEY by default; when NIMCache is HF (or NIMService has hf:// model name) - # it injects HF_TOKEN from authSecret instead. We use Files placeholder secret when we have NIMCache (always HF); - # otherwise use NGC secret so the NIM can pull weights from NGC at runtime. - auth_secret = backend_config.files_auth_secret if nimcache_name else backend_config.auth_secret - - sidecar_containers = [] - if nim_config.lora_enabled: - sidecar_containers = [ - ContainerSpec( - name=_get_k8s_safe_name( - resource_name, max_length=63, suffix="-lora-sidecar", name_type="label", include_hash=False - ), - image=Image( - repository=f"{platform_config.image_registry}/{backend_config.lora_sidecar_image_name}", - tag=platform_config.image_tag, - pullPolicy="IfNotPresent", - pullSecrets=image_pull_secrets if image_pull_secrets else None, - ), - command=backend_config.lora_sidecar_command, - args=backend_config.lora_sidecar_args if backend_config.lora_sidecar_args else None, - env=sidecar_env_vars, - ) - ] - spec_labels = _merge_model_labels( - backend_config, - { - "app.kubernetes.io/name": resource_name, - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - "nmp.nvidia.com/deployment-workspace": deployment.workspace, - "nmp.nvidia.com/deployment-name": deployment.name, - }, - ) - spec_annotations = _merge_default_annotations(backend_config.default_annotations) - - spec = Spec( - authSecret=auth_secret, - image=Image( - repository=nim_config.image_name or backend_config.default_nimservice_image, - tag=nim_config.image_tag or backend_config.default_nimservice_image_tag, - pullPolicy="IfNotPresent", - pullSecrets=image_pull_secrets if image_pull_secrets else None, - ), - resources=_compile_resources(nim_config.gpu), - storage=_compile_storage(backend_config, resource_name, pvc_size, nimcache_name), - expose=_compile_expose(), - env=env_vars, - startupProbe=_compile_startup_probe(startup_grace_seconds), - replicas=1, - labels=spec_labels, - annotations=spec_annotations, - userID=backend_config.default_user_id, - groupID=backend_config.default_group_id, - initContainers=plugin_containers, - sidecarContainers=sidecar_containers, - ) - - # Apply configuration in precedence order: backend defaults < per-deployment k8s_nim_operator_config < override_config - - # Apply backend config defaults - backend_defaults = {} - if backend_config.default_resources: - backend_defaults["resources"] = backend_config.default_resources - if backend_config.default_tolerations: - backend_defaults["tolerations"] = backend_config.default_tolerations - if backend_config.default_node_selector: - backend_defaults["node_selector"] = backend_config.default_node_selector - - spec = _apply_k8s_nim_operator_config(spec, backend_defaults) - - # Apply per-deployment k8s_nim_operator_config (overrides backend defaults) - if nim_config.k8s_nim_operator_config: - spec = _apply_k8s_nim_operator_config(spec, nim_config.k8s_nim_operator_config) - - # Apply override_config (final override) - if nim_config.override_config: - spec = _apply_override_config(spec, nim_config.override_config) - - nimservice_metadata: dict[str, Any] = { - "name": resource_name, - "namespace": k8s_namespace, - "labels": _merge_model_labels( - backend_config, - { - "app.kubernetes.io/name": resource_name, - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - "nmp.nvidia.com/deployment-workspace": deployment.workspace, - "nmp.nvidia.com/deployment-name": deployment.name, - }, - ), - } - nimservice_cr_annotations = _merge_default_annotations(backend_config.default_annotations) - if nimservice_cr_annotations: - nimservice_metadata["annotations"] = nimservice_cr_annotations - - nimservice = NIMService( - apiVersion="apps.nvidia.com/v1alpha1", - kind="NIMService", - metadata=nimservice_metadata, - spec=spec, - ) - - logger.info(f"Compiled NIMService {k8s_namespace}/{resource_name}") - return nimservice - - -def _compile_resources(gpu_count: int) -> Resources: - return Resources( - limits={"nvidia.com/gpu": Limits1(root=str(gpu_count))}, - requests={"cpu": Requests1(root="1000m")}, - ) - - -def _compile_storage( - backend_config: K8sNimOperatorConfig, resource_name: str, disk_size: str, nimcache_name: str | None = None -) -> Storage: - """Compile storage configuration for the NIMService. - - If nimcache_name is provided, configures the NIMService to use the NIMCache. - Otherwise, creates a standard PVC. - """ - if nimcache_name: - logger.info(f"Configuring NIMService to use NIMCache: {nimcache_name}") - return Storage( - nimCache={ - "name": nimcache_name, - "profile": "", - } - ) - else: - return Storage( - pvc={ - "create": True, - "name": resource_name, - "size": disk_size, - "storageClass": backend_config.default_storage_class or None, - "volumeAccessMode": "ReadWriteOnce", - } - ) - - -def _compile_expose() -> Expose: - return Expose(service={"type": "ClusterIP", "port": 8000}) - - -def _compile_startup_probe(grace_seconds: int | None = None) -> StartupProbe: - """Compile startup probe configuration for NIM containers. - - NIMs can take several minutes to start up (downloading/loading models). - This configures a 10-minute grace period by default. - - Args: - grace_seconds: Optional grace period in seconds. If provided, failureThreshold - is calculated by dividing by 10 (rounded up). Defaults to 600 seconds (10 minutes). - """ - # Default to 600 seconds (10 minutes) if not specified - if grace_seconds is None: - grace_seconds = 600 - - # Calculate failureThreshold by dividing grace_seconds by periodSeconds (10s), rounding up - # Using math.ceil equivalent: (grace_seconds + 9) // 10 - failure_threshold = (grace_seconds + 9) // 10 - - return StartupProbe( - enabled=True, - probe=Probe2( - httpGet=HttpGet2( - path="/v1/health/ready", - port=8000, - ), - periodSeconds=10, - timeoutSeconds=5, - failureThreshold=failure_threshold, - successThreshold=1, - ), - ) - - -def _compile_env_vars( - backend_config: K8sNimOperatorConfig, - nim_config: DeploymentConfigView, - nimcache_name: str | None = None, - model_entity: ModelEntity | None = None, - tool_call_plugin_path: str | None = None, -) -> list[EnvItem]: - """Compile environment variables for the NIMService. - - Args: - backend_config: Backend configuration - nim_config: NIM deployment configuration - nimcache_name: Optional NIMCache name when pulling weights from Files service (weights in /model-store). - model_entity: Optional model entity this deployment references. - tool_call_plugin_path: Optional resolved plugin path for NIM_TOOL_PARSER_PLUGIN. - Returns: - List of environment variables - """ - default_envs = {"NIM_GUIDED_DECODING_BACKEND": backend_config.nim_guided_decoding_backend} - env_items = [] - - model_fqdn: str | None = None - if nim_config.model_name: - parsed_namespace, parsed_name, parsed_revision = parse_model_name_revision( - model_namespace=nim_config.model_namespace, - model_name=nim_config.model_name, - model_revision=nim_config.model_revision, - ) - if parsed_namespace and parsed_name: - model_fqdn = f"{parsed_namespace}/{parsed_name}" - elif parsed_name: - model_fqdn = parsed_name - if model_fqdn and parsed_revision: - model_fqdn += f"@{parsed_revision}" - - if model_fqdn: - default_envs["NIM_MODEL_NAME"] = model_fqdn - default_envs["NIM_SERVED_MODEL_NAME"] = model_fqdn - - if nimcache_name: - # NIMCache is used to pull Files service weights; they're in /model-store. - default_envs["NIM_MODEL_NAME"] = "/model-store" - - if nim_config.lora_enabled: - # default_envs["NIM_PEFT_SOURCE"] = backend_config.peft_source - default_envs["NIM_PEFT_SOURCE"] = "/scratch/loras" - default_envs["NIM_PEFT_REFRESH_INTERVAL"] = str(backend_config.peft_refresh_interval) - - # Only set NIM_FT_MODEL for model-specific NIM (not multi-LLM). When nimcache_name is set, - # weights are in /model-store; multi-LLM uses only NIM_MODEL_NAME (see NIM fine-tuned model docs). - effective_image = nim_config.image_name or backend_config.default_nimservice_image - if nimcache_name and not is_multi_llm_image(effective_image): - logger.info("Adding fine-tuned model environment variables for model-specific container") - default_envs["NIM_FT_MODEL"] = "/model-store" - default_envs["NIM_CUSTOM_MODEL"] = "/model-store" - default_envs["NIM_MODEL_PATH"] = "/model-store" # NIM LLM 2.0 expected parameter - - if model_entity: - default_envs["NMP_MODEL_ENTITY_WORKSPACE"] = model_entity.workspace - default_envs["NMP_MODEL_ENTITY_NAME"] = model_entity.name - if model_entity.trust_remote_code: - default_envs["NIM_FORCE_TRUST_REMOTE_CODE"] = "1" - default_envs["NIM_TRUST_CUSTOM_CODE"] = "1" # NIM LLM 2.0 expected parameter - - # Set NIM env vars from model entity spec (base layer) - if model_entity.spec: - if model_entity.spec.chat_template: - default_envs["NIM_CHAT_TEMPLATE"] = model_entity.spec.chat_template - - if model_entity.spec.tool_call_config: - tool_cfg = model_entity.spec.tool_call_config - if tool_cfg.tool_call_parser: - default_envs["NIM_TOOL_CALL_PARSER"] = tool_cfg.tool_call_parser - if tool_call_plugin_path: - # Point to the actual .py file discovered in the pulled fileset - default_envs["NIM_TOOL_PARSER_PLUGIN"] = tool_call_plugin_path - if tool_cfg.auto_tool_choice is not None: - default_envs["NIM_ENABLE_AUTO_TOOL_CHOICE"] = "1" if tool_cfg.auto_tool_choice else "0" - - # Deployment-level overrides (highest priority). - if nim_config.chat_template: - default_envs["NIM_CHAT_TEMPLATE"] = nim_config.chat_template - - deploy_tool_cfg = nim_config.tool_call_config - if deploy_tool_cfg: - if deploy_tool_cfg.tool_call_parser: - default_envs["NIM_TOOL_CALL_PARSER"] = deploy_tool_cfg.tool_call_parser - if deploy_tool_cfg.tool_call_plugin: - if tool_call_plugin_path: - default_envs["NIM_TOOL_PARSER_PLUGIN"] = tool_call_plugin_path - else: - logger.warning( - "Deployment tool_call_config.tool_call_plugin is set but no plugin path was prepared by init " - "containers." - ) - if deploy_tool_cfg.auto_tool_choice is not None: - default_envs["NIM_ENABLE_AUTO_TOOL_CHOICE"] = "1" if deploy_tool_cfg.auto_tool_choice else "0" - - if nim_config.additional_envs: - default_envs.update(nim_config.additional_envs) - - for key, value in default_envs.items(): - env_items.append(EnvItem(name=key, value=str(value))) - - # When using NIMCache (Files service), also pass NGC_API_KEY from auth secret for parity with Docker. - if nimcache_name: - env_items.append( - EnvItem( - name="NGC_API_KEY", - valueFrom=ValueFrom(secretKeyRef=SecretKeyRef(name=backend_config.auth_secret, key="NGC_API_KEY")), - ) - ) - - return env_items - - -def _apply_k8s_nim_operator_config(spec: Spec, k8s_config: Any) -> Spec: - """Apply k8s_nim_operator_config to the NIMService spec. - - Converts snake_case Python field names to camelCase Kubernetes field names. - """ - logger.info("Applying k8s_nim_operator_config to NIMService spec") - - # Convert k8s_config to dict, excluding None values - if hasattr(k8s_config, "model_dump"): - k8s_config_dict = k8s_config.model_dump(exclude_none=True) - elif isinstance(k8s_config, dict): - # Handle if it's already a dict - k8s_config_dict = {k: v for k, v in k8s_config.items() if v is not None} - else: - # Handle unexpected types gracefully - logger.warning(f"Unexpected k8s_config type: {type(k8s_config)}, skipping") - return spec - - if not k8s_config_dict: - return spec - - # Map snake_case Python field names to camelCase Kubernetes field names - # Note: startup_probe_grace_seconds is handled separately in _compile_startup_probe - field_mapping = { - "resources": "resources", - "tolerations": "tolerations", - "node_selector": "nodeSelector", - } - - # Convert to camelCase for Kubernetes - k8s_spec_updates = {} - for python_field, k8s_field in field_mapping.items(): - if python_field in k8s_config_dict: - k8s_spec_updates[k8s_field] = k8s_config_dict[python_field] - - # Apply to spec - spec_dict = spec.model_dump(exclude_none=True) - _deep_merge(spec_dict, k8s_spec_updates) - - return Spec(**spec_dict) - - -def _apply_override_config(spec: Spec, override_config: dict[str, Any]) -> Spec: - """Apply override configuration to the NIMService spec.""" - logger.info("Applying override configuration to NIMService spec") - - spec_dict = spec.model_dump(exclude_none=True) - _deep_merge(spec_dict, override_config) - return Spec(**spec_dict) - - -def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> None: - for key, value in override.items(): - if key in base and isinstance(base[key], dict) and isinstance(value, dict): - _deep_merge(base[key], value) - else: - base[key] = value - - -def _merge_model_labels(backend_config: K8sNimOperatorConfig, base: dict[str, str]) -> dict[str, str]: - """Merge configured model labels with base; base takes precedence on conflict.""" - out = dict(backend_config.default_labels or {}) - out.update(backend_config.model_labels or {}) - out.update(base) - return out - - -def _merge_default_annotations(default_annotations: Optional[dict[str, str]]) -> Optional[dict[str, str]]: - """Return default annotations dict or None if empty.""" - if not default_annotations: - return None - return dict(default_annotations) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/base.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/base.py deleted file mode 100644 index 5e150fc142..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/base.py +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Reconciler interface + the pre-resolved inputs a reconciler operates on. - -The k8s service backend splits responsibilities: - -* ``K8sNimOperatorServiceBackend`` (the ``ServiceBackend``) owns the - ``nemo_platform`` SDK, determines the current state of the *API object* - (ModelDeployment / ModelDeploymentConfig), resolves all inputs a reconciler - needs (weight source, resource names, Files endpoint), selects the correct - reconciler by engine, and delegates. -* A ``Reconciler`` reconciles desired state (the API object) against the actual - state of *backend infra resources*. It does NOT call the ``nemo_platform`` SDK - and does NOT infer API-object state; it receives everything pre-resolved in a - :class:`ResolvedDeployment` and talks only to the Kubernetes API. - -Two reconcilers implement this interface: - -* ``NimOperatorReconciler`` -- emits ``NIMService`` / ``NIMCache`` CRs and lets the - in-cluster k8s-nim-operator do the actual reconciliation; status is propagated - upward from the operator-created resources. -* ``K8sReconciler`` -- emits native Kubernetes objects directly (PVC / Job / - Deployment / Service) and drives a staged rollout itself, advancing the - deployment one phase at a time as it is polled via ``get_status``. - -``Reconciler`` is a pure interface: shared read-side logic (status projection) and -delete semantics are *composed* in via :class:`StatusProjector` and -:class:`ResourceDeleter` rather than inherited, so each reconciler declares -exactly the collaborators it needs. -""" - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import Optional - -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig -from nemo_platform.types.models.model_entity import ModelEntity -from nmp.core.models.app import ModelWeightsType -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.common import DeploymentConfigView - - -@dataclass -class ResolvedDeployment: - """Everything a reconciler needs, pre-resolved by the ServiceBackend. - - The ServiceBackend computes these (the SDK/entity-shaping and API-object work) - and hands them to a reconciler so the reconciler can stay infra-only: it never - calls the ``nemo_platform`` SDK and never re-derives names or the weight - source. Fields not relevant to a given engine are simply left unset. - """ - - deployment: ModelDeployment - config: ModelDeploymentConfig - model_entity: Optional[ModelEntity] - view: DeploymentConfigView - - # k8s resource name for the deployment (NIMService / vLLM Deployment / PVC). - resource_name: str - # k8s resource name for the NIMCache (NIM path only; reserves the "-job" suffix). - nimcache_resource_name: str - - # Resolved weight source. - weights_type: ModelWeightsType - model_namespace: Optional[str] = None - model_name: Optional[str] = None - model_revision: Optional[str] = None - - # Cluster-routable Files HF endpoint for the in-cluster weight puller (vLLM). - files_hf_url: Optional[str] = None - # Image used to pull weights (NIMCache modelPuller / vLLM puller Job). - huggingface_model_puller: Optional[str] = None - - -class Reconciler(ABC): - """Reconciles desired deployment state against actual backend resources. - - Implementations talk only to Kubernetes. Shared status projection and delete - semantics are composed in (see :class:`StatusProjector` / - :class:`ResourceDeleter`), not inherited. - """ - - @abstractmethod - async def create(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - """Reconcile toward the desired state for a newly-created deployment.""" - ... - - @abstractmethod - async def update(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - """Reconcile toward the desired state for an updated deployment.""" - ... - - @abstractmethod - async def get_status(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - """Project the actual state of backend resources into a status update. - - Reconcilers MAY advance creation here (the direct-emission reconciler - drives its staged rollout from this method); the operator reconciler just - reads operator-reported status. - """ - ... - - @abstractmethod - async def delete(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete the backend resources this reconciler owns (idempotent).""" - ... - - @abstractmethod - async def list_managed_deployment_names(self) -> list[str]: - """List ``workspace/name`` for deployments this reconciler manages.""" - ... diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/k8s.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/k8s.py deleted file mode 100644 index 1108761348..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/k8s.py +++ /dev/null @@ -1,792 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Direct-emission Kubernetes reconciler for the vLLM and generic engines. - -Emits native Kubernetes objects (PVC / weight-puller Job / Deployment / Service) -directly -- there is no operator. Whether creation is staged depends on whether -the deployment has platform-managed weights (a fileset-backed model), NOT on the -engine: - -* **Weighted** (vLLM always; generic when a fileset is present): a staged rollout. - * P0 (``create``): emit the PVC + weight-puller Job. The serving Deployment + - Service are intentionally NOT created yet so the controller can gate on weight - readiness. - * P3 (in ``get_status``, once the puller Job succeeds): delete the completed - puller Job to release its ReadWriteOnce volume, then emit the serving - Deployment + Service with ownerReferences so a later delete cascades. -* **Weightless** (generic with no fileset): the serving Deployment + Service are - emitted immediately at ``create`` -- no PVC, no puller Job, no ``/model-store`` - mount; the container runs purely from its image. - -The engine selects only the compiler (image/args/env), the pod uid/gid, and -whether the LoRA sidecar is wired; the staged-vs-immediate decision is driven by -weight presence. Inputs arrive pre-resolved on a :class:`ResolvedDeployment` (the -ServiceBackend does the SDK / entity-shaping work); this reconciler talks only to -Kubernetes. -""" - -from logging import getLogger -from typing import Any, Optional - -from kubernetes import client as k8s_client -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.models.model_entity import ModelEntity -from nmp.common.config import get_platform_config -from nmp.core.models.app import ModelWeightsType, get_deployment_resource_name -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.controllers.backends import generic_compiler, vllm_compiler -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.common import DeploymentConfigView -from nmp.core.models.controllers.backends.engine import ( - ENGINE_GENERIC, - ENGINE_VLLM, - config_engine, - resolve_health_path, -) -from nmp.core.models.controllers.backends.k8s_nim_operator import vllm_k8s_compiler -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.base import ( - Reconciler, - ResolvedDeployment, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector - -logger = getLogger(__name__) - - -class K8sReconciler(Reconciler): - """Reconciles a deployment by emitting native Kubernetes objects directly. - - Handles the ``vllm`` and ``generic`` engines, which share this direct-emission - path. The rollout shape is chosen by **weight presence**, not engine: - - * Weighted (vLLM always; generic with a fileset) -- a staged rollout - (PVC + weight-puller Job -> serving Deployment + Service), advanced one phase - at a time as it is polled via :meth:`get_status`. - * Weightless (generic with no fileset) -- the serving Deployment + Service are - emitted immediately at create, with no PVC/puller and no ``/model-store`` - mount. - - The engine (:func:`config_engine`) selects only the compiler, uid/gid, and - LoRA wiring (see :meth:`_serving_plan`). Holds its own typed API clients - (CoreV1 / AppsV1 / BatchV1), composes a :class:`StatusProjector` (serving-pod - readiness/diagnostics) and a :class:`ResourceDeleter`. - """ - - def __init__( - self, - k8s_client_: k8s_client.ApiClient, - backend_config: K8sNimOperatorConfig, - k8s_namespace: str, - huggingface_model_puller: str, - status: StatusProjector, - deleter: ResourceDeleter, - ) -> None: - self._k8s_client = k8s_client_ - self._backend_config = backend_config - self._k8s_namespace = k8s_namespace - self._core_v1 = k8s_client.CoreV1Api(k8s_client_) - self._apps_v1 = k8s_client.AppsV1Api(k8s_client_) - self._batch_v1 = k8s_client.BatchV1Api(k8s_client_) - self._huggingface_model_puller = huggingface_model_puller - self._status = status - self._deleter = deleter - - # ------------------------------------------------------------------ - # Reconciler interface - # ------------------------------------------------------------------ - - @staticmethod - def _has_weights(resolved: ResolvedDeployment) -> bool: - """True when the platform pulls weights for this deployment. - - vLLM always pulls weights (it serves a model from the Files service). - Generic is weightless by default and only pulls weights when its config - resolves to a Files-service model (a fileset-backed entity). This -- not - the engine alone -- decides staged vs. immediate rollout. - """ - if config_engine(resolved.config) == ENGINE_GENERIC: - return resolved.weights_type == ModelWeightsType.FILES_SERVICE - # vLLM (the only other engine routed here) is always weighted. - return True - - async def create(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - """Create the deployment's backend resources. - - Weighted deployments start at phase P0: emit the PVC + weight-puller Job - (the Deployment + Service are created later by the status path once the Job - completes -- controller-side weight-readiness gating). Weightless generic - deployments have nothing to pull, so the serving Deployment + Service are - emitted immediately. - """ - if not self._has_weights(resolved): - return self._create_serving_objects(resolved) - - deployment = resolved.deployment - engine = config_engine(resolved.config) - logger.info( - f"Creating {engine} deployment: {deployment.workspace}/{deployment.name} " - f"(version: {deployment.entity_version})" - ) - try: - resource_name = resolved.resource_name - view = resolved.view - model_repo, source_tag = self._model_source(resolved) - disk_size = view.disk_size or self._backend_config.default_pvc_size - if resolved.files_hf_url is None: - raise ValueError(f"Cannot create {engine} deployment: Files HF endpoint was not resolved") - - user_id, group_id = self._pod_user(engine, view) - pvc = vllm_k8s_compiler.compile_pvc( - resource_name=resource_name, - workspace=deployment.workspace, - name=deployment.name, - engine=engine, - disk_size=disk_size, - storage_class=self._backend_config.default_storage_class, - model_source=source_tag, - namespace=self._k8s_namespace, - annotations=self._backend_config.default_annotations, - extra_labels=self._backend_config.model_labels, - ) - job = vllm_k8s_compiler.compile_puller_job( - resource_name=resource_name, - workspace=deployment.workspace, - name=deployment.name, - engine=engine, - image=self._huggingface_model_puller, - container_args=["download", model_repo, "--local-dir", vllm_k8s_compiler.MODEL_STORE_PATH], - env={"HF_ENDPOINT": resolved.files_hf_url, "HF_TOKEN": "service:models"}, - gpu=view.gpu, - namespace=self._k8s_namespace, - service_account_name=self._backend_config.service_account_name, - image_pull_secret=self._backend_config.huggingface_model_puller_image_pull_secret, - user_id=user_id, - group_id=group_id, - model_source=source_tag, - extra_labels=self._backend_config.model_labels, - ) - - self._create_or_skip(self._core_v1.create_namespaced_persistent_volume_claim, pvc, "PVC") - self._create_or_skip(self._batch_v1.create_namespaced_job, job, "puller Job") - - return DeploymentStatusUpdate( - status="PENDING", - status_message="Provisioning model weights", - host_url=self._status.host_url(resource_name), - ) - except Exception as e: - logger.error(f"Failed to create {engine} deployment for {deployment.workspace}/{deployment.name}: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to create deployment {deployment.workspace}/{deployment.name} due to a service backend error", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - - async def update(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - """Update a deployment. - - For weighted deployments, weights are only re-pulled when the model source - (name/revision) changes: a changed source deletes the objects (cascading - PVC + Job) and drops back to the phased create; an unchanged source patches - the serving Deployment + Service in place if they exist, else lets the - status path create them at P3. - - Weightless generic deployments have no weight source, so update just - patches (or creates) the serving Deployment + Service from the latest - config. - """ - deployment = resolved.deployment - engine = config_engine(resolved.config) - resource_name = resolved.resource_name - logger.info( - f"Updating {engine} deployment: {deployment.workspace}/{deployment.name} " - f"(version: {deployment.entity_version})" - ) - try: - if not self._has_weights(resolved): - # No weights => no PVC/puller; the serving objects are the whole - # deployment. Re-apply them (patch in place if present). - self._apply_serving_objects(resolved) - return DeploymentStatusUpdate( - status="PENDING", - status_message="Update accepted", - host_url=self._status.host_url(resource_name), - ) - - _, source_tag = self._model_source(resolved) - existing_source = self._existing_model_source(resource_name) - if existing_source is not None and existing_source != source_tag: - logger.info( - f"Model source changed ({existing_source} -> {source_tag}); re-pulling weights for {resource_name}" - ) - self._delete_serving_resources(resource_name) - return await self.create(resolved) - - # Unchanged source: patch the serving Deployment + Service in place if - # present. - if self._serving_deployment_exists(resource_name): - self._apply_serving_objects(resolved) - return DeploymentStatusUpdate( - status="PENDING", - status_message="Update accepted", - host_url=self._status.host_url(resource_name), - ) - # No serving Deployment yet. If the puller Job is still present we're - # mid-pull; the status path will emit the serving objects at P3, so the - # update is a no-op (re-running create() here would re-assert the PVC + - # Job needlessly). Only fall back to create() if the pull objects are - # gone (genuine drift). - if self._puller_job_exists(resource_name): - return DeploymentStatusUpdate( - status="PENDING", - status_message="Update accepted", - host_url=self._status.host_url(resource_name), - ) - return await self.create(resolved) - except Exception as e: - logger.error(f"Failed to update {engine} deployment for {deployment.workspace}/{deployment.name}: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to update deployment {deployment.workspace}/{deployment.name} due to a service backend error", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - - async def get_status(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - """Project status, driving the phased lifecycle for weighted deployments. - - Weightless generic deployments have no puller phase: the Deployment is - created at ``create`` time, so a 404 means it was deleted externally (LOST). - - Weighted deployments read the puller Job + (once created) the Deployment; - when the Job has completed and the Deployment doesn't exist yet, this - advances creation (phase P3) by emitting the Deployment + Service. - """ - resource_name = resolved.resource_name - - # The serving Deployment is the source of truth once it exists. - try: - self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - deployment_exists = True - except k8s_client.exceptions.ApiException as e: - if e.status != 404: - raise - deployment_exists = False - - if deployment_exists: - return self._project_deployment_readiness(resource_name) - - # No Deployment. For weightless deployments it should have been created at - # create-time, so its absence is external deletion -> LOST. - if not self._has_weights(resolved): - return DeploymentStatusUpdate( - status="LOST", - status_message="Serving Deployment not found; resources may have been deleted externally.", - host_url=None, - ) - - # Weighted: we're still in the pull phase. Consult the puller Job. - job_name = vllm_k8s_compiler.pull_job_name(resource_name) - try: - job = self._batch_v1.read_namespaced_job(name=job_name, namespace=self._k8s_namespace) - except k8s_client.exceptions.ApiException as e: - if e.status != 404: - raise - # Job absent. This is one of: - # (a) the transient P3 window after we deleted a *succeeded* puller - # Job to release the RWO volume (the PVC still exists and holds the - # weights -> resume P3 by creating the serving objects); or - # (b) genuine drift (PVC also gone -> LOST). - if self._pvc_exists(resource_name): - return self._create_serving_objects(resolved) - return DeploymentStatusUpdate( - status="LOST", - status_message="Weight-puller Job and PVC not found; resources may have been deleted externally.", - host_url=None, - ) - - job_status = job.status - if job_status and job_status.failed and job_status.failed >= 1 and not (job_status.succeeded or 0): - pod_name = self._find_job_pod_name(job_name) - logs = self._status.fetch_pod_logs(pod_name) if pod_name else "" - return DeploymentStatusUpdate( - status="ERROR", - status_message="Model weight download failed.", - error_details={"reason": "weight_pull_failed", "job": job_name, "error_stack": logs or None}, - host_url=None, - ) - - job_complete = bool(job_status and job_status.succeeded and job_status.succeeded >= 1) - if not job_complete: - return DeploymentStatusUpdate(status="PENDING", status_message="Downloading model weights", host_url=None) - - # Job complete and no Deployment yet: phase P3 -- create the serving objects. - return self._create_serving_objects(resolved) - - async def delete(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete the directly-emitted vLLM objects this reconciler owns. - - Returns an aggregated update; the ServiceBackend combines this with the - other reconciler's delete result. - """ - resource_name = get_deployment_resource_name(workspace, name) - errors = self._delete_serving_resources(resource_name) - if errors: - summary = "; ".join(errors) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to fully delete deployment {workspace}/{name}: {summary}", - error_details={"errors": errors}, - host_url=None, - ) - return DeploymentStatusUpdate( - status="DELETED", - status_message="Deployment deletion initiated successfully", - host_url=None, - ) - - async def list_managed_deployment_names(self) -> list[str]: - """List ``workspace/name`` for directly-emitted Deployments we manage.""" - label_selector = f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}" - seen: set[str] = set() - try: - deployments = self._apps_v1.list_namespaced_deployment( - namespace=self._k8s_namespace, label_selector=label_selector - ) - for dep in deployments.items: - labels = (dep.metadata.labels or {}) if dep.metadata else {} - workspace = labels.get(vllm_k8s_compiler.DEPLOYMENT_WORKSPACE_LABEL) - name = labels.get(vllm_k8s_compiler.DEPLOYMENT_NAME_LABEL) - if workspace and name: - seen.add(f"{workspace}/{name}") - except Exception as e: - logger.warning(f"Failed to list vLLM Deployments for orphan reconciliation: {e}") - return sorted(seen) - - # ------------------------------------------------------------------ - # vLLM-specific helpers (moved verbatim) - # ------------------------------------------------------------------ - - @staticmethod - def _model_source(resolved: ResolvedDeployment) -> tuple[str, str]: - """Resolve the puller's model repo (``namespace/name``) and a source tag. - - The source tag (``namespace/name@revision``) is stamped on the PVC + Job so - the update path can detect a weight-source change and decide to re-pull. - """ - namespace = resolved.model_namespace - name = resolved.model_name - revision = resolved.model_revision - if not namespace or not name: - raise ValueError(f"Cannot resolve model source for vLLM deployment: namespace='{namespace}', name='{name}'") - model_repo = f"{namespace}/{name}" - source_tag = f"{model_repo}@{revision}" if revision else model_repo - return model_repo, source_tag - - def _serving_deployment_exists(self, resource_name: str) -> bool: - """True if the serving Deployment for this resource already exists.""" - try: - self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - return True - except k8s_client.exceptions.ApiException as e: - if e.status == 404: - return False - raise - - def _pvc_exists(self, resource_name: str) -> bool: - """True if the model-weights PVC for this deployment exists.""" - try: - self._core_v1.read_namespaced_persistent_volume_claim( - name=vllm_k8s_compiler.pvc_name(resource_name), namespace=self._k8s_namespace - ) - return True - except k8s_client.exceptions.ApiException as e: - if e.status == 404: - return False - raise - - def _puller_job_exists(self, resource_name: str) -> bool: - """True if the weight-puller Job for this deployment still exists. - - Used by the update path to detect the mid-pull window (PVC + Job created, - serving Deployment not yet emitted) so an unchanged-source update is a - no-op rather than re-running create(). - """ - try: - self._batch_v1.read_namespaced_job( - name=vllm_k8s_compiler.pull_job_name(resource_name), namespace=self._k8s_namespace - ) - return True - except k8s_client.exceptions.ApiException as e: - if e.status == 404: - return False - raise - - def _create_or_skip(self, create_fn, body, kind: str) -> None: - """Create a namespaced object, tolerating 409 Conflict (already exists).""" - try: - create_fn(namespace=self._k8s_namespace, body=body) - logger.info(f"Created {kind} {body.metadata.name} in {self._k8s_namespace}") - except k8s_client.exceptions.ApiException as e: - if e.status == 409: - logger.info(f"{kind} {body.metadata.name} already exists, skipping creation") - return - raise - - def _project_deployment_readiness(self, resource_name: str) -> DeploymentStatusUpdate: - """Map the serving Deployment's status to a DeploymentStatusUpdate.""" - deployment = self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - ready = (deployment.status.ready_replicas or 0) if deployment.status else 0 - if ready >= 1: - return DeploymentStatusUpdate( - status="READY", status_message="", host_url=self._status.host_url(resource_name) - ) - # Not ready yet: reuse the pod-drilldown (crash loop, image pull, events). - return self._status.pod_status_from_deployment(resource_name) - - # ------------------------------------------------------------------ - # Engine-parameterized serving objects (shared by vLLM + generic) - # ------------------------------------------------------------------ - - def _pod_user(self, engine: str, view: DeploymentConfigView) -> tuple[Optional[int], Optional[int]]: - """Pod securityContext uid/gid for a deployment. - - An explicit ``executor_config.run_as_user`` / ``run_as_group`` always wins - (each independently). Otherwise the engine default applies: vLLM pins its - image's user (2000/0); a generic container runs as its own image's user - (unset), since we can't assume an arbitrary image tolerates a forced - uid/gid. - """ - if engine == ENGINE_VLLM: - default_user: Optional[int] = self._backend_config.default_vllm_user_id - default_group: Optional[int] = self._backend_config.default_vllm_group_id - else: - default_user = None - default_group = None - - user_id = view.run_as_user if view.run_as_user is not None else default_user - group_id = view.run_as_group if view.run_as_group is not None else default_group - return user_id, group_id - - def _serving_spec( - self, - resolved: ResolvedDeployment, - *, - mount_model_store: bool, - ) -> tuple[k8s_client.V1Deployment, k8s_client.V1Service]: - """Compile the serving Deployment + Service for the deployment's engine. - - The engine selects the compiler (image/args/env), uid/gid, and LoRA wiring; - ``mount_model_store`` controls whether the model-weights PVC is mounted - (True for weighted deployments, False for a weightless generic container). - """ - deployment = resolved.deployment - view = resolved.view - engine = config_engine(resolved.config) - resource_name = resolved.resource_name - health_path = resolve_health_path(engine, view) - startup_grace = self._backend_config.default_startup_probe_grace_period_seconds or 600 - user_id, group_id = self._pod_user(engine, view) - - if engine == ENGINE_GENERIC: - image_name, image_tag = generic_compiler.resolve_generic_image(view) - args = generic_compiler.compile_generic_args(view) - env = generic_compiler.compile_generic_env_vars(view) - init_containers: Optional[list] = None - sidecar_containers: Optional[list] = None - else: - image_name, image_tag = vllm_compiler.resolve_vllm_image( - view, self._backend_config.default_vllm_image, self._backend_config.default_vllm_image_tag - ) - args = vllm_compiler.compile_vllm_args(view, resolved.model_entity) - env = vllm_compiler.compile_vllm_env_vars(view) - init_containers, sidecar_containers = self._build_lora_containers(deployment, view, resolved.model_entity) - - dep_obj = vllm_k8s_compiler.compile_deployment( - resource_name=resource_name, - workspace=deployment.workspace, - name=deployment.name, - engine=engine, - image=f"{image_name}:{image_tag}", - args=args, - health_path=health_path, - env=env, - gpu=view.gpu, - namespace=self._k8s_namespace, - service_account_name=self._backend_config.service_account_name, - user_id=user_id, - group_id=group_id, - shared_memory_size_limit=self._backend_config.default_shared_memory_size_limit, - startup_grace_seconds=startup_grace, - init_containers=init_containers, - sidecar_containers=sidecar_containers, - extra_labels=self._backend_config.model_labels, - mount_model_store=mount_model_store, - ) - svc_obj = vllm_k8s_compiler.compile_service( - resource_name=resource_name, - workspace=deployment.workspace, - name=deployment.name, - engine=engine, - namespace=self._k8s_namespace, - extra_labels=self._backend_config.model_labels, - ) - return dep_obj, svc_obj - - def _create_serving_objects(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - """Create the serving Deployment + Service for an engine. - - For weighted deployments this is phase P3: the completed puller Job is - deleted first so its pod releases the ReadWriteOnce PVC's volume attachment - (a completed pod keeps the volume attached to its node, which would - otherwise block the server pod with a Multi-Attach error if it scheduled - onto a different node). For a weightless generic deployment there is no - Job/PVC, so this runs straight through. - - Sets ownerReferences (PVC + Service -> Deployment) so deleting the - Deployment cascades the rest. - """ - resource_name = resolved.resource_name - has_weights = self._has_weights(resolved) - - if has_weights: - # Release the RWO volume from the completed puller before the server - # needs it. Idempotent: a missing Job/pod counts as released. - if not self._delete_puller_job(resource_name): - return DeploymentStatusUpdate( - status="PENDING", - status_message="Releasing model weights volume", - host_url=self._status.host_url(resource_name), - ) - - dep_obj, svc_obj = self._serving_spec(resolved, mount_model_store=has_weights) - - try: - created_dep = self._apps_v1.create_namespaced_deployment(namespace=self._k8s_namespace, body=dep_obj) - logger.info(f"Created Deployment {resource_name} in {self._k8s_namespace}") - except k8s_client.exceptions.ApiException as e: - if e.status != 409: - raise - created_dep = self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - - # Owner reference -> Deployment, so PVC/Service cascade on delete. - owner_ref = k8s_client.V1OwnerReference( - api_version="apps/v1", - kind="Deployment", - name=created_dep.metadata.name, - uid=created_dep.metadata.uid, - controller=True, - block_owner_deletion=True, - ) - svc_obj.metadata.owner_references = [owner_ref] - self._create_or_skip(self._core_v1.create_namespaced_service, svc_obj, "Service") - if has_weights: - self._set_owner_reference_on_pvc(resource_name, owner_ref) - - return DeploymentStatusUpdate(status="PENDING", status_message="Starting server", host_url=None) - - def _apply_serving_objects(self, resolved: ResolvedDeployment) -> None: - """Re-apply the serving Deployment + Service for an update. - - Patches the Deployment and Service when they already exist (so changed - image/args/env/health/gpu/labels actually take effect), and creates them - when they don't. Used by the update path; the PVC (if any) is unaffected. - """ - resource_name = resolved.resource_name - has_weights = self._has_weights(resolved) - dep_obj, svc_obj = self._serving_spec(resolved, mount_model_store=has_weights) - - if self._serving_deployment_exists(resource_name): - self._apps_v1.patch_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace, body=dep_obj) - logger.info(f"Patched Deployment {resource_name} in {self._k8s_namespace}") - try: - self._core_v1.patch_namespaced_service(name=resource_name, namespace=self._k8s_namespace, body=svc_obj) - except k8s_client.exceptions.ApiException as e: - if e.status != 404: - raise - self._core_v1.create_namespaced_service(namespace=self._k8s_namespace, body=svc_obj) - return - - # Deployment absent: create both (idempotent on Service). - self._apps_v1.create_namespaced_deployment(namespace=self._k8s_namespace, body=dep_obj) - self._create_or_skip(self._core_v1.create_namespaced_service, svc_obj, "Service") - - def _delete_puller_job(self, resource_name: str) -> bool: - """Delete the puller Job and confirm its pod is gone (releases RWO volume). - - Deletes the Job with foreground/background propagation so its pod is - removed, freeing the volume attachment for the server pod. Returns True - once no puller pod remains; False if a pod is still terminating (caller - should retry on the next poll). Idempotent: a missing Job/pod counts as - released. - """ - job_name = vllm_k8s_compiler.pull_job_name(resource_name) - try: - self._batch_v1.delete_namespaced_job( - name=job_name, - namespace=self._k8s_namespace, - propagation_policy="Background", - ) - logger.info(f"Deleted puller Job {job_name} to release the model-weights volume") - except k8s_client.exceptions.ApiException as e: - if e.status != 404: - raise - - # The volume stays attached until the pod object is gone, so confirm. - try: - pods = self._core_v1.list_namespaced_pod( - namespace=self._k8s_namespace, label_selector=f"job-name={job_name}" - ) - except Exception: - return True - return len(pods.items) == 0 - - def _build_lora_containers( - self, deployment: ModelDeployment, view: Any, model_entity: Optional[ModelEntity] - ) -> tuple[Optional[list], Optional[list]]: - """Build the LoRA init container + adapter sidecar for a vLLM Deployment. - - Returns ``(init_containers, sidecar_containers)``; both ``None`` when LoRA - is not enabled. - - - The init container pre-creates ``/scratch/loras`` (vLLM's filesystem - resolver validates the dir exists at startup). - - The sidecar runs the engine-agnostic ``nmp-api`` adapters controller, - pointed at the same dir, rewriting each adapter's base-model name to the - served model path (``VLLM_LORA_BASE_MODEL_OVERRIDE=/model-store``). - """ - if not view.lora_enabled: - return None, None - - lora_dir = vllm_compiler.VLLM_LORA_CACHE_DIR - platform_config = get_platform_config() - sidecar_image = f"{platform_config.image_registry}/nmp-api:{platform_config.image_tag}" - - init_container = k8s_client.V1Container( - name="lora-cache-init", - image=f"{self._backend_config.busybox_image}:{self._backend_config.busybox_image_tag}", - command=["sh", "-c", f"mkdir -p {lora_dir} && chmod -R 777 {lora_dir}"], - volume_mounts=[k8s_client.V1VolumeMount(name="scratch", mount_path=vllm_k8s_compiler.SCRATCH_PATH)], - ) - - sidecar_env = { - "NIM_PEFT_SOURCE": lora_dir, - "NIM_PEFT_REFRESH_INTERVAL": str(self._backend_config.peft_refresh_interval), - "VLLM_LORA_BASE_MODEL_OVERRIDE": vllm_compiler.MODEL_STORE_PATH, - "NMP_MODEL_ENTITY_WORKSPACE": deployment.workspace, - "NMP_MODEL_ENTITY_NAME": deployment.name, - } - if model_entity is not None: - sidecar_env["NMP_MODEL_ENTITY_WORKSPACE"] = model_entity.workspace - sidecar_env["NMP_MODEL_ENTITY_NAME"] = model_entity.name - sidecar_env.update(platform_config.to_shared_envvars()) - - sidecar = k8s_client.V1Container( - name="lora-sidecar", - image=sidecar_image, - image_pull_policy="IfNotPresent", - command=["nemo", "services", "run", "--sidecars", "adapters"], - env=[k8s_client.V1EnvVar(name=k, value=str(v)) for k, v in sidecar_env.items()], - volume_mounts=[ - k8s_client.V1VolumeMount( - name="model-store", mount_path=vllm_k8s_compiler.MODEL_STORE_PATH, read_only=True - ), - k8s_client.V1VolumeMount(name="scratch", mount_path=vllm_k8s_compiler.SCRATCH_PATH), - ], - ) - # NOTE: the sidecar image comes from the platform registry, but - # imagePullSecrets are pod-level (not per-container), so we don't set them - # on the sidecar here. The pod relies on the models ServiceAccount's pull - # secret, which is applied at the chart level. - return [init_container], [sidecar] - - def _set_owner_reference_on_pvc(self, resource_name: str, owner_ref: k8s_client.V1OwnerReference) -> None: - """Patch the PVC to be owned by the Deployment (best-effort). - - The puller Job is deleted before the Deployment is created (to release the - RWO volume), so only the PVC needs an ownerRef here; the Service gets its - ownerRef at create time. - """ - patch = {"metadata": {"ownerReferences": [self._k8s_client.sanitize_for_serialization(owner_ref)]}} - try: - self._core_v1.patch_namespaced_persistent_volume_claim( - name=vllm_k8s_compiler.pvc_name(resource_name), namespace=self._k8s_namespace, body=patch - ) - except Exception as e: - logger.warning(f"Failed to set ownerReference on PVC for {resource_name}: {e}") - - def _find_job_pod_name(self, job_name: str) -> str | None: - """Find the most recent pod for a Job (best-effort, for failure logs).""" - try: - pods = self._core_v1.list_namespaced_pod( - namespace=self._k8s_namespace, label_selector=f"job-name={job_name}" - ) - if not pods.items: - return None - return max(pods.items, key=lambda p: p.metadata.creation_timestamp).metadata.name - except Exception: - return None - - def _existing_model_source(self, resource_name: str) -> str | None: - """Read the recorded model-source for a deployment, from the Job or the PVC. - - The model source is stamped as an annotation on BOTH the puller Job and - the PVC at create time. The Job is the primary source while it exists, but - it is deleted at P3 (to release the RWO volume) once weights are pulled -- - so for an already-serving deployment the Job is gone and we must read the - annotation from the (long-lived) PVC instead. Without the PVC fallback a - later model-revision change would read ``None`` here, skip the re-pull - branch, and serve stale weights. - - Returns ``None`` only when neither object carries the annotation (or - neither exists). Non-404 API errors are propagated rather than swallowed. - """ - # Primary: the puller Job (present during/just after the pull phase). - try: - job = self._batch_v1.read_namespaced_job( - name=vllm_k8s_compiler.pull_job_name(resource_name), namespace=self._k8s_namespace - ) - annotations = (job.metadata.annotations or {}) if job.metadata else {} - source = annotations.get(vllm_k8s_compiler.MODEL_SOURCE_ANNOTATION) - if source: - return source - except k8s_client.exceptions.ApiException as e: - if e.status != 404: - raise - - # Fallback: the PVC (survives Job deletion at P3). - try: - pvc = self._core_v1.read_namespaced_persistent_volume_claim( - name=vllm_k8s_compiler.pvc_name(resource_name), namespace=self._k8s_namespace - ) - except k8s_client.exceptions.ApiException as e: - if e.status == 404: - return None - raise - annotations = (pvc.metadata.annotations or {}) if pvc.metadata else {} - return annotations.get(vllm_k8s_compiler.MODEL_SOURCE_ANNOTATION) - - def _delete_serving_resources(self, resource_name: str) -> list[str]: - """Delete the directly-emitted objects by name (idempotent). - - Covers the Deployment + Service plus the (optional) puller Job + PVC; a - weightless generic deployment simply has no Job/PVC, so those deletes are - 404-tolerant no-ops. Returns concise error strings for any real (non-404) - failures; empty when everything was deleted or already absent. - """ - deleters = [ - (self._apps_v1.delete_namespaced_deployment, "Deployment", resource_name), - (self._core_v1.delete_namespaced_service, "Service", resource_name), - (self._batch_v1.delete_namespaced_job, "puller Job", vllm_k8s_compiler.pull_job_name(resource_name)), - (self._core_v1.delete_namespaced_persistent_volume_claim, "PVC", vllm_k8s_compiler.pvc_name(resource_name)), - ] - errors: list[str] = [] - for delete_fn, kind, obj_name in deleters: - err = self._deleter.delete_one(delete_fn, kind, obj_name) - if err: - errors.append(err) - return errors diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/nim_operator.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/nim_operator.py deleted file mode 100644 index 19c86bccee..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/nim_operator.py +++ /dev/null @@ -1,460 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NIM-operator reconciler: emits NIMService / NIMCache CRs. - -This reconciler delegates the actual reconciliation to the in-cluster -k8s-nim-operator. It creates/updates/deletes ``NIMService`` and ``NIMCache`` -custom resources and projects status by reading the operator-reported -``NIMService.status`` (drilling into the operator-created Deployment's pods when -the operator reports ``NotReady``). - -Inputs arrive pre-resolved on a :class:`ResolvedDeployment` (the ServiceBackend -does the SDK / entity-shaping work); this reconciler talks only to Kubernetes. -Shared status projection and delete semantics are composed in via -:class:`StatusProjector` and :class:`ResourceDeleter`. -""" - -from logging import getLogger - -from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic import exceptions as k8s_dynamic_exceptions -from nmp.core.models.app import ( - ModelWeightsType, - get_deployment_resource_name, - get_nimcache_resource_name, -) -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler import ( - compile_nimcache, - compile_nimservice, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.base import ( - Reconciler, - ResolvedDeployment, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector - -logger = getLogger(__name__) - -NIM_OPERATOR_GROUP = "apps.nvidia.com" -NIMSERVICE_VERSION = "v1alpha1" -NIMSERVICE_API_VERSION = f"{NIM_OPERATOR_GROUP}/{NIMSERVICE_VERSION}" -NIMSERVICE_PLURAL = "nimservices" - -NIMCACHE_VERSION = "v1alpha1" -NIMCACHE_API_VERSION = f"{NIM_OPERATOR_GROUP}/{NIMCACHE_VERSION}" -NIMCACHE_PLURAL = "nimcaches" - -# Labels stamped by the NIMService compiler for orphan reconciliation. -NIMSERVICE_DEPLOYMENT_WORKSPACE_LABEL = "nmp.nvidia.com/deployment-workspace" -NIMSERVICE_DEPLOYMENT_NAME_LABEL = "nmp.nvidia.com/deployment-name" - - -class NimOperatorReconciler(Reconciler): - """Reconciles a deployment by emitting NIMService / NIMCache CRs. - - Holds its own dynamic client (NIM CRDs are accessed via API discovery) and - composes a :class:`StatusProjector` (for the operator-created Deployment's pod - status when the operator reports ``NotReady``) and a :class:`ResourceDeleter`. - """ - - def __init__( - self, - dynamic_client: DynamicClient, - backend_config: K8sNimOperatorConfig, - k8s_namespace: str, - huggingface_model_puller: str, - status: StatusProjector, - deleter: ResourceDeleter, - ) -> None: - self._dynamic_client = dynamic_client - self._backend_config = backend_config - self._k8s_namespace = k8s_namespace - self._huggingface_model_puller = huggingface_model_puller - self._status = status - self._deleter = deleter - - # ------------------------------------------------------------------ - # Reconciler interface - # ------------------------------------------------------------------ - - async def create(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - deployment = resolved.deployment - config = resolved.config - model_entity = resolved.model_entity - - logger.info( - "Creating NIMService", - extra={ - "workspace": deployment.workspace, - "deployment_name": deployment.name, - "version": deployment.entity_version, - }, - ) - - # Check if Files service model (SFT or fileset) and create NIMCache if needed. - nimcache_name, error = await self._ensure_nimcache(resolved, action="creating") - if error is not None: - return error - - try: - resource_name = resolved.resource_name - - # Compile NIMService with optional NIMCache reference (env vars depend on nimcache_name + image type) - nimservice = compile_nimservice( - deployment=deployment, - config=config, - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - resource_name=resource_name, - nimcache_name=nimcache_name, - model_entity=model_entity, - huggingface_model_puller=self._huggingface_model_puller, - ) - - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - try: - created = nimservice_api.create( - body=nimservice_dict, - namespace=self._k8s_namespace, - ) - logger.info( - "Successfully created NIMService", - extra={ - "namespace": self._k8s_namespace, - "resource_name": resource_name, - "uid": created.metadata.uid, - }, - ) - except k8s_dynamic_exceptions.ConflictError: - # NIMService already exists, just return PENDING and let status check handle it - logger.info("NIMService already exists, skipping creation", extra={"resource_name": resource_name}) - - return DeploymentStatusUpdate( - status="PENDING", - status_message="NIMService creation initiated successfully", - host_url=self._status.host_url(resource_name), - ) - - except Exception as e: - logger.error( - "Failed to create NIMService", - extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "error": str(e)}, - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to create deployment {deployment.workspace}/{deployment.name} due to a service backend error", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - - async def update(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - deployment = resolved.deployment - config = resolved.config - model_entity = resolved.model_entity - - logger.info( - "Updating NIMService", - extra={ - "workspace": deployment.workspace, - "deployment_name": deployment.name, - "version": deployment.entity_version, - }, - ) - - # Check if Files service model (SFT or fileset) and create/update NIMCache if needed. - nimcache_name, error = await self._ensure_nimcache(resolved, action="updating") - if error is not None: - return error - - try: - resource_name = resolved.resource_name - - # Compile NIMService with optional NIMCache reference (env vars depend on nimcache_name + image type) - nimservice = compile_nimservice( - deployment=deployment, - config=config, - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - resource_name=resource_name, - nimcache_name=nimcache_name, - model_entity=model_entity, - huggingface_model_puller=self._huggingface_model_puller, - ) - - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - updated = nimservice_api.replace( - body=nimservice_dict, - name=resource_name, - namespace=self._k8s_namespace, - ) - - logger.info( - "Successfully updated NIMService", - extra={"namespace": self._k8s_namespace, "resource_name": resource_name, "uid": updated.metadata.uid}, - ) - - return DeploymentStatusUpdate( - status="PENDING", - status_message="NIMService update initiated successfully", - host_url=self._status.host_url(resource_name), - ) - - except k8s_dynamic_exceptions.NotFoundError: - logger.warning( - "NIMService not found, treating as create operation", extra={"resource_name": resolved.resource_name} - ) - return await self.create(resolved) - - except Exception as e: - logger.error( - "Failed to update NIMService", - extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "error": str(e)}, - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to update deployment {deployment.workspace}/{deployment.name} due to a service backend error", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - - async def get_status(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: - return self._get_nimservice_status(resolved.resource_name) - - async def delete(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete the NIMService / NIMCache CRs this reconciler owns (idempotent). - - Returns an aggregated update; the ServiceBackend combines this with the - other reconciler's delete result. - """ - nimservice_name = get_deployment_resource_name(workspace, name) - nimcache_name = get_nimcache_resource_name(workspace, name) - errors: list[str] = [] - - for api_version, kind, cr_name in ( - (NIMSERVICE_API_VERSION, "NIMService", nimservice_name), - (NIMCACHE_API_VERSION, "NIMCache", nimcache_name), - ): - try: - cr_api = self._dynamic_client.resources.get(api_version=api_version, kind=kind) - except Exception as e: - errors.append(f"error resolving {kind} API: {e}") - continue - err = self._deleter.delete_one( - lambda name, namespace, _api=cr_api: _api.delete(name=name, namespace=namespace), - kind, - cr_name, - ) - if err: - errors.append(err) - - if errors: - summary = "; ".join(errors) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to fully delete deployment {workspace}/{name}: {summary}", - error_details={"errors": errors}, - host_url=None, - ) - return DeploymentStatusUpdate( - status="DELETED", - status_message="Deployment deletion initiated successfully", - host_url=None, - ) - - async def list_managed_deployment_names(self) -> list[str]: - """List ``workspace/name`` for NIMServices this reconciler manages.""" - label_selector = f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}" - seen: set[str] = set() - try: - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - result = nimservice_api.get(namespace=self._k8s_namespace, label_selector=label_selector) - for item in getattr(result, "items", None) or []: - labels = getattr(getattr(item, "metadata", None), "labels", None) or {} - if isinstance(labels, dict): - workspace = labels.get(NIMSERVICE_DEPLOYMENT_WORKSPACE_LABEL) - name = labels.get(NIMSERVICE_DEPLOYMENT_NAME_LABEL) - if workspace and name: - seen.add(f"{workspace}/{name}") - except k8s_dynamic_exceptions.ForbiddenError: - # No RBAC for the NIM CRDs (e.g. a vLLM-only deployment). Not an error. - logger.debug("No access to NIMServices for orphan reconciliation; skipping NIM path") - except Exception as e: - logger.warning("Failed to list NIMServices for orphan reconciliation", extra={"error": str(e)}) - return sorted(seen) - - # ------------------------------------------------------------------ - # NIM-specific helpers - # ------------------------------------------------------------------ - - async def _ensure_nimcache( - self, resolved: ResolvedDeployment, action: str - ) -> tuple[str | None, DeploymentStatusUpdate | None]: - """Create the NIMCache for a Files-service model, if applicable. - - Returns ``(nimcache_name, None)`` on success (``nimcache_name`` is ``None`` - when the model is not a Files-service model), or ``(None, error_update)`` - when NIMCache creation should abort the create/update. - """ - deployment = resolved.deployment - - if resolved.weights_type != ModelWeightsType.FILES_SERVICE: - logger.debug( - "No Files service model detected", - extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "action": action}, - ) - return None, None - - logger.info( - "Files service model detected, creating NIMCache", - extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "action": action}, - ) - - model_namespace = resolved.model_namespace - model_name = resolved.model_name - if not model_namespace or not model_name: - logger.error( - "Files service model detected but missing model namespace or name in config", - extra={"model_namespace": model_namespace, "model_name": model_name}, - ) - return None, DeploymentStatusUpdate( - status="ERROR", - status_message="Cannot create NIMCache for Files service model: missing model namespace or name in configuration", - error_details={ - "error": "Missing required model namespace or name for Files service model", - "model_namespace": model_namespace, - "model_name": model_name, - }, - host_url=None, - ) - - view = resolved.view - pvc_size = view.disk_size if view.disk_size else self._backend_config.default_pvc_size - - try: - nimcache = compile_nimcache( - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - resource_name=resolved.nimcache_resource_name, - model_namespace=model_namespace, - model_name=model_name, - pvc_size=pvc_size, - huggingface_model_puller=self._huggingface_model_puller, - model_revision=resolved.model_revision, - ) - await self._create_nimcache(nimcache) - logger.info("NIMCache created successfully", extra={"resource_name": resolved.nimcache_resource_name}) - return resolved.nimcache_resource_name, None - except Exception as e: - logger.error("Failed to create NIMCache for Files service model", extra={"error": str(e)}) - return None, DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to create NIMCache for Files service model: {str(e)}", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - - async def _create_nimcache(self, nimcache) -> None: - """Create a NIMCache CR in Kubernetes. - - Args: - nimcache: The NIMCache CR to create - """ - try: - nimcache_api = self._dynamic_client.resources.get( - api_version=NIMCACHE_API_VERSION, - kind="NIMCache", - ) - - nimcache_dict = nimcache.model_dump(exclude_none=True, by_alias=True) - - created = nimcache_api.create( - body=nimcache_dict, - namespace=self._k8s_namespace, - ) - logger.info( - "Successfully created NIMCache", - extra={ - "namespace": self._k8s_namespace, - "resource_name": nimcache.metadata["name"], - "uid": created.metadata.uid, - }, - ) - except k8s_dynamic_exceptions.ConflictError: - logger.info( - "NIMCache already exists, skipping creation", extra={"resource_name": nimcache.metadata["name"]} - ) - except Exception as e: - logger.error( - "Failed to create NIMCache", extra={"resource_name": nimcache.metadata["name"], "error": str(e)} - ) - raise - - def _get_nimservice_status(self, resource_name: str) -> DeploymentStatusUpdate: - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - - try: - nimservice = nimservice_api.get(name=resource_name, namespace=self._k8s_namespace) - except k8s_dynamic_exceptions.NotFoundError: - logger.warning( - "NIMService not found in cluster; may have been deleted externally", - extra={"resource_name": resource_name}, - ) - return DeploymentStatusUpdate( - status="LOST", - status_message="NIMService not found in cluster. Resource may have been deleted externally.", - host_url=None, - ) - - # ``status`` / ``status.state`` may be absent or explicitly null while the - # operator is still populating them -- coerce to "" so .lower() is safe. - nim_status = nimservice.get("status") or {} - state = (nim_status.get("state") or "").lower() - - match state: - case "ready": - return DeploymentStatusUpdate( - status="READY", - status_message="", - host_url=self._status.host_url(resource_name), - ) - case "notready": - conditions = nim_status.get("conditions", []) - logger.info("NIMService is NotReady", extra={"resource_name": resource_name, "conditions": conditions}) - return self._status.pod_status_from_deployment(resource_name) - case "failed": - conditions = nim_status.get("conditions", []) - logger.error("NIMService has failed", extra={"resource_name": resource_name, "conditions": conditions}) - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"NIMService failed: {conditions}", - host_url=None, - ) - case _: - return DeploymentStatusUpdate( - status="PENDING", - status_message=f"NIMService in {state or 'unknown'} state", - host_url=None, - ) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/resource_deleter.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/resource_deleter.py deleted file mode 100644 index 5421dcdb05..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/resource_deleter.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Idempotent, 404-tolerant Kubernetes object deletion. - -Teardown deletes every resource type a deployment could own *by name* (no engine -detection), so it can self-heal partial-deletion states and is safe to call for -orphan reconciliation. :class:`ResourceDeleter` owns that single-object delete -semantics so both reconcilers can *compose* it rather than inherit it. -""" - -from logging import getLogger -from typing import Optional - -from kubernetes import client as k8s_client -from kubernetes.dynamic import exceptions as k8s_dynamic_exceptions - -logger = getLogger(__name__) - - -class ResourceDeleter: - """Deletes namespaced Kubernetes objects by name, tolerating "already gone".""" - - def __init__(self, k8s_namespace: str) -> None: - self._k8s_namespace = k8s_namespace - - def delete_one(self, delete_fn, kind: str, obj_name: str) -> Optional[str]: - """Delete a single namespaced object by name, tolerating "already gone". - - A 404 (object absent) is success. Any other failure is logged concisely - (no stack trace) and returned as a short error string so the caller can - aggregate and surface it (we must NOT mark a deployment DELETED if cluster - resources may remain). - """ - try: - delete_fn(name=obj_name, namespace=self._k8s_namespace) - logger.info(f"Deleted {kind} {self._k8s_namespace}/{obj_name}") - return None - except (k8s_client.exceptions.ApiException, k8s_dynamic_exceptions.NotFoundError) as e: - # NotFound (typed status 404 or dynamic NotFoundError) -> already gone. - if isinstance(e, k8s_dynamic_exceptions.NotFoundError) or getattr(e, "status", None) == 404: - logger.debug(f"{kind} {obj_name} not found, already deleted") - return None - return self._classify_delete_error(e, kind, obj_name) - except Exception as e: - # Any other failure (forbidden, connection/transport error, dynamic API - # error, ...) must be classified and returned -- never raised -- so the - # caller's per-resource delete loop continues and aggregates failures - # rather than aborting cleanup partway and risking a false DELETED. - return self._classify_delete_error(e, kind, obj_name) - - @staticmethod - def _classify_delete_error(e: Exception, kind: str, obj_name: str) -> str: - """Concise, human-readable delete failure (no stack trace) for aggregation.""" - status = getattr(e, "status", None) - is_forbidden = status == 403 or isinstance(e, k8s_dynamic_exceptions.ForbiddenError) - if is_forbidden: - # With the models ServiceAccount RBAC in place this should not happen; - # if it does, the SA is missing delete on this resource type. - msg = f"forbidden to delete {kind} {obj_name} (ServiceAccount lacks RBAC)" - logger.error(msg) - return msg - msg = f"error deleting {kind} {obj_name}: {status or type(e).__name__}" - logger.warning(msg) - return msg diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/status_projector.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/status_projector.py deleted file mode 100644 index d1de87d960..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/status_projector.py +++ /dev/null @@ -1,320 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Engine-agnostic Kubernetes status projection. - -Both reconcilers (operator-driven and direct-emission) ultimately observe the -same underlying Kubernetes objects -- a Deployment, its pods, and their events -- -when reporting status. :class:`StatusProjector` owns that shared read-side logic -(pod log fetch, crash-loop detection, pod-status drill-down, the host URL, and -the PENDING-timeout / crash-loop error builders) so it can be *composed* into a -reconciler rather than inherited. - -It talks only to Kubernetes via an injected ``ApiClient`` and never mutates -cluster state -- it just projects what it sees into a -:class:`DeploymentStatusUpdate`. -""" - -from logging import getLogger -from typing import Any, Dict - -from kubernetes import client as k8s_client -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.common import ( - LOG_MAX_CHARS, - LOG_TAIL_LINES, - format_duration, -) -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig - -logger = getLogger(__name__) - -# Maximum length of a recent-event message surfaced in ``status_message``. The -# value is persisted as the deployment's status and shown in the UI/CLI status -# history, so we cap it to keep history entries readable (not a protocol limit). -MAX_EVENT_MESSAGE_CHARS = 200 - -POD_EVENT_TO_MESSAGE_MAP = { - "startup probe failed": "Waiting for pod to finish startup", -} - - -class StatusProjector: - """Reads a Deployment + its pods/events and projects a status update. - - Engine-agnostic: composed into both reconcilers (and used directly by the - ServiceBackend to enforce the PENDING-timeout policy). - """ - - def __init__( - self, - k8s_client_: k8s_client.ApiClient, - backend_config: K8sNimOperatorConfig, - k8s_namespace: str, - ) -> None: - self._k8s_client = k8s_client_ - self._backend_config = backend_config - self._k8s_namespace = k8s_namespace - - def host_url(self, resource_name: str) -> str: - """Generate the Kubernetes service host URL for a deployment.""" - return f"http://{resource_name}.{self._k8s_namespace}.svc.cluster.local:8000" - - # Pod log fetching and pod lookup (best-effort diagnostics) - - def fetch_pod_logs(self, pod_name: str) -> str: - """Fetch recent pod logs for error reporting, truncated to LOG_MAX_CHARS.""" - try: - core_v1 = k8s_client.CoreV1Api(self._k8s_client) - logs = core_v1.read_namespaced_pod_log( - name=pod_name, - namespace=self._k8s_namespace, - tail_lines=LOG_TAIL_LINES, - ) - if len(logs) > LOG_MAX_CHARS: - logs = logs[-LOG_MAX_CHARS:] - return logs - except Exception as e: - logger.warning( - "Failed to retrieve pod logs for error report", extra={"pod_name": pod_name, "error": str(e)} - ) - return "" - - def find_pod_name(self, resource_name: str) -> str | None: - """Find the most recent pod name for a k8s Deployment (best-effort).""" - try: - apps_v1 = k8s_client.AppsV1Api(self._k8s_client) - core_v1 = k8s_client.CoreV1Api(self._k8s_client) - - try: - deployment = apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - except k8s_client.exceptions.ApiException: - return None - - if not deployment.spec.selector or not deployment.spec.selector.match_labels: - return None - - label_selector = ",".join([f"{k}={v}" for k, v in deployment.spec.selector.match_labels.items()]) - pods = core_v1.list_namespaced_pod(namespace=self._k8s_namespace, label_selector=label_selector) - - if not pods.items: - return None - - pod = max(pods.items, key=lambda p: p.metadata.creation_timestamp) - return pod.metadata.name - except Exception: - return None - - # Crash loop and pending timeout error builders - - def build_pending_timeout_error( - self, - resource_name: str, - elapsed: float, - pod_name: str | None, - ) -> DeploymentStatusUpdate: - """Build ERROR status update for a PENDING timeout.""" - error_stack = self.fetch_pod_logs(pod_name) if pod_name else "" - kubectl_target = pod_name if pod_name else f"deployment/{resource_name}" - status_msg = ( - f"Deployment timed out after {format_duration(elapsed)} waiting for NIM " - f"to pass health checks (timeout: {format_duration(self._backend_config.pending_timeout_seconds)}).\n\n" - f"Inspect the model deployment's pod logs with:\n" - f" kubectl logs -n {self._k8s_namespace} {kubectl_target}" - ) - error_details: Dict[str, Any] = { - "reason": "pending_timeout", - "elapsed_seconds": int(elapsed), - "timeout_seconds": self._backend_config.pending_timeout_seconds, - "resource_name": resource_name, - "namespace": self._k8s_namespace, - "error_stack": error_stack if error_stack else None, - } - if pod_name: - error_details["pod_name"] = pod_name - return DeploymentStatusUpdate( - status="ERROR", - status_message=status_msg, - error_details=error_details, - host_url=None, - ) - - def build_crash_loop_error( - self, - resource_name: str, - pod_name: str, - restart_count: int, - ) -> DeploymentStatusUpdate: - """Build ERROR status update for a crash loop.""" - error_stack = self.fetch_pod_logs(pod_name) - status_msg = ( - f"Deployment entered crash loop after {restart_count} container restarts " - f"(max: {self._backend_config.max_restart_count}).\n\n" - f"Inspect the model deployment's pod logs with:\n" - f" kubectl logs -n {self._k8s_namespace} {pod_name}" - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message=status_msg, - error_details={ - "reason": "crash_loop", - "restart_count": restart_count, - "max_restart_count": self._backend_config.max_restart_count, - "pod_name": pod_name, - "namespace": self._k8s_namespace, - "resource_name": resource_name, - "error_stack": error_stack if error_stack else None, - }, - host_url=None, - ) - - # Pod status helpers - - @staticmethod - def _get_pod_restart_count(pod: k8s_client.V1Pod) -> int: - """Get the maximum restart count across all containers in a pod.""" - if not pod.status.container_statuses: - return 0 - return max((cs.restart_count or 0) for cs in pod.status.container_statuses) - - @staticmethod - def _with_restart_info(status_msg: str, restart_count: int) -> str: - """Append restart count to a status message when restarts > 0.""" - if restart_count > 0: - return f"{status_msg}, restarts: {restart_count}" - return status_msg - - def check_crash_loop(self, pod: k8s_client.V1Pod, resource_name: str) -> DeploymentStatusUpdate | None: - """Check if a pod is in a crash loop (restart count >= max_restart_count and waiting). - - Returns a DeploymentStatusUpdate with ERROR if crash loop detected, else None. - """ - pod_name = pod.metadata.name - logger.debug("Checking pod for crash loop", extra={"pod": pod_name, "phase": pod.status.phase}) - - if not pod.status.container_statuses: - logger.debug("Pod has no container statuses", extra={"pod": pod_name}) - return None - - max_restarts = self._backend_config.max_restart_count - - for idx, container_status in enumerate(pod.status.container_statuses): - restart_count = container_status.restart_count or 0 - logger.debug( - "Container status check", - extra={"pod": pod_name, "container_index": idx, "restart_count": restart_count}, - ) - - if restart_count >= max_restarts: - if container_status.state and container_status.state.waiting: - waiting_reason = container_status.state.waiting.reason - logger.warning( - "Pod entered crash loop", - extra={ - "pod": pod_name, - "restart_count": restart_count, - "max_restarts": max_restarts, - "waiting_reason": waiting_reason, - }, - ) - return self.build_crash_loop_error(resource_name, pod_name, restart_count) - else: - logger.debug( - "Pod has restarts above threshold but is not in waiting state", - extra={"pod": pod_name, "container_index": idx, "restart_count": restart_count}, - ) - - logger.debug("Crash loop check complete, no crash loop detected", extra={"pod": pod_name}) - return None - - def pod_status_from_deployment(self, resource_name: str) -> DeploymentStatusUpdate: - """Get status message from pod events for a deployment. - - Returns: - DeploymentStatusUpdate with status (PENDING or ERROR) and descriptive message. - Crash loop detection is performed here; PENDING timeout is handled by the caller. - """ - logger.info(f"Getting pod status for deployment: {resource_name}") - try: - apps_v1 = k8s_client.AppsV1Api(self._k8s_client) - core_v1 = k8s_client.CoreV1Api(self._k8s_client) - - try: - deployment = apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - except k8s_client.exceptions.ApiException as e: - if e.status == 404: - return DeploymentStatusUpdate( - status="PENDING", status_message="Waiting for k8s deployment to be created", host_url=None - ) - raise - - if not deployment.spec.selector or not deployment.spec.selector.match_labels: - return DeploymentStatusUpdate( - status="PENDING", - status_message="Waiting for k8s deployment - invalid selector configuration", - host_url=None, - ) - - label_selector = ",".join([f"{k}={v}" for k, v in deployment.spec.selector.match_labels.items()]) - pods = core_v1.list_namespaced_pod(namespace=self._k8s_namespace, label_selector=label_selector) - - if not pods.items: - logger.info(f"No pods found for deployment {resource_name}") - return DeploymentStatusUpdate( - status="PENDING", status_message="Waiting for k8s deployment - no pods created yet", host_url=None - ) - - logger.info(f"Found {len(pods.items)} pod(s) for deployment {resource_name}") - - pod: k8s_client.V1Pod = max(pods.items, key=lambda p: p.metadata.creation_timestamp) - logger.info(f"Checking most recent pod: {pod.metadata.name}") - - crash_result = self.check_crash_loop(pod, resource_name) - if crash_result: - return crash_result - - restart_count = self._get_pod_restart_count(pod) - - events = core_v1.list_namespaced_event( - namespace=self._k8s_namespace, field_selector=f"involvedObject.name={pod.metadata.name}" - ) - - if not events.items: - if pod.status.phase == "Pending" and pod.status.container_statuses: - for container_status in pod.status.container_statuses: - if container_status.state and container_status.state.waiting: - reason = container_status.state.waiting.reason - message = container_status.state.waiting.message or "" - status_msg = f"{reason}: {message}" if message else reason - status_msg = self._with_restart_info(status_msg, restart_count) - return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) - pod_status = pod.status.phase.lower() if pod.status.phase else "unknown" - status_msg = f"Waiting for k8s deployment - pod status is {pod_status}" - status_msg = self._with_restart_info(status_msg, restart_count) - return DeploymentStatusUpdate( - status="PENDING", - status_message=status_msg, - host_url=None, - ) - - recent_event = max( - events.items, key=lambda e: e.last_timestamp or e.event_time or e.metadata.creation_timestamp - ) - - reason = recent_event.reason - message = recent_event.message - - for search_string, return_message in POD_EVENT_TO_MESSAGE_MAP.items(): - if search_string in message.lower(): - status_msg = self._with_restart_info(return_message, restart_count) - return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) - - if len(message) > MAX_EVENT_MESSAGE_CHARS: - message = message[: MAX_EVENT_MESSAGE_CHARS - 3] + "..." - - status_msg = self._with_restart_info(f"{reason}: {message}", restart_count) - return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) - - except Exception as e: - logger.warning(f"Failed to get pod status for deployment {resource_name}: {e}") - return DeploymentStatusUpdate(status="PENDING", status_message="Waiting for k8s deployment", host_url=None) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/README.md b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/README.md deleted file mode 100644 index 9ef8fb2cdb..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/README.md +++ /dev/null @@ -1,145 +0,0 @@ -# NIM Operator Types - -This directory contains Pydantic models automatically generated from the NVIDIA k8s-nim-operator Kubernetes CRD definitions. - -## Overview - -These models provide native Python/Pydantic representations of the `NIMService` and `NIMCache` Custom Resource Definitions (CRDs) from the [k8s-nim-operator](https://github.com/NVIDIA/k8s-nim-operator) project. This allows us to: - -1. **Type-safe model definitions**: Use Pydantic for validation and IDE autocomplete when working with NIM operator resources -2. **Compile ModelDeploymentConfigs to NIMService**: Transform our internal ModelDeploymentConfig specs into valid NIMService Kubernetes resources -3. **Stay in sync**: Regenerate types when the upstream operator CRDs change - -## Generated Files - -- `nimservice.py` - Pydantic models for the `NIMService` CRD (~2700 lines) -- `nimcache.py` - Pydantic models for the `NIMCache` CRD (~580 lines) -- `__init__.py` - Package exports - -## Regenerating Types - -When the k8s-nim-operator CRDs are updated, you can regenerate the Pydantic models using the provided script. - -### List Available Versions - -To see all available versions (tags) from the k8s-nim-operator repository: - -```bash -cd services/core/infrastructure/models -./scripts/update-types.sh -``` - -This will clone the repository (if needed), list the most recent versions, and leave the repository for inspection. - -### Generate from Specific Version - -To generate types from a specific version: - -```bash -cd services/core/infrastructure/models - -# Generate types from a specific version -uv run ./scripts/update-types.sh --version v2.0.2 -``` - -This will: - -1. Clone or update the k8s-nim-operator repository -2. Checkout the specified version tag -3. Extract the OpenAPI v3 schemas from the CRD YAML files -4. Use `datamodel-code-generator` to create Pydantic v2 models -5. Clean up the cloned repository - -### Importing the Models - -```python -from models.nim_operator_types import NIMService, NIMCache -``` - -### Creating a NIMService - -```python -from models.nim_operator_types import NIMService - -nim_service = NIMService( - apiVersion="apps.nvidia.com/v1alpha1", - kind="NIMService", - metadata={ - "name": "my-llm-service", - "namespace": "default" - }, - spec={ - "image": { - "repository": "nvcr.io/nim/meta/llama-3.1-8b-instruct", - "tag": "1.0.0" - }, - "authSecret": "ngc-api-key", - "replicas": 2, - "resources": { - "limits": { - "nvidia.com/gpu": "1" - } - } - } -) - -# Serialize to JSON for Kubernetes API -k8s_manifest = nim_service.model_dump_json(exclude_none=True) -``` - -### Creating a NIMCache - -```python -from models.nim_operator_types import NIMCache - -nim_cache = NIMCache( - apiVersion="apps.nvidia.com/v1alpha1", - kind="NIMCache", - metadata={ - "name": "my-llm-cache", - "namespace": "default" - }, - spec={ - "source": { - "ngc": { - "authSecret": "ngc-api-key", - "modelPuller": "nvcr.io/nim/meta/llama-3.1-8b-instruct:1.0.0", - "model": { - "profiles": ["profile-a"] - } - } - }, - "storage": { - "pvc": { - "create": True, - "size": "50Gi", - "storageClass": "fast-ssd" - } - } - } -) -``` - - -### Testing the Generated Models - -After regenerating, test that the models work correctly: - -```bash -uv run python test_nim_types.py -``` - -## Upstream Source - -- **Repository**: https://github.com/NVIDIA/k8s-nim-operator -- **CRD Definitions**: `config/crd/bases/` - - `apps.nvidia.com_nimservices.yaml` - - `apps.nvidia.com_nimcaches.yaml` - -## Development Dependencies - -The generation process requires: -- `datamodel-code-generator>=0.26.2` (for generating Pydantic models from OpenAPI schemas) -- `pyyaml>=6.0.2` (for parsing CRD YAML files) - -These are included in the `dev` dependency group in `pyproject.toml`. diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/__init__.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/__init__.py deleted file mode 100644 index f3019aa403..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Generated from k8s-nim-operator v2.0.2 -# Source: https://github.com/NVIDIA/k8s-nim-operator -"""Generated Pydantic models from k8s-nim-operator CRDs.""" - -from .nimcache import NIMCache as NIMCache -from .nimservice import NIMService as NIMService diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/nimcache.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/nimcache.py deleted file mode 100644 index a5a3f64b5d..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/nimcache.py +++ /dev/null @@ -1,514 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Generated from k8s-nim-operator v2.0.2 -# Source: https://github.com/NVIDIA/k8s-nim-operator -# generated by datamodel-codegen: -# filename: temp_schema.json -# timestamp: 2025-10-24T18:28:58+00:00 - -from __future__ import annotations - -from enum import Enum -from typing import Any, Optional, Union - -from pydantic import AwareDatetime, BaseModel, Field, RootModel - - -class CertConfig(BaseModel): - """ - CertConfig is the name of the ConfigMap containing the custom certificates. - for secure communication. - Deprecated: use `Proxy` instead to configure custom certificates for using proxy. - """ - - mountPath: str = Field( - ..., - description="MountPath is the path where the certificates should be mounted in the container.", - ) - name: str = Field(..., description="Name of the ConfigMap containing the certificate data.") - - -class ConfigMapKeyRef(BaseModel): - """ - Selects a key of a ConfigMap. - """ - - key: str = Field(..., description="The key to select.") - name: Optional[str] = Field( - "", - description="Name of the referent.\nThis field is effectively required, but due to backwards compatibility is\nallowed to be empty. Instances of this type with an empty value here are\nalmost certainly wrong.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", - ) - optional: Optional[bool] = Field(None, description="Specify whether the ConfigMap or its key must be defined") - - -class FieldRef(BaseModel): - """ - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - """ - - apiVersion: Optional[str] = Field( - None, - description='Version of the schema the FieldPath is written in terms of, defaults to "v1".', - ) - fieldPath: str = Field(..., description="Path of the field to select in the specified API version.") - - -class Divisor(RootModel[int]): - root: int - - -class Divisor1(RootModel[str]): - root: str - - -class ResourceFieldRef(BaseModel): - """ - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - """ - - containerName: Optional[str] = Field( - None, description="Container name: required for volumes, optional for env vars" - ) - divisor: Optional[Union[Divisor, Divisor1]] = Field( - None, - description='Specifies the output format of the exposed resources, defaults to "1"', - ) - resource: str = Field(..., description="Required: resource to select") - - -class SecretKeyRef(BaseModel): - """ - Selects a key of a secret in the pod's namespace - """ - - key: str = Field( - ..., - description="The key of the secret to select from. Must be a valid secret key.", - ) - name: Optional[str] = Field( - "", - description="Name of the referent.\nThis field is effectively required, but due to backwards compatibility is\nallowed to be empty. Instances of this type with an empty value here are\nalmost certainly wrong.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", - ) - optional: Optional[bool] = Field(None, description="Specify whether the Secret or its key must be defined") - - -class ValueFrom(BaseModel): - """ - Source for the environment variable's value. Cannot be used if value is not empty. - """ - - configMapKeyRef: Optional[ConfigMapKeyRef] = Field(None, description="Selects a key of a ConfigMap.") - fieldRef: Optional[FieldRef] = Field( - None, - description="Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`,\nspec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.", - ) - resourceFieldRef: Optional[ResourceFieldRef] = Field( - None, - description="Selects a resource of the container: only resources limits and requests\n(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", - ) - secretKeyRef: Optional[SecretKeyRef] = Field(None, description="Selects a key of a secret in the pod's namespace") - - -class EnvItem(BaseModel): - """ - EnvVar represents an environment variable present in a Container. - """ - - name: str = Field(..., description="Name of the environment variable. Must be a C_IDENTIFIER.") - value: Optional[str] = Field( - None, - description='Variable references $(VAR_NAME) are expanded\nusing the previously defined environment variables in the container and\nany service environment variables. If a variable cannot be resolved,\nthe reference in the input string will be unchanged. Double $$ are reduced\nto a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.\n"$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)".\nEscaped references will never be expanded, regardless of whether the variable\nexists or not.\nDefaults to "".', - ) - valueFrom: Optional[ValueFrom] = Field( - None, - description="Source for the environment variable's value. Cannot be used if value is not empty.", - ) - - -class Proxy(BaseModel): - """ - ProxySpec defines the proxy configuration for NIMService. - """ - - certConfigMap: Optional[str] = None - httpProxy: Optional[str] = None - httpsProxy: Optional[str] = None - noProxy: Optional[str] = None - - -class Cpu(RootModel[int]): - root: int - - -class Cpu1(RootModel[str]): - root: str - - -class Memory(RootModel[int]): - root: int - - -class Memory1(RootModel[str]): - root: str - - -class Resources(BaseModel): - """ - Resources defines the minimum resources required for the caching job to run(cpu, memory, gpu). - """ - - cpu: Optional[Union[Cpu, Cpu1]] = Field( - None, - description="CPU indicates the minimum number of CPUs to use while caching NIM", - ) - memory: Optional[Union[Memory, Memory1]] = Field( - None, - description='Memory indicates the minimum amount of memory to use while caching NIM\nValid values are numbers followed by one of the suffixes Ki, Mi, Gi, or Ti (e.g. "4Gi", "4096Mi").', - ) - - -class DataStore(BaseModel): - """ - DataStore represents models stored in NVIDIA NeMo DataStore service - """ - - authSecret: str = Field( - ..., - description='AuthSecret is the name of the secret containing the "HF_TOKEN" token', - min_length=1, - ) - datasetName: Optional[str] = Field(None, description="DatasetName is the name of the dataset") - endpoint: str = Field( - ..., - description="Endpoint is the HuggingFace endpoint from NeMo DataStore", - pattern="^https?://.*/v1/hf/?$", - ) - modelName: Optional[str] = Field(None, description="ModelName is the name of the model") - modelPuller: str = Field( - ..., - description="ModelPuller is the containerized huggingface-cli image to pull the data", - min_length=1, - ) - namespace: str = Field(..., description="Namespace is the namespace within NeMo DataStore") - pullSecret: str = Field( - ..., - description="PullSecret is the name of the image pull secret for the modelPuller image", - min_length=1, - ) - revision: Optional[str] = Field( - None, - description="Revision is the revision of the object to be cached. This is either a commit hash, branch name or tag.", - min_length=1, - ) - - -class Hf(BaseModel): - """ - HuggingFaceHub represents models stored in HuggingFace Hub - """ - - authSecret: str = Field( - ..., - description='AuthSecret is the name of the secret containing the "HF_TOKEN" token', - min_length=1, - ) - datasetName: Optional[str] = Field(None, description="DatasetName is the name of the dataset") - endpoint: str = Field(..., description="Endpoint is the HuggingFace endpoint", pattern="^https?://.*$") - modelName: Optional[str] = Field(None, description="ModelName is the name of the model") - modelPuller: str = Field( - ..., - description="ModelPuller is the containerized huggingface-cli image to pull the data", - min_length=1, - ) - namespace: str = Field( - ..., - description="Namespace is the namespace within the HuggingFace Hub", - min_length=1, - ) - pullSecret: str = Field( - ..., - description="PullSecret is the name of the image pull secret for the modelPuller image", - min_length=1, - ) - revision: Optional[str] = Field( - None, - description="Revision is the revision of the object to be cached. This is either a commit hash, branch name or tag.", - min_length=1, - ) - - -class Gpu(BaseModel): - """ - GPUSpec is the spec required to cache models for selected gpu type. - """ - - ids: Optional[list[str]] = Field(None, description="IDs are the device-ids for a specific GPU SKU") - product: Optional[str] = Field(None, description="Product is the GPU product string (h100, a100, l40s)") - - -class Model(BaseModel): - """ - Model spec for caching - """ - - buildable: Optional[bool] = Field( - None, - description="Buildable indicates generic model profiles that can be optimized with an NVIDIA engine for any GPUs", - ) - engine: Optional[str] = Field(None, description="Engine is the backend engine (tensorrt_llm, vllm)") - gpus: Optional[list[Gpu]] = Field( - None, - description="GPU is the spec for matching GPUs for caching optimized models", - ) - lora: Optional[bool] = Field(None, description="Lora indicates a finetuned model with LoRa adapters") - precision: Optional[str] = Field(None, description="Precision is the precision for model quantization") - profiles: Optional[list[str]] = Field( - None, - description="Profiles are the specific model profiles to cache. When these are provided, rest of the model parameters for profile selection are ignored", - ) - qosProfile: Optional[str] = Field( - None, - description="QoSProfile is the supported QoS profile types for the models (throughput, latency)", - ) - tensorParallelism: Optional[str] = Field( - None, - description="TensorParallelism is the minimum GPUs required for the model computations", - ) - - -class Ngc(BaseModel): - """ - NGCSource represents models stored in NGC - """ - - authSecret: str = Field( - ..., - description="The name of an existing pull secret containing the NGC_API_KEY", - ) - model: Optional[Model] = Field(None, description="Model spec for caching") - modelPuller: str = Field(..., description="ModelPuller is the container image that can pull the model") - pullSecret: Optional[str] = Field(None, description="PullSecret to pull the model puller image") - - -class Source(BaseModel): - """ - Source is the NIM model source to cache - """ - - dataStore: Optional[DataStore] = Field( - None, - description="DataStore represents models stored in NVIDIA NeMo DataStore service", - ) - hf: Optional[Hf] = Field(None, description="HuggingFaceHub represents models stored in HuggingFace Hub") - ngc: Optional[Ngc] = Field(None, description="NGCSource represents models stored in NGC") - - -class Pvc(BaseModel): - """ - PersistentVolumeClaim is the pvc volume used for caching NIM - """ - - annotations: Optional[dict[str, str]] = Field(None, description="Annotations for the PVC") - create: Optional[bool] = Field( - None, - description="Create specifies whether to create a new PersistentVolumeClaim (PVC).\nIf set to false, an existing PVC must be referenced via the `Name` field.", - ) - name: Optional[str] = Field( - None, - description="Name of the PVC to use. Required if `Create` is false (i.e., using an existing PVC).", - ) - size: Optional[str] = Field(None, description="Size of the NIM cache in Gi, used during PVC creation") - storageClass: Optional[str] = Field( - None, - description="StorageClass to be used for PVC creation. Leave it as empty if the PVC is already created or\na default storage class is set in the cluster.", - ) - subPath: Optional[str] = Field(None, description="SubPath is the path inside the PVC that should be mounted") - volumeAccessMode: Optional[str] = Field(None, description="VolumeAccessMode is the volume access mode of the PVC") - - -class Storage(BaseModel): - """ - Storage is the target storage for caching NIM model - """ - - hostPath: Optional[str] = Field( - None, - description="HostPath is the host path volume for caching NIM\n\nDeprecated: use PVC instead.", - ) - pvc: Optional[Pvc] = Field(None, description="PersistentVolumeClaim is the pvc volume used for caching NIM") - - -class Toleration(BaseModel): - """ - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - """ - - effect: Optional[str] = Field( - None, - description="Effect indicates the taint effect to match. Empty means match all taint effects.\nWhen specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", - ) - key: Optional[str] = Field( - None, - description="Key is the taint key that the toleration applies to. Empty means match all taint keys.\nIf the key is empty, operator must be Exists; this combination means to match all values and all keys.", - ) - operator: Optional[str] = Field( - None, - description="Operator represents a key's relationship to the value.\nValid operators are Exists and Equal. Defaults to Equal.\nExists is equivalent to wildcard for value, so that a pod can\ntolerate all taints of a particular category.", - ) - tolerationSeconds: Optional[int] = Field( - None, - description="TolerationSeconds represents the period of time the toleration (which must be\nof effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,\nit is not set, which means tolerate the taint forever (do not evict). Zero and\nnegative values will be treated as 0 (evict immediately) by the system.", - ) - value: Optional[str] = Field( - None, - description="Value is the taint value the toleration matches to.\nIf the operator is Exists, the value should be empty, otherwise just a regular string.", - ) - - -class Spec(BaseModel): - """ - NIMCacheSpec defines the desired state of NIMCache. - """ - - certConfig: Optional[CertConfig] = Field( - None, - description="CertConfig is the name of the ConfigMap containing the custom certificates.\nfor secure communication.\nDeprecated: use `Proxy` instead to configure custom certificates for using proxy.", - ) - env: Optional[list[EnvItem]] = Field( - None, - description="Env are the additional custom environment variabes for the caching job", - ) - groupID: Optional[int] = Field(None, description="GroupID is the group ID for the caching job") - nodeSelector: Optional[dict[str, str]] = Field( - None, - description="NodeSelector is the node selector labels to schedule the caching job.", - ) - proxy: Optional[Proxy] = Field(None, description="ProxySpec defines the proxy configuration for NIMService.") - resources: Optional[Resources] = Field( - None, - description="Resources defines the minimum resources required for the caching job to run(cpu, memory, gpu).", - ) - runtimeClassName: Optional[str] = Field( - None, description="RuntimeClassName is the runtimeclass for the caching job" - ) - source: Source = Field(..., description="Source is the NIM model source to cache") - storage: Storage = Field(..., description="Storage is the target storage for caching NIM model") - tolerations: Optional[list[Toleration]] = Field( - None, description="Tolerations for running the job to cache the NIM model" - ) - userID: Optional[int] = Field(None, description="UserID is the user ID for the caching job") - - initContainers: Optional[list[ContainerSpec]] = Field(None, description="Init containers to add to the NIMCache") - - -class Image(BaseModel): - """ - Image defines image attributes. - """ - - pullPolicy: Optional[str] = None - pullSecrets: Optional[list[str]] = None - repository: str - tag: str - - -class ContainerSpec(BaseModel): - """ - ContainerSpec defines a container specification for init containers. - """ - - name: str = Field(..., description="Name of the container") - image: Image = Field(..., description="Image defines image attributes.") - command: Optional[list[str]] = Field(None, description="Command to execute in the container") - args: Optional[list[str]] = Field(None, description="Arguments to pass to the command") - env: Optional[list[EnvItem]] = Field(None, description="Environment variables for the container") - workingDir: Optional[str] = Field(None, description="Working directory for the container") - - -class Status1(Enum): - """ - status of the condition, one of True, False, Unknown. - """ - - True_ = "True" - False_ = "False" - Unknown = "Unknown" - - -class Condition(BaseModel): - """ - Condition contains details for one aspect of the current state of this API Resource. - """ - - lastTransitionTime: AwareDatetime = Field( - ..., - description="lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", - ) - message: str = Field( - ..., - description="message is a human readable message indicating details about the transition.\nThis may be an empty string.", - max_length=32768, - ) - observedGeneration: Optional[int] = Field( - None, - description="observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.", - ge=0, - ) - reason: str = Field( - ..., - description="reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.", - max_length=1024, - min_length=1, - pattern="^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", - ) - status: Status1 = Field(..., description="status of the condition, one of True, False, Unknown.") - type: str = Field( - ..., - description="type of condition in CamelCase or in foo.example.com/CamelCase.", - max_length=316, - pattern="^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", - ) - - -class Profile(BaseModel): - """ - NIMProfile defines the profiles that were cached. - """ - - config: Optional[dict[str, str]] = None - model: Optional[str] = None - name: Optional[str] = None - release: Optional[str] = None - - -class Status(BaseModel): - """ - NIMCacheStatus defines the observed state of NIMCache. - """ - - conditions: Optional[list[Condition]] = None - profiles: Optional[list[Profile]] = None - pvc: Optional[str] = None - state: Optional[str] = None - - -class NIMCache(BaseModel): - """ - NIMCache is the Schema for the nimcaches API. - """ - - apiVersion: Optional[str] = Field( - None, - description="APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", - ) - kind: Optional[str] = Field( - None, - description="Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", - ) - metadata: Optional[dict[str, Any]] = None - spec: Optional[Spec] = Field(None, description="NIMCacheSpec defines the desired state of NIMCache.") - status: Optional[Status] = Field(None, description="NIMCacheStatus defines the observed state of NIMCache.") diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/vllm_k8s_compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/vllm_k8s_compiler.py deleted file mode 100644 index 227991ffc8..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/vllm_k8s_compiler.py +++ /dev/null @@ -1,439 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Engine-agnostic compiler for directly-emitted Kubernetes objects. - -For the k8s service backend's non-operator path, this module compiles the four -native Kubernetes objects a model deployment needs: - -* ``V1PersistentVolumeClaim`` -- holds the model weights. -* ``V1Job`` -- weight puller (populates the PVC, then exits 0). -* ``V1Deployment`` -- the inference server (mounts the PVC, serves the model). -* ``V1Service`` -- ClusterIP exposing the server port for IGW routing. - -The builders are intentionally **engine-agnostic**: every value the -k8s-nim-operator used to hardcode or derive (image, command/args, env, -securityContext, probes, resources, shared memory, service account, labels) is a -parameter the caller supplies. The vLLM path passes vLLM's values (via the shared -``vllm_compiler``); when NIM migrates onto this emission path it will pass NIM's -values through the same builders. Keep this module free of engine-specific logic. - -These functions are pure (no Kubernetes I/O); the backend applies the returned -objects via the typed Kubernetes API clients. - -FUTURE / NIM migration (dropping k8s-nim-operator -- see the Deployments Plugin -RFC): - When NIM is cut over to emit these raw objects instead of NIMService/NIMCache - CRs, route the NIM path through these same builders -- but DO NOT reuse vLLM's - values. The footgun is the securityContext ``user_id`` / ``group_id`` params: - they are engine-specific on purpose. The vLLM path passes - ``default_vllm_user_id`` / ``default_vllm_group_id`` (2000/0) because that is - the user the ``vllm/vllm-openai`` image ships with an ``/etc/passwd`` entry - (an arbitrary uid like 1000 crashes torch/inductor's ``getpass.getuser()``). - NIM images expect the operator's historical 1000/2000. So the NIM path must - pass its own uid/gid (e.g. the existing ``default_user_id`` / - ``default_group_id`` config, defaulting to the NIM-appropriate values) -- NOT - the ``default_vllm_*`` fields. Same reasoning applies to image, args/command - (NIM is env-configured; vLLM is arg-configured), and env. Pick per engine at - the call site; never hardcode either engine's value in this module. -""" - -from logging import getLogger -from typing import Optional - -from kubernetes import client as k8s_client -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER - -logger = getLogger(__name__) - -# Label keys (shared with the operator path for orphan reconciliation / listing). -DEPLOYMENT_WORKSPACE_LABEL = "nmp.nvidia.com/deployment-workspace" -DEPLOYMENT_NAME_LABEL = "nmp.nvidia.com/deployment-name" -# Records the resolved model source on the PVC + Job so update can detect a change -# and decide whether to re-pull weights (see backend re-pull policy). This is an -# ANNOTATION, not a label: the value is "/@" which contains '/' -# and ':' and is therefore not a valid label value. -MODEL_SOURCE_ANNOTATION = "nmp.nvidia.com/model-source" - -# In-pod paths. -MODEL_STORE_PATH = "/model-store" -SCRATCH_PATH = "/scratch" -DSHM_PATH = "/dev/shm" - -# Resource-name suffixes derived from the deployment resource name. -PVC_SUFFIX = "-pvc" -PULL_JOB_SUFFIX = "-pull" - -# Defaults mirroring the k8s-nim-operator. -DEFAULT_BACKOFF_LIMIT = 5 -DEFAULT_TTL_SECONDS_AFTER_FINISHED = 600 -DEFAULT_USER_ID = 1000 -DEFAULT_GROUP_ID = 2000 -SERVER_PORT_NAME = "api" - - -def pvc_name(resource_name: str) -> str: - """PVC name derived from the deployment resource name.""" - return f"{resource_name}{PVC_SUFFIX}" - - -def pull_job_name(resource_name: str) -> str: - """Weight-puller Job name derived from the deployment resource name.""" - return f"{resource_name}{PULL_JOB_SUFFIX}" - - -def common_labels( - workspace: str, - name: str, - engine: str, - *, - extra: Optional[dict[str, str]] = None, -) -> dict[str, str]: - """Labels stamped on every emitted object for management + orphan listing.""" - labels = { - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - DEPLOYMENT_WORKSPACE_LABEL: workspace, - DEPLOYMENT_NAME_LABEL: name, - "nmp.nvidia.com/engine": engine, - } - if extra: - labels.update(extra) - return labels - - -def _merge_annotations( - base: Optional[dict[str, str]], - model_source: Optional[str], -) -> Optional[dict[str, str]]: - """Merge caller annotations with the model-source annotation (re-pull marker).""" - annotations = dict(base) if base else {} - if model_source: - annotations[MODEL_SOURCE_ANNOTATION] = model_source - return annotations or None - - -def _gpu_resources(gpu: int) -> Optional[k8s_client.V1ResourceRequirements]: - """GPU resource requirements (requests == limits). None when gpu == 0.""" - if gpu < 1: - return None - quantity = {"nvidia.com/gpu": str(gpu)} - return k8s_client.V1ResourceRequirements(requests=dict(quantity), limits=dict(quantity)) - - -def _pod_security_context( - user_id: Optional[int], - group_id: Optional[int], -) -> Optional[k8s_client.V1PodSecurityContext]: - """Pod securityContext from explicitly-configured uid/gid only. - - Returns ``None`` when neither is set, so the pod runs as the container image's - default user. We intentionally do NOT force the operator's 1000/2000 default: - some images (e.g. vLLM) lack an ``/etc/passwd`` entry for uid 1000, which makes - libraries that call ``getpass.getuser()`` (torch inductor) crash with - ``getpwuid(): uid not found``. NIM can opt into a uid/gid via config. - """ - if user_id is None and group_id is None: - return None - return k8s_client.V1PodSecurityContext( - run_as_user=user_id, - run_as_group=group_id, - fs_group=group_id, - ) - - -def compile_pvc( - *, - resource_name: str, - workspace: str, - name: str, - engine: str, - disk_size: str, - storage_class: Optional[str] = None, - access_modes: Optional[list[str]] = None, - model_source: Optional[str] = None, - namespace: Optional[str] = None, - annotations: Optional[dict[str, str]] = None, - extra_labels: Optional[dict[str, str]] = None, -) -> k8s_client.V1PersistentVolumeClaim: - """Compile the model-weights PVC. - - ``access_modes`` defaults to ``["ReadWriteOnce"]`` (single-pod; the puller and - server co-locate). ``model_source`` is stamped as an annotation so the - backend's update path can detect a weight-source change and decide whether to - re-pull. - """ - return k8s_client.V1PersistentVolumeClaim( - metadata=k8s_client.V1ObjectMeta( - name=pvc_name(resource_name), - namespace=namespace, - labels=common_labels(workspace, name, engine, extra=extra_labels), - annotations=_merge_annotations(annotations, model_source), - ), - spec=k8s_client.V1PersistentVolumeClaimSpec( - access_modes=access_modes or ["ReadWriteOnce"], - resources=k8s_client.V1VolumeResourceRequirements(requests={"storage": disk_size}), - storage_class_name=storage_class, - ), - ) - - -def compile_puller_job( - *, - resource_name: str, - workspace: str, - name: str, - engine: str, - image: str, - container_args: list[str], - env: Optional[dict[str, str]] = None, - gpu: int = 0, - namespace: Optional[str] = None, - service_account_name: Optional[str] = None, - image_pull_secret: Optional[str] = None, - user_id: Optional[int] = None, - group_id: Optional[int] = None, - model_source: Optional[str] = None, - backoff_limit: int = DEFAULT_BACKOFF_LIMIT, - ttl_seconds_after_finished: int = DEFAULT_TTL_SECONDS_AFTER_FINISHED, - annotations: Optional[dict[str, str]] = None, - extra_labels: Optional[dict[str, str]] = None, -) -> k8s_client.V1Job: - """Compile the weight-puller Job. - - Mirrors the docker puller: a single container running ``hf download - --local-dir /model-store [...]`` against ``image`` (the platform nmp-api - image), mounting the PVC at ``/model-store``. ``command=["hf"]`` overrides the - image ENTRYPOINT (nmp-api's is ``nemo services run``) to the Hugging Face CLI, - and ``container_args`` (e.g. ``["download", "", "--local-dir", - "/model-store"]``) are the CLI arguments. The puller requests the same ``gpu`` - as the server -- - not for compute, but to pin it into GPU topology so the shared RWO PVC binds - where the server can mount it (correct across any StorageClass - ``volumeBindingMode``). - """ - labels = common_labels(workspace, name, engine, extra=extra_labels) - job_annotations = _merge_annotations(annotations, model_source) - - env_list = [k8s_client.V1EnvVar(name=k, value=str(v)) for k, v in (env or {}).items()] - - container = k8s_client.V1Container( - name="weight-puller", - image=image, - command=["hf"], - args=container_args, - env=env_list or None, - resources=_gpu_resources(gpu), - security_context=k8s_client.V1SecurityContext( - allow_privilege_escalation=False, - run_as_non_root=True, - run_as_user=user_id if user_id is not None else DEFAULT_USER_ID, - run_as_group=group_id if group_id is not None else DEFAULT_GROUP_ID, - capabilities=k8s_client.V1Capabilities(drop=["ALL"]), - ), - volume_mounts=[ - k8s_client.V1VolumeMount(name="model-store", mount_path=MODEL_STORE_PATH), - ], - ) - - # The puller writes to a freshly-provisioned PVC, so it needs fsGroup to own - # the volume's filesystem (without it, a non-root puller can't create files at - # the PVC root -> PermissionError on /model-store). Default to 1000/2000; the - # nmp-api puller image runs as the 'nvs' user (uid/gid 1000). - puller_security_context = k8s_client.V1PodSecurityContext( - run_as_user=user_id if user_id is not None else DEFAULT_USER_ID, - run_as_group=group_id if group_id is not None else DEFAULT_GROUP_ID, - fs_group=group_id if group_id is not None else DEFAULT_GROUP_ID, - ) - pod_spec = k8s_client.V1PodSpec( - restart_policy="Never", - service_account_name=service_account_name, - security_context=puller_security_context, - image_pull_secrets=([k8s_client.V1LocalObjectReference(name=image_pull_secret)] if image_pull_secret else None), - containers=[container], - volumes=[ - k8s_client.V1Volume( - name="model-store", - persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( - claim_name=pvc_name(resource_name), - ), - ), - ], - ) - - return k8s_client.V1Job( - metadata=k8s_client.V1ObjectMeta( - name=pull_job_name(resource_name), - namespace=namespace, - labels=labels, - annotations=job_annotations, - ), - spec=k8s_client.V1JobSpec( - backoff_limit=backoff_limit, - ttl_seconds_after_finished=ttl_seconds_after_finished, - template=k8s_client.V1PodTemplateSpec( - metadata=k8s_client.V1ObjectMeta(labels=labels), - spec=pod_spec, - ), - ), - ) - - -def _probe(health_path: str, port: int, *, failure_threshold: int, period_seconds: int = 10) -> k8s_client.V1Probe: - return k8s_client.V1Probe( - http_get=k8s_client.V1HTTPGetAction(path=health_path, port=port), - period_seconds=period_seconds, - timeout_seconds=5, - failure_threshold=failure_threshold, - ) - - -def compile_deployment( - *, - resource_name: str, - workspace: str, - name: str, - engine: str, - image: str, - args: list[str], - health_path: str, - port: int = 8000, - env: Optional[dict[str, str]] = None, - gpu: int = 0, - namespace: Optional[str] = None, - service_account_name: Optional[str] = None, - image_pull_secret: Optional[str] = None, - user_id: Optional[int] = None, - group_id: Optional[int] = None, - shared_memory_size_limit: Optional[str] = None, - startup_grace_seconds: int = 600, - init_containers: Optional[list[k8s_client.V1Container]] = None, - sidecar_containers: Optional[list[k8s_client.V1Container]] = None, - extra_labels: Optional[dict[str, str]] = None, - mount_model_store: bool = True, -) -> k8s_client.V1Deployment: - """Compile the inference-server Deployment. - - ``args`` is the server arg vector (e.g. from ``compile_vllm_args``), appended - to the image's entrypoint; ``command`` is intentionally left unset so the - upstream image entrypoint (``vllm serve``) runs. ``health_path`` drives the - startup/readiness probes. A ``dshm`` emptyDir is always mounted at - ``/dev/shm`` (vLLM uses it for tensor-parallel NCCL); ``scratch`` is mounted - for the LoRA cache dir. - - ``mount_model_store`` controls whether the ``model-store`` PVC volume + mount - are attached. The vLLM/NIM weight-pull paths set it ``True`` (the PVC holds - the pulled weights). The ``generic`` engine pulls no weights and has no PVC, - so it passes ``False`` -- the container runs purely from its image. - """ - selector_labels = {"app": resource_name} - pod_labels = { - **selector_labels, - **common_labels(workspace, name, engine), - } - if extra_labels: - pod_labels.update(extra_labels) - - env_list = [k8s_client.V1EnvVar(name=k, value=str(v)) for k, v in (env or {}).items()] - period = 10 - failure_threshold = max(1, -(-startup_grace_seconds // period)) # ceil - - volume_mounts = [ - k8s_client.V1VolumeMount(name="scratch", mount_path=SCRATCH_PATH), - k8s_client.V1VolumeMount(name="dshm", mount_path=DSHM_PATH), - ] - if mount_model_store: - volume_mounts.insert( - 0, k8s_client.V1VolumeMount(name="model-store", mount_path=MODEL_STORE_PATH, read_only=True) - ) - - container = k8s_client.V1Container( - name=f"{resource_name}-ctr", - image=image, - args=args or None, - env=env_list or None, - ports=[k8s_client.V1ContainerPort(container_port=port, name=SERVER_PORT_NAME)], - resources=_gpu_resources(gpu), - startup_probe=_probe(health_path, port, failure_threshold=failure_threshold, period_seconds=period), - readiness_probe=_probe(health_path, port, failure_threshold=3, period_seconds=period), - volume_mounts=volume_mounts, - ) - - containers = [container] - if sidecar_containers: - containers.extend(sidecar_containers) - - volumes = [ - k8s_client.V1Volume(name="scratch", empty_dir=k8s_client.V1EmptyDirVolumeSource()), - k8s_client.V1Volume( - name="dshm", - empty_dir=k8s_client.V1EmptyDirVolumeSource(medium="Memory", size_limit=shared_memory_size_limit), - ), - ] - if mount_model_store: - volumes.insert( - 0, - k8s_client.V1Volume( - name="model-store", - persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( - claim_name=pvc_name(resource_name), - read_only=True, - ), - ), - ) - - pod_spec = k8s_client.V1PodSpec( - service_account_name=service_account_name, - security_context=_pod_security_context(user_id, group_id), - image_pull_secrets=([k8s_client.V1LocalObjectReference(name=image_pull_secret)] if image_pull_secret else None), - init_containers=init_containers or None, - containers=containers, - volumes=volumes, - ) - - return k8s_client.V1Deployment( - metadata=k8s_client.V1ObjectMeta( - name=resource_name, - namespace=namespace, - labels=common_labels(workspace, name, engine, extra=extra_labels), - ), - spec=k8s_client.V1DeploymentSpec( - replicas=1, - selector=k8s_client.V1LabelSelector(match_labels=selector_labels), - template=k8s_client.V1PodTemplateSpec( - metadata=k8s_client.V1ObjectMeta(labels=pod_labels), - spec=pod_spec, - ), - ), - ) - - -def compile_service( - *, - resource_name: str, - workspace: str, - name: str, - engine: str, - port: int = 8000, - namespace: Optional[str] = None, - extra_labels: Optional[dict[str, str]] = None, -) -> k8s_client.V1Service: - """Compile the ClusterIP Service exposing the server port for IGW routing.""" - return k8s_client.V1Service( - metadata=k8s_client.V1ObjectMeta( - name=resource_name, - namespace=namespace, - labels=common_labels(workspace, name, engine, extra=extra_labels), - ), - spec=k8s_client.V1ServiceSpec( - type="ClusterIP", - selector={"app": resource_name}, - ports=[ - k8s_client.V1ServicePort( - name=SERVER_PORT_NAME, - port=port, - target_port=SERVER_PORT_NAME, - protocol="TCP", - ), - ], - ), - ) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/none_backend.py b/services/core/models/src/nmp/core/models/controllers/backends/none_backend.py deleted file mode 100644 index 9f5e42560b..0000000000 --- a/services/core/models/src/nmp/core/models/controllers/backends/none_backend.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""None service backend.""" - -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend -from nmp.core.models.controllers.context import ModelContext - - -class NoneServiceBackend(ServiceBackend): - """None service backend.""" - - def init(self) -> None: - """Initialize None service backend.""" - ... - - def shutdown(self) -> None: - """Shutdown None service backend.""" - ... - - async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Create a new model deployment.""" - raise NotImplementedError("NoneServiceBackend does not support deployments") - - async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Update a model deployment.""" - raise NotImplementedError("NoneServiceBackend does not support deployments") - - async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: - """Get the status of a model deployment.""" - return DeploymentStatusUpdate( - status="UNKNOWN", - status_message="NoneServiceBackend does not support deployments", - error_details={"error": "NoneServiceBackend does not support deployments"}, - ) - - async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete a model deployment.""" - raise NotImplementedError("NoneServiceBackend does not support deployments") - - async def list_managed_deployment_names(self) -> list[str]: - """Return an empty list — NoneServiceBackend manages no deployments. - - The orphan-reconciliation loop calls this to find deployments the - backend owns. The ``none`` backend can't own any, so it has nothing - to report. - """ - return [] diff --git a/services/core/models/src/nmp/core/models/controllers/backends/registry.py b/services/core/models/src/nmp/core/models/controllers/backends/registry.py index f17cc61009..2010772345 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/registry.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/registry.py @@ -4,7 +4,7 @@ """Backend registry for Models Controller service.""" from logging import getLogger -from typing import Dict, Self, Union +from typing import Dict, Self from nemo_platform import AsyncNeMoPlatform from nmp.core.models.controllers.backends.backends import ServiceBackend @@ -14,62 +14,19 @@ # `nemo_deployments_plugin` dependency. The backend class itself is resolved # lazily in `from_config` only when the deployments_plugin backend is selected. from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginBackendConfigModel -from nmp.core.models.controllers.backends.docker import DockerBackendConfig as DockerConfig -from nmp.core.models.controllers.backends.docker import DockerServiceBackend -from nmp.core.models.controllers.backends.k8s_nim_operator import K8sNimOperatorConfig, K8sNimOperatorServiceBackend -from nmp.core.models.controllers.backends.none_backend import NoneServiceBackend -from pydantic import BaseModel, Field logger = getLogger(__name__) - -class K8sNimOperatorBackendConfigModel(K8sNimOperatorConfig): - """Configuration for Kubernetes NIM Operator backend (flat: enabled + operator fields at top level).""" - - enabled: bool = Field(default=False, description="Whether this backend is enabled") - - -class DockerBackendConfigModel(DockerConfig): - """Configuration for Docker backend (flat: enabled + Docker config fields at top level).""" - - enabled: bool = Field(default=False, description="Whether this backend is enabled") - - -class NoneBackendConfigModel(BaseModel): - """Configuration for the ``none`` backend (no deployment substrate). - - Used when ``platform.runtime`` is ``none``. The backend is a deliberate - no-op: create/update/delete raise ``NotImplementedError``, status returns - ``UNKNOWN``, and orphan reconciliation sees no managed deployments. - """ - - enabled: bool = Field(default=False, description="Whether this backend is enabled") - - -# Union of all backend configurations (no discriminator needed since dict key is the backend name) -BackendConfig = Union[ - DockerBackendConfigModel, - K8sNimOperatorBackendConfigModel, - DeploymentsPluginBackendConfigModel, - NoneBackendConfigModel, -] - +# Union of all backend configurations (dict key is the backend name). +BackendConfig = DeploymentsPluginBackendConfigModel # Type alias for the backend name BackendName = str -# Global registry of always-importable backend implementations. The -# `deployments_plugin` backend is intentionally excluded here because it imports -# the optional `nemo_deployments_plugin` package; it is resolved lazily by -# `_resolve_backend_class` when selected. -backend_classes: Dict[BackendName, type[ServiceBackend]] = { - "docker": DockerServiceBackend, - "nim_operator": K8sNimOperatorServiceBackend, - "none": NoneServiceBackend, -} - -# Backends whose implementation lives behind an optional dependency and must be -# imported lazily. +# The deployments_plugin backend is resolved lazily because it imports the +# optional `nemo_deployments_plugin` package. +backend_classes: Dict[BackendName, type[ServiceBackend]] = {} + _LAZY_BACKEND_NAMES = frozenset({"deployments_plugin"}) _DEPLOYMENTS_PLUGIN_IMPORT_ERROR = ( @@ -128,7 +85,7 @@ def from_config( nmp_sdk: AsyncNeMoPlatform, backend_configs: Dict[BackendName, BackendConfig], huggingface_model_puller: str, - available_backends: Dict[BackendName, type[ServiceBackend]] = backend_classes, + available_backends: Dict[BackendName, type[ServiceBackend]] | None = None, ) -> Self: """Create a BackendRegistry from backend configurations. @@ -145,6 +102,9 @@ def from_config( KeyError: If a backend configuration references an unknown backend type ValueError: If zero or multiple backends are enabled """ + if available_backends is None: + available_backends = backend_classes + if not backend_configs: raise ValueError("At least one backend must be configured") @@ -166,10 +126,7 @@ def from_config( logger.info(f"Initializing backend: {backend_name}") config_dict = backend_config.model_dump(exclude={"enabled"}) - if backend_name in {"nim_operator", "deployments_plugin"}: - registry[backend_name] = backend_class(nmp_sdk, config_dict, huggingface_model_puller) - else: - registry[backend_name] = backend_class(nmp_sdk, config_dict) + registry[backend_name] = backend_class(nmp_sdk, config_dict, huggingface_model_puller) logger.info(f"Backend registry initialized with {len(registry)} backend(s)") return cls(registry) @@ -178,7 +135,7 @@ def get_backend(self, name: str | None = None) -> ServiceBackend: """Retrieve a configured backend by name. Args: - name: The backend name (e.g., "docker", "nim_operator"). + name: The backend name (e.g., "deployments_plugin"). If None, returns the default backend. Returns: diff --git a/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py b/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py index 6c2dad5c63..e60a596616 100644 --- a/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py +++ b/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py @@ -17,6 +17,7 @@ from nmp.common.entities.utils import parse_entity_ref from nmp.core.models.config import ControllerConfig from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend +from nmp.core.models.controllers.backends.common import deleting_elapsed_seconds from nmp.core.models.controllers.backends.registry import BackendRegistry from nmp.core.models.controllers.context import ModelContext @@ -228,7 +229,11 @@ async def reconcile_deployments(self, deployment_contexts: list[ModelContext]) - case "DELETING": await self._reconcile_individual_deployment( deployment, - lambda d: backend.delete_model_deployment(d.workspace, d.name), + lambda d: backend.delete_model_deployment( + d.workspace, + d.name, + deleting_elapsed_seconds=deleting_elapsed_seconds(d), + ), "delete", existing_provider=ctx.model_provider, ) diff --git a/services/core/models/src/nmp/core/models/schemas.py b/services/core/models/src/nmp/core/models/schemas.py index d1edb28202..2769c2a624 100644 --- a/services/core/models/src/nmp/core/models/schemas.py +++ b/services/core/models/src/nmp/core/models/schemas.py @@ -1452,11 +1452,14 @@ class ContainerExecutorConfig(BaseModel): "Applied after defaults but before override_config. Ignored by non-NIM engines.", ) - # Raw NIMService spec override (NIM engine on k8s only). + # Raw per-container override map (NIM engine on k8s only). override_config: Optional[Dict[str, Any]] = Field( default=None, - description="Raw NIMService spec configuration that takes precedence over generated config (NIM engine " - "on k8s). Allows advanced configuration options directly. Ignored by non-NIM engines.", + description="Partial NIMService Spec fragments deep-merged after generated defaults and " + "k8s_nim_operator_config (NIM engine on k8s only). Supported keys: image, command, args, " + "resources, env, readinessProbe, livenessProbe, startupProbe, nodeSelector, tolerations, " + "userID, groupID, labels, initContainers, sidecarContainers. Unsupported keys are rejected " + "at compile time. Ignored by non-NIM engines and docker runtime.", ) diff --git a/services/core/models/tests/integration/README.md b/services/core/models/tests/integration/README.md index fda7cadc2c..03d1fa8609 100644 --- a/services/core/models/tests/integration/README.md +++ b/services/core/models/tests/integration/README.md @@ -46,7 +46,7 @@ uv run pytest services/core/models/tests/integration/ -v -n 4 --dist loadscope To run specific tests: ```bash -uv run pytest services/core/models/tests/integration/test_models_controller.py::test_docker_deployment_lifecycle -v +uv run pytest services/core/models/tests/integration/test_deployments_plugin_lifecycle.py::test_deployments_plugin_docker_lifecycle -v ``` ### Cleanup diff --git a/services/core/models/tests/integration/conftest.py b/services/core/models/tests/integration/conftest.py index be86ea273b..0ac087d285 100644 --- a/services/core/models/tests/integration/conftest.py +++ b/services/core/models/tests/integration/conftest.py @@ -10,14 +10,18 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from nemo_deployments_plugin.config import ControllerConfig, DeploymentsConfig, ExecutorConfigEntry +from nemo_deployments_plugin.controller import DeploymentsController from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform.types.inference.model_deployment import ModelDeployment from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig from nemo_platform.types.models.model_entity import ModelEntity +from nmp.common.config import Runtime from nmp.common.secrets.encryption import get_base64_encoded_random_bytes from nmp.core.files.app.backends.base import FileInfo from nmp.core.files.app.backends.huggingface import HuggingfaceStorageImpl from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend +from nmp.core.models.controllers.backends.deployments_plugin.backend import DeploymentsPluginServiceBackend from nmp.core.models.controllers.backends.registry import BackendRegistry from nmp.core.models.controllers.context import ModelContext from nmp.core.models.controllers.models_controller import ModelsController @@ -178,7 +182,13 @@ async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStat self.status_calls.append(deployment) return self.status_responses.get(deployment.name, self.default_status_response) - async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: + async def delete_model_deployment( + self, + workspace: str, + name: str, + *, + deleting_elapsed_seconds: float | None = None, + ) -> DeploymentStatusUpdate: """Record call and return configured response.""" self.delete_calls.append((workspace, name)) return self.delete_response @@ -353,6 +363,116 @@ def docker_backend_config( } +# ============================================================================= +# Deployments-plugin Docker Integration Fixtures +# ============================================================================= + + +@pytest.fixture +def deployments_plugin_backend_config(worker_id: str) -> dict[str, Any]: + """Flat deployments_plugin backend config for integration tests.""" + return { + "docker_executor": "local-docker", + "default_executor": "local-docker", + "deleting_timeout_seconds": 300, + } + + +@pytest.fixture +def deployments_plugin_platform_config(worker_id: str) -> MagicMock: + """Platform config with docker runtime and worker-scoped executor ports.""" + start_port, end_port = get_worker_port_range(worker_id) + mock_platform_config = MagicMock() + mock_platform_config.models_url = "http://testserver" + mock_platform_config.base_url = "http://testserver" + mock_platform_config.runtime = Runtime.DOCKER + mock_platform_config.get_service_url.return_value = "http://testserver" + mock_platform_config.service_discovery = {"files": "http://testserver"} + mock_platform_config._deployments_config = DeploymentsConfig( + executors=[ + ExecutorConfigEntry( + name="local-docker", + backend="docker", + config={ + "pull_images": False, + "port_range_start": start_port, + "port_range_end": end_port, + }, + ) + ], + default_executor="local-docker", + controller=ControllerConfig( + interval_seconds=1, + orphan_cleanup_interval_seconds=0, + ), + ) + return mock_platform_config + + +@pytest.fixture +def controller_with_deployments_plugin( + test_clients, + docker_client, # noqa: ARG001 - ensures docker is available + mock_nim_image, + docker_test_context, + models_controller_container_cleanup, + deployments_plugin_backend_config, + deployments_plugin_platform_config, +): + """Models controller + deployments plugin controller on real Docker.""" + mock_platform_config = deployments_plugin_platform_config + deployments_config = mock_platform_config._deployments_config + + plugin_backend = DeploymentsPluginServiceBackend( + nmp_sdk=test_clients.async_sdk, + config=deployments_plugin_backend_config, + huggingface_model_puller="alpine:3.20", + ) + plugin_backend.init() + backend_registry = BackendRegistry(registry={"deployments_plugin": plugin_backend}) + + deployments_controller = DeploymentsController() + + def reconcile_stack(models_controller: ModelsController) -> None: + models_controller.step() + models_controller._loop.run_until_complete(deployments_controller.reconcile()) + + with ( + patch("nmp.core.models.config.get_platform_config", return_value=mock_platform_config), + patch("nmp.core.models.controllers.main.get_platform_config", return_value=mock_platform_config), + patch( + "nmp.core.models.controllers.backends.deployments_plugin.resolve.get_platform_config", + return_value=mock_platform_config, + ), + patch("nmp.core.models.controllers.models_controller.get_async_platform_sdk") as mock_models_sdk, + patch("nemo_platform_plugin.sdk_provider.get_async_platform_sdk") as mock_sdk, + patch("nemo_deployments_plugin.config.DeploymentsConfig.get", return_value=deployments_config), + ): + mock_models_sdk.return_value = test_clients.async_sdk + mock_sdk.return_value = test_clients.async_sdk + + models_controller = ModelsController( + backend_registry=backend_registry, + stop_signal=None, + ) + models_controller._provider_reconciler.reconcile_model_providers = AsyncMock(return_value=None) + models_controller._loop.run_until_complete(deployments_controller.on_startup()) + + yield ( + models_controller, + deployments_controller, + test_clients.sdk, + mock_nim_image, + docker_test_context, + reconcile_stack, + ) + + try: + models_controller._loop.run_until_complete(deployments_controller.on_shutdown()) + finally: + models_controller.shutdown() + + # ============================================================================= # Pytest Hooks # ============================================================================= diff --git a/services/core/models/tests/integration/test_chat_template_tool_calling.py b/services/core/models/tests/integration/test_chat_template_tool_calling.py index 31d8aa889a..576d5e7a4e 100644 --- a/services/core/models/tests/integration/test_chat_template_tool_calling.py +++ b/services/core/models/tests/integration/test_chat_template_tool_calling.py @@ -18,7 +18,7 @@ 2. Model spec task updates model entity via API → retrieve preserves spec 3. _compile_env_vars produces correct NIM_* env vars from real entities -Unlike the unit tests in test_docker_backend.py (which use MagicMock), +Unlike the removed docker backend unit tests (which used MagicMock), these tests use: - Real Pydantic ModelSpec / ToolCallConfig objects for the merge step - The actual Models API (via in-memory test client) for the CRUD step @@ -32,15 +32,18 @@ from unittest.mock import MagicMock, patch import pytest +from nmp.common.config import Runtime from nmp.common.files.metadata import FilesetMetadata, ModelMetadataContent, ToolCallingMetadataContent +from nmp.core.models.app import ModelWeightsType, get_model_weights_type from nmp.core.models.config import config as models_config -from nmp.core.models.controllers.backends.docker.backend import DockerServiceBackend -from nmp.core.models.controllers.backends.docker.creation_reconciler import DockerDeploymentCreationReconciler -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler import ( - TOOL_CALL_PLUGIN_PATH, - compile_nimservice, +from nmp.core.models.controllers.backends.common import deployment_config_view +from nmp.core.models.controllers.backends.deployments_plugin.compiler import compile_model_deployment +from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginConfig +from nmp.core.models.controllers.backends.deployments_plugin.nim_compiler import ( + _TOOL_CALL_PLUGIN_PATH, + compile_nim_server_env, ) +from nmp.core.models.controllers.backends.deployments_plugin.resolve import ResolvedPluginDeployment from nmp.core.models.schemas import ( ContainerExecutorConfig, ModelDeploymentConfigModelSpec, @@ -109,36 +112,71 @@ def _set_llama_config(config, *, chat_template=None, tool_call_config=None) -> N ) +def _resolved_plugin( + config, model_entity, deployment, *, runtime: Runtime = Runtime.DOCKER +) -> ResolvedPluginDeployment: + view = deployment_config_view(config) + return ResolvedPluginDeployment( + deployment=deployment, + config=config, + model_entity=model_entity, + view=view, + weights_type=get_model_weights_type( + model_deployment=deployment, + model_deployment_config=config, + model_entity=model_entity, + ), + model_namespace=view.model_namespace, + model_name=view.model_name, + model_revision=view.model_revision, + files_hf_url="http://testserver/apis/files/v2/hf", + huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", + runtime=runtime, + ) + + +def _compile_nim_env( + config, + model_entity, + deployment, + *, + weighted: bool = False, + tool_call_plugin_path: str | None = None, +) -> dict[str, str]: + return compile_nim_server_env( + _resolved_plugin(config, model_entity, deployment), + DeploymentsPluginConfig(), + weighted=weighted, + tool_call_plugin_path=tool_call_plugin_path, + ) + + +def _compile_k8s_deployment(config, model_entity, deployment): + view = deployment_config_view(config) + resolved = ResolvedPluginDeployment( + deployment=deployment, + config=config, + model_entity=model_entity, + view=view, + weights_type=ModelWeightsType.FILES_SERVICE, + model_namespace=view.model_namespace, + model_name=view.model_name, + model_revision=view.model_revision, + files_hf_url="http://testserver/apis/files/v2/hf", + huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", + runtime=Runtime.KUBERNETES, + ) + return compile_model_deployment( + resolved, + DeploymentsPluginConfig(busybox_image="busybox", busybox_image_tag="latest"), + ) + + # ============================================================================ # Fixtures # ============================================================================ -@pytest.fixture -def docker_backend(): - """Create a DockerServiceBackend with mocked internals.""" - with ( - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock_docker, - patch("nmp.core.models.controllers.backends.docker.backend.SharedResourceManager"), - ): - mock_docker.return_value = MagicMock() - backend = DockerServiceBackend.__new__(DockerServiceBackend) - backend._client = mock_docker.return_value - backend._backend_config = MagicMock() - backend._backend_config.nim_guided_decoding_backend = "outlines" - backend._backend_config.peft_source = "/scratch/loras" - backend._backend_config.peft_refresh_interval = 60 - backend._backend_config.models_docker_host_service_name = "localhost" - backend._gpu_pool = None - backend._reconciler = DockerDeploymentCreationReconciler( - client=backend._client, - backend_config=backend._backend_config, - nmp_sdk=MagicMock(), - gpu_pool=backend._gpu_pool, - ) - yield backend - - @pytest.fixture def sample_deployment(): """Minimal deployment mock.""" @@ -556,8 +594,7 @@ def test_merge_keeps_tool_call_plugin_when_enabled(self): # ============================================================================ -@pytest.mark.asyncio -async def test_pipeline_fileset_metadata_to_env_vars(docker_backend, sample_deployment): +def test_pipeline_fileset_metadata_to_env_vars(sample_deployment): """End-to-end: fileset metadata → ModelSpec merge → _compile_env_vars. 1. Start with a bare ModelSpec. @@ -597,11 +634,11 @@ async def test_pipeline_fileset_metadata_to_env_vars(docker_backend, sample_depl config = MagicMock() _set_llama_config(config) - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, + env_vars = _compile_nim_env( config, - model_entity=model_entity, - is_multi_llm=True, + model_entity, + sample_deployment, + weighted=True, ) assert env_vars["NIM_CHAT_TEMPLATE"] == SAMPLE_CHAT_TEMPLATE @@ -610,8 +647,7 @@ async def test_pipeline_fileset_metadata_to_env_vars(docker_backend, sample_depl assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars -@pytest.mark.asyncio -async def test_pipeline_deployment_overrides_fileset_values(docker_backend, sample_deployment): +def test_pipeline_deployment_overrides_fileset_values(sample_deployment): """End-to-end: deployment config overrides values merged from fileset.""" model_spec = ModelSpec(**MINIMAL_SPEC) fileset = SimpleNamespace( @@ -644,11 +680,11 @@ async def test_pipeline_deployment_overrides_fileset_values(docker_backend, samp }, ) - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, + env_vars = _compile_nim_env( config, - model_entity=model_entity, - is_multi_llm=True, + model_entity, + sample_deployment, + weighted=True, ) # Deployment-level values should win over fileset values @@ -657,8 +693,7 @@ async def test_pipeline_deployment_overrides_fileset_values(docker_backend, samp assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" -@pytest.mark.asyncio -async def test_pipeline_mixed_sources(docker_backend, sample_deployment): +def test_pipeline_mixed_sources(sample_deployment): """End-to-end: chat_template from fileset, tool_call_config from deployment.""" model_spec = ModelSpec(**MINIMAL_SPEC) @@ -685,11 +720,11 @@ async def test_pipeline_mixed_sources(docker_backend, sample_deployment): tool_call_config={"tool_call_parser": "hermes", "auto_tool_choice": True}, ) - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, + env_vars = _compile_nim_env( config, - model_entity=model_entity, - is_multi_llm=True, + model_entity, + sample_deployment, + weighted=True, ) # chat_template from fileset (via model spec), tool config from deployment @@ -698,8 +733,7 @@ async def test_pipeline_mixed_sources(docker_backend, sample_deployment): assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" -@pytest.mark.asyncio -async def test_pipeline_plugin_path_flows_through(docker_backend, sample_deployment): +def test_pipeline_plugin_path_flows_through(sample_deployment): """End-to-end: tool_call_plugin from fileset metadata → env var with plugin path.""" model_spec = ModelSpec(**MINIMAL_SPEC) @@ -727,11 +761,11 @@ async def test_pipeline_plugin_path_flows_through(docker_backend, sample_deploym config = MagicMock() _set_llama_config(config) - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, + env_vars = _compile_nim_env( config, - model_entity=model_entity, - is_multi_llm=True, + model_entity, + sample_deployment, + weighted=True, tool_call_plugin_path="/model-store/tool_call_plugin/my_plugin.py", ) @@ -741,8 +775,7 @@ async def test_pipeline_plugin_path_flows_through(docker_backend, sample_deploym assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" -@pytest.mark.asyncio -async def test_pipeline_no_fileset_metadata_no_deployment_overrides(docker_backend, sample_deployment): +def test_pipeline_no_fileset_metadata_no_deployment_overrides(sample_deployment): """End-to-end: no fileset metadata, no deployment overrides → no tool env vars.""" model_spec = ModelSpec(**MINIMAL_SPEC) @@ -760,11 +793,11 @@ async def test_pipeline_no_fileset_metadata_no_deployment_overrides(docker_backe config = MagicMock() _set_llama_config(config) - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, + env_vars = _compile_nim_env( config, - model_entity=model_entity, - is_multi_llm=True, + model_entity, + sample_deployment, + weighted=True, ) assert "NIM_CHAT_TEMPLATE" not in env_vars @@ -794,40 +827,19 @@ def test_k8s_nimservice_adds_plugin_init_containers_from_model_entity(sample_dep config.entity_version = "v1" _set_llama_config(config) - platform_config = SimpleNamespace( - image_pull_secrets=[], - to_shared_envvars=lambda: {}, - get_service_url=lambda _svc: "http://files:8000", - ) - with ( - patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ), - ): - nimservice = compile_nimservice( - deployment=sample_deployment, - config=config, - backend_config=K8sNimOperatorConfig(busybox_image="busybox", busybox_image_tag="latest"), - k8s_namespace="default", - resource_name="md-integ-k8s-entity-plugin", - model_entity=model_entity, - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - ) + compiled = _compile_k8s_deployment(config, model_entity, sample_deployment) - assert nimservice.spec.initContainers is not None - assert len(nimservice.spec.initContainers) == 3 - pull_container = nimservice.spec.initContainers[1] + assert len(compiled.server_config.init_containers) == 3 + pull_container = compiled.server_config.init_containers[1] assert pull_container.command == ["download", "default/entity-plugin-fileset", "--local-dir", "/scratch/plugin"] - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - assert env_dict["NIM_GUIDED_DECODING_BACKEND"] == "outlines" - assert env_dict["NIM_MODEL_NAME"] == "meta/llama-3.2-1b-instruct" + env_dict = {item.name: item.value for item in compiled.server_config.containers[0].env} + assert env_dict["NIM_MODEL_NAME"] == "/model-store" assert env_dict["NIM_SERVED_MODEL_NAME"] == "meta/llama-3.2-1b-instruct" assert env_dict["NMP_MODEL_ENTITY_WORKSPACE"] == DEFAULT_WORKSPACE assert env_dict["NMP_MODEL_ENTITY_NAME"] == "integ-k8s-entity-plugin-model" assert env_dict["NIM_TOOL_CALL_PARSER"] == "entity-parser" - assert env_dict["NIM_TOOL_PARSER_PLUGIN"] == TOOL_CALL_PLUGIN_PATH + assert env_dict["NIM_TOOL_PARSER_PLUGIN"] == _TOOL_CALL_PLUGIN_PATH assert env_dict["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" @@ -859,38 +871,17 @@ def test_k8s_nimservice_deployment_tool_config_takes_priority(sample_deployment) }, ) - platform_config = SimpleNamespace( - image_pull_secrets=[], - to_shared_envvars=lambda: {}, - get_service_url=lambda _svc: "http://files:8000", - ) - with ( - patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ), - ): - nimservice = compile_nimservice( - deployment=sample_deployment, - config=config, - backend_config=K8sNimOperatorConfig(busybox_image="busybox", busybox_image_tag="latest"), - k8s_namespace="default", - resource_name="md-integ-k8s-priority", - model_entity=model_entity, - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - ) + compiled = _compile_k8s_deployment(config, model_entity, sample_deployment) - assert nimservice.spec.initContainers is not None - assert len(nimservice.spec.initContainers) == 3 - pull_container = nimservice.spec.initContainers[1] + assert len(compiled.server_config.init_containers) == 3 + pull_container = compiled.server_config.init_containers[1] assert pull_container.command == ["download", "default/deployment-plugin-fileset", "--local-dir", "/scratch/plugin"] - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - assert env_dict["NIM_GUIDED_DECODING_BACKEND"] == "outlines" - assert env_dict["NIM_MODEL_NAME"] == "meta/llama-3.2-1b-instruct" + env_dict = {item.name: item.value for item in compiled.server_config.containers[0].env} + assert env_dict["NIM_MODEL_NAME"] == "/model-store" assert env_dict["NIM_SERVED_MODEL_NAME"] == "meta/llama-3.2-1b-instruct" assert env_dict["NMP_MODEL_ENTITY_WORKSPACE"] == DEFAULT_WORKSPACE assert env_dict["NMP_MODEL_ENTITY_NAME"] == "integ-k8s-priority-model" assert env_dict["NIM_TOOL_CALL_PARSER"] == "deployment-parser" - assert env_dict["NIM_TOOL_PARSER_PLUGIN"] == TOOL_CALL_PLUGIN_PATH + assert env_dict["NIM_TOOL_PARSER_PLUGIN"] == _TOOL_CALL_PLUGIN_PATH assert env_dict["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" diff --git a/services/core/models/tests/integration/test_deployments_plugin_lifecycle.py b/services/core/models/tests/integration/test_deployments_plugin_lifecycle.py new file mode 100644 index 0000000000..e0826fbe8e --- /dev/null +++ b/services/core/models/tests/integration/test_deployments_plugin_lifecycle.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for models controller via deployments_plugin on real Docker.""" + +from __future__ import annotations + +import uuid + +import pytest +from docker.errors import NotFound +from nemo_deployments_plugin.backends.labels import container_name +from nemo_platform import NotFoundError +from nmp.core.models.controllers.backends.deployments_plugin.naming import entity_names +from tenacity import retry, stop_after_delay, wait_fixed + +import docker + +try: + docker.from_env().ping() + _DOCKER_AVAILABLE = True +except Exception: + _DOCKER_AVAILABLE = False + +skip_without_docker = pytest.mark.skipif(not _DOCKER_AVAILABLE, reason="Docker daemon not available") + +pytestmark = [ + skip_without_docker, + pytest.mark.xdist_group("nemo_deployments_docker_integration"), +] + + +def test_deployments_plugin_docker_lifecycle(controller_with_deployments_plugin, docker_client: docker.DockerClient): + """Full stack: models → deployments_plugin entities → plugin docker backend. + + Tests: create → PENDING → READY → delete → cleanup + Also verifies ModelProvider creation and deletion. + """ + controller, _, sdk, mock_nim_image, ctx, reconcile = controller_with_deployments_plugin + test_uuid = uuid.uuid4().hex[:8] + config_name = f"test-plugin-lifecycle-{test_uuid}" + deployment_name = f"test-plugin-lifecycle-{test_uuid}" + workspace = "default" + server_entity = entity_names(deployment_name).server + server_container_name = container_name(workspace, server_entity) + + ctx.register_container(server_container_name) + + image_name, image_tag = mock_nim_image.rsplit(":", 1) + sdk.inference.deployment_configs.create( + name=config_name, + workspace=workspace, + engine="nim", + model_spec={}, + executor_config={ + "gpu": 0, + "image_name": image_name, + "image_tag": image_tag, + }, + ) + sdk.inference.deployments.create( + name=deployment_name, + workspace=workspace, + config=config_name, + ) + + @retry(stop=stop_after_delay(30), wait=wait_fixed(0.2), reraise=True) + def wait_for_container_created(): + reconcile(controller) + container = docker_client.containers.get(server_container_name) + assert container.status in ["created", "running"], f"Unexpected status: {container.status}" + return container + + container = wait_for_container_created() + + @retry(stop=stop_after_delay(15), wait=wait_fixed(0.2), reraise=True) + def wait_for_container_running(): + container.reload() + assert container.status == "running", f"Container not running: {container.status}" + + wait_for_container_running() + + @retry(stop=stop_after_delay(45), wait=wait_fixed(0.2), reraise=True) + def wait_for_deployment_ready(): + reconcile(controller) + dep = sdk.inference.deployments.retrieve(deployment_name, workspace=workspace) + assert dep.status == "READY", f"Deployment not READY: {dep.status} ({dep.status_message})" + return dep + + deployment = wait_for_deployment_ready() + + provider_id = deployment.model_provider_id + assert provider_id is not None, "ModelProvider should be created when deployment becomes READY" + provider_workspace, provider_name = provider_id.split("/") + provider = sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) + assert provider.host_url is not None + assert provider.status == "READY" + + reconcile(controller) + sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) + + sdk.inference.deployments.delete(deployment_name, workspace=workspace) + reconcile(controller) + + @retry(stop=stop_after_delay(30), wait=wait_fixed(0.2), reraise=True) + def wait_for_delete_complete(): + reconcile(controller) + try: + container = docker_client.containers.get(server_container_name) + container.reload() + if container.status not in ["exited", "removing", "dead"]: + raise AssertionError(f"Container still running: {container.status}") + except NotFound: + pass + try: + sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) + except NotFoundError: + return + raise AssertionError("ModelProvider still exists after deployment delete") + + wait_for_delete_complete() + + with pytest.raises(NotFoundError): + sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) diff --git a/services/core/models/tests/integration/test_models.py b/services/core/models/tests/integration/test_models.py index bcd458fe8d..2c23c70437 100644 --- a/services/core/models/tests/integration/test_models.py +++ b/services/core/models/tests/integration/test_models.py @@ -16,12 +16,8 @@ from unittest.mock import AsyncMock, patch from nemo_platform import ConflictError -from nmp.core.models.config import BackendName, ControllerConfig, ModelsConfig -from nmp.core.models.controllers.backends.registry import ( - BackendRegistry, - DockerBackendConfigModel, - K8sNimOperatorBackendConfigModel, -) +from nmp.core.models.config import ControllerConfig, ModelsConfig +from nmp.core.models.controllers.backends.registry import BackendRegistry from nmp.testing import ClientContext # Default workspace for tests @@ -1519,53 +1515,18 @@ def test_deployment_workspace_isolation(test_clients: ClientContext): # "nim_operator" that would cause service startup failures. -def test_backend_config_key_docker_works_end_to_end(): - """Verify 'docker' key in ModelsConfig works with BackendRegistry. +def test_backend_config_key_deployments_plugin_works_end_to_end(): + """Verify deployments_plugin key in ModelsConfig works with BackendRegistry.""" + from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginBackendConfigModel - This test validates the end-to-end path from config to registry: - 1. ModelsConfig validates the key against BackendName type - 2. BackendRegistry.from_config() validates key exists in backend_classes - - If these definitions drift apart, this test will fail. - """ - # Create config using ModelsConfig (validates BackendName type) - config = ModelsConfig(controller=ControllerConfig(backends={"docker": DockerBackendConfigModel(enabled=True)})) - - # Mock Docker client to avoid needing actual Docker daemon - with patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env"): - registry = BackendRegistry.from_config( - nmp_sdk=AsyncMock(), - backend_configs=config.controller.backends, - huggingface_model_puller=config.huggingface_model_puller, - ) - - assert "docker" in registry.list_backends() - - -def test_backend_config_key_k8s_works_end_to_end(): - """Verify K8s backend key in ModelsConfig works with BackendRegistry. - - This test validates that: - 1. The K8s backend key in BackendName type is valid - 2. The same key exists in BackendRegistry.backend_classes - - If these definitions use different key names, this test will fail. - """ - # Get the K8s backend key from the BackendName type - # This ensures we use whatever key the config expects - k8s_key = [k for k in BackendName.__args__ if k != "docker"][0] - - # Create config using ModelsConfig (validates BackendName type) config = ModelsConfig( - controller=ControllerConfig(backends={k8s_key: K8sNimOperatorBackendConfigModel(enabled=True)}) + controller=ControllerConfig( + backends={"deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True)}, + ) ) - # Mock K8s client to avoid needing actual K8s cluster - with ( - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_config.load_incluster_config"), - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_config.load_kube_config"), - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_client.ApiClient"), - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.DynamicClient"), + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.backend.NemoEntitiesClient", ): registry = BackendRegistry.from_config( nmp_sdk=AsyncMock(), @@ -1573,6 +1534,4 @@ def test_backend_config_key_k8s_works_end_to_end(): huggingface_model_puller=config.huggingface_model_puller, ) - # Verify the registry was created with the expected backend - backends = registry.list_backends() - assert len(backends) == 1 + assert registry.list_backends() == ["deployments_plugin"] diff --git a/services/core/models/tests/integration/test_models_controller.py b/services/core/models/tests/integration/test_models_controller.py index 01cc6934ec..cfd05f8d58 100644 --- a/services/core/models/tests/integration/test_models_controller.py +++ b/services/core/models/tests/integration/test_models_controller.py @@ -3,26 +3,18 @@ """Integration tests for Models Controller. -This module contains two categories of tests: -1. Backend-agnostic tests - Test controller logic with mock backend (no Docker required) -2. Docker integration tests - Test with real Docker backend (requires --run-docker-tests flag) +Backend-agnostic tests use a mock backend (no Docker required). Full-stack +docker coverage lives in test_deployments_plugin_lifecycle.py. """ from __future__ import annotations -import os import uuid -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock import pytest -from docker.errors import NotFound from nemo_platform import NotFoundError -from nmp.core.models.app.utils import get_docker_container_name, get_docker_volume_name from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.docker import DockerServiceBackend -from nmp.core.models.controllers.backends.registry import BackendRegistry -from nmp.core.models.controllers.models_controller import ModelsController -from tenacity import retry, stop_after_delay, wait_fixed # ============================================================================= # Backend-Agnostic Tests (Mock Backend) @@ -384,225 +376,3 @@ def test_controller_deletes_model_provider_on_delete(controller_with_mock_backen # Verify provider was deleted with pytest.raises(NotFoundError): sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) - - -# ============================================================================= -# Docker Integration Tests -# ============================================================================= - - -def _get_worker_port_range(worker_id: str, ports_per_worker: int = 100) -> tuple[int, int]: - """Calculate unique port range for a pytest-xdist worker. - - Args: - worker_id: The xdist worker ID ("master", "gw0", "gw1", etc.) - ports_per_worker: Number of ports to allocate per worker - - Returns: - Tuple of (start_port, end_port) - - Environment Variables: - MODELS_DOCKER_PORT_RANGE_START: Override the base port (default: 49152) - """ - # Use IANA ephemeral port range (49152-65535) to avoid conflicts with system services - # Can be overridden via environment variable for DinD testing - base_port = int(os.environ.get("MODELS_DOCKER_PORT_RANGE_START", "49152")) - if worker_id == "master": - worker_num = 0 - else: - # Extract number from "gw0", "gw1", etc. - worker_num = int(worker_id.replace("gw", "")) - - start_port = base_port + (worker_num * ports_per_worker) - end_port = start_port + ports_per_worker - 1 - return start_port, end_port - - -@pytest.fixture -def docker_backend_config(worker_id, docker_owner_labels): - """Configuration for Docker backend in tests. - - Uses worker_id from pytest-xdist to allocate unique port ranges - per worker, enabling parallel test execution. - """ - start_port, end_port = _get_worker_port_range(worker_id) - return { - "models_docker_port_range_start": start_port, - "models_docker_port_range_end": end_port, - "docker_timeout": 60, - "models_docker_host_service_name": "localhost", - "model_labels": docker_owner_labels, - } - - -@pytest.fixture -def controller_with_docker( - test_clients, - docker_client, # noqa: ARG001 - dependency ensures docker is available - mock_nim_image, - mock_sidecar_image, - docker_backend_config, - docker_test_context, - models_controller_container_cleanup, -): - """Create controller with real Docker backend.""" - from nemo_platform_plugin.jobs.image import get_qualified_image as real_get_qualified_image - - def patched_get_qualified_image(name: str, tag=None, registry=None): - if name in ["nmp-core", "nmp-api"]: - return mock_sidecar_image - return real_get_qualified_image(name, tag=tag, registry=registry) - - # Create Docker backend with test config - docker_backend = DockerServiceBackend( - nmp_sdk=test_clients.async_sdk, - config=docker_backend_config, - ) - - # Create registry with Docker backend - backend_registry = BackendRegistry(registry={"docker": docker_backend}) - - # Create controller with patched SDK factory and get_qualified_image so backend uses our mock sidecar - mock_platform_config = MagicMock() - mock_platform_config.models_url = "http://testserver" - mock_platform_config.get_service_url.return_value = "http://testserver" - with ( - patch("nmp.core.models.config.get_platform_config", return_value=mock_platform_config), - patch("nmp.core.models.controllers.main.get_platform_config", return_value=mock_platform_config), - patch("nmp.core.models.controllers.models_controller.get_async_platform_sdk") as mock_sdk_factory, - patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.get_qualified_image", - side_effect=patched_get_qualified_image, - ), - ): - mock_sdk_factory.return_value = test_clients.async_sdk - - controller = ModelsController( - backend_registry=backend_registry, - stop_signal=None, - ) - - yield controller, docker_backend, test_clients.sdk, mock_nim_image, docker_test_context - - # Clean up controller resources (event loop, backend registry, etc.) - controller.shutdown() - - -def test_docker_deployment_lifecycle(controller_with_docker, docker_client): - """Test full Docker deployment lifecycle with provider reconciliation. - - Tests: create → PENDING → READY → delete → cleanup - Also verifies: ModelProvider creation, served_models autodiscovery, provider deletion - """ - controller, _, sdk, mock_nim_image, ctx = controller_with_docker - test_uuid = uuid.uuid4().hex[:8] - config_name = f"test-docker-lifecycle-{test_uuid}" - deployment_name = f"test-docker-lifecycle-{test_uuid}" - workspace = "default" - container_name = get_docker_container_name(workspace, deployment_name) - volume_name = get_docker_volume_name(workspace, deployment_name) - - # Register container for cleanup - ctx.register_container(container_name) - ctx.register_volume(volume_name) - - # === Phase 1: Create config and deployment === - # Parse image name and tag - use rsplit to handle registry URLs with port numbers - # e.g., "registry.example.com/nemo-platform/mock-nim:1.0.0" - image_name, image_tag = mock_nim_image.rsplit(":", 1) - sdk.inference.deployment_configs.create( - name=config_name, - workspace="default", - engine="nim", - model_spec={}, - executor_config={ - "gpu": 0, - "image_name": image_name, - "image_tag": image_tag, - }, - ) - - sdk.inference.deployments.create( - name=deployment_name, - workspace="default", - config=config_name, - ) - - # === Phase 2: Controller advances creation pipeline (CREATED -> PENDING -> container) === - # The staged creation pipeline needs multiple controller steps: - # step 1: CREATED→PENDING (registers deployment, starts image pull) - # step 2+: advance through PULLING_NIM_IMAGE → CREATING_CONTAINER - @retry(stop=stop_after_delay(15), wait=wait_fixed(0.1), reraise=True) - def wait_for_container_created(): - controller.step() - container = docker_client.containers.get(container_name) - assert container.status in ["created", "running"], f"Unexpected status: {container.status}" - return container - - container = wait_for_container_created() - - # === Phase 3: Wait for container to start, then PENDING -> READY === - @retry(stop=stop_after_delay(10), wait=wait_fixed(0.1), reraise=True) - def wait_for_container_running(): - container.reload() - assert container.status == "running", f"Container not running: {container.status}" - - wait_for_container_running() - - # Controller polls health check and marks READY. - # The mock NIM may need a moment to start responding to health checks, - # so we poll multiple times until READY or timeout. - @retry(stop=stop_after_delay(30), wait=wait_fixed(0.1), reraise=True) - def wait_for_deployment_ready(): - controller.step() - dep = sdk.inference.deployments.retrieve(deployment_name, workspace="default") - assert dep.status == "READY", f"Deployment not READY: {dep.status}" - return dep - - deployment = wait_for_deployment_ready() - - # === Phase 3b: Verify ModelProvider was created === - provider_id = deployment.model_provider_id - assert provider_id is not None, "ModelProvider should be created when deployment becomes READY" - - provider_workspace, provider_name = provider_id.split("/") - provider = sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) - assert provider.host_url is not None, "Provider should have host_url set" - assert provider.status == "READY", "Provider should be READY when deployment is READY" - - # === Phase 3c: Run another step to trigger provider reconciliation === - # Note: Autodiscovery of served_models requires Inference Gateway which is not - # set up in this test. The provider_reconciler will log a warning but continue. - # Key functionality (provider creation/deletion) is already verified above. - controller.step() - - # Verify provider still exists after reconciliation step (no errors) - sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) - - # === Phase 4: Delete deployment and verify cleanup === - sdk.inference.deployments.delete(deployment_name, workspace="default") - - # Controller processes deletion - controller.step() - - # Poll for container to be removed or stopped (DinD may be slow) - @retry(stop=stop_after_delay(15), wait=wait_fixed(0.1), reraise=True) - def wait_for_container_deleted(): - try: - c = docker_client.containers.get(container_name) - c.reload() - if c.status in ["exited", "removing", "dead"]: - return # Container is stopping/stopped - raise AssertionError(f"Container still running: {c.status}") - except NotFound: - return # Container was removed, which is expected - - try: - wait_for_container_deleted() - except AssertionError: - # After all retries, log but don't fail - provider deletion is the key check - print(f"Warning: Container {container_name} still running after retries") - - # === Phase 4b: Verify ModelProvider was deleted === - with pytest.raises(NotFoundError): - sdk.inference.providers.retrieve(provider_name, workspace=provider_workspace) diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_backend.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_backend.py index 9e99e0ad46..408e7e5692 100644 --- a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_backend.py +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_backend.py @@ -249,9 +249,9 @@ async def test_pending_timeout_escalates_stuck_deployment() -> None: @pytest.mark.asyncio -async def test_delete_waits_for_server_before_config() -> None: +async def test_delete_returns_deleting_when_server_still_exists() -> None: backend = DeploymentsPluginServiceBackend(AsyncMock(), {}, "puller:latest") - backend._backend_config = DeploymentsPluginConfig(delete_wait_seconds=0.02, delete_poll_seconds=0.005) + backend.init() backend._entities = AsyncMock() server = Deployment( name="my-dep-server", @@ -261,8 +261,10 @@ async def test_delete_waits_for_server_before_config() -> None: ) async def _get(entity_type: type, name: str, workspace: str | None = None) -> Deployment: - del entity_type, name, workspace - return server + del entity_type, workspace + if name == "my-dep-server": + return server + raise NemoEntityNotFoundError("missing") backend._entities.get = AsyncMock(side_effect=_get) backend._entities.update = AsyncMock(side_effect=lambda entity: entity) @@ -271,3 +273,107 @@ async def _get(entity_type: type, name: str, workspace: str | None = None) -> De result = await backend.delete_model_deployment("default", "my-dep") assert result.status == "DELETING" backend._entities.delete.assert_not_called() + backend._entities.update.assert_awaited_once() + assert server.status == "DELETING" + assert server.desired_state == "STOPPED" + + +@pytest.mark.asyncio +async def test_delete_returns_deleting_without_blocking_poll() -> None: + backend = DeploymentsPluginServiceBackend(AsyncMock(), {}, "puller:latest") + backend.init() + backend._entities = AsyncMock() + server = Deployment( + name="my-dep-server", + workspace="default", + deployment_config="my-dep-server", + status="READY", + ) + backend._entities.get = AsyncMock(return_value=server) + backend._entities.update = AsyncMock(side_effect=lambda entity: entity) + backend._entities.delete = AsyncMock() + + with patch("asyncio.sleep", AsyncMock()) as sleep_mock: + result = await backend.delete_model_deployment("default", "my-dep") + + assert result.status == "DELETING" + sleep_mock.assert_not_called() + backend._entities.delete.assert_not_called() + + +@pytest.mark.asyncio +async def test_delete_retries_on_next_call_when_deployment_still_exists() -> None: + backend = DeploymentsPluginServiceBackend(AsyncMock(), {}, "puller:latest") + backend.init() + backend._entities = AsyncMock() + server = Deployment( + name="my-dep-server", + workspace="default", + deployment_config="my-dep-server", + status="DELETING", + desired_state="STOPPED", + ) + seen_server = False + + async def _get(entity_type: type, name: str, workspace: str | None = None) -> Deployment: + del entity_type, workspace + nonlocal seen_server + if name == "my-dep-server": + if not seen_server: + seen_server = True + return server + raise NemoEntityNotFoundError("missing") + raise NemoEntityNotFoundError("missing") + + backend._entities.get = AsyncMock(side_effect=_get) + backend._entities.update = AsyncMock(side_effect=lambda entity: entity) + backend._entities.delete = AsyncMock() + + first = await backend.delete_model_deployment("default", "my-dep") + assert first.status == "DELETING" + + second = await backend.delete_model_deployment("default", "my-dep") + assert second.status == "DELETED" + + +@pytest.mark.asyncio +async def test_delete_completes_when_plugin_deployment_failed() -> None: + backend = DeploymentsPluginServiceBackend(AsyncMock(), {}, "puller:latest") + backend.init() + backend._entities = AsyncMock() + server = Deployment( + name="my-dep-server", + workspace="default", + deployment_config="my-dep-server", + status="FAILED", + desired_state="STOPPED", + ) + backend._entities.get = AsyncMock(return_value=server) + backend._entities.delete = AsyncMock() + + result = await backend.delete_model_deployment("default", "my-dep") + + assert result.status == "DELETED" + backend._entities.delete.assert_any_await(Deployment, name="my-dep-server", workspace="default") + + +@pytest.mark.asyncio +async def test_delete_escalates_to_error_after_deleting_timeout() -> None: + backend = DeploymentsPluginServiceBackend(AsyncMock(), {}, "puller:latest") + backend.init() + backend._backend_config = DeploymentsPluginConfig(deleting_timeout_seconds=60) + backend._entities = AsyncMock() + server = Deployment( + name="my-dep-server", + workspace="default", + deployment_config="my-dep-server", + status="DELETING", + ) + backend._entities.get = AsyncMock(return_value=server) + backend._entities.update = AsyncMock(side_effect=lambda entity: entity) + + result = await backend.delete_model_deployment("default", "my-dep", deleting_elapsed_seconds=120) + assert result.status == "ERROR" + assert result.error_details is not None + assert result.error_details["reason"] == "deleting_timeout" + assert result.error_details["timeout_seconds"] == 60 diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py index f6794faab0..8d4334379a 100644 --- a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py @@ -2,14 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch +from nemo_platform.types.inference.k8s_nim_operator_config import K8sNIMOperatorConfig from nmp.common.config import Runtime from nmp.core.models.app import ModelWeightsType from nmp.core.models.controllers.backends.common import DeploymentConfigView from nmp.core.models.controllers.backends.deployments_plugin.compiler import compile_model_deployment from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginConfig from nmp.core.models.controllers.backends.deployments_plugin.resolve import ResolvedPluginDeployment +from nmp.core.models.controllers.backends.vllm_compiler import MODEL_STORE_PATH def _resolved(engine: str, *, lora: bool = False, runtime: Runtime = Runtime.KUBERNETES) -> ResolvedPluginDeployment: @@ -37,6 +39,64 @@ def test_vllm_weighted_chain_has_on_failure_puller_and_always_server() -> None: assert compiled.server_config.containers[0].volume_mounts[0].read_only is True +def test_vllm_server_command_image_args_and_gpu() -> None: + resolved = _resolved("vllm") + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView(model_namespace="org", model_name="model", gpu=1), + weights_type=resolved.weights_type, + model_namespace=resolved.model_namespace, + model_name=resolved.model_name, + model_revision=resolved.model_revision, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller=resolved.huggingface_model_puller, + runtime=resolved.runtime, + ) + compiled = compile_model_deployment(resolved, DeploymentsPluginConfig()) + server = compiled.server_config.containers[0] + assert server.command == ["vllm", "serve"] + assert server.args[0] == MODEL_STORE_PATH + assert server.image == "docker.io/vllm/vllm-openai:v0.22.1" + assert server.resources is not None + assert server.resources.limits["nvidia.com/gpu"] == "1" + assert compiled.puller_config is not None + puller = compiled.puller_config.containers[0] + puller_env = {item.name: item.value for item in puller.env} + assert puller_env["HF_ENDPOINT"] == resolved.files_hf_url + assert puller_env["HF_TOKEN"] == "service:models" + assert puller.resources is not None + assert puller.resources.limits["nvidia.com/gpu"] == "1" + + +def test_nim_gpu_resources_without_override() -> None: + resolved = _resolved("nim") + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="org", + model_name="model", + gpu=1, + image_name="nvcr.io/nim/meta/llama-3.1-8b-instruct", + image_tag="1.8.5", + ), + weights_type=ModelWeightsType.BAKED_CONTAINER, + model_namespace=None, + model_name=None, + model_revision=None, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller=resolved.huggingface_model_puller, + runtime=Runtime.KUBERNETES, + ) + compiled = compile_model_deployment(resolved, DeploymentsPluginConfig()) + server = compiled.server_config.containers[0] + assert server.resources is not None + assert server.resources.limits["nvidia.com/gpu"] == "1" + + def test_nim_weighted_chain_sets_model_path_env() -> None: compiled = compile_model_deployment(_resolved("nim"), DeploymentsPluginConfig()) assert compiled.volume is not None @@ -45,6 +105,143 @@ def test_nim_weighted_chain_sets_model_path_env() -> None: assert env["NIM_MODEL_NAME"] == "/model-store" assert env["NIM_MODEL_PATH"] == "/model-store" assert env["NIM_SERVED_MODEL_NAME"] == "org/model" + assert env["NIM_FT_MODEL"] == "/model-store" + assert env["NIM_CUSTOM_MODEL"] == "/model-store" + + +def test_nim_multi_llm_weighted_omits_ft_model_env() -> None: + resolved = _resolved("nim") + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="nvidia", + model_name="Llama-3.1-Nemotron-Nano-4B-v1.1", + image_name="nvcr.io/nim/nvidia/llm-nim", + ), + weights_type=resolved.weights_type, + model_namespace="nvidia", + model_name="Llama-3.1-Nemotron-Nano-4B-v1.1", + model_revision=resolved.model_revision, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller=resolved.huggingface_model_puller, + runtime=resolved.runtime, + ) + compiled = compile_model_deployment(resolved, DeploymentsPluginConfig()) + env = {item.name: item.value for item in compiled.server_config.containers[0].env} + assert env["NIM_MODEL_NAME"] == "/model-store" + assert "NIM_FT_MODEL" not in env + assert "NIM_CUSTOM_MODEL" not in env + assert env["NIM_SERVED_MODEL_NAME"] == "nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1" + + +def test_nim_tool_call_plugin_adds_init_containers_and_env() -> None: + resolved = _resolved("nim") + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="org", + model_name="model", + tool_call_config=SimpleNamespace( + tool_call_plugin="test-ws/my-plugin-fileset", + tool_call_parser=None, + auto_tool_choice=None, + ), + ), + weights_type=resolved.weights_type, + model_namespace=resolved.model_namespace, + model_name=resolved.model_name, + model_revision=resolved.model_revision, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", + runtime=resolved.runtime, + ) + compiled = compile_model_deployment(resolved, DeploymentsPluginConfig()) + assert len(compiled.server_config.init_containers) == 3 + pull = compiled.server_config.init_containers[1] + assert pull.command == ["download", "test-ws/my-plugin-fileset", "--local-dir", "/scratch/plugin"] + pull_env = {item.name: item.value for item in pull.env} + assert pull_env["HF_TOKEN"] == "service:models" + assert pull_env["HF_ENDPOINT"] == resolved.files_hf_url + server_env = {item.name: item.value for item in compiled.server_config.containers[0].env} + assert server_env["NIM_TOOL_PARSER_PLUGIN"] == "/model-store/plugin/plugin.py" + + +def test_nim_k8s_nim_operator_config_maps_tolerations_and_resources() -> None: + resolved = _resolved("nim") + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="org", + model_name="model", + k8s_nim_operator_config=K8sNIMOperatorConfig( + tolerations=[{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + node_selector={"node-type": "gpu-node"}, + resources={"requests": {"cpu": "4"}, "limits": {"memory": "32Gi"}}, + startup_probe_grace_seconds=600, + ), + ), + weights_type=resolved.weights_type, + model_namespace=resolved.model_namespace, + model_name=resolved.model_name, + model_revision=resolved.model_revision, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller=resolved.huggingface_model_puller, + runtime=Runtime.KUBERNETES, + ) + compiled = compile_model_deployment(resolved, DeploymentsPluginConfig()) + k8s = compiled.server_config.backend_config.k8s + assert k8s is not None + assert len(k8s.tolerations) == 1 + assert k8s.tolerations[0].key == "nvidia.com/gpu" + assert k8s.affinity is not None + server = compiled.server_config.containers[0] + assert server.resources.requests["cpu"] == "4" + assert server.resources.limits["memory"] == "32Gi" + assert server.readiness_probe is not None + assert server.readiness_probe.failure_threshold == 60 + assert k8s.security_context is not None + assert k8s.security_context.run_as_user == 1000 + assert k8s.security_context.run_as_group == 2000 + + +def test_nim_override_config_wins_over_k8s_nim_operator_config() -> None: + resolved = _resolved("nim") + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="org", + model_name="model", + k8s_nim_operator_config=K8sNIMOperatorConfig( + node_selector={"zone": "a"}, + resources={"requests": {"cpu": "2"}}, + ), + override_config={ + "nodeSelector": {"zone": "override-zone"}, + "resources": {"requests": {"cpu": "16"}}, + }, + ), + weights_type=ModelWeightsType.BAKED_CONTAINER, + model_namespace=None, + model_name=None, + model_revision=None, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller=resolved.huggingface_model_puller, + runtime=Runtime.KUBERNETES, + ) + compiled = compile_model_deployment(resolved, DeploymentsPluginConfig()) + server = compiled.server_config.containers[0] + assert server.resources.requests["cpu"] == "16" + k8s = compiled.server_config.backend_config.k8s + assert k8s is not None + assert k8s.affinity is not None def test_generic_weightless_is_server_only() -> None: @@ -53,7 +250,11 @@ def test_generic_weightless_is_server_only() -> None: deployment=resolved.deployment, config=resolved.config, model_entity=None, - view=DeploymentConfigView(image_name="custom/image", image_tag="1"), + view=DeploymentConfigView( + image_name="custom/image", + image_tag="1", + additional_args=["python3", "-m", "http.server", "8000"], + ), weights_type=ModelWeightsType.BAKED_CONTAINER, model_namespace=None, model_name=None, @@ -67,13 +268,25 @@ def test_generic_weightless_is_server_only() -> None: assert compiled.puller_config is None assert compiled.puller_prerequisite is False assert compiled.server_config.containers[0].image == "custom/image:1" + assert compiled.server_config.containers[0].args == ["python3", "-m", "http.server", "8000"] + assert compiled.server_config.containers[0].command == [] + k8s = compiled.server_config.backend_config.k8s + assert k8s is None def test_lora_uses_native_sidecar_on_k8s_and_container_on_docker() -> None: config = DeploymentsPluginConfig() - with patch( - "nmp.core.models.controllers.backends.deployments_plugin.compiler.get_qualified_image", - return_value="registry/nmp-api:tag", + platform = MagicMock() + platform.base_url = "http://platform.example:8080" + with ( + patch( + "nmp.core.models.controllers.backends.deployments_plugin.compiler.get_qualified_image", + return_value="registry/nmp-api:tag", + ), + patch( + "nmp.core.models.controllers.backends.deployments_plugin.compiler.get_platform_config", + return_value=platform, + ), ): k8s = compile_model_deployment(_resolved("vllm", lora=True), config) docker = compile_model_deployment(_resolved("vllm", lora=True, runtime=Runtime.DOCKER), config) @@ -88,4 +301,6 @@ def test_lora_uses_native_sidecar_on_k8s_and_container_on_docker() -> None: env = {item.name: item.value for item in sidecar.env} assert env["NIM_PEFT_SOURCE"] == "/scratch/loras" assert env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store" + assert env["NMP_BASE_URL"] == "http://platform.example:8080" + assert env["VLLM_ENDPOINT"] == "http://127.0.0.1:8000" assert len(docker.server_config.containers) == 2 diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_nim_compiler.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_nim_compiler.py new file mode 100644 index 0000000000..28db45644a --- /dev/null +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_nim_compiler.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +import pytest +from nemo_deployments_plugin.entities import ( + Container, + DeploymentBackendConfig, + K8sDeploymentConfig, + Probe, + ResourceRequirements, + Toleration, + VolumeMount, +) +from nemo_platform.types.inference.k8s_nim_operator_config import K8sNIMOperatorConfig +from nmp.common.config import Runtime +from nmp.core.models.app import ModelWeightsType +from nmp.core.models.controllers.backends.common import DeploymentConfigView +from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginConfig +from nmp.core.models.controllers.backends.deployments_plugin.nim_compiler import ( + apply_container_resources, + apply_k8s_nim_operator_container_overrides, + apply_nim_override_config, + build_k8s_deployment_backend_config, + compile_nim_server_env, + k8s_backend_config_from_nim_operator, + pod_security_context_for_engine, + startup_probe_failure_threshold, + tool_call_plugin_init_containers, +) +from nmp.core.models.controllers.backends.deployments_plugin.resolve import ResolvedPluginDeployment + + +def _resolved() -> ResolvedPluginDeployment: + return ResolvedPluginDeployment( + deployment=SimpleNamespace(name="my-dep", workspace="default"), + config=SimpleNamespace(engine="nim"), + model_entity=None, + view=DeploymentConfigView(model_namespace="org", model_name="model"), + weights_type=ModelWeightsType.FILES_SERVICE, + model_namespace="org", + model_name="model", + model_revision=None, + files_hf_url="http://files/hf", + huggingface_model_puller="puller:latest", + runtime=Runtime.KUBERNETES, + ) + + +def test_compile_nim_server_env_sets_ft_model_for_model_specific_image() -> None: + env = compile_nim_server_env(_resolved(), DeploymentsPluginConfig(), weighted=True, tool_call_plugin_path=None) + assert env["NIM_FT_MODEL"] == "/model-store" + assert env["NIM_CUSTOM_MODEL"] == "/model-store" + + +def test_compile_nim_server_env_omits_ft_model_for_multi_llm_image() -> None: + resolved = _resolved() + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="org", + model_name="model", + image_name="nvcr.io/nim/nvidia/llm-nim", + ), + weights_type=resolved.weights_type, + model_namespace=resolved.model_namespace, + model_name=resolved.model_name, + model_revision=resolved.model_revision, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller=resolved.huggingface_model_puller, + runtime=resolved.runtime, + ) + env = compile_nim_server_env(resolved, DeploymentsPluginConfig(), weighted=True, tool_call_plugin_path=None) + assert "NIM_FT_MODEL" not in env + + +def test_tool_call_plugin_init_containers_require_puller_tag() -> None: + resolved = _resolved() + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="org", + model_name="model", + tool_call_config=SimpleNamespace( + tool_call_plugin="ws/fileset", + tool_call_parser=None, + auto_tool_choice=None, + ), + ), + weights_type=resolved.weights_type, + model_namespace=resolved.model_namespace, + model_name=resolved.model_name, + model_revision=resolved.model_revision, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller="registry:5000/puller", + runtime=resolved.runtime, + ) + assert ( + tool_call_plugin_init_containers( + resolved, DeploymentsPluginConfig(), names_volume="vol", names_scratch="scr", weighted=True + ) + is None + ) + + +def test_k8s_backend_config_maps_node_selector_to_affinity() -> None: + view = DeploymentConfigView( + k8s_nim_operator_config=K8sNIMOperatorConfig(node_selector={"zone": "us-west1-a"}), + ) + k8s = k8s_backend_config_from_nim_operator(view) + assert k8s is not None + assert k8s.affinity is not None + + +def test_startup_probe_failure_threshold_rounds_up() -> None: + view = DeploymentConfigView(k8s_nim_operator_config=K8sNIMOperatorConfig(startup_probe_grace_seconds=605)) + assert startup_probe_failure_threshold(view, period_seconds=10) == 61 + + +def test_apply_k8s_nim_operator_container_overrides_updates_resources_and_probe() -> None: + container = Container(name="server", image="nim:latest") + probe = Probe() + view = DeploymentConfigView( + k8s_nim_operator_config=K8sNIMOperatorConfig( + resources={"requests": {"cpu": "2"}}, + startup_probe_grace_seconds=30, + ) + ) + apply_k8s_nim_operator_container_overrides(container, probe, view) + assert container.resources.requests["cpu"] == "2" + assert probe.failure_threshold == 3 + + +def test_pod_security_context_uses_nim_defaults() -> None: + view = DeploymentConfigView() + config = DeploymentsPluginConfig(default_user_id=1000, default_group_id=2000) + security_context = pod_security_context_for_engine("nim", view, config) + assert security_context is not None + assert security_context.run_as_user == 1000 + assert security_context.run_as_group == 2000 + + +def test_pod_security_context_generic_uses_image_user_by_default() -> None: + view = DeploymentConfigView() + config = DeploymentsPluginConfig(default_user_id=1000, default_group_id=2000) + assert pod_security_context_for_engine("generic", view, config) is None + + +def test_pod_security_context_generic_honors_explicit_run_as() -> None: + view = DeploymentConfigView(run_as_user=0, run_as_group=0) + config = DeploymentsPluginConfig(default_user_id=1000, default_group_id=2000) + security_context = pod_security_context_for_engine("generic", view, config) + assert security_context is not None + assert security_context.run_as_user == 0 + assert security_context.run_as_group == 0 + + +def test_build_k8s_deployment_backend_config_merges_security_context() -> None: + view = DeploymentConfigView() + config = DeploymentsPluginConfig(default_user_id=1000, default_group_id=2000) + backend = build_k8s_deployment_backend_config("nim", view, config) + assert backend.k8s is not None + assert backend.k8s.security_context is not None + assert backend.k8s.security_context.run_as_user == 1000 + + +def test_build_k8s_deployment_backend_config_ignores_nim_operator_fields_for_vllm() -> None: + view = DeploymentConfigView( + k8s_nim_operator_config=K8sNIMOperatorConfig(node_selector={"zone": "us-west1-a"}), + ) + backend = build_k8s_deployment_backend_config("vllm", view, DeploymentsPluginConfig()) + assert backend.k8s is not None + assert backend.k8s.affinity is None + + +def test_apply_container_resources_deep_merges_existing_values() -> None: + container = Container( + name="server", + image="nim:1.0", + resources=ResourceRequirements(requests={"cpu": "1", "memory": "8Gi"}, limits={"memory": "16Gi"}), + ) + apply_container_resources(container, {"requests": {"cpu": "4"}, "limits": {"cpu": "2"}}) + assert container.resources.requests == {"cpu": "4", "memory": "8Gi"} + assert container.resources.limits == {"memory": "16Gi", "cpu": "2"} + + +def test_tool_call_plugin_init_containers_use_scratch_path_when_unweighted() -> None: + resolved = _resolved() + resolved = ResolvedPluginDeployment( + deployment=resolved.deployment, + config=resolved.config, + model_entity=resolved.model_entity, + view=DeploymentConfigView( + model_namespace="org", + model_name="model", + tool_call_config=SimpleNamespace( + tool_call_plugin="ws/fileset", + tool_call_parser=None, + auto_tool_choice=None, + ), + ), + weights_type=ModelWeightsType.BAKED_CONTAINER, + model_namespace=resolved.model_namespace, + model_name=resolved.model_name, + model_revision=resolved.model_revision, + files_hf_url=resolved.files_hf_url, + huggingface_model_puller="puller:latest", + runtime=resolved.runtime, + ) + inits = tool_call_plugin_init_containers( + resolved, DeploymentsPluginConfig(), names_volume="vol", names_scratch="scr", weighted=False + ) + assert inits is not None + assert inits[0].volume_mounts == [VolumeMount(name="scr", mountPath="/scratch")] + assert "/scratch/plugin/plugin.py" in inits[0].command[2] + finalize_script = inits[2].command[2] + assert 'if [ "$plugin_file" != "/scratch/plugin/plugin.py" ]; then' in finalize_script + + +def test_apply_nim_override_config_overrides_operator_defaults() -> None: + from nemo_deployments_plugin.entities import DeploymentConfig + + container = Container(name="server", image="nim:1.0") + container.resources = ResourceRequirements(requests={"cpu": "1"}) + container.readiness_probe = Probe() + server_config = DeploymentConfig( + name="dep-server", + workspace="default", + containers=[container], + backendConfig=DeploymentBackendConfig( + k8s=K8sDeploymentConfig( + tolerations=[Toleration(key="old-key")], + ) + ), + ) + view = DeploymentConfigView( + k8s_nim_operator_config=K8sNIMOperatorConfig(resources={"requests": {"cpu": "2"}}), + override_config={ + "resources": {"requests": {"cpu": "8"}, "limits": {"memory": "64Gi"}}, + "nodeSelector": {"zone": "b"}, + "env": [{"name": "EXTRA", "value": "1"}], + "image": {"repository": "nvcr.io/override", "tag": "2.0"}, + "readinessProbe": { + "enabled": True, + "probe": {"httpGet": {"path": "/ready", "port": 8080}, "failureThreshold": 12}, + }, + "userID": 42, + "groupID": 84, + "labels": {"custom": "label"}, + }, + ) + apply_nim_override_config( + container, + server_config, + view, + engine="nim", + runtime=Runtime.KUBERNETES, + ) + assert container.image == "nvcr.io/override:2.0" + assert container.resources.requests["cpu"] == "8" + assert container.resources.limits["memory"] == "64Gi" + assert {item.name: item.value for item in container.env}["EXTRA"] == "1" + assert container.readiness_probe is not None + assert container.readiness_probe.http_get is not None + assert container.readiness_probe.http_get.path == "/ready" + assert container.readiness_probe.http_get.port == 8080 + assert container.readiness_probe.failure_threshold == 12 + k8s = server_config.backend_config.k8s + assert k8s is not None + assert k8s.affinity is not None + assert k8s.security_context is not None + assert k8s.security_context.run_as_user == 42 + assert server_config.labels["custom"] == "label" + + +def test_apply_nim_override_config_ignored_on_docker() -> None: + from nemo_deployments_plugin.entities import DeploymentConfig + + container = Container(name="server", image="nim:1.0") + server_config = DeploymentConfig(name="dep-server", workspace="default", containers=[container]) + view = DeploymentConfigView(override_config={"image": {"repository": "override", "tag": "x"}}) + apply_nim_override_config( + container, + server_config, + view, + engine="nim", + runtime=Runtime.DOCKER, + ) + assert container.image == "nim:1.0" + + +def test_apply_nim_override_config_rejects_unsupported_keys() -> None: + from nemo_deployments_plugin.entities import DeploymentConfig + + container = Container(name="server", image="nim:1.0") + server_config = DeploymentConfig(name="dep-server", workspace="default", containers=[container]) + view = DeploymentConfigView(override_config={"authSecret": "secret", "replicas": 2}) + with pytest.raises(ValueError, match="unsupported keys: authSecret, replicas"): + apply_nim_override_config( + container, + server_config, + view, + engine="nim", + runtime=Runtime.KUBERNETES, + ) diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_status.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_status.py index 5429c5db4b..375385f271 100644 --- a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_status.py +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_status.py @@ -4,9 +4,12 @@ import pytest from nemo_deployments_plugin.entities import Deployment, Volume from nemo_deployments_plugin.types import Endpoint +from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate from nmp.core.models.controllers.backends.deployments_plugin.status import ( aggregate_status, + apply_deleting_timeout, apply_pending_timeout, + build_deleting_timeout_error, build_pending_timeout_error, map_status, project_host_url, @@ -105,3 +108,37 @@ def test_build_pending_timeout_error_includes_substrate() -> None: assert result.status == "ERROR" assert result.error_details is not None assert result.error_details["substrate"] == substrate + + +def test_apply_deleting_timeout_escalates_deleting_only() -> None: + deleting = DeploymentStatusUpdate(status="DELETING", status_message="waiting") + assert ( + apply_deleting_timeout( + deleting, + elapsed_seconds=30, + timeout_seconds=60, + deployment_name="my-dep", + ).status + == "DELETING" + ) + timed_out = apply_deleting_timeout( + deleting, + elapsed_seconds=90, + timeout_seconds=60, + deployment_name="my-dep", + ) + assert timed_out.status == "ERROR" + assert timed_out.error_details is not None + assert timed_out.error_details["reason"] == "deleting_timeout" + + +def test_build_deleting_timeout_error_message() -> None: + result = build_deleting_timeout_error( + deployment_name="my-dep", + elapsed_seconds=120, + timeout_seconds=60, + ) + assert result.status == "ERROR" + assert "my-dep" in result.status_message + assert result.error_details is not None + assert result.error_details["reason"] == "deleting_timeout" diff --git a/services/core/models/tests/unit/controllers/backends/test_vllm_k8s_compiler.py b/services/core/models/tests/unit/controllers/backends/test_vllm_k8s_compiler.py deleted file mode 100644 index 88dc6ed14a..0000000000 --- a/services/core/models/tests/unit/controllers/backends/test_vllm_k8s_compiler.py +++ /dev/null @@ -1,387 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for the engine-agnostic k8s object compiler (vLLM raw-object path).""" - -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.controllers.backends.k8s_nim_operator import vllm_k8s_compiler as c - -# --------------------------------------------------------------------------- -# Naming + labels -# --------------------------------------------------------------------------- - - -def test_resource_name_suffixes(): - assert c.pvc_name("md-default-qwen") == "md-default-qwen-pvc" - assert c.pull_job_name("md-default-qwen") == "md-default-qwen-pull" - - -def test_common_labels(): - labels = c.common_labels("default", "qwen", "vllm") - assert labels[MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert labels[c.DEPLOYMENT_WORKSPACE_LABEL] == "default" - assert labels[c.DEPLOYMENT_NAME_LABEL] == "qwen" - assert labels["nmp.nvidia.com/engine"] == "vllm" - - -# --------------------------------------------------------------------------- -# PVC -# --------------------------------------------------------------------------- - - -def test_compile_pvc_basic(): - pvc = c.compile_pvc( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - disk_size="50Gi", - namespace="nemo", - ) - assert pvc.metadata.name == "md-default-qwen-pvc" - assert pvc.metadata.namespace == "nemo" - assert pvc.spec.access_modes == ["ReadWriteOnce"] - assert pvc.spec.resources.requests["storage"] == "50Gi" - assert pvc.spec.storage_class_name is None - - -def test_model_labels_apply_to_raw_k8s_object_metadata(): - labels = {"example.com/test-label": "A", "example.com/test-suite": "B"} - - pvc = c.compile_pvc( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - disk_size="50Gi", - extra_labels=labels, - ) - job = c.compile_puller_job( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - image="hf-cli:25.10", - container_args=["download", "default/qwen", "--local-dir", "/model-store"], - extra_labels=labels, - ) - deployment = c.compile_deployment( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - image="vllm:v1", - args=["default/qwen"], - health_path="/v1/health/ready", - extra_labels=labels, - ) - service = c.compile_service( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - extra_labels=labels, - ) - assert pvc.metadata is not None and pvc.metadata.labels is not None - assert job.metadata is not None and job.metadata.labels is not None - assert deployment.metadata is not None and deployment.metadata.labels is not None - assert ( - deployment.spec is not None - and deployment.spec.template is not None - and deployment.spec.template.metadata is not None - and deployment.spec.template.metadata.labels is not None - ) - assert service.metadata is not None and service.metadata.labels is not None - assert pvc.metadata.labels["example.com/test-label"] == "A" - assert job.metadata.labels["example.com/test-suite"] == "B" - assert deployment.metadata.labels["example.com/test-label"] == "A" - assert deployment.spec.template.metadata.labels["example.com/test-suite"] == "B" - assert service.metadata.labels["example.com/test-label"] == "A" - - -def test_compile_pvc_storage_class_and_model_source(): - pvc = c.compile_pvc( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - disk_size="100Gi", - storage_class="fast-ssd", - model_source="default/qwen@main", - ) - assert pvc.spec.storage_class_name == "fast-ssd" - # model source is an annotation (its value contains '/' and '@', invalid for labels). - assert pvc.metadata.annotations[c.MODEL_SOURCE_ANNOTATION] == "default/qwen@main" - - -def test_compile_pvc_custom_access_modes(): - pvc = c.compile_pvc( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - disk_size="10Gi", - access_modes=["ReadWriteMany"], - ) - assert pvc.spec.access_modes == ["ReadWriteMany"] - - -# --------------------------------------------------------------------------- -# Puller Job -# --------------------------------------------------------------------------- - - -def test_compile_puller_job_basic(): - job = c.compile_puller_job( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - image="hf-cli:25.10", - container_args=["download", "default/qwen", "--local-dir", "/model-store"], - env={"HF_ENDPOINT": "http://files/apis/files/v2/hf", "HF_TOKEN": "service:models"}, - gpu=2, - namespace="nemo", - service_account_name="nemo-models-sa", - image_pull_secret="nvcrimagepullsecret", - model_source="default/qwen@main", - ) - assert job.metadata.name == "md-default-qwen-pull" - assert job.spec.backoff_limit == c.DEFAULT_BACKOFF_LIMIT - assert job.spec.ttl_seconds_after_finished == c.DEFAULT_TTL_SECONDS_AFTER_FINISHED - - pod = job.spec.template.spec - assert pod.restart_policy == "Never" - assert pod.service_account_name == "nemo-models-sa" - assert pod.image_pull_secrets[0].name == "nvcrimagepullsecret" - - ctr = pod.containers[0] - assert ctr.args == ["download", "default/qwen", "--local-dir", "/model-store"] - # Entrypoint overridden to the HF CLI (nmp-api's image entrypoint is `nemo - # services run`); args run as `hf download ...`. - assert ctr.command == ["hf"] - env = {e.name: e.value for e in ctr.env} - assert env["HF_ENDPOINT"] == "http://files/apis/files/v2/hf" - assert env["HF_TOKEN"] == "service:models" - # GPU request pins the puller into GPU topology for PVC binding. - assert ctr.resources.requests["nvidia.com/gpu"] == "2" - assert ctr.resources.limits["nvidia.com/gpu"] == "2" - assert ctr.volume_mounts[0].mount_path == "/model-store" - # Job annotation carries the model source for the re-pull policy. - assert job.metadata.annotations[c.MODEL_SOURCE_ANNOTATION] == "default/qwen@main" - - -def test_compile_puller_job_cpu_only_no_gpu_request(): - job = c.compile_puller_job( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="hf-cli", - container_args=["download", "w/n", "--local-dir", "/model-store"], - gpu=0, - ) - assert job.spec.template.spec.containers[0].resources is None - - -def test_compile_puller_job_no_image_pull_secret(): - job = c.compile_puller_job( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="hf-cli", - container_args=["download"], - ) - assert job.spec.template.spec.image_pull_secrets is None - - -# --------------------------------------------------------------------------- -# Deployment -# --------------------------------------------------------------------------- - - -def test_compile_deployment_basic(): - dep = c.compile_deployment( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - image="vllm/vllm-openai:v0.22.1", - args=["/model-store", "--served-model-name", "default/qwen"], - health_path="/health", - gpu=2, - namespace="nemo", - service_account_name="nemo-models-sa", - ) - assert dep.metadata.name == "md-default-qwen" - assert dep.spec.replicas == 1 - assert dep.spec.selector.match_labels == {"app": "md-default-qwen"} - - pod = dep.spec.template.spec - assert pod.service_account_name == "nemo-models-sa" - ctr = pod.containers[0] - # command unset -> image entrypoint (vllm serve) runs; args appended. - assert ctr.command is None - assert ctr.args == ["/model-store", "--served-model-name", "default/qwen"] - assert ctr.ports[0].container_port == 8000 - assert ctr.resources.limits["nvidia.com/gpu"] == "2" - assert ctr.startup_probe.http_get.path == "/health" - assert ctr.readiness_probe.http_get.path == "/health" - - # PVC mounted read-only at /model-store; scratch + dshm present. - mounts = {m.name: m for m in ctr.volume_mounts} - assert mounts["model-store"].mount_path == "/model-store" - assert mounts["model-store"].read_only is True - assert mounts["scratch"].mount_path == "/scratch" - assert mounts["dshm"].mount_path == "/dev/shm" - - vols = {v.name: v for v in pod.volumes} - assert vols["model-store"].persistent_volume_claim.claim_name == "md-default-qwen-pvc" - assert vols["dshm"].empty_dir.medium == "Memory" - - -def test_compile_deployment_no_model_store_for_generic(): - """mount_model_store=False omits the model-store PVC volume + mount (generic engine).""" - dep = c.compile_deployment( - resource_name="md-default-jb", - workspace="default", - name="jb", - engine="generic", - image="nvcr.io/nim/nvidia/nemoguard-jailbreak-detect:1.10.1", - args=["--port", "8000"], - health_path="/v1/health/ready", - gpu=0, - mount_model_store=False, - ) - pod = dep.spec.template.spec - ctr = pod.containers[0] - - mount_names = {m.name for m in ctr.volume_mounts} - assert "model-store" not in mount_names - # scratch + dshm are still present (harmless emptyDirs). - assert "scratch" in mount_names - assert "dshm" in mount_names - - vol_names = {v.name for v in pod.volumes} - assert "model-store" not in vol_names - assert "scratch" in vol_names - assert "dshm" in vol_names - - -def test_compile_deployment_cpu_only_no_gpu(): - dep = c.compile_deployment( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="img", - args=["/model-store"], - health_path="/health", - gpu=0, - ) - assert dep.spec.template.spec.containers[0].resources is None - - -def test_compile_deployment_startup_grace_to_failure_threshold(): - dep = c.compile_deployment( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="img", - args=[], - health_path="/health", - startup_grace_seconds=600, - ) - # ceil(600 / 10) == 60 - assert dep.spec.template.spec.containers[0].startup_probe.failure_threshold == 60 - - -def test_compile_deployment_shared_memory_size_limit(): - dep = c.compile_deployment( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="img", - args=[], - health_path="/health", - shared_memory_size_limit="8Gi", - ) - vols = {v.name: v for v in dep.spec.template.spec.volumes} - assert vols["dshm"].empty_dir.size_limit == "8Gi" - - -def test_compile_deployment_security_context_set_when_uid_gid_given(): - """When uid/gid are provided, the server pod gets that securityContext.""" - dep = c.compile_deployment( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="img", - args=[], - health_path="/health", - user_id=2000, - group_id=0, - ) - sc = dep.spec.template.spec.security_context - assert sc.run_as_user == 2000 - assert sc.run_as_group == 0 - assert sc.fs_group == 0 - - -def test_compile_deployment_no_security_context_when_uid_gid_unset(): - """No uid/gid -> no forced securityContext (runs as the image's default user).""" - dep = c.compile_deployment( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="img", - args=[], - health_path="/health", - ) - assert dep.spec.template.spec.security_context is None - - -def test_compile_deployment_sidecars_and_init_containers(): - from kubernetes import client as k8s_client - - sidecar = k8s_client.V1Container(name="lora-sidecar", image="nmp-api") - init = k8s_client.V1Container(name="lora-cache-init", image="busybox") - dep = c.compile_deployment( - resource_name="r", - workspace="w", - name="n", - engine="vllm", - image="img", - args=[], - health_path="/health", - init_containers=[init], - sidecar_containers=[sidecar], - ) - pod = dep.spec.template.spec - assert pod.init_containers[0].name == "lora-cache-init" - assert [ctr.name for ctr in pod.containers] == ["r-ctr", "lora-sidecar"] - - -# --------------------------------------------------------------------------- -# Service -# --------------------------------------------------------------------------- - - -def test_compile_service_basic(): - svc = c.compile_service( - resource_name="md-default-qwen", - workspace="default", - name="qwen", - engine="vllm", - namespace="nemo", - ) - assert svc.spec.type == "ClusterIP" - assert svc.spec.selector == {"app": "md-default-qwen"} - assert svc.spec.ports[0].port == 8000 - assert svc.spec.ports[0].target_port == c.SERVER_PORT_NAME - assert svc.metadata.labels[MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER diff --git a/services/core/models/tests/unit/controllers/test_backend_config_fields.py b/services/core/models/tests/unit/controllers/test_backend_config_fields.py deleted file mode 100644 index 96ffbcc6ae..0000000000 --- a/services/core/models/tests/unit/controllers/test_backend_config_fields.py +++ /dev/null @@ -1,679 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for backend config field usage in NIMService compilation.""" - -from unittest.mock import MagicMock, patch - -import pytest -from nmp.common.config import ImagePullSecret, PlatformConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler import compile_nimservice - - -@pytest.fixture -def sample_deployment(): - """Create a sample ModelDeployment for testing.""" - deployment = MagicMock() - deployment.workspace = "test-ns" - deployment.name = "test-deployment" - deployment.entity_version = "v1" - return deployment - - -@pytest.fixture -def minimal_nim_config(): - """Create minimal NIM deployment config.""" - config = MagicMock() - config.workspace = "test-ns" - config.name = "test-config" - config.entity_version = "v1" - config.engine = "nim" - config.model_spec = MagicMock() - config.model_spec.lora_enabled = False - config.model_spec.model_name = None - config.model_spec.model_namespace = None - config.model_spec.model_revision = None - config.model_spec.tool_call_config = None - config.executor_config = MagicMock() - config.executor_config.image_name = "nvcr.io/nim/test" - config.executor_config.image_tag = "1.0.0" - config.executor_config.gpu = 1 - config.executor_config.disk_size = None # Will use backend default - config.executor_config.additional_envs = {} - config.executor_config.k8s_nim_operator_config = None - config.executor_config.override_config = {} - return config - - -def test_default_storage_class_is_used(sample_deployment, minimal_nim_config): - """Test that default_storage_class from backend config is used in PVC.""" - backend_config = K8sNimOperatorConfig(default_storage_class="fast-ssd") - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.storage.pvc.storageClass == "fast-ssd" - - -def test_default_pvc_size_is_used_when_disk_size_not_specified(sample_deployment, minimal_nim_config): - """Test that default_pvc_size is used when deployment config doesn't specify disk_size.""" - backend_config = K8sNimOperatorConfig(default_pvc_size="500Gi") - minimal_nim_config.executor_config.disk_size = None - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.storage.pvc.size == "500Gi" - - -def test_deployment_disk_size_overrides_default_pvc_size(sample_deployment, minimal_nim_config): - """Test that deployment config disk_size takes precedence over default_pvc_size.""" - backend_config = K8sNimOperatorConfig(default_pvc_size="500Gi") - minimal_nim_config.executor_config.disk_size = "100Gi" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.storage.pvc.size == "100Gi" - - -def test_peft_source_only_used_when_lora_enabled(sample_deployment, minimal_nim_config): - """Test that peft_source is only used when lora_enabled is true.""" - backend_config = K8sNimOperatorConfig(peft_source="http://custom-peft-source:8000") - - # Test with lora disabled - minimal_nim_config.model_spec.lora_enabled = False - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert "NIM_PEFT_SOURCE" not in env_vars - - # Test with lora enabled - minimal_nim_config.model_spec.lora_enabled = True - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert env_vars["NIM_PEFT_SOURCE"] == "/scratch/loras" - - -def test_peft_refresh_interval_only_used_when_lora_enabled(sample_deployment, minimal_nim_config): - """Test that peft_refresh_interval is only used when lora_enabled is true.""" - backend_config = K8sNimOperatorConfig(peft_refresh_interval=60) - - # Test with lora disabled - minimal_nim_config.model_spec.lora_enabled = False - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert "NIM_PEFT_REFRESH_INTERVAL" not in env_vars - - # Test with lora enabled - minimal_nim_config.model_spec.lora_enabled = True - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert env_vars["NIM_PEFT_REFRESH_INTERVAL"] == "60" - - -def test_image_pull_secrets_included_when_set(sample_deployment, minimal_nim_config): - """Test that image_pull_secrets are included when set in platform config.""" - backend_config = K8sNimOperatorConfig() - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - models_url="http://models-service:8000", - image_pull_secrets=[ - ImagePullSecret(name="secret1"), - ImagePullSecret(name="secret2"), - ImagePullSecret(name="secret3"), - ], - ) - - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.image.pullSecrets == ["secret1", "secret2", "secret3"] - - -def test_image_pull_secrets_not_included_when_none(sample_deployment, minimal_nim_config): - """Test that image_pull_secrets are not included when not set in platform config.""" - backend_config = K8sNimOperatorConfig() - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - models_url="http://models-service:8000", - image_pull_secrets=[], - ) - - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.image.pullSecrets is None - - -def test_image_pull_secrets_from_platform_config(sample_deployment, minimal_nim_config): - """Test that image_pull_secrets are correctly read from platform config.""" - backend_config = K8sNimOperatorConfig() - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - models_url="http://models-service:8000", - image_pull_secrets=[ - ImagePullSecret(name="secret1"), - ImagePullSecret(name="secret2"), - ImagePullSecret(name="secret3"), - ], - ) - - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.image.pullSecrets == ["secret1", "secret2", "secret3"] - - -def test_default_user_id_included_when_set(sample_deployment, minimal_nim_config): - """Test that default_user_id is included when set.""" - backend_config = K8sNimOperatorConfig(default_user_id=1000) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.userID == 1000 - - -def test_default_user_id_not_included_when_none(sample_deployment, minimal_nim_config): - """Test that default_user_id is not included when not set.""" - backend_config = K8sNimOperatorConfig(default_user_id=None) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.userID is None - - -def test_default_group_id_included_when_set(sample_deployment, minimal_nim_config): - """Test that default_group_id is included when set.""" - backend_config = K8sNimOperatorConfig(default_group_id=1000) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.groupID == 1000 - - -def test_default_group_id_not_included_when_none(sample_deployment, minimal_nim_config): - """Test that default_group_id is not included when not set.""" - backend_config = K8sNimOperatorConfig(default_group_id=None) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.groupID is None - - -def test_both_user_and_group_id_can_be_set(sample_deployment, minimal_nim_config): - """Test that both user_id and group_id can be set together.""" - backend_config = K8sNimOperatorConfig(default_user_id=1000, default_group_id=2000) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.userID == 1000 - assert nimservice.spec.groupID == 2000 - - -def test_auth_secret_is_always_included(sample_deployment, minimal_nim_config): - """Test that authSecret is set from auth_secret when no NIMCache, and from files_auth_secret when NIMCache.""" - backend_config = K8sNimOperatorConfig(auth_secret="my-custom-ngc-secret") - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - assert nimservice.spec.authSecret == "my-custom-ngc-secret" - - backend_config_files = K8sNimOperatorConfig(files_auth_secret="my-custom-files-token") - nimservice_with_cache = compile_nimservice( - backend_config=backend_config_files, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - nimcache_name="test-resource", - ) - assert nimservice_with_cache.spec.authSecret == "my-custom-files-token" - - -def test_auth_secret_default_value(sample_deployment, minimal_nim_config): - """Test that authSecret uses auth_secret default (ngc-api) when no NIMCache.""" - backend_config = K8sNimOperatorConfig() # Use defaults - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.authSecret == "ngc-api" - - -def test_all_backend_config_fields_together(sample_deployment, minimal_nim_config): - """Test that all backend config fields work together correctly.""" - backend_config = K8sNimOperatorConfig( - default_storage_class="premium-ssd", - default_pvc_size="1Ti", - peft_source="http://custom-peft:8000", - peft_refresh_interval=45, - default_user_id=1001, - default_group_id=2001, - auth_secret="custom-ngc-api", - ) - - # Enable lora to test peft fields - minimal_nim_config.model_spec.lora_enabled = True - minimal_nim_config.executor_config.disk_size = None # Use backend default - - platform_config = PlatformConfig( # type: ignore[abstract] - base_url="http://platform-service:8080", - image_pull_secrets=[ - ImagePullSecret(name="secret1"), - ImagePullSecret(name="secret2"), - ], - ) - - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - # Verify storage - assert nimservice.spec.storage.pvc.storageClass == "premium-ssd" - assert nimservice.spec.storage.pvc.size == "1Ti" - - # Verify peft env vars - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert env_vars["NIM_PEFT_SOURCE"] == "/scratch/loras" - assert env_vars["NIM_PEFT_REFRESH_INTERVAL"] == "45" - - # Verify image pull secrets (from platform config) - assert nimservice.spec.image.pullSecrets == ["secret1", "secret2"] - - # Verify security context - assert nimservice.spec.userID == 1001 - assert nimservice.spec.groupID == 2001 - - # Verify auth (NGC secret when no NIMCache) - assert nimservice.spec.authSecret == "custom-ngc-api" - - assert nimservice.spec.sidecarContainers is not None - assert len(nimservice.spec.sidecarContainers) == 1 - assert nimservice.spec.sidecarContainers[0].name == "test-resource-lora-sidecar" - assert len(nimservice.spec.sidecarContainers[0].env) >= 1 - - keys = set([e.name for e in nimservice.spec.sidecarContainers[0].env]) - assert "NMP_FILES_URL" in keys - assert "NMP_MODELS_URL" in keys - - -def test_default_resources_applied(sample_deployment, minimal_nim_config): - """Test that default_resources from backend config is applied to NIMService.""" - backend_config = K8sNimOperatorConfig( - default_resources={ - "requests": {"cpu": "2", "memory": "8Gi"}, - "limits": {"memory": "16Gi"}, - } - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - # Should have default resources merged with GPU limits (from gpu=1) - assert nimservice.spec.resources.requests["cpu"].root == "2" - assert nimservice.spec.resources.requests["memory"].root == "8Gi" - assert nimservice.spec.resources.limits["memory"].root == "16Gi" - # GPU should still be there (deployment gpu count wins over default_resources) - assert "nvidia.com/gpu" in nimservice.spec.resources.limits - assert nimservice.spec.resources.limits["nvidia.com/gpu"].root == "1" - - -def test_default_tolerations_applied(sample_deployment, minimal_nim_config): - """Test that default_tolerations from backend config is applied to NIMService.""" - backend_config = K8sNimOperatorConfig( - default_tolerations=[ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, - ] - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.tolerations is not None - assert len(nimservice.spec.tolerations) == 1 - assert nimservice.spec.tolerations[0].key == "nvidia.com/gpu" - assert nimservice.spec.tolerations[0].operator == "Exists" - assert nimservice.spec.tolerations[0].effect == "NoSchedule" - - -def test_default_node_selector_applied(sample_deployment, minimal_nim_config): - """Test that default_node_selector from backend config is applied to NIMService.""" - backend_config = K8sNimOperatorConfig(default_node_selector={"node-type": "gpu-node", "zone": "us-west1-a"}) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.spec.nodeSelector is not None - assert nimservice.spec.nodeSelector["node-type"] == "gpu-node" - assert nimservice.spec.nodeSelector["zone"] == "us-west1-a" - - -def test_default_startup_probe_grace_period_seconds_applied(sample_deployment, minimal_nim_config): - """Test that default_startup_probe_grace_period_seconds from backend config is applied.""" - backend_config = K8sNimOperatorConfig(default_startup_probe_grace_period_seconds=1200) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - # 1200 seconds / 10 = 120 failureThreshold - assert nimservice.spec.startupProbe.probe.failureThreshold == 120 - assert nimservice.spec.startupProbe.probe.periodSeconds == 10 - - -def test_per_deployment_config_overrides_backend_defaults(sample_deployment, minimal_nim_config): - """Test that per-deployment k8s_nim_operator_config overrides backend defaults.""" - backend_config = K8sNimOperatorConfig( - default_resources={"requests": {"cpu": "2", "memory": "8Gi"}}, - default_node_selector={"zone": "us-west1-a"}, - default_startup_probe_grace_period_seconds=600, - ) - - # Per-deployment config overrides - minimal_nim_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_nim_config.executor_config.k8s_nim_operator_config.startup_probe_grace_seconds = 1200 - minimal_nim_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "resources": {"requests": {"cpu": "4"}}, # Override CPU - "node_selector": {"zone": "us-east1-b"}, # Override zone - "startup_probe_grace_seconds": 1200, # Override grace period - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - # Per-deployment overrides should win - assert nimservice.spec.resources.requests["cpu"].root == "4" - assert nimservice.spec.nodeSelector["zone"] == "us-east1-b" - # 1200 seconds / 10 = 120 - assert nimservice.spec.startupProbe.probe.failureThreshold == 120 - - -def test_backend_defaults_not_applied_when_none(sample_deployment, minimal_nim_config): - """Test that backend defaults are not applied when set to None.""" - backend_config = K8sNimOperatorConfig( - default_resources=None, - default_tolerations=None, - default_node_selector=None, - default_startup_probe_grace_period_seconds=None, - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - # Should use defaults from compiler, not backend config - # Default startup probe is 600 seconds = 60 failureThreshold - assert nimservice.spec.startupProbe.probe.failureThreshold == 60 - - -def test_default_labels_applied_to_nimservice_metadata_and_spec(sample_deployment, minimal_nim_config): - """Test that default_labels from backend config are applied to NIMService CR metadata and spec (pods).""" - backend_config = K8sNimOperatorConfig( - default_labels={"team": "ml-platform", "environment": "prod"}, - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - # CR metadata labels: defaults + controller labels (controller wins on conflict) - meta_labels = nimservice.metadata["labels"] - assert meta_labels["team"] == "ml-platform" - assert meta_labels["environment"] == "prod" - assert meta_labels["app.kubernetes.io/name"] == "test-resource" - - # Spec labels (pod): same merge - spec_labels = nimservice.spec.labels - assert spec_labels["team"] == "ml-platform" - assert spec_labels["environment"] == "prod" - assert spec_labels["nmp.nvidia.com/deployment-workspace"] == sample_deployment.workspace - - -def test_model_labels_applied_to_nimservice_metadata_and_spec(sample_deployment, minimal_nim_config): - """Test that model_labels from backend config are applied to NIMService CR metadata and spec (pods).""" - backend_config = K8sNimOperatorConfig( - model_labels={"nmp.nvidia.com/test-run": "run-1", "nmp.nvidia.com/test-worker": "gw0"}, - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - meta_labels = nimservice.metadata["labels"] - assert meta_labels["nmp.nvidia.com/test-run"] == "run-1" - assert meta_labels["nmp.nvidia.com/test-worker"] == "gw0" - assert meta_labels["app.kubernetes.io/name"] == "test-resource" - - spec_labels = nimservice.spec.labels - assert spec_labels["nmp.nvidia.com/test-run"] == "run-1" - assert spec_labels["nmp.nvidia.com/test-worker"] == "gw0" - assert spec_labels["nmp.nvidia.com/deployment-workspace"] == sample_deployment.workspace - - -def test_default_annotations_applied_to_nimservice_metadata_and_spec(sample_deployment, minimal_nim_config): - """Test that default_annotations from backend config are applied to NIMService CR metadata and spec (pods).""" - backend_config = K8sNimOperatorConfig( - default_annotations={"prometheus.io/scrape": "true", "custom/key": "value"}, - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_nim_config, - k8s_namespace="default", - resource_name="test-resource", - ) - - assert nimservice.metadata["annotations"] == {"prometheus.io/scrape": "true", "custom/key": "value"} - assert nimservice.spec.annotations == {"prometheus.io/scrape": "true", "custom/key": "value"} - - -# --------------------------------------------------------------------------- -# vLLM-on-k8s config fields (raw-object emission path) -# --------------------------------------------------------------------------- - - -def test_default_vllm_image_default_value(): - """default_vllm_image / _tag fall back to the upstream vLLM image.""" - backend_config = K8sNimOperatorConfig() - assert backend_config.default_vllm_image == "vllm/vllm-openai" - assert backend_config.default_vllm_image_tag == "v0.22.1" - - -def test_default_vllm_image_override(): - """default_vllm_image / _tag can be repointed at a mirror.""" - backend_config = K8sNimOperatorConfig( - default_vllm_image="my-registry/vllm-openai", - default_vllm_image_tag="v0.99.0", - ) - assert backend_config.default_vllm_image == "my-registry/vllm-openai" - assert backend_config.default_vllm_image_tag == "v0.99.0" - - -def test_service_account_name_defaults_to_none(): - """service_account_name defaults to None (namespace default ServiceAccount).""" - assert K8sNimOperatorConfig().service_account_name is None - - -def test_service_account_name_override(): - """service_account_name can be set to a shared models ServiceAccount.""" - backend_config = K8sNimOperatorConfig(service_account_name="nemo-models-sa") - assert backend_config.service_account_name == "nemo-models-sa" - - -def test_default_shared_memory_size_limit_defaults_to_none(): - """default_shared_memory_size_limit defaults to None (node default /dev/shm).""" - assert K8sNimOperatorConfig().default_shared_memory_size_limit is None - - -def test_default_shared_memory_size_limit_override(): - """default_shared_memory_size_limit can be set for vLLM tensor-parallel NCCL.""" - backend_config = K8sNimOperatorConfig(default_shared_memory_size_limit="8Gi") - assert backend_config.default_shared_memory_size_limit == "8Gi" - - -def test_default_vllm_uid_gid_match_image_user(): - """vLLM uid/gid default to the upstream image's 'vllm' user (2000) / root group (0).""" - backend_config = K8sNimOperatorConfig() - assert backend_config.default_vllm_user_id == 2000 - assert backend_config.default_vllm_group_id == 0 - - -def test_default_vllm_uid_gid_override(): - backend_config = K8sNimOperatorConfig(default_vllm_user_id=1234, default_vllm_group_id=5678) - assert backend_config.default_vllm_user_id == 1234 - assert backend_config.default_vllm_group_id == 5678 diff --git a/services/core/models/tests/unit/controllers/test_backend_registry.py b/services/core/models/tests/unit/controllers/test_backend_registry.py index 5a1e6ecc66..70c33a23f2 100644 --- a/services/core/models/tests/unit/controllers/test_backend_registry.py +++ b/services/core/models/tests/unit/controllers/test_backend_registry.py @@ -6,94 +6,63 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -from nmp.core.models.controllers.backends.registry import ( - BackendRegistry, - DockerBackendConfigModel, - K8sNimOperatorBackendConfigModel, -) - - -@pytest.fixture(autouse=True) -def mock_docker_client(): - """Mock Docker client to avoid needing actual Docker daemon.""" - with patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock: - client = MagicMock() - mock.return_value = client - - # Setup default behaviors - client.login = MagicMock() - client.api = MagicMock() - client.api.timeout = 600 - client.containers = MagicMock() - client.images = MagicMock() - client.volumes = MagicMock() - - yield client - - -@pytest.fixture(autouse=True) -def mock_k8s_config(): - """Mock kubernetes config loading to avoid needing actual k8s config.""" - with ( - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_config.load_incluster_config"), - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_config.load_kube_config"), - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_client.ApiClient"), - patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.DynamicClient"), - ): - yield +from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginBackendConfigModel +from nmp.core.models.controllers.backends.registry import BackendRegistry @pytest.fixture def mock_nmp_sdk(): """Create a mock AsyncNeMoPlatform SDK.""" - mock = AsyncMock() - return mock + return AsyncMock() @pytest.fixture def sample_backend_configs(): """Create sample backend configurations.""" return { - "docker": DockerBackendConfigModel(enabled=True), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True), } def test_backend_registry_from_config(mock_nmp_sdk, sample_backend_configs): """Test creating BackendRegistry from configuration.""" - registry = BackendRegistry.from_config( - nmp_sdk=mock_nmp_sdk, - backend_configs=sample_backend_configs, - huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", - ) + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.backend.NemoEntitiesClient", + ): + registry = BackendRegistry.from_config( + nmp_sdk=mock_nmp_sdk, + backend_configs=sample_backend_configs, + huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", + ) - # Verify registry was created assert isinstance(registry, BackendRegistry) - - # Verify backend can be retrieved - docker_backend = registry.get_backend("docker") - assert docker_backend is not None + assert registry.get_backend("deployments_plugin") is not None def test_backend_registry_get_default_backend(mock_nmp_sdk, sample_backend_configs): """Test getting default backend (the single enabled one).""" - registry = BackendRegistry.from_config( - nmp_sdk=mock_nmp_sdk, - backend_configs=sample_backend_configs, - huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", - ) + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.backend.NemoEntitiesClient", + ): + registry = BackendRegistry.from_config( + nmp_sdk=mock_nmp_sdk, + backend_configs=sample_backend_configs, + huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", + ) - # Get default backend (should be the enabled one - docker) - default_backend = registry.get_backend() - assert default_backend is not None + assert registry.get_backend() is not None def test_backend_registry_get_backend_not_found(mock_nmp_sdk, sample_backend_configs): """Test that KeyError is raised for unknown backend.""" - registry = BackendRegistry.from_config( - nmp_sdk=mock_nmp_sdk, - backend_configs=sample_backend_configs, - huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", - ) + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.backend.NemoEntitiesClient", + ): + registry = BackendRegistry.from_config( + nmp_sdk=mock_nmp_sdk, + backend_configs=sample_backend_configs, + huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", + ) with pytest.raises(KeyError, match="Backend 'unknown' not found"): registry.get_backend("unknown") @@ -101,15 +70,17 @@ def test_backend_registry_get_backend_not_found(mock_nmp_sdk, sample_backend_con def test_backend_registry_list_backends(mock_nmp_sdk, sample_backend_configs): """Test listing all registered backends.""" - registry = BackendRegistry.from_config( - nmp_sdk=mock_nmp_sdk, - backend_configs=sample_backend_configs, - huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", - ) + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.backend.NemoEntitiesClient", + ): + registry = BackendRegistry.from_config( + nmp_sdk=mock_nmp_sdk, + backend_configs=sample_backend_configs, + huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", + ) backends = registry.list_backends() - assert "docker" in backends - assert len(backends) == 1 + assert backends == ["deployments_plugin"] def test_backend_registry_empty_config_raises_error(mock_nmp_sdk): @@ -130,9 +101,9 @@ def test_backend_registry_init_with_empty_dict_raises_error(): def test_backend_registry_unknown_backend_type(mock_nmp_sdk): """Test that unknown backend type raises KeyError when backend class not in registry.""" - bad_config = {"docker": DockerBackendConfigModel(enabled=True)} + bad_config = {"legacy-docker": DeploymentsPluginBackendConfigModel(enabled=True)} - with pytest.raises(KeyError, match="Unknown backend 'docker'"): + with pytest.raises(KeyError, match="Unknown backend 'legacy-docker'"): BackendRegistry.from_config( nmp_sdk=mock_nmp_sdk, backend_configs=bad_config, @@ -141,86 +112,36 @@ def test_backend_registry_unknown_backend_type(mock_nmp_sdk): ) -def test_backend_registry_single_backend(mock_nmp_sdk): - """Test registry with only one backend.""" - single_config = {"docker": DockerBackendConfigModel(enabled=True)} - - registry = BackendRegistry.from_config( - nmp_sdk=mock_nmp_sdk, - backend_configs=single_config, - huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", - ) - - # Verify only one backend - backends = registry.list_backends() - assert len(backends) == 1 - assert backends[0] == "docker" - - # Verify default backend is set - default_backend = registry.get_backend() - assert default_backend is not None - - -def test_backend_config_discriminated_union_parsing(): - """Test that backend configs can be parsed from dict (simulating YAML loading).""" - from nmp.core.models.controllers.backends.registry import BackendConfig - from pydantic import TypeAdapter - - docker_dict = {"enabled": True} - k8s_nim_dict = { - "enabled": True, - "default_storage_class": "fast-ssd", - "default_pvc_size": "500Gi", - "peft_source": "http://custom-entity-store:8000", - } - - adapter = TypeAdapter(BackendConfig) - - docker_config = adapter.validate_python(docker_dict) - assert docker_config.enabled is True - assert isinstance(docker_config, DockerBackendConfigModel) - assert hasattr(docker_config, "default_nimservice_image") - - k8s_config = adapter.validate_python(k8s_nim_dict) - assert k8s_config.enabled is True - assert isinstance(k8s_config, K8sNimOperatorBackendConfigModel) - assert k8s_config.default_storage_class == "fast-ssd" - assert k8s_config.default_pvc_size == "500Gi" - assert k8s_config.peft_source == "http://custom-entity-store:8000" - - def test_backend_config_from_yaml_to_registry(mock_nmp_sdk): """Test end-to-end: parse backend configs from dicts (like YAML) and use them with registry.""" from nmp.core.models.config import ControllerConfig yaml_config = { - "docker": {"enabled": False}, - "nim_operator": {"enabled": True, "default_pvc_size": "100Gi"}, + "deployments_plugin": {"enabled": True, "default_pvc_size": "100Gi"}, } - # Use ControllerConfig so validate_backends uses backend key to pick the right model controller_config = ControllerConfig(backends=yaml_config) parsed_configs = controller_config.backends - assert isinstance(parsed_configs["docker"], DockerBackendConfigModel) - assert isinstance(parsed_configs["nim_operator"], K8sNimOperatorBackendConfigModel) - assert parsed_configs["nim_operator"].default_pvc_size == "100Gi" + assert isinstance(parsed_configs["deployments_plugin"], DeploymentsPluginBackendConfigModel) + assert parsed_configs["deployments_plugin"].default_pvc_size == "100Gi" - registry = BackendRegistry.from_config( - nmp_sdk=mock_nmp_sdk, - backend_configs=parsed_configs, - huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", - ) + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.backend.NemoEntitiesClient", + ): + registry = BackendRegistry.from_config( + nmp_sdk=mock_nmp_sdk, + backend_configs=parsed_configs, + huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", + ) - assert len(registry.list_backends()) == 1 - assert "nim_operator" in registry.list_backends() + assert registry.list_backends() == ["deployments_plugin"] def test_backend_registry_no_enabled_backends_raises_error(mock_nmp_sdk): """Test that having no enabled backends raises ValueError.""" config_with_no_enabled = { - "docker": DockerBackendConfigModel(enabled=False), - "nim_operator": K8sNimOperatorBackendConfigModel(enabled=False), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=False), } with pytest.raises(ValueError, match="No backends are enabled"): @@ -234,8 +155,8 @@ def test_backend_registry_no_enabled_backends_raises_error(mock_nmp_sdk): def test_backend_registry_multiple_enabled_backends_raises_error(mock_nmp_sdk): """Test that having multiple enabled backends raises ValueError.""" config_with_multiple_enabled = { - "docker": DockerBackendConfigModel(enabled=True), - "nim_operator": K8sNimOperatorBackendConfigModel(enabled=True), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True), + "deployments_plugin_shadow": DeploymentsPluginBackendConfigModel(enabled=True), } with pytest.raises(ValueError, match="Multiple backends are enabled"): @@ -250,7 +171,6 @@ def test_deployments_plugin_missing_package_raises_guidance(mock_nmp_sdk): """Missing nemo-deployments-plugin should surface install guidance.""" import builtins - from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginBackendConfigModel from nmp.core.models.controllers.backends.registry import _resolve_backend_class, backend_classes real_import = builtins.__import__ @@ -266,7 +186,6 @@ def mock_import(name: str, *args: object, **kwargs: object) -> object: ): _resolve_backend_class("deployments_plugin", backend_classes) - # Enabled config path should fail the same way during registry init. with ( patch("builtins.__import__", side_effect=mock_import), pytest.raises(ImportError, match="nemo-deployments-plugin"), @@ -276,3 +195,20 @@ def mock_import(name: str, *args: object, **kwargs: object) -> object: backend_configs={"deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True)}, huggingface_model_puller="puller:latest", ) + + +def test_backend_registry_shutdown_calls_backend_shutdown(mock_nmp_sdk, sample_backend_configs): + """Test that shutdown_all_backends calls shutdown on each backend.""" + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.backend.NemoEntitiesClient", + ): + registry = BackendRegistry.from_config( + nmp_sdk=mock_nmp_sdk, + backend_configs=sample_backend_configs, + huggingface_model_puller="puller:latest", + ) + + backend = registry.get_backend() + backend.shutdown = MagicMock() + registry.shutdown_all_backends() + backend.shutdown.assert_called_once() diff --git a/services/core/models/tests/unit/controllers/test_deployment_reconciler.py b/services/core/models/tests/unit/controllers/test_deployment_reconciler.py index a295a2d1bc..9d52834e29 100644 --- a/services/core/models/tests/unit/controllers/test_deployment_reconciler.py +++ b/services/core/models/tests/unit/controllers/test_deployment_reconciler.py @@ -232,6 +232,31 @@ async def test_reconcile_individual_deployment_error_fallback_conflict_is_noop( assert call_kwargs["status"] == "ERROR" +@pytest.mark.asyncio +async def test_reconcile_created_docker_lora_error_persisted(reconciler, mock_backend_registry, make_deployment): + """CREATED + docker LoRA backend ERROR is persisted with a clear single-container message.""" + deployment = make_deployment(status="CREATED") + mock_backend = MagicMock() + mock_backend.create_model_deployment = AsyncMock( + return_value=DeploymentStatusUpdate( + status="ERROR", + status_message=( + "LoRA serving is not supported on the docker runtime yet " + "(deployments-plugin docker is single-container). Deploy LoRA " + "models on the kubernetes runtime instead." + ), + ) + ) + mock_backend_registry.get_backend.return_value = mock_backend + reconciler._models_sdk.inference.deployments.update_status = AsyncMock() + + await reconciler._reconcile_individual_deployment(deployment, mock_backend.create_model_deployment, "create") + + call_kwargs = reconciler._models_sdk.inference.deployments.update_status.call_args.kwargs + assert call_kwargs["status"] == "ERROR" + assert "single-container" in call_kwargs["status_message"] + + @pytest.mark.asyncio async def test_reconcile_deployments_with_created_status(reconciler, mock_backend_registry, make_deployment): """Test processing deployments calls handler for CREATED deployments.""" diff --git a/services/core/models/tests/unit/controllers/test_docker_backend.py b/services/core/models/tests/unit/controllers/test_docker_backend.py deleted file mode 100644 index 2b74ec592e..0000000000 --- a/services/core/models/tests/unit/controllers/test_docker_backend.py +++ /dev/null @@ -1,4800 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for DockerServiceBackend.""" - -import asyncio -from datetime import datetime, timedelta, timezone -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from docker.errors import ImageNotFound, NotFound -from nmp.common.config import PlatformConfig -from nmp.core.models.app import ModelWeightsType -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.app.utils import ( - get_docker_container_name, - get_docker_plugin_puller_container_name, - get_docker_volume_name, -) -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.docker import DockerServiceBackend -from nmp.core.models.controllers.backends.docker.creation_reconciler import ( - CreationStage, - CreationState, - _compute_multi_gpu_shm_size, -) -from nmp.core.models.controllers.context import ModelContext -from nmp.core.models.schemas import ( - ContainerExecutorConfig, - Engine, - ModelDeploymentConfigModelSpec, -) - -_MODEL_SPEC_FIELDS = { - "model_type", - "model_namespace", - "model_name", - "model_revision", - "model_provider", - "chat_template", - "tool_call_config", - "lora_enabled", -} -_EXECUTOR_FIELDS = { - "gpu", - "disk_size", - "image_name", - "image_tag", - "health_check_path", - "additional_envs", - "additional_args", - "k8s_nim_operator_config", - "override_config", -} - - -def set_deployment_config(config, engine: str = "nim", **kwargs) -> None: - """Populate a config mock with the engine-split deployment shape. - - Splits flat NIMDeployment-style kwargs into a real ``ModelDeploymentConfigModelSpec`` - and ``ContainerExecutorConfig`` so the flattened ``deployment_config_view`` (which uses - ``getattr``) reads real attributes rather than MagicMock children. - """ - model_spec_kwargs = {k: v for k, v in kwargs.items() if k in _MODEL_SPEC_FIELDS} - executor_kwargs = {k: v for k, v in kwargs.items() if k in _EXECUTOR_FIELDS} - unknown = set(kwargs) - _MODEL_SPEC_FIELDS - _EXECUTOR_FIELDS - if unknown: - raise ValueError(f"Unknown deployment config fields: {unknown}") - config.engine = Engine(engine) - config.model_spec = ModelDeploymentConfigModelSpec(**model_spec_kwargs) - config.executor_config = ContainerExecutorConfig(**executor_kwargs) - - -async def drive_creation_to_completion(backend: DockerServiceBackend, deployment) -> DeploymentStatusUpdate: - """Drive the staged creation pipeline to completion by advancing all stages. - - After create_model_deployment() starts the pipeline, this helper awaits - background tasks and advances each stage until the deployment exits - the creation pipeline (either successfully or with an error). - """ - key = backend._get_deployment_key(deployment) - last_status: DeploymentStatusUpdate | None = None - for _ in range(20): - if key not in backend._reconciler._creation_states: - break - state = backend._reconciler._creation_states[key] - if state.task and not state.task.done(): - try: - await state.task - except Exception: - pass - last_status = await backend._reconciler.advance(key) - if last_status.status == "ERROR" or key not in backend._reconciler._creation_states: - return last_status - if last_status is not None: - return last_status - return await backend.get_model_deployment_status(ModelContext(model_deployment=deployment)) - - -@pytest.fixture -def backend_with_mock_client(): - """Create a DockerServiceBackend with a mocked Docker client.""" - with ( - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock_from_env, - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config"), - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect, - ): - mock_resource_config.return_value.docker = MagicMock() - mock_resource_config.return_value.docker.get_reserved_gpu_ids.return_value = [0] - mock_detect.return_value = [0] - client = MagicMock() - mock_from_env.return_value = client - backend = DockerServiceBackend(nmp_sdk=AsyncMock(), config={}) - backend._client = client - yield backend, client - - -class TestComputeMultiGpuShmSize: - """Unit tests for multi-GPU shm_size calculation.""" - - def test_2_gpus_default_per_gpu(self): - """2 GPUs × 1024 MB per GPU = 2048m.""" - assert _compute_multi_gpu_shm_size("", 1024, 2) == "2048m" - - def test_4_gpus_default_per_gpu(self): - """4 GPUs × 1024 MB per GPU = 4096m.""" - assert _compute_multi_gpu_shm_size("", 1024, 4) == "4096m" - - def test_2_gpus_fixed_overrides(self): - """Fixed total is used when set.""" - assert _compute_multi_gpu_shm_size("4g", 1024, 2) == "4g" - assert _compute_multi_gpu_shm_size("2g", 512, 4) == "2g" - - def test_4_gpus_custom_per_gpu(self): - """4 GPUs × 512 MB per GPU = 2048m.""" - assert _compute_multi_gpu_shm_size("", 512, 4) == "2048m" - - def test_2_gpus_custom_per_gpu(self): - """2 GPUs × 512 MB per GPU = 1024m.""" - assert _compute_multi_gpu_shm_size("", 512, 2) == "1024m" - - def test_empty_fixed_uses_per_gpu_calculation(self): - """Empty fixed uses per_gpu_mb * gpu_count.""" - assert _compute_multi_gpu_shm_size("", 2048, 2) == "4096m" - assert _compute_multi_gpu_shm_size("", 1024, 8) == "8192m" - - -class TestPullImageIfNotLocal: - """Unit tests for _pull_image_if_not_local.""" - - def test_image_found_locally_does_not_pull(self, backend_with_mock_client): - """When image exists locally, _pull_image should not be called.""" - backend, client = backend_with_mock_client - client.images.get.return_value = MagicMock() - - backend._reconciler.pull_image_if_not_local("myregistry/myimage:v1") - - client.images.get.assert_called_once_with("myregistry/myimage:v1") - client.images.pull.assert_not_called() - - def test_image_not_found_locally_triggers_pull(self, backend_with_mock_client): - """When image is not local, it should be pulled via _pull_image.""" - backend, client = backend_with_mock_client - client.images.get.side_effect = ImageNotFound("not found") - client.images.pull.return_value = MagicMock() - - backend._reconciler.pull_image_if_not_local("myregistry/myimage:v1") - - client.images.get.assert_called_once_with("myregistry/myimage:v1") - client.images.pull.assert_called_once_with("myregistry/myimage:v1", tag=None) - - def test_pull_failure_propagates(self, backend_with_mock_client): - """When image is not local and pull fails, the exception should propagate.""" - backend, client = backend_with_mock_client - client.images.get.side_effect = ImageNotFound("not found") - client.images.pull.side_effect = Exception("registry unreachable") - - with pytest.raises(Exception, match="registry unreachable"): - backend._reconciler.pull_image_if_not_local("myregistry/myimage:v1") - - def test_image_without_tag(self, backend_with_mock_client): - """Image string without a tag should be handled correctly.""" - backend, client = backend_with_mock_client - client.images.get.side_effect = ImageNotFound("not found") - client.images.pull.return_value = MagicMock() - - backend._reconciler.pull_image_if_not_local("myregistry/myimage") - - client.images.get.assert_called_once_with("myregistry/myimage") - client.images.pull.assert_called_once_with("myregistry/myimage", tag=None) - - def test_non_image_not_found_exception_propagates(self, backend_with_mock_client): - """Exceptions other than ImageNotFound from images.get should propagate.""" - backend, client = backend_with_mock_client - client.images.get.side_effect = ConnectionError("docker daemon unavailable") - - with pytest.raises(ConnectionError, match="docker daemon unavailable"): - backend._reconciler.pull_image_if_not_local("myregistry/myimage:v1") - - client.images.pull.assert_not_called() - - -class TestPullImage: - """Unit tests for _pull_image with optional tag.""" - - def test_pull_with_name_and_tag(self, backend_with_mock_client): - """Explicit name + tag should be forwarded to client.images.pull.""" - backend, client = backend_with_mock_client - backend._reconciler.pull_image("myregistry/myimage", "v1") - client.images.pull.assert_called_once_with("myregistry/myimage", tag="v1") - - def test_pull_with_name_only(self, backend_with_mock_client): - """Calling with only the image name should pass tag=None.""" - backend, client = backend_with_mock_client - backend._reconciler.pull_image("myregistry/myimage:v1") - client.images.pull.assert_called_once_with("myregistry/myimage:v1", tag=None) - - def test_pull_with_explicit_none_tag(self, backend_with_mock_client): - """Explicitly passing None as tag should work.""" - backend, client = backend_with_mock_client - backend._reconciler.pull_image("myregistry/myimage:v1", None) - client.images.pull.assert_called_once_with("myregistry/myimage:v1", tag=None) - - -def create_mock_docker_config(reserved_gpu_ids: str = "all") -> MagicMock: - """Create a mock DockerConfig with the given reserved_gpu_device_ids value.""" - mock_docker_config = MagicMock() - mock_docker_config.reserved_gpu_device_ids = reserved_gpu_ids - - def get_reserved_gpu_ids(): - if reserved_gpu_ids.lower() == "all": - return None - if reserved_gpu_ids.lower() == "none" or not reserved_gpu_ids: - return [] - return [int(p.strip()) for p in reserved_gpu_ids.split(",") if p.strip()] - - mock_docker_config.get_reserved_gpu_ids = get_reserved_gpu_ids - return mock_docker_config - - -@pytest.fixture -def mock_nmp_sdk(): - """Create a mock AsyncNeMoPlatform SDK.""" - mock = AsyncMock() - return mock - - -@pytest.fixture -def mock_docker_client(): - """Create a mock Docker client.""" - with ( - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock, - patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.DockerDeploymentCreationReconciler._is_port_free" - ) as mock_is_port_free, - patch( - "nmp.core.models.controllers.backends.docker.backend.DockerServiceBackend._probe_nim_health" - ) as probe_nim_health, - ): - mock_is_port_free.return_value = True - probe_nim_health.return_value = (True, "") # Returns tuple (is_healthy, failure_reason) - client = MagicMock() - mock.return_value = client - - # Setup default behaviors - client.login = MagicMock() - client.containers.get = MagicMock(side_effect=NotFound("Container not found")) - client.containers.run = MagicMock() - client.images.pull = MagicMock() - client.volumes.create = MagicMock() - client.volumes.get = MagicMock(side_effect=NotFound("Volume not found")) - mock_is_port_free.return_value = True - - yield client - - -@pytest.fixture -def reset_shared_resource_manager_base(): - """Reset SharedResourceManager singleton before and after test.""" - from nmp.common.resources import SharedResourceManager - - SharedResourceManager.reset_instance() - yield - SharedResourceManager.reset_instance() - - -@pytest.fixture -def docker_backend(mock_nmp_sdk, mock_docker_client, reset_shared_resource_manager_base): - """Create a DockerServiceBackend instance for testing with mocked GPU detection.""" - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect_gpu_device_ids, - ): - mock_resource_config.return_value.docker = create_mock_docker_config("0,1,2,3") - mock_detect_gpu_device_ids.return_value = [0, 1, 2, 3] - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config={}) - backend._client = mock_docker_client - return backend - - -@pytest.fixture -def sample_deployment(): - """Create a sample ModelDeployment for testing.""" - deployment = MagicMock() - deployment.workspace = "default" - deployment.name = "test-deployment" - deployment.entity_version = 1 - deployment.status = "CREATED" - deployment.created_at = datetime.now(timezone.utc) - return deployment - - -@pytest.fixture -def sample_config(): - """Create a sample ModelDeploymentConfig for testing.""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - ) - return config - - -@pytest.fixture -def sample_resource_names(docker_backend, sample_deployment): - """Computed Docker resource names for the default sample deployment.""" - reconciler = docker_backend._reconciler - ws, name = sample_deployment.workspace, sample_deployment.name - container = reconciler.get_container_name(ws, name) - volume = reconciler.get_volume_name(ws, name) - return { - "container": container, - "volume": volume, - "scratch_volume": f"{volume}-scratch", - "puller": reconciler.get_puller_container_name(ws, name), - "plugin": reconciler.get_plugin_puller_container_name(ws, name), - "sidecar": f"{container}-sidecar", - "host_url": f"http://{container}:8000", - } - - -@pytest.mark.asyncio -async def test_docker_backend_create_model_deployment( - docker_backend, sample_deployment, sample_config, mock_docker_client, sample_resource_names -): - """Test creating a model deployment with Docker backend.""" - # Setup mock container - mock_container = MagicMock() - docker_backend._backend_config.models_docker_networking_mode = "dond" - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() # Mock the start method - mock_docker_client.containers.create.return_value = mock_container - - # Enable lora to trigger sidecar creation (2 containers) - sample_config.model_spec.lora_enabled = True - - # Mock image not found locally (will trigger pull) - from docker.errors import ImageNotFound as DockerImageNotFound - - mock_docker_client.images.get.side_effect = DockerImageNotFound("Image not found") - mock_docker_client.images.pull.return_value = MagicMock() # Pull succeeds - mock_docker_client.containers.list.return_value = [] - - # create_model_deployment now starts the image pull and returns PENDING immediately - initial_status = await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - assert initial_status.status == "PENDING" - assert "pulling container image" in initial_status.status_message.lower() - - # Drive creation to completion (image pull -> container creation) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update is not None - assert status_update.status == "PENDING" - assert "container created" in status_update.status_message.lower() - assert status_update.host_url == sample_resource_names["host_url"] - - # Verify Docker calls (image check + pull for both NIM and sidecar images) - assert mock_docker_client.images.get.call_count >= 2 - assert mock_docker_client.images.pull.call_count >= 2 - assert mock_docker_client.volumes.create.call_count == 2 - assert mock_docker_client.volumes.create.call_args_list[0][0][0] == sample_resource_names["volume"] - assert mock_docker_client.volumes.create.call_args_list[1][0][0] == sample_resource_names["scratch_volume"] - - assert mock_docker_client.containers.create.call_count == 2 - nim_create_args = mock_docker_client.containers.create.call_args_list[0][1] - sidecar_create_args = mock_docker_client.containers.create.call_args_list[1][1] - assert nim_create_args["name"] == sample_resource_names["container"] - assert sidecar_create_args["name"] == sample_resource_names["sidecar"] - assert nim_create_args["labels"][MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert sidecar_create_args["labels"][MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert mock_container.start.call_count == 2 - - -@pytest.mark.asyncio -async def test_docker_backend_adds_configured_labels_to_managed_containers( - mock_nmp_sdk, - mock_docker_client, - reset_shared_resource_manager_base, - sample_deployment, - sample_config, -): - """Configured owner labels are copied to managed NIM and sidecar containers.""" - owner_labels = { - "nmp.nvidia.com/test-run": "run-1", - "nmp.nvidia.com/test-worker": "gw0", - } - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect_gpu_device_ids, - ): - mock_resource_config.return_value.docker = create_mock_docker_config("0,1,2,3") - mock_detect_gpu_device_ids.return_value = [0, 1, 2, 3] - backend = DockerServiceBackend( - nmp_sdk=mock_nmp_sdk, - config={ - "model_labels": owner_labels, - "models_docker_networking_mode": "dond", - }, - ) - - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - sample_config.model_spec.lora_enabled = True - - await backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(backend, sample_deployment) - - assert mock_docker_client.containers.create.call_count == 2 - for call in mock_docker_client.containers.create.call_args_list: - labels = call.kwargs["labels"] - assert labels[MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert labels["nmp.nvidia.com/test-run"] == "run-1" - assert labels["nmp.nvidia.com/test-worker"] == "gw0" - - -def _make_vllm_config(*, lora_enabled: bool = False, image_name=None, image_tag=None): - config = MagicMock() - kwargs = dict( - engine="vllm", - model_namespace="default", - model_name="qwen-2-5-1-5b", - gpu=1, - lora_enabled=lora_enabled, - ) - if image_name is not None: - kwargs["image_name"] = image_name - if image_tag is not None: - kwargs["image_tag"] = image_tag - set_deployment_config(config, **kwargs) - return config - - -def _vllm_model_entity(): - """A Files-service-backed model entity (drives the puller path).""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "qwen-2-5-1-5b" - model_entity.spec = None - model_entity.trust_remote_code = False - model_entity.fileset = "hf://default/qwen-2-5-1-5b" - return model_entity - - -async def _drive_vllm_with_puller(docker_backend, sample_deployment, mock_docker_client, config): - """Run the vLLM create pipeline through the puller stage and return container create args.""" - docker_backend._backend_config.models_docker_networking_mode = "dond" - - mock_puller_container = MagicMock() - mock_puller_container.id = "puller123456789" - mock_puller_container.wait.return_value = {"StatusCode": 0} - mock_puller_container.status = "exited" - mock_puller_container.attrs = {"State": {"ExitCode": 0}} - mock_puller_container.name = docker_backend._reconciler.get_puller_container_name( - sample_deployment.workspace, sample_deployment.name - ) - - mock_vllm_container = MagicMock() - mock_vllm_container.id = "vllm1234567890a" - mock_vllm_container.start = MagicMock() - - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.run.return_value = mock_puller_container - mock_docker_client.containers.create.return_value = mock_vllm_container - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, model_deployment_config=config, model_entity=_vllm_model_entity() - ) - ) - - def get_container_side_effect(name): - if "puller" in name: - return mock_puller_container - raise NotFound("Container not found") - - mock_docker_client.containers.get.side_effect = get_container_side_effect - - return await drive_creation_to_completion(docker_backend, sample_deployment) - - -@pytest.mark.asyncio -async def test_docker_backend_create_vllm_deployment( - docker_backend, sample_deployment, mock_docker_client, sample_resource_names -): - """Engine=vllm produces a vLLM container with serve args, engine label, and default image.""" - config = _make_vllm_config() - status_update = await _drive_vllm_with_puller(docker_backend, sample_deployment, mock_docker_client, config) - - assert status_update.status == "PENDING" - assert status_update.host_url == sample_resource_names["host_url"] - - create_args = mock_docker_client.containers.create.call_args_list[0][1] - # Default vLLM image is used when none specified (exact version is config-driven). - cfg = docker_backend._backend_config - assert create_args["image"] == f"{cfg.default_vllm_image}:{cfg.default_vllm_image_tag}" - # vLLM serve args are passed as the container command. - command = create_args["command"] - assert command[0] == "/model-store" - assert command[command.index("--served-model-name") + 1] == "default/qwen-2-5-1-5b" - # Engine label recorded for the health-probe selection. - assert create_args["labels"]["nmp.nvidia.com/engine"] == "vllm" - # No LoRA env when lora is disabled. - assert "VLLM_PLUGINS" not in create_args["environment"] - - -@pytest.mark.asyncio -async def test_docker_backend_create_vllm_lora_sidecar( - docker_backend, sample_deployment, mock_docker_client, sample_resource_names -): - """Engine=vllm with lora_enabled wires the adapter sidecar and vLLM LoRA env/args.""" - config = _make_vllm_config(lora_enabled=True) - await _drive_vllm_with_puller(docker_backend, sample_deployment, mock_docker_client, config) - - # Two containers: vLLM server + adapter sidecar. - assert mock_docker_client.containers.create.call_count == 2 - vllm_args = mock_docker_client.containers.create.call_args_list[0][1] - sidecar_args = mock_docker_client.containers.create.call_args_list[1][1] - assert vllm_args["name"] == sample_resource_names["container"] - assert sidecar_args["name"] == sample_resource_names["sidecar"] - # vLLM LoRA hot-reload env + serve flag. - env = vllm_args["environment"] - assert env["VLLM_PLUGINS"] == "lora_filesystem_resolver" - assert env["VLLM_LORA_RESOLVER_CACHE_DIR"] == "/scratch/loras" - assert env["VLLM_ALLOW_RUNTIME_LORA_UPDATING"] == "True" - assert "--enable-lora" in vllm_args["command"] - # The adapters sidecar is engine-agnostic and reads NIM_PEFT_SOURCE + - # the model-entity identity. For vLLM these must point at the same - # directory vLLM's filesystem resolver watches, or the sidecar crashes - # on startup and never delivers adapters. - sidecar_env = sidecar_args["environment"] - assert sidecar_env["NIM_PEFT_SOURCE"] == "/scratch/loras" - assert sidecar_env["NMP_MODEL_ENTITY_WORKSPACE"] == "default" - assert sidecar_env["NMP_MODEL_ENTITY_NAME"] == "qwen-2-5-1-5b" - # vLLM's filesystem resolver requires the adapter's base_model_name_or_path to - # equal vLLM's --model value, so the sidecar is told to rewrite it to /model-store. - assert sidecar_env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store" - # vLLM's filesystem resolver validates VLLM_LORA_RESOLVER_CACHE_DIR exists at - # startup, so the controller pre-creates it in the scratch volume via a busybox - # run before launching the vLLM container (otherwise vLLM crash-loops). - run_commands = [ - c.kwargs.get("command") for c in mock_docker_client.containers.run.call_args_list if c.kwargs.get("command") - ] - assert any(isinstance(cmd, list) and "mkdir -p /scratch/loras" in " ".join(cmd) for cmd in run_commands), ( - f"expected a busybox mkdir for the LoRA cache dir, got run commands: {run_commands}" - ) - - -@pytest.mark.asyncio -async def test_docker_backend_create_generic_deployment(docker_backend, sample_deployment, mock_docker_client): - """Engine=generic runs the user's image + raw args/env verbatim, no puller, no LoRA sidecar.""" - config = MagicMock() - set_deployment_config( - config, - engine="generic", - gpu=0, - image_name="nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", - image_tag="1.10.1", - health_check_path="/v1/health/ready", - additional_args=["--port", "8000"], - additional_envs={"FOO": "bar"}, - ) - - mock_container = MagicMock() - mock_container.id = "generic12345678" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - status_update = await drive_creation_to_completion_after_create( - docker_backend, sample_deployment, config, mock_docker_client - ) - - assert status_update.status == "PENDING" - - # Self-contained image: no model puller ran (generic has no model weights). - run_commands = [c.kwargs.get("command") for c in mock_docker_client.containers.run.call_args_list] - assert not any(isinstance(cmd, list) and any("download" in str(p) for p in cmd) for cmd in run_commands) - - # Exactly one container (no LoRA sidecar). - assert mock_docker_client.containers.create.call_count == 1 - create_args = mock_docker_client.containers.create.call_args_list[0][1] - assert create_args["image"] == "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect:1.10.1" - # Raw additional_args become the container command verbatim. - assert create_args["command"] == ["--port", "8000"] - # Raw additional_envs become the env verbatim. - assert create_args["environment"]["FOO"] == "bar" - # Engine + explicit health-path labels recorded for status-time probe selection. - assert create_args["labels"]["nmp.nvidia.com/engine"] == "generic" - assert create_args["labels"]["nmp.nvidia.com/health-path"] == "/v1/health/ready" - # Weightless generic runs raw: no platform volumes mounted (they'd shadow the image). - assert create_args["volumes"] == {} - - -async def drive_creation_to_completion_after_create(docker_backend, sample_deployment, config, mock_docker_client): - """Start creation for a no-puller config and drive it to completion.""" - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - mock_docker_client.containers.get.side_effect = NotFound("Container not found") - return await drive_creation_to_completion(docker_backend, sample_deployment) - - -@pytest.mark.asyncio -async def test_docker_backend_create_generic_with_fileset_pulls_and_mounts( - docker_backend, sample_deployment, mock_docker_client -): - """Engine=generic with a fileset-backed model runs the puller and mounts platform volumes.""" - config = MagicMock() - set_deployment_config( - config, - engine="generic", - gpu=0, - image_name="my/custom-server", - image_tag="1.0", - health_check_path="/healthz", - model_namespace="default", - model_name="qwen-2-5-1-5b", - additional_args=["--model-dir", "/model-store"], - ) - - status_update = await _drive_vllm_with_puller(docker_backend, sample_deployment, mock_docker_client, config) - assert status_update.status == "PENDING" - - # Generic + fileset => the puller ran and the platform volumes are mounted so - # the pulled weights are available to the user's container at /model-store. - create_args = mock_docker_client.containers.create.call_args_list[0][1] - assert create_args["image"] == "my/custom-server:1.0" - assert create_args["command"] == ["--model-dir", "/model-store"] - assert create_args["volumes"][docker_backend._reconciler.get_volume_name("default", "test-deployment")] == { - "bind": "/model-store", - "mode": "rw", - } - assert create_args["labels"]["nmp.nvidia.com/engine"] == "generic" - assert create_args["labels"]["nmp.nvidia.com/health-path"] == "/healthz" - - -def test_get_health_path_from_container_vllm(docker_backend): - """vLLM containers probe /health.""" - container = MagicMock() - container.labels = {"nmp.nvidia.com/engine": "vllm"} - assert docker_backend._reconciler.get_health_path_from_container(container) == "/health" - - -def test_get_health_path_from_container_nim(docker_backend): - """NIM containers probe /v1/health/ready.""" - container = MagicMock() - container.labels = {"nmp.nvidia.com/engine": "nim"} - assert docker_backend._reconciler.get_health_path_from_container(container) == "/v1/health/ready" - - -def test_get_health_path_from_container_defaults_to_nim(docker_backend): - """Containers without an engine label default to the NIM probe path.""" - container = MagicMock() - container.labels = {} - assert docker_backend._reconciler.get_health_path_from_container(container) == "/v1/health/ready" - - -def test_get_health_path_from_container_explicit_override(docker_backend): - """An explicit health-path label (from executor_config.health_check_path) wins over the engine default.""" - container = MagicMock() - container.labels = {"nmp.nvidia.com/engine": "generic", "nmp.nvidia.com/health-path": "/custom/ready"} - assert docker_backend._reconciler.get_health_path_from_container(container) == "/custom/ready" - - -def test_resolve_health_path_prefers_explicit(): - """_resolve_health_path uses executor_config.health_check_path when set, else the engine default.""" - from nmp.core.models.controllers.backends.common import DeploymentConfigView - from nmp.core.models.controllers.backends.docker.creation_reconciler import ( - ENGINE_NIM, - ENGINE_VLLM, - _resolve_health_path, - ) - - assert _resolve_health_path(ENGINE_VLLM, DeploymentConfigView()) == "/health" - assert _resolve_health_path(ENGINE_NIM, DeploymentConfigView()) == "/v1/health/ready" - assert _resolve_health_path("generic", DeploymentConfigView(health_check_path="/ping")) == "/ping" - # generic with no explicit path falls back to the NIM path. - assert _resolve_health_path("generic", DeploymentConfigView()) == "/v1/health/ready" - - -@pytest.mark.asyncio -async def test_docker_backend_create_sft_model_success( - docker_backend, sample_deployment, sample_config, mock_docker_client -): - """Test creating an SFT model with full weights runs puller then NIM.""" - # Mock model entity with SFT full weights and fileset - model_entity = MagicMock() - model_entity.workspace = "test" - model_entity.name = "sft-model" - model_entity.spec = None - model_entity.finetuning_type = "all_weights" - peft_mock = MagicMock() - peft_mock.finetuning_type = "all_weights" - model_entity.peft = peft_mock - - # Add fileset (required for SFT model deployment) - model_entity.fileset = "hf://test/sft-model-weights" - - # Enable lora to trigger sidecar creation (2 containers) - sample_config.model_spec.lora_enabled = True - - # Setup mock puller container - mock_puller_container = MagicMock() - mock_puller_container.id = "puller123456789" - mock_puller_container.wait.return_value = {"StatusCode": 0} # Success - mock_puller_container.remove = MagicMock() - - # Setup mock NIM container - mock_nim_container = MagicMock() - mock_nim_container.id = "nim1234567890ab" - mock_nim_container.start = MagicMock() - - # Mock images.get to return image (no pull needed) - mock_docker_client.images.get.return_value = MagicMock() - - # Mock containers.run for puller, containers.create for NIM - mock_docker_client.containers.run.return_value = mock_puller_container - mock_docker_client.containers.create.return_value = mock_nim_container - - # Mock containers.list to return empty (no ports in use) - mock_docker_client.containers.list.return_value = [] - - # create_model_deployment starts the pipeline; drive it to completion - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity - ) - ) - - # Mock puller container for get/reload polling - mock_puller_container.status = "exited" - mock_puller_container.attrs = {"State": {"ExitCode": 0}} - mock_puller_container.name = docker_backend._reconciler.get_puller_container_name( - sample_deployment.workspace, sample_deployment.name - ) - - def get_container_side_effect(name): - if "puller" in name: - return mock_puller_container - raise NotFound("Container not found") - - mock_docker_client.containers.get.side_effect = get_container_side_effect - - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - # Verify status update returns PENDING (NIM starting) - assert status_update is not None - assert status_update.status == "PENDING" - assert "container created" in status_update.status_message.lower() - - # Verify puller container was run (busybox for permissions + puller + busybox for chown) - run_calls = mock_docker_client.containers.run.call_args_list - puller_calls = [ - c for c in run_calls if c[1].get("labels", {}).get("nmp.nvidia.com/container-type") == "model-puller" - ] - assert len(puller_calls) == 1 - assert puller_calls[0][1]["command"][0] == "download" - # Entrypoint overridden to the HF CLI (puller image is nmp-api, whose - # entrypoint is `nemo services run`); command runs as `hf download ...`. - assert puller_calls[0][1]["entrypoint"] == ["hf"] - - # Verify NIM and sidecar containers were created with managed-by label - assert mock_docker_client.containers.create.call_count == 2 - nim_call_args = mock_docker_client.containers.create.call_args_list[0][1] - sidecar_call_args = mock_docker_client.containers.create.call_args_list[1][1] - assert nim_call_args["labels"][MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert sidecar_call_args["labels"][MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert nim_call_args["environment"]["NIM_FT_MODEL"] == "/model-store" - assert nim_call_args["environment"]["NIM_CUSTOM_MODEL"] == "/model-store" - - -@pytest.mark.asyncio -async def test_docker_backend_create_sft_model_puller_fails( - docker_backend, sample_deployment, sample_config, mock_docker_client, sample_resource_names -): - """Test that SFT model deployment fails gracefully when puller fails.""" - # Mock model entity with SFT full weights and artifact - model_entity = MagicMock() - model_entity.workspace = "test" - model_entity.name = "sft-model" - model_entity.spec = None - model_entity.finetuning_type = "all_weights" - peft_mock = MagicMock() - peft_mock.finetuning_type = "all_weights" - model_entity.peft = peft_mock - - # Add fileset (required for SFT model deployment) - model_entity.fileset = "hf://test/sft-model-weights" - - # Setup mock puller container that fails - mock_puller_container = MagicMock() - mock_puller_container.id = "puller123456789" - mock_puller_container.name = sample_resource_names["puller"] - mock_puller_container.logs.return_value = b"Error: Failed to download model" - mock_puller_container.status = "exited" - mock_puller_container.attrs = {"State": {"ExitCode": 1}} - - # Mock images.get to return image (no pull needed) - mock_docker_client.images.get.return_value = MagicMock() - - # Mock containers.run for puller - mock_docker_client.containers.run.return_value = mock_puller_container - - # Mock containers.list to return empty (no ports in use) - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity - ) - ) - - # Mock get for puller container polling - def get_container_side_effect(name): - if "puller" in name: - return mock_puller_container - raise NotFound("Container not found") - - mock_docker_client.containers.get.side_effect = get_container_side_effect - - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - # Verify status update returns ERROR - assert status_update is not None - assert status_update.status == "ERROR" - assert "model puller failed" in status_update.status_message.lower() - assert status_update.error_details["stage"] == "model_puller" - - # Verify NIM container was NOT created - mock_docker_client.containers.create.assert_not_called() - - -@pytest.mark.asyncio -async def test_docker_backend_get_model_deployment_status_running( - docker_backend, sample_deployment, mock_docker_client, sample_resource_names -): - """Test getting status of a running deployment.""" - # Setup mock container in running state - mock_container = MagicMock() - mock_container.status = "running" - mock_container.id = "1234567890abcdef" - docker_backend._backend_config.models_docker_networking_mode = "dond" - mock_docker_client.containers.get.return_value = mock_container - mock_docker_client.containers.get.side_effect = None - - status_update = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - # Verify status update - assert status_update is not None - assert status_update.status == "READY" - assert "running" in status_update.status_message.lower() - assert status_update.host_url == sample_resource_names["host_url"] - - -@pytest.mark.asyncio -async def test_docker_backend_get_model_deployment_status_not_found( - docker_backend, sample_deployment, mock_docker_client -): - """Test getting status when container is not found.""" - mock_docker_client.containers.get.side_effect = NotFound("Container not found") - - status_update = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - # Verify status update - assert status_update is not None - assert status_update.status == "LOST" - assert "not found" in status_update.status_message.lower() - - -@pytest.mark.asyncio -async def test_docker_backend_delete_model_deployment(docker_backend, sample_deployment, mock_docker_client): - """Test deleting a model deployment.""" - # Setup mock NIM container - mock_nim_container = MagicMock() - mock_nim_container.id = "1234567890abcdef" - - # Setup mock puller container (should be cleaned up too) - mock_puller_container = MagicMock() - mock_puller_container.id = "puller12345678" - - # containers.get returns different containers based on name - def mock_get_container(name): - if "puller" in name: - return mock_puller_container - return mock_nim_container - - mock_docker_client.containers.get.side_effect = mock_get_container - - # Setup mock volume - mock_volume = MagicMock() - mock_docker_client.volumes.get.return_value = mock_volume - mock_docker_client.volumes.get.side_effect = None - - status_update = await docker_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) - - # Verify deletion - assert status_update is not None - assert status_update.status == "DELETED" - # Both containers should be stopped and removed - assert mock_nim_container.stop.call_count == 2 - assert mock_nim_container.remove.call_count == 2 - assert mock_puller_container.stop.call_count == 1 - assert mock_puller_container.remove.call_count == 1 - assert mock_volume.remove.call_count == 2 - - -@pytest.mark.asyncio -async def test_docker_backend_delete_cleans_up_puller_even_when_not_found( - docker_backend, sample_deployment, mock_docker_client -): - """Test that delete handles missing puller container gracefully.""" - # Setup mock NIM container - mock_nim_container = MagicMock() - mock_nim_container.id = "1234567890abcdef" - - # containers.get raises NotFound for puller but returns NIM container - def mock_get_container(name): - if "puller" in name: - raise NotFound("Puller not found") - return mock_nim_container - - mock_docker_client.containers.get.side_effect = mock_get_container - - # Setup mock volume - mock_volume = MagicMock() - mock_docker_client.volumes.get.return_value = mock_volume - mock_docker_client.volumes.get.side_effect = None - - status_update = await docker_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) - - # Verify deletion still succeeds - assert status_update is not None - assert status_update.status == "DELETED" - assert mock_nim_container.stop.call_count == 2 - assert mock_nim_container.remove.call_count == 2 - assert mock_volume.remove.call_count == 2 - - -def test_docker_backend_initialization(mock_nmp_sdk, mock_docker_client): - """Test Docker backend initializes correctly.""" - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config): - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config={}) - - # Verify backend was created - assert backend is not None - assert backend._nmp_sdk == mock_nmp_sdk - - -def test_docker_backend_container_naming(docker_backend, sample_deployment, sample_resource_names): - """Test container naming convention.""" - container_name = docker_backend._reconciler.get_container_name(sample_deployment.workspace, sample_deployment.name) - assert container_name == sample_resource_names["container"] - assert container_name == get_docker_container_name("default", "test-deployment") - assert container_name == "md-default-test-deployment-a47830f1" - assert container_name.startswith("md-default-test-deployment-") - assert len(f"{container_name}-sidecar") <= 63 - - -def test_docker_backend_volume_naming(docker_backend, sample_deployment, sample_resource_names): - """Test volume naming convention.""" - volume_name = docker_backend._reconciler.get_volume_name(sample_deployment.workspace, sample_deployment.name) - assert volume_name == sample_resource_names["volume"] - assert volume_name == get_docker_volume_name("default", "test-deployment") - assert volume_name == "nim-cache-default-test-deployment-a47830f1" - assert volume_name.startswith("nim-cache-default-test-deployment-") - - -def test_docker_backend_deployment_key(docker_backend, sample_deployment): - """Test deployment key generation.""" - deployment_key = docker_backend._reconciler.get_deployment_key(sample_deployment.workspace, sample_deployment.name) - assert deployment_key == "default/test-deployment" - - -def test_docker_backend_puller_container_naming(docker_backend, sample_deployment, sample_resource_names): - """Test puller container naming convention.""" - puller_name = docker_backend._reconciler.get_puller_container_name( - sample_deployment.workspace, sample_deployment.name - ) - assert puller_name == sample_resource_names["puller"] - assert puller_name.startswith("md-puller-default-test-deployment-") - - -def test_docker_backend_plugin_puller_container_naming(docker_backend, sample_deployment, sample_resource_names): - """Test plugin puller container naming convention.""" - plugin_name = docker_backend._reconciler.get_plugin_puller_container_name( - sample_deployment.workspace, sample_deployment.name - ) - assert plugin_name == sample_resource_names["plugin"] - assert plugin_name == get_docker_plugin_puller_container_name("default", "test-deployment") - assert plugin_name == "md-plugin-default-test-deployment-a47830f1" - assert plugin_name.startswith("md-plugin-default-test-deployment-") - - -# ============================================================================= -# Tests for DNS label truncation (RFC 1035 - max 63 characters) -# Uses _get_k8s_safe_name from utils.py for consistent naming across backends -# ============================================================================= - - -def test_docker_backend_container_naming_truncates_long_names(docker_backend): - """Test that _get_container_name_for_model_deployment_id truncates names exceeding DNS limit.""" - # Create deployment with long names that would exceed 63 chars - deployment = MagicMock() - deployment.workspace = "e2e-customization-test" - deployment.name = "customization-bf0cc3016831-deployment-1770393778" - - container_name = docker_backend._reconciler.get_container_name(deployment.workspace, deployment.name) - - # Should be truncated to 63 chars max - assert len(container_name) <= 63 - # Should start with expected prefix (normalized to lowercase) - assert container_name.startswith("md-e2e-customization-test-") - - -def test_docker_backend_puller_naming_truncates_long_names(docker_backend): - """Test that _get_puller_container_name_for_model_deployment_id truncates names exceeding DNS limit.""" - # Create deployment with long names - deployment = MagicMock() - deployment.workspace = "e2e-customization-test" - deployment.name = "customization-bf0cc3016831-deployment-1770393778" - - puller_name = docker_backend._reconciler.get_puller_container_name(deployment.workspace, deployment.name) - - # Should be truncated to 63 chars max - assert len(puller_name) <= 63 - # Should start with expected prefix - assert puller_name.startswith("md-puller-e2e-customization") - - -def test_docker_backend_volume_naming_truncates_long_names(docker_backend): - """Test that _get_volume_name_for_model_deployment_id truncates names exceeding DNS limit.""" - deployment = MagicMock() - deployment.workspace = "e2e-customization-test" - deployment.name = "customization-bf0cc3016831-deployment-1770393778" - - volume_name = docker_backend._reconciler.get_volume_name(deployment.workspace, deployment.name) - - # Should be truncated to 63 chars max - assert len(volume_name) <= 63 - # Should start with expected prefix - assert volume_name.startswith("nim-cache-e2e-customization") - - -def test_docker_backend_naming_deterministic(docker_backend): - """Test that container naming is deterministic (same input = same output).""" - deployment = MagicMock() - deployment.workspace = "e2e-customization-test" - deployment.name = "customization-bf0cc3016831-deployment-1770393778" - - result1 = docker_backend._reconciler.get_container_name(deployment.workspace, deployment.name) - result2 = docker_backend._reconciler.get_container_name(deployment.workspace, deployment.name) - - assert result1 == result2 - - -def test_docker_backend_naming_unique_for_different_deployments(docker_backend): - """Test that different deployments get different container names.""" - deployment1 = MagicMock() - deployment1.workspace = "e2e-customization-test" - deployment1.name = "customization-bf0cc3016831-deployment-1770393778" - - deployment2 = MagicMock() - deployment2.workspace = "e2e-customization-test" - deployment2.name = "customization-bf0cc3016831-deployment-9999999999" - - name1 = docker_backend._reconciler.get_container_name(deployment1.workspace, deployment1.name) - name2 = docker_backend._reconciler.get_container_name(deployment2.workspace, deployment2.name) - - # Both should be 63 chars max - assert len(name1) <= 63 - assert len(name2) <= 63 - # But they should be different (different hash suffixes) - assert name1 != name2 - - -@pytest.mark.parametrize( - "files_url,expected", - [ - ("http://localhost:8080", "http://localhost:8080/apis/files/v2/hf"), - ("http://localhost:8080/", "http://localhost:8080/apis/files/v2/hf"), - ("http://localhost:8080/v2/hf", "http://localhost:8080/apis/files/v2/hf"), - ("http://localhost:8080/v2/hf/", "http://localhost:8080/apis/files/v2/hf"), - ("http://localhost:8080/other/path", "http://localhost:8080/apis/files/v2/hf"), - ], -) -def test_docker_backend_get_hf_compatible_files_url(mock_nmp_sdk, mock_docker_client, files_url, expected): - """Test HF-compatible files URL generation with various inputs.""" - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": files_url}, - ) - with patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config): - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config={}) - assert backend._reconciler._get_hf_compatible_files_url() == expected - - -@pytest.fixture -def docker_backend_with_dind_mode(mock_nmp_sdk, mock_docker_client, reset_shared_resource_manager_base): - """Create a DockerServiceBackend instance with DinD networking mode.""" - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000"}, - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect_gpu_device_ids, - ): - # Use explicit GPU list (simulating reserved_gpu_device_ids: "0,1,2,3") - mock_resource_config.return_value.docker = create_mock_docker_config("0,1,2,3") - mock_detect_gpu_device_ids.return_value = [0, 1, 2, 3] - config = { - "models_docker_networking_mode": "dind", - "models_docker_host_service_name": "docker", - "models_docker_port_range_start": 49152, - "models_docker_port_range_end": 49652, - } - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - backend._client = mock_docker_client - - # Mock containers.list to return empty list by default (no ports in use) - mock_docker_client.containers.list.return_value = [] - - return backend - - -def test_docker_backend_dind_mode_initialization(docker_backend_with_dind_mode): - """Test Docker backend with DinD mode initializes correctly.""" - assert docker_backend_with_dind_mode._backend_config.models_docker_networking_mode == "dind" - assert docker_backend_with_dind_mode._backend_config.models_docker_host_service_name == "docker" - assert docker_backend_with_dind_mode._backend_config.models_docker_port_range_start == 49152 - assert docker_backend_with_dind_mode._backend_config.models_docker_port_range_end == 49652 - - -@pytest.mark.asyncio -async def test_find_available_port_ignores_containers_removed_during_list(docker_backend_with_dind_mode): - """Port allocation tolerates another worker removing a container during Docker list hydration.""" - reconciler = docker_backend_with_dind_mode._reconciler - - with patch.object(reconciler, "list_containers", return_value=[]) as list_containers: - port = await reconciler.find_available_port() - - assert port == 49152 - list_containers.assert_called_once_with( - all=True, - filters={"label": f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}"}, - ignore_removed=True, - ) - - -def test_docker_backend_local_mode_initialization(mock_nmp_sdk, mock_docker_client): - """Test Docker backend with local mode (default) initializes correctly.""" - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000"}, - ) - with patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config): - config = {"models_docker_networking_mode": "local"} - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - backend._client = mock_docker_client - - assert backend._backend_config.models_docker_networking_mode == "local" - - -@pytest.mark.asyncio -async def test_docker_backend_create_with_port_forwarding( - docker_backend_with_dind_mode, sample_deployment, sample_config, mock_docker_client -): - """Test creating a deployment with port forwarding enabled.""" - # Setup mock container - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - - # Mock image found locally - mock_docker_client.images.get.return_value = MagicMock() - - # Mock containers.list to return empty (no ports in use) - mock_docker_client.containers.list.return_value = [] - - await docker_backend_with_dind_mode.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status_update = await drive_creation_to_completion(docker_backend_with_dind_mode, sample_deployment) - - # Verify status update - assert status_update is not None - assert status_update.status == "PENDING" - - # Host URL should use docker service name and first available port (8000) - assert status_update.host_url == "http://docker:49152" - - # Verify container was created with port mapping - call_args = mock_docker_client.containers.create.call_args_list[0] - assert "ports" in call_args[1] - assert "8000/tcp" in call_args[1]["ports"] - assert call_args[1]["ports"]["8000/tcp"] == 49152 - - -@pytest.mark.asyncio -async def test_docker_backend_get_status_with_port_forwarding( - docker_backend_with_dind_mode, sample_deployment, mock_docker_client -): - """Test getting deployment status with port forwarding uses correct URL.""" - # Mock a running container with port 8001 mapped - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.status = "running" - mock_container.attrs = {"State": {"StartedAt": "2024-01-01T00:00:00Z"}} - mock_container.ports = {"8000/tcp": [{"HostIp": "0.0.0.0", "HostPort": "49152"}]} - mock_container.reload = MagicMock() # Mock reload method - - # Clear side_effect and set return_value - mock_docker_client.containers.get.side_effect = None - mock_docker_client.containers.get.return_value = mock_container - - status_update = await docker_backend_with_dind_mode.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment) - ) - - # Verify status and URL uses the port from container bindings - assert status_update.status == "READY" - assert status_update.host_url == "http://docker:49152" - - -@pytest.mark.asyncio -async def test_docker_backend_delete_with_port_forwarding_releases_port( - docker_backend_with_dind_mode, sample_deployment, sample_config, mock_docker_client -): - """Test deleting a deployment releases the allocated port (port is freed automatically when container is removed).""" - # Mock NIM container - mock_nim_container = MagicMock() - mock_nim_container.id = "1234567890abcdef" - mock_nim_container.stop = MagicMock() - mock_nim_container.remove = MagicMock() - - # Mock puller container (not found - common case when NIM already started) - def mock_get_container(name): - if "puller" in name: - raise NotFound("Puller not found") - return mock_nim_container - - mock_docker_client.containers.get.side_effect = mock_get_container - - mock_volume = MagicMock() - mock_docker_client.volumes.get.side_effect = None - mock_docker_client.volumes.get.return_value = mock_volume - - status_update = await docker_backend_with_dind_mode.delete_model_deployment( - sample_deployment.workspace, sample_deployment.name - ) - - # Verify deletion succeeded - assert status_update.status == "DELETED" - - # Verify NIM container was stopped and removed - assert mock_nim_container.stop.call_count == 2 - assert mock_nim_container.remove.call_count == 2 - - -@pytest.mark.asyncio -async def test_docker_backend_port_exhaustion_error( - docker_backend_with_dind_mode, sample_deployment, sample_config, mock_docker_client -): - """Test that deployment fails gracefully when ports are exhausted.""" - # Mock image found locally - mock_docker_client.images.get.return_value = MagicMock() - - # Mock containers.list to return containers using all ports in range (8000-9000) - # Create mock containers with all ports allocated - mock_containers = [] - for port in range(49152, 49652 + 1): # All ports in range - mock_cont = MagicMock() - mock_cont.ports = {"8000/tcp": [{"HostIp": "0.0.0.0", "HostPort": str(port)}]} - mock_containers.append(mock_cont) - - mock_docker_client.containers.list.return_value = mock_containers - - await docker_backend_with_dind_mode.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status_update = await drive_creation_to_completion(docker_backend_with_dind_mode, sample_deployment) - - # Should return ERROR status (port exhaustion happens during container creation stage) - assert status_update.status == "ERROR" - assert "port" in status_update.status_message.lower() - assert "no ports available" in status_update.status_message.lower() - - -@pytest.mark.asyncio -async def test_docker_backend_multiple_deployments_unique_ports( - docker_backend_with_dind_mode, sample_config, mock_docker_client -): - """Test that multiple deployments get unique ports.""" - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - - # Track allocated containers to simulate real Docker state - allocated_containers = [] - - def mock_containers_list(**kwargs): - return allocated_containers - - mock_docker_client.containers.list.side_effect = mock_containers_list - - # Create three different deployments - deployments = [] - ports = [] - - for i in range(3): - deployment = MagicMock() - deployment.workspace = "test" - deployment.name = f"model-{i}" - deployments.append(deployment) - - await docker_backend_with_dind_mode.create_model_deployment( - ModelContext(model_deployment=deployment, model_deployment_config=sample_config) - ) - status = await drive_creation_to_completion(docker_backend_with_dind_mode, deployment) - - # Extract port from host_url (format: http://docker:PORT) - print(f"status.host_url: {status.host_url}") - port = int(status.host_url.split(":")[-1]) - ports.append(port) - - # Add mock container with allocated port to the list - mock_cont = MagicMock() - mock_cont.ports = {"8000/tcp": [{"HostIp": "0.0.0.0", "HostPort": str(port)}]} - allocated_containers.append(mock_cont) - - # Verify all ports are unique, valid, and sequential starting from 8000 - assert len(set(ports)) == 3 - assert ports == [49152, 49153, 49154] - - -# ============================================================================= -# Tests for multi-LLM image detection and SFT model handling -# ============================================================================= - - -@pytest.fixture -def multi_llm_config(): - """Create a config using the default multi-LLM image (no image_name specified).""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name=None, # Will use default multi-LLM image - image_tag=None, - model_name="test-sft-model", - model_namespace="default", - lora_enabled=False, - additional_envs=None, - ) - return config - - -@pytest.fixture -def explicit_multi_llm_config(): - """Create a config explicitly using the multi-LLM image.""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/nvidia/llm-nim", # Explicit multi-LLM image - image_tag="latest", - model_name="test-sft-model", - model_namespace="default", - lora_enabled=False, - additional_envs=None, - ) - return config - - -@pytest.fixture -def model_specific_nim_config(): - """Create a config using a model-specific NIM image (not multi-LLM).""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", # Model-specific NIM - image_tag="1.8.6", - model_name="test-sft-model", - model_namespace="default", - lora_enabled=False, - additional_envs=None, - ) - return config - - -@pytest.fixture -def sft_model_entity(): - """Create a mock SFT model entity with full weights.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-sft-model" - model_entity.spec = None - model_entity.finetuning_type = "all_weights" - peft_mock = MagicMock() - peft_mock.finetuning_type = "all_weights" - model_entity.peft = peft_mock - return model_entity - - -def _setup_puller_mock_for_polling(mock_docker_client, deployment, exit_code=0): - """Configure mock containers for the polled puller flow. - - Sets up containers.run to return a puller mock, and containers.get - to return it for reload-based polling (exited with given exit_code). - For non-puller containers, falls back to NotFound (the default mock_docker_client behavior). - """ - mock_puller = MagicMock() - mock_puller.id = "puller123456789" - mock_puller.name = f"md-puller-{deployment.workspace}-{deployment.name}" - mock_puller.status = "exited" - mock_puller.attrs = {"State": {"ExitCode": exit_code}} - mock_puller.reload = MagicMock() - mock_puller.remove = MagicMock() - mock_puller.logs = MagicMock(return_value=b"puller output") - - mock_docker_client.containers.run.return_value = mock_puller - - def get_container_side_effect(name): - if "puller" in name: - return mock_puller - raise NotFound("Container not found") - - mock_docker_client.containers.get.side_effect = get_container_side_effect - return mock_puller - - -@pytest.mark.asyncio -async def test_multi_llm_sft_model_now_runs_puller_old_test_updated( - docker_backend, sample_deployment, multi_llm_config, sft_model_entity, mock_docker_client -): - """Test that multi-LLM image with SFT model NOW runs the model puller (updated behavior).""" - multi_llm_config.model_spec.lora_enabled = True - - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=multi_llm_config, - model_entity=sft_model_entity, - ) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update is not None - assert status_update.status == "PENDING" - - puller_calls = [ - call - for call in mock_docker_client.containers.run.call_args_list - if call[1].get("labels", {}).get("nmp.nvidia.com/container-type") == "model-puller" - ] - assert len(puller_calls) == 1, "Model puller SHOULD run for multi-LLM deployments" - - assert mock_docker_client.containers.create.call_count == 2, "NIM container and sidecar should be created" - - nim_call_args = mock_docker_client.containers.create.call_args_list[0] - env_vars = nim_call_args[1]["environment"] - assert env_vars["NIM_MODEL_NAME"] == "/model-store" - assert "NIM_FT_MODEL" not in env_vars - assert "HF_ENDPOINT" not in env_vars - - sidecar_call_args = mock_docker_client.containers.create.call_args_list[1] - sidecar_env_vars = sidecar_call_args[1]["environment"] - assert "NMP_MODELS_URL" in sidecar_env_vars - assert "NMP_FILES_URL" in sidecar_env_vars - - -@pytest.mark.asyncio -async def test_model_specific_nim_sft_model_runs_puller( - docker_backend, sample_deployment, model_specific_nim_config, sft_model_entity, mock_docker_client -): - """Test that model-specific NIM with SFT model DOES run the model puller.""" - model_specific_nim_config.model_spec.lora_enabled = True - - mock_nim_container = MagicMock() - mock_nim_container.id = "nim1234567890ab" - mock_nim_container.start = MagicMock() - - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.create.return_value = mock_nim_container - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=model_specific_nim_config, - model_entity=sft_model_entity, - ) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update is not None - assert status_update.status == "PENDING" - - puller_calls = [ - call - for call in mock_docker_client.containers.run.call_args_list - if call[1].get("labels", {}).get("nmp.nvidia.com/container-type") == "model-puller" - ] - assert len(puller_calls) == 1 - - assert mock_docker_client.containers.create.call_count == 2 - nim_call_args = mock_docker_client.containers.create.call_args_list[0] - env_vars = nim_call_args[1]["environment"] - assert env_vars.get("NIM_FT_MODEL") == "/model-store" - sidecar_call_args = mock_docker_client.containers.create.call_args_list[1] - sidecar_env_vars = sidecar_call_args[1]["environment"] - assert "NMP_MODELS_URL" in sidecar_env_vars - assert "NMP_FILES_URL" in sidecar_env_vars - - -@pytest.mark.asyncio -async def test_explicit_multi_llm_image_now_runs_puller( - docker_backend, sample_deployment, explicit_multi_llm_config, sft_model_entity_with_artifact, mock_docker_client -): - """Test that explicitly specifying the multi-LLM image now runs the puller.""" - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=explicit_multi_llm_config, - model_entity=sft_model_entity_with_artifact, - ) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update is not None - assert status_update.status == "PENDING" - - puller_calls = [ - call - for call in mock_docker_client.containers.run.call_args_list - if call[1].get("labels", {}).get("nmp.nvidia.com/container-type") == "model-puller" - ] - assert len(puller_calls) == 1 - - -@pytest.mark.asyncio -async def test_multi_llm_non_sft_model_fails_without_supported_weights_type( - docker_backend, sample_deployment, multi_llm_config, mock_docker_client -): - """Test that multi-LLM with non-SFT model (no model entity) fails due to unsupported weights type.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=multi_llm_config, model_entity=None) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update is not None - assert status_update.status == "ERROR" - assert ( - "model puller" in status_update.status_message.lower() - or "model weights" in status_update.status_message.lower() - or "unsupported" in status_update.status_message.lower() - ) - - -def test_default_multi_llm_image_config(): - """Test that the default NIM image is the multi-LLM image.""" - from nmp.core.models.controllers.backends.docker.config import DockerBackendConfig - - config = DockerBackendConfig() - assert config.default_nimservice_image == "nvcr.io/nim/nvidia/llm-nim", ( - "Default NIM image should be the multi-LLM image" - ) - - -def test_default_vllm_image_config(): - """Test that a default vLLM image and tag are configured.""" - from nmp.core.models.controllers.backends.docker.config import DockerBackendConfig - - config = DockerBackendConfig() - assert config.default_vllm_image == "vllm/vllm-openai", "Default vLLM image should be the vllm-openai image" - # The exact tag is config-driven and bumped over time; just assert one is set. - assert config.default_vllm_image_tag, "A default vLLM image tag should be configured" - - -# ============================================================================= -# Tests for model puller with different model weights types -# ============================================================================= - - -@pytest.fixture -def sft_model_entity_with_artifact(): - """Create a mock SFT model entity with fileset.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-sft-model" - model_entity.spec = None - model_entity.finetuning_type = "all_weights" - # Setup PEFT mock for SFT model - peft_mock = MagicMock() - peft_mock.finetuning_type = "all_weights" - model_entity.peft = peft_mock - # Setup fileset - model_entity.fileset = "hf://workspace/test-fileset" - return model_entity - - -@pytest.fixture -def huggingface_model_config(): - """Create a config for HuggingFace model deployment.""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - model_revision="main", - lora_enabled=False, - additional_envs=None, - ) - return config - - -@pytest.mark.asyncio -async def test_multi_llm_now_runs_puller( - docker_backend, sample_deployment, multi_llm_config, sft_model_entity_with_artifact, mock_docker_client -): - """Test that multi-LLM deployments now run the model puller (updated behavior).""" - multi_llm_config.model_spec.lora_enabled = True - - mock_nim_container = MagicMock() - mock_nim_container.id = "nim1234567890ab" - mock_nim_container.start = MagicMock() - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.create.return_value = mock_nim_container - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=multi_llm_config, - model_entity=sft_model_entity_with_artifact, - ) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update is not None - assert status_update.status == "PENDING" - - puller_calls = [ - call - for call in mock_docker_client.containers.run.call_args_list - if call[1].get("labels", {}).get("nmp.nvidia.com/container-type") == "model-puller" - ] - assert len(puller_calls) == 1 - - assert mock_docker_client.containers.create.call_count == 2 - - nim_call_args = mock_docker_client.containers.create.call_args - env_vars = nim_call_args[1]["environment"] - assert env_vars.get("NIM_MODEL_NAME") == "/model-store" - assert "HF_ENDPOINT" not in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_multi_llm_updated_behavior(docker_backend, sample_deployment, multi_llm_config): - """Test that _compile_env_vars for multi-LLM uses updated behavior.""" - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - multi_llm_config, - model_entity=None, - model_weights_type=ModelWeightsType.FILES_SERVICE, - is_multi_llm=True, - ) - - # Multi-LLM should set NIM_MODEL_NAME to /model-store - assert env_vars.get("NIM_MODEL_NAME") == "/model-store" - - # Should NOT have HF_ENDPOINT (removed in updated behavior) - assert "HF_ENDPOINT" not in env_vars - - # Should have NIM_SERVED_MODEL_NAME (multi_llm_config has model_name set) - assert "NIM_SERVED_MODEL_NAME" in env_vars - - -@pytest.fixture -def nim_only_config(): - """Create a deployment config with only NIM image info, no model_name/model_namespace. - - This simulates deployments like: - nemo inference deployment-configs create \ - --name "nemoguard-jailbreak-config" \ - --nim-deployment '{"gpu": 1, "image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", "image_tag": "1.10.1"}' - """ - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", - image_tag="1.10.1", - model_name=None, # Not set - model_namespace=None, # Not set - lora_enabled=False, - additional_envs=None, - ) - return config - - -@pytest.mark.asyncio -async def test_compile_env_vars_without_model_name(docker_backend, sample_deployment, nim_only_config): - """Test that _compile_env_vars works when model_name is not provided. - - Regression test: Previously this would raise NameError because model_fqdn - was only defined inside the 'if nim_config.model_name:' block. - """ - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - nim_only_config, - model_entity=None, - model_weights_type=ModelWeightsType.BAKED_CONTAINER, - is_multi_llm=False, - ) - - # Should NOT have NIM_SERVED_MODEL_NAME when model_name is not provided - assert "NIM_SERVED_MODEL_NAME" not in env_vars - - # Should still have other required env vars - assert "NIM_GUIDED_DECODING_BACKEND" in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_with_trust_remote_code_false(docker_backend, sample_deployment, sample_config): - """Test that _compile_env_vars does NOT set NIM_FORCE_TRUST_REMOTE_CODE when trust_remote_code is False.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.trust_remote_code = False - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - model_weights_type=ModelWeightsType.BAKED_CONTAINER, - is_multi_llm=False, - ) - - # Should have model entity env vars - assert env_vars.get("NMP_MODEL_ENTITY_WORKSPACE") == "default" - assert env_vars.get("NMP_MODEL_ENTITY_NAME") == "test-model" - - # Should NOT have NIM_FORCE_TRUST_REMOTE_CODE when trust_remote_code is False - assert "NIM_FORCE_TRUST_REMOTE_CODE" not in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_with_trust_remote_code_true(docker_backend, sample_deployment, sample_config): - """Test that _compile_env_vars sets NIM_FORCE_TRUST_REMOTE_CODE when trust_remote_code is True.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.trust_remote_code = True - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - model_weights_type=ModelWeightsType.BAKED_CONTAINER, - is_multi_llm=False, - ) - - # Should have model entity env vars - assert env_vars.get("NMP_MODEL_ENTITY_WORKSPACE") == "default" - assert env_vars.get("NMP_MODEL_ENTITY_NAME") == "test-model" - - # Should have NIM_FORCE_TRUST_REMOTE_CODE set to "1" when trust_remote_code is True - assert env_vars.get("NIM_FORCE_TRUST_REMOTE_CODE") == "1" - - -@pytest.mark.asyncio -async def test_compile_env_vars_without_model_entity(docker_backend, sample_deployment, sample_config): - """Test that _compile_env_vars works correctly when model_entity is None.""" - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=None, - model_weights_type=ModelWeightsType.BAKED_CONTAINER, - is_multi_llm=False, - ) - - # Should NOT have model entity env vars when model_entity is None - assert "NMP_MODEL_ENTITY_WORKSPACE" not in env_vars - assert "NMP_MODEL_ENTITY_NAME" not in env_vars - assert "NIM_FORCE_TRUST_REMOTE_CODE" not in env_vars - - -# ============================================================================= -# Tests for retry logic -# ============================================================================= - - -def test_should_retry_docker_error_with_retryable_errors(): - """Test that _should_retry_docker_error returns True for retryable errors.""" - from docker.errors import APIError - from nmp.core.models.controllers.backends.docker.creation_reconciler import _should_retry_docker_error - from requests.exceptions import ConnectionError as RequestsConnectionError - from requests.exceptions import ReadTimeout - from urllib3.exceptions import ReadTimeoutError as Urllib3ReadTimeoutError - - # These should be retried - assert _should_retry_docker_error(APIError("API error")) is True - assert _should_retry_docker_error(ReadTimeout("Read timeout")) is True - assert _should_retry_docker_error(Urllib3ReadTimeoutError(None, None, "timeout")) is True - assert _should_retry_docker_error(RequestsConnectionError("Connection error")) is True - assert _should_retry_docker_error(TimeoutError("Timeout")) is True - - -def test_should_retry_docker_error_with_non_retryable_errors(): - """Test that _should_retry_docker_error returns False for non-retryable errors.""" - from docker.errors import ImageNotFound, NotFound - from nmp.core.models.controllers.backends.docker.creation_reconciler import _should_retry_docker_error - - # These should NOT be retried (resource doesn't exist) - assert _should_retry_docker_error(NotFound("Container not found")) is False - assert _should_retry_docker_error(ImageNotFound("Image not found")) is False - - # Other exceptions should not be retried - assert _should_retry_docker_error(ValueError("Value error")) is False - assert _should_retry_docker_error(KeyError("Key error")) is False - - -# ============================================================================= -# Tests for remote Docker host detection -# ============================================================================= - - -def test_is_remote_docker_host_with_tcp(docker_backend): - """Test _is_remote_docker_host returns True for TCP Docker host.""" - with patch.dict("os.environ", {"DOCKER_HOST": "tcp://docker:2375"}): - assert docker_backend._reconciler._is_remote_docker_host() is True - - -def test_is_remote_docker_host_with_unix_socket(docker_backend): - """Test _is_remote_docker_host returns False for Unix socket.""" - with patch.dict("os.environ", {"DOCKER_HOST": "unix:///var/run/docker.sock"}): - assert docker_backend._reconciler._is_remote_docker_host() is False - - -def test_is_remote_docker_host_with_no_env(docker_backend): - """Test _is_remote_docker_host returns False when DOCKER_HOST not set.""" - with patch.dict("os.environ", {}, clear=True): - assert docker_backend._reconciler._is_remote_docker_host() is False - - -def test_is_port_free_skips_check_for_remote_docker(docker_backend): - """Test _is_port_free skips local port check for remote Docker host.""" - with patch.dict("os.environ", {"DOCKER_HOST": "tcp://docker:2375"}): - # Should return True immediately without attempting to bind - assert docker_backend._reconciler._is_port_free(8000) is True - - -# ============================================================================= -# Tests for host URL generation -# ============================================================================= - - -def test_get_host_url_dond_mode(): - """Test _get_host_url returns container name URL in DonD mode.""" - from unittest.mock import MagicMock, patch - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock_docker, - ): - mock_docker.return_value = MagicMock() - - mock_nmp_sdk = AsyncMock() - config = {"models_docker_networking_mode": "dond"} - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - - url = backend._reconciler.get_host_url(container_name="test-container", host_port=8500) - assert url == "http://test-container:8000" - - -def test_get_host_url_dind_mode(): - """Test _get_host_url returns Docker service URL with port in DinD mode.""" - from unittest.mock import AsyncMock, MagicMock, patch - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock_docker, - ): - mock_docker.return_value = MagicMock() - - mock_nmp_sdk = AsyncMock() - config = { - "models_docker_networking_mode": "dind", - "models_docker_host_service_name": "docker", - } - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - - url = backend._reconciler.get_host_url(container_name="test-container", host_port=8500) - assert url == "http://docker:8500" - - -def test_get_host_url_local_mode(): - """Test _get_host_url returns localhost URL in local mode.""" - from unittest.mock import AsyncMock, MagicMock, patch - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock_docker, - ): - mock_docker.return_value = MagicMock() - - mock_nmp_sdk = AsyncMock() - config = {"models_docker_networking_mode": "local"} - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - - url = backend._reconciler.get_host_url(container_name="test-container", host_port=8500) - assert url == "http://localhost:8500" - - -# ============================================================================= -# Tests for Files service URL handling in DonD mode -# ============================================================================= - - -def test_get_hf_compatible_files_url_replaces_localhost_in_dond(): - """Test _get_hf_compatible_files_url replaces localhost with container name in DonD mode.""" - from unittest.mock import AsyncMock, MagicMock, patch - - # Default base_url is http://localhost:8080; get_service_url("files") returns base_url - platform_config = PlatformConfig() # type: ignore[abstract] - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock_docker, - ): - mock_docker.return_value = MagicMock() - - mock_nmp_sdk = AsyncMock() - config = { - "models_docker_networking_mode": "dond", - "models_docker_container_name": "nmp-container", - } - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - - url = backend._reconciler._get_hf_compatible_files_url() - assert url == "http://nmp-container:8080/apis/files/v2/hf" - - -def test_get_hf_compatible_files_url_no_replacement_in_local_mode(): - """Test _get_hf_compatible_files_url doesn't replace localhost in local mode.""" - from unittest.mock import AsyncMock, MagicMock, patch - - # Default base_url is http://localhost:8080 - platform_config = PlatformConfig() # type: ignore[abstract] - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.core.models.controllers.backends.docker.backend.docker.from_env") as mock_docker, - ): - mock_docker.return_value = MagicMock() - - mock_nmp_sdk = AsyncMock() - config = {"models_docker_networking_mode": "local"} - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - - url = backend._reconciler._get_hf_compatible_files_url() - assert url == "http://localhost:8080/apis/files/v2/hf" - - -@pytest.mark.asyncio -async def test_compile_env_vars_multi_llm_updated(docker_backend, sample_deployment, multi_llm_config): - """Test that _compile_env_vars sets correct env vars for multi-LLM deployments (updated behavior).""" - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - multi_llm_config, - model_entity=None, - model_weights_type=ModelWeightsType.FILES_SERVICE, - is_multi_llm=True, - ) - - # Multi-LLM should NOT have HF_ENDPOINT anymore (updated behavior) - assert "HF_ENDPOINT" not in env_vars, "Multi-LLM should NOT have HF_ENDPOINT in NIM container (updated behavior)" - assert "NIM_MODEL_NAME" in env_vars - assert env_vars["NIM_MODEL_NAME"] == "/model-store", "Multi-LLM should use /model-store (updated behavior)" - assert "NIM_SERVED_MODEL_NAME" in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_model_specific_nim_sft(docker_backend, sample_deployment, model_specific_nim_config): - """Test that _compile_env_vars sets NIM_FT_MODEL for model-specific NIM with SFT.""" - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - model_specific_nim_config, - model_entity=None, - model_weights_type=ModelWeightsType.FILES_SERVICE, - is_multi_llm=False, - ) - - assert env_vars.get("NIM_FT_MODEL") == "/model-store", "Model-specific NIM SFT should have NIM_FT_MODEL" - assert env_vars.get("NIM_CUSTOM_MODEL") == "/model-store", "Model-specific NIM SFT should have NIM_CUSTOM_MODEL" - # Should NOT have HF_ENDPOINT (model-specific NIMs don't use it) - assert "HF_ENDPOINT" not in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_multi_llm_sft_no_ft_model(docker_backend, sample_deployment, multi_llm_config): - """Test that _compile_env_vars does NOT set NIM_FT_MODEL for multi-LLM even with SFT flag.""" - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - multi_llm_config, - model_entity=None, - model_weights_type=ModelWeightsType.FILES_SERVICE, - is_multi_llm=True, - ) - - # Multi-LLM should NOT have NIM_FT_MODEL even when puller ran (weights from Files). - assert "NIM_FT_MODEL" not in env_vars, "Multi-LLM should NOT have NIM_FT_MODEL" - assert "NIM_CUSTOM_MODEL" not in env_vars - # Updated behavior: HF_ENDPOINT is not set in NIM container for multi-LLM (only in puller) - assert "HF_ENDPOINT" not in env_vars, "Multi-LLM should NOT have HF_ENDPOINT in NIM container (updated behavior)" - assert env_vars.get("NIM_MODEL_NAME") == "/model-store", "Multi-LLM should use /model-store" - - -# ============================================================================= -# Tests for DonD networking mode (quickstart setup) -# ============================================================================= - - -@pytest.fixture -def docker_backend_with_dond_mode(mock_nmp_sdk, mock_docker_client, reset_shared_resource_manager_base): - """Create a DockerServiceBackend instance with DonD networking mode. - - This simulates the DonD (Docker-on-Docker) quickstart setup where NIMs need to - join the same network as the NeMo Platform container. - """ - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect_gpu_device_ids, - ): - # Use explicit GPU list (simulating reserved_gpu_device_ids: "0,1,2,3") - mock_resource_config.return_value.docker = create_mock_docker_config("0,1,2,3") - mock_detect_gpu_device_ids.return_value = [0, 1, 2, 3] - config = { - "models_docker_networking_mode": "dond", - "models_docker_network": "nmp-quickstart-network", - "models_docker_port_range_start": 49152, - "models_docker_port_range_end": 49652, - } - backend = DockerServiceBackend(nmp_sdk=mock_nmp_sdk, config=config) - backend._client = mock_docker_client - - # Mock containers.list to return empty list by default (no ports in use) - mock_docker_client.containers.list.return_value = [] - - return backend - - -def test_should_attach_network_with_dond_mode(docker_backend_with_dond_mode): - """Test that _should_attach_network returns True when using DonD mode.""" - assert docker_backend_with_dond_mode._reconciler._should_attach_network() is True - - -def test_should_attach_network_with_local_mode(docker_backend): - """Test that _should_attach_network returns False when using local mode (default).""" - assert docker_backend._reconciler._should_attach_network() is False - - -def test_should_attach_network_with_dind_mode(docker_backend_with_dind_mode): - """Test that _should_attach_network returns False when using DinD mode.""" - assert docker_backend_with_dind_mode._reconciler._should_attach_network() is False - - -@pytest.mark.asyncio -async def test_docker_backend_create_with_dond_mode_uses_container_name_url( - docker_backend_with_dond_mode, sample_deployment, sample_config, mock_docker_client, sample_resource_names -): - """Test that create_model_deployment uses container name URL when using DonD mode. - - In DonD mode, containers communicate via the shared network using container names. - """ - # Setup mock container - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - - # Mock image found locally - mock_docker_client.images.get.return_value = MagicMock() - - # Mock containers.list to return empty (no ports in use) - mock_docker_client.containers.list.return_value = [] - - await docker_backend_with_dond_mode.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status_update = await drive_creation_to_completion(docker_backend_with_dond_mode, sample_deployment) - - # Verify status update - assert status_update is not None - assert status_update.status == "PENDING" - - # Host URL should use container name (DonD mode uses container names) - assert status_update.host_url == sample_resource_names["host_url"] - - # Verify container was created with the network - call_args = mock_docker_client.containers.create.call_args_list[0] - assert call_args[1]["network"] == "nmp-quickstart-network" - - -@pytest.mark.asyncio -async def test_docker_backend_get_status_with_dond_mode_uses_container_name_url( - docker_backend_with_dond_mode, sample_deployment, mock_docker_client, sample_resource_names -): - """Test that get_model_deployment_status uses container name URL when using DonD mode.""" - # Mock a running container with port mapping (should be ignored for URL in DonD mode) - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.status = "running" - mock_container.attrs = {"State": {"StartedAt": "2024-01-01T00:00:00Z"}} - mock_container.ports = {"8000/tcp": [{"HostIp": "0.0.0.0", "HostPort": "49200"}]} - mock_container.reload = MagicMock() - - # Clear side_effect and set return_value - mock_docker_client.containers.get.side_effect = None - mock_docker_client.containers.get.return_value = mock_container - - status_update = await docker_backend_with_dond_mode.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment) - ) - - # Verify status and URL uses container name (DonD mode ignores port bindings) - assert status_update.status == "READY" - assert status_update.host_url == sample_resource_names["host_url"] - - -@pytest.mark.asyncio -async def test_docker_backend_dond_mode_container_joins_network( - docker_backend_with_dond_mode, sample_deployment, sample_config, mock_docker_client -): - """Test that NIM container is created with the network attached in DonD mode.""" - # Enable lora to trigger sidecar creation (2 containers) - sample_config.model_spec.lora_enabled = True - - # Setup mock container - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await docker_backend_with_dond_mode.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend_with_dond_mode, sample_deployment) - - # Verify every container was created with network attached - assert mock_docker_client.containers.create.call_count == 2, "Should create 2 containers" - call_args = mock_docker_client.containers.create.call_args_list - - assert "network" in call_args[0][1], "Container should have network specified in DonD mode" - assert call_args[0][1]["network"] == "nmp-quickstart-network" - - -def test_docker_backend_dond_mode_initialization(docker_backend_with_dond_mode): - """Test Docker backend with DonD mode initializes correctly.""" - assert docker_backend_with_dond_mode._backend_config.models_docker_networking_mode == "dond" - assert docker_backend_with_dond_mode._backend_config.models_docker_network == "nmp-quickstart-network" - - -# ============================================================================= -# GPU Pool Tests -# ============================================================================= - - -@pytest.fixture -def reset_shared_resource_manager(): - """Reset SharedResourceManager singleton before and after test.""" - from nmp.common.resources import SharedResourceManager - - SharedResourceManager.reset_instance() - yield - SharedResourceManager.reset_instance() - - -@pytest.fixture -def docker_backend_with_gpu_pool(mock_nmp_sdk, mock_docker_client, reset_shared_resource_manager): - """Create a DockerServiceBackend instance with GPU pool enabled via explicit config.""" - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect_gpu_device_ids, - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - ): - # Configure mocks to simulate 4 GPUs via explicit config - mock_detect_gpu_device_ids.return_value = [0, 1, 2, 3] - mock_resource_config.return_value.docker = create_mock_docker_config("0,1,2,3") - backend = DockerServiceBackend( - nmp_sdk=mock_nmp_sdk, - config={}, - ) - backend._client = mock_docker_client - return backend - - -@pytest.fixture -def docker_backend_without_gpu_pool(mock_nmp_sdk, mock_docker_client, reset_shared_resource_manager): - """Create a DockerServiceBackend instance without GPU pool (empty config).""" - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect_gpu_device_ids, - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - ): - mock_detect_gpu_device_ids.return_value = None - # Empty string means no GPUs configured - mock_resource_config.return_value.docker = create_mock_docker_config("") - backend = DockerServiceBackend( - nmp_sdk=mock_nmp_sdk, - config={}, - ) - backend._client = mock_docker_client - return backend - - -def test_docker_backend_gpu_pool_initialization(docker_backend_with_gpu_pool): - """Test that GPU pool is initialized when GPUs are detected.""" - assert docker_backend_with_gpu_pool._gpu_pool is not None - assert docker_backend_with_gpu_pool._gpu_pool.num_reserved_gpus == 4 - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - -def test_docker_backend_no_gpu_pool_without_gpus(docker_backend_without_gpu_pool): - """Test that GPU pool is None when no GPUs are detected.""" - assert docker_backend_without_gpu_pool._gpu_pool is None - - -@pytest.mark.asyncio -async def test_docker_backend_allocates_gpu_from_pool( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that creating a deployment allocates GPUs from the pool.""" - # Setup mock container - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - # Initially all GPUs available - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - - # One GPU should be allocated - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 - - # Verify container was created with specific device_ids - call_args = mock_docker_client.containers.create.call_args_list[0] - device_requests = call_args[1]["device_requests"] - assert len(device_requests) == 1 - # Should have driver="nvidia" and specific device_ids (not count) - assert device_requests[0].driver == "nvidia" - assert device_requests[0].device_ids is not None - assert len(device_requests[0].device_ids) == 1 - - -@pytest.mark.asyncio -async def test_docker_backend_fails_without_gpu_pool( - docker_backend_without_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that without GPU pool (no GPUs detected), deployment returns ERROR.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await docker_backend_without_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status_update = await drive_creation_to_completion(docker_backend_without_gpu_pool, sample_deployment) - - assert status_update.status == "ERROR" - assert "no gpus available" in status_update.status_message.lower() - - mock_docker_client.containers.create.assert_not_called() - - -@pytest.mark.asyncio -async def test_docker_backend_releases_gpu_on_delete( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that deleting a deployment releases GPUs back to the pool.""" - # Setup mock container for creation - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_container.stop = MagicMock() - mock_container.remove = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 - - # Setup mock for deletion - mock_docker_client.containers.get.side_effect = None - mock_docker_client.containers.get.return_value = mock_container - - mock_volume = MagicMock() - mock_docker_client.volumes.get.side_effect = None - mock_docker_client.volumes.get.return_value = mock_volume - - # Delete deployment - should release GPU - await docker_backend_with_gpu_pool.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) - - # GPU should be released back to pool - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - -@pytest.mark.asyncio -async def test_docker_backend_gpu_allocation_failure( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that deployment fails gracefully when no GPUs are available.""" - # Setup mock container - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - # Allocate all GPUs first - for i in range(4): - docker_backend_with_gpu_pool._gpu_pool.allocate_gpu(f"other-workload-{i}", num_requested=1) - - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 0 - - # Try to create deployment - GPU allocation failure happens during container creation stage - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status_update = await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - - assert status_update.status == "ERROR" - assert "GPU" in status_update.status_message - assert status_update.error_details["stage"] == "gpu_allocation" - - -@pytest.mark.asyncio -async def test_docker_backend_releases_gpu_on_port_allocation_failure( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that GPUs are released when port allocation fails after GPU allocation. - - This test verifies the fix for the GPU leak bug where GPUs were allocated - but not released when subsequent port allocation failed. - """ - # Setup mocks - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - # Make _find_available_port return None to simulate port exhaustion - docker_backend_with_gpu_pool._reconciler.find_available_port = AsyncMock(return_value=None) - - # Initial pool should have 4 GPUs available - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - # Try to create deployment - port allocation failure happens during container creation stage - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status_update = await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - - assert status_update.status == "ERROR" - assert "port" in status_update.status_message.lower() - assert status_update.error_details["error"] == "Port allocation failed" - - # Critical: GPU should be released back to the pool (not leaked) - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - -@pytest.mark.asyncio -async def test_docker_backend_releases_gpu_on_container_creation_failure( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that GPUs are released when container creation fails after GPU allocation. - - This test verifies the fix for the GPU leak bug where GPUs were allocated - but not released when Docker container creation raised an exception. - """ - from docker.errors import APIError - - # Setup mocks - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - # Make containers.create raise an APIError - mock_docker_client.containers.create.side_effect = APIError("Docker API error: container creation failed") - - # Initial pool should have 4 GPUs available - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - # Try to create deployment - container creation failure happens during CREATING_CONTAINER stage - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status_update = await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - - assert status_update.status == "ERROR" - assert "Docker API error" in status_update.status_message - - # Critical: GPU should be released back to the pool (not leaked) - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - -@pytest.mark.asyncio -async def test_docker_backend_multi_gpu_allocation(mock_nmp_sdk, mock_docker_client, reset_shared_resource_manager): - """Test that multi-GPU deployments allocate the correct number of GPUs.""" - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with ( - patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config), - patch("nmp.common.resources.manager.get_platform_config") as mock_resource_config, - patch("nmp.common.resources.manager.detect_gpu_device_ids") as mock_detect_gpu_device_ids, - ): - mock_resource_config.return_value.docker = create_mock_docker_config("0,1,2,3") - mock_detect_gpu_device_ids.return_value = [0, 1, 2, 3] - backend = DockerServiceBackend( - nmp_sdk=mock_nmp_sdk, - config={}, - ) - backend._client = mock_docker_client - - # Create deployment config requesting 2 GPUs - multi_gpu_config = MagicMock() - set_deployment_config( - multi_gpu_config, - gpu=2, # Request 2 GPUs - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - ) - - sample_deployment = MagicMock() - sample_deployment.workspace = "default" - sample_deployment.name = "multi-gpu-deployment" - sample_deployment.entity_version = 1 - sample_deployment.status = "CREATED" - - # Setup mock container - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=multi_gpu_config) - ) - await drive_creation_to_completion(backend, sample_deployment) - - # Should have allocated 2 GPUs - assert backend._gpu_pool.get_available_count() == 2 - - # Verify container was created with 2 device_ids - call_args = mock_docker_client.containers.create.call_args_list[0] - device_requests = call_args[1]["device_requests"] - assert len(device_requests[0].device_ids) == 2 - - -# ============================================================================= -# Tests for GPU release during get_model_deployment_status -# ============================================================================= - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "container_state", - [ - pytest.param("exited", id="exited_container"), - pytest.param("dead", id="dead_container"), - ], -) -async def test_docker_backend_releases_gpu_on_status_check_terminated( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client, container_state -): - """Test that GPUs are released when status check finds container in exited/dead state. - - This ensures that GPU resources are reclaimed when a container has terminated - unexpectedly and the status check discovers this. - """ - # Setup mock container for creation - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_container.reload = MagicMock() - mock_container.logs = MagicMock(return_value=b"Container terminated") - mock_container.attrs = {"State": {"ExitCode": 1}} - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 - - # Simulate container in terminated state - mock_container.status = container_state - mock_container.ports = {} - mock_docker_client.containers.get.side_effect = None - mock_docker_client.containers.get.return_value = mock_container - - # Check status - should release GPU when container is terminated - status_update = await docker_backend_with_gpu_pool.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment) - ) - - # Verify ERROR status returned - assert status_update.status == "ERROR" - assert "exited" in status_update.status_message.lower() - - # GPU should be released back to pool - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - -@pytest.mark.asyncio -async def test_docker_backend_releases_gpu_on_status_check_lost( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that GPUs are released when status check finds container is missing (LOST). - - This ensures that GPU resources are reclaimed when a container has been - manually deleted or otherwise disappeared. - """ - # Setup mock container for creation - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 - - # Simulate container not found (was deleted externally) - mock_docker_client.containers.get.side_effect = NotFound("Container not found") - - # Check status - should release GPU when container is missing - status_update = await docker_backend_with_gpu_pool.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment) - ) - - # Verify LOST status returned - assert status_update.status == "LOST" - assert "not found" in status_update.status_message.lower() - - # GPU should be released back to pool - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - - -@pytest.mark.asyncio -async def test_docker_backend_status_check_does_not_release_gpu_when_running( - docker_backend_with_gpu_pool, sample_deployment, sample_config, mock_docker_client -): - """Test that GPUs are NOT released when container is still running. - - This ensures we don't accidentally release GPUs for healthy deployments. - """ - # Setup mock container for creation - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_container.reload = MagicMock() - mock_container.attrs = {"State": {"StartedAt": "2024-01-01T00:00:00Z"}} - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - # Set networking mode to dond for predictable URL - docker_backend_with_gpu_pool._backend_config.models_docker_networking_mode = "dond" - - # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 - - # Simulate container still running - mock_container.status = "running" - mock_container.ports = {"8000/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8000"}]} - mock_docker_client.containers.get.side_effect = None - mock_docker_client.containers.get.return_value = mock_container - - # Check status - should NOT release GPU when container is running - status_update = await docker_backend_with_gpu_pool.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment) - ) - - # Verify READY status returned - assert status_update.status == "READY" - - # GPU should still be allocated (not released) - assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 - - -# ============================================================================= -# Tests for FILES_SERVICE weights type (non-SFT models with model_name) -# ============================================================================= - - -@pytest.fixture -def multi_llm_config_with_model_name(): - """Create a config for multi-LLM image with model_name (no model entity). - - This simulates the deployment: - nemo inference deployment-configs create \ - --name "multi-nim-nemotron-nano-9b-config" \ - --nim-deployment '{ - "gpu": 1, - "image_name": "nvcr.io/nim/nvidia/llm-nim", - "image_tag": "1.13.1", - "model_name": "nvidia/NVIDIA-Nemotron-Nano-9B-v2" - }' - """ - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/nvidia/llm-nim", # Multi-LLM image - image_tag="1.13.1", - model_name="nvidia/NVIDIA-Nemotron-Nano-9B-v2", # Model name for Files service - model_namespace=None, # Namespace is in model_name - lora_enabled=False, - additional_envs=None, - ) - return config - - -@pytest.mark.asyncio -async def test_multi_llm_files_service_deployment_succeeds( - docker_backend, mock_docker_client, multi_llm_config_with_model_name -): - """Test multi-LLM deployment with FILES_SERVICE weights type succeeds (bug #3759). - - Regression test for the scenario: - - Use multi-LLM image (nvcr.io/nim/nvidia/llm-nim) - - Provide model_name (nvidia/NVIDIA-Nemotron-Nano-9B-v2) - - No model entity - - This triggers ModelWeightsType.FILES_SERVICE which was previously not supported. - """ - # Setup deployment without model entity - deployment = MagicMock() - deployment.workspace = "default" - deployment.name = "nemotron-deployment" - deployment.entity_version = 1 - deployment.status = "CREATED" - - # Setup mocks for puller and NIM containers - mock_puller_container = MagicMock() - mock_puller_container.id = "puller123456789" - mock_puller_container.wait.return_value = {"StatusCode": 0} - mock_puller_container.remove = MagicMock() - - mock_nim_container = MagicMock() - mock_nim_container.id = "nim1234567890ab" - mock_nim_container.start = MagicMock() - - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.run.return_value = mock_puller_container - mock_docker_client.containers.create.return_value = mock_nim_container - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, deployment, exit_code=0) - - # No model entity (matches the bug scenario) - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=deployment, model_deployment_config=multi_llm_config_with_model_name, model_entity=None - ) - ) - status_update = await drive_creation_to_completion(docker_backend, deployment) - - # Should succeed with PENDING status - assert status_update.status == "PENDING", ( - f"Expected PENDING but got {status_update.status}: {status_update.status_message}" - ) - assert "container created" in status_update.status_message.lower() - - # Verify model puller was called - puller_calls = [ - call - for call in mock_docker_client.containers.run.call_args_list - if call[1].get("labels", {}).get("nmp.nvidia.com/container-type") == "model-puller" - ] - assert len(puller_calls) == 1, "Model puller should run for multi-LLM FILES_SERVICE deployment" - - # Verify puller used correct model repo (from model_name) - puller_command = puller_calls[0][1]["command"] - assert puller_command[0] == "download" - assert puller_command[1] == "nvidia/NVIDIA-Nemotron-Nano-9B-v2" - - # Verify puller configured for Files service (HF_ENDPOINT) - puller_env = puller_calls[0][1]["environment"] - assert "HF_ENDPOINT" in puller_env - assert "/apis/files/v2/hf" in puller_env["HF_ENDPOINT"] - - -def _puller_state(deployment, config): - """Build a minimal CreationState for exercising _start_model_puller_container directly.""" - return CreationState( - stage=CreationStage.RUNNING_PULLER, - deployment=deployment, - config=config, - model_entity=None, - model_weights_type=ModelWeightsType.FILES_SERVICE, - volume_name="vol-test", - ) - - -@pytest.mark.asyncio -async def test_model_puller_env_override_merged(docker_backend, sample_deployment, sample_config, mock_docker_client): - """huggingface_model_puller_env is merged into the puller env, with operator values winning.""" - reconciler = docker_backend._reconciler - reconciler._backend_config.huggingface_model_puller_env = { - "HF_HUB_ENABLE_HF_TRANSFER": "0", - "HF_ENDPOINT": "https://override.example", # must win over the hardcoded default - } - - puller_container = MagicMock() - puller_container.id = "puller123456789" - reconciler.run_container = MagicMock(return_value=puller_container) - reconciler._get_model_repo_from_entity = MagicMock(return_value="nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16") - - await reconciler._start_model_puller_container(_puller_state(sample_deployment, sample_config)) - - reconciler.run_container.assert_called_once() - env = reconciler.run_container.call_args[1]["environment"] - # Operator-provided override is applied - assert env["HF_HUB_ENABLE_HF_TRANSFER"] == "0" - # Operator value wins over the hardcoded HF_ENDPOINT default - assert env["HF_ENDPOINT"] == "https://override.example" - # Untouched default remains - assert env["HF_TOKEN"] == "service:models" - - -@pytest.mark.asyncio -async def test_model_puller_env_no_override_uses_defaults( - docker_backend, sample_deployment, sample_config, mock_docker_client -): - """With no huggingface_model_puller_env configured, the puller env keeps only its defaults.""" - reconciler = docker_backend._reconciler - assert reconciler._backend_config.huggingface_model_puller_env == {} - - puller_container = MagicMock() - puller_container.id = "puller123456789" - reconciler.run_container = MagicMock(return_value=puller_container) - reconciler._get_model_repo_from_entity = MagicMock(return_value="nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16") - - await reconciler._start_model_puller_container(_puller_state(sample_deployment, sample_config)) - - reconciler.run_container.assert_called_once() - env = reconciler.run_container.call_args[1]["environment"] - assert "HF_HUB_ENABLE_HF_TRANSFER" not in env - assert env["HF_TOKEN"] == "service:models" - - -@pytest.mark.asyncio -async def test_get_model_repo_from_entity_with_files_service(docker_backend, multi_llm_config_with_model_name): - """Test _get_model_repo_from_entity handles FILES_SERVICE type. - - FILES_SERVICE (non-SFT) should extract model_name from nim_config. - """ - from nmp.core.models.app import ModelWeightsType - - # Model entity with fileset but no PEFT (non-SFT) - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = None - model_entity.peft = None - model_entity.fileset = "hf://workspace/fileset" - - # Should succeed with FILES_SERVICE (previously raised ValueError) - model_repo = docker_backend._reconciler._get_model_repo_from_entity( - model_entity=model_entity, - model_weights_type=ModelWeightsType.FILES_SERVICE, - nim_config=multi_llm_config_with_model_name.model_spec, - ) - - # Should extract from fileset when available - assert model_repo == "workspace/fileset" - - -@pytest.mark.asyncio -async def test_get_model_repo_from_entity_files_service_uses_nim_config_when_no_artifact( - docker_backend, multi_llm_config_with_model_name -): - """Test _get_model_repo_from_entity falls back to nim_config for FILES_SERVICE. - - When model_entity has no fileset, should use model_name from nim_config. - """ - from nmp.core.models.app import ModelWeightsType - - # Model entity without fileset - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = None - model_entity.peft = None - model_entity.fileset = None # No fileset - - # Should fall back to nim_config.model_name - model_repo = docker_backend._reconciler._get_model_repo_from_entity( - model_entity=model_entity, - model_weights_type=ModelWeightsType.FILES_SERVICE, - nim_config=multi_llm_config_with_model_name.model_spec, - ) - - assert model_repo == "nvidia/NVIDIA-Nemotron-Nano-9B-v2" - - -@pytest.mark.asyncio -async def test_get_model_repo_from_entity_files_service_no_entity_uses_nim_config( - docker_backend, multi_llm_config_with_model_name -): - """Test _get_model_repo_from_entity uses nim_config when no entity for FILES_SERVICE.""" - from nmp.core.models.app import ModelWeightsType - - # No model entity at all - model_repo = docker_backend._reconciler._get_model_repo_from_entity( - model_entity=None, - model_weights_type=ModelWeightsType.FILES_SERVICE, - nim_config=multi_llm_config_with_model_name.model_spec, - ) - - assert model_repo == "nvidia/NVIDIA-Nemotron-Nano-9B-v2" - - -@pytest.mark.asyncio -async def test_multi_llm_huggingface_deployment_succeeds_with_hf_token( - docker_backend, mock_docker_client, mock_nmp_sdk -): - """Test multi-LLM deployment with HuggingFace token succeeds (bug #3716). - - Regression test for the scenario from docs/run-inference/tutorials/deploy-models.md: - - Omit image_name to auto-select multi-LLM image - - Provide hf_token_secret_name for HuggingFace authentication - - No model entity (deploying directly from HuggingFace) - - Before the fix, this failed with "ModelWeightsType.UNKNOWN is not supported" - because get_model_weights_type() was called without deployment/config context. - """ - # Setup deployment with HF token (matches docs example) - deployment = MagicMock() - deployment.workspace = "default" - deployment.name = "qwen-deployment" - deployment.entity_version = 1 - deployment.status = "CREATED" - deployment.hf_token_secret_name = "hf-token-secret" # KEY: HF token provided - - # Config without image_name (multi-LLM) but with model_name (matches docs) - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name=None, # KEY: Omitted to use multi-LLM - image_tag=None, - model_name="Qwen2.5-1.5B-Instruct", - model_namespace="Qwen", - lora_enabled=False, - additional_envs=None, - ) - - # Setup mocks for puller and NIM containers - mock_puller_container = MagicMock() - mock_puller_container.id = "puller123456789" - mock_puller_container.wait.return_value = {"StatusCode": 0} - mock_puller_container.remove = MagicMock() - - mock_nim_container = MagicMock() - mock_nim_container.id = "nim1234567890ab" - mock_nim_container.start = MagicMock() - - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.create.return_value = mock_nim_container - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, deployment, exit_code=0) - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=deployment, model_deployment_config=config, model_entity=None) - ) - status_update = await drive_creation_to_completion(docker_backend, deployment) - - assert status_update.status == "PENDING", ( - f"Expected PENDING but got {status_update.status}: {status_update.status_message}" - ) - assert "container created" in status_update.status_message.lower() - - puller_calls = [ - call - for call in mock_docker_client.containers.run.call_args_list - if call[1].get("labels", {}).get("nmp.nvidia.com/container-type") == "model-puller" - ] - assert len(puller_calls) == 1 - - puller_command = puller_calls[0][1]["command"] - assert puller_command[0] == "download" - assert "Qwen" in puller_command[1] - - -# ============================================================================ -# GPU Cleanup for Drift Recovery Tests -# ============================================================================ - - -@pytest.mark.asyncio -async def test_create_releases_stale_gpu_allocation_single( - docker_backend, sample_deployment, sample_config, mock_docker_client -): - """Test that create_model_deployment releases stale GPU allocation for single GPU.""" - from nmp.common.docker.gpu_pool import DockerGPUPool - - # Create a real GPU pool with 2 GPUs - gpu_pool = DockerGPUPool(reserved_gpu_device_ids=[0, 1]) - docker_backend._gpu_pool = gpu_pool - docker_backend._reconciler._gpu_pool = gpu_pool - - # Pre-allocate GPU 0 to this deployment (simulating stale allocation from lost container) - deployment_key = f"{sample_deployment.workspace}/{sample_deployment.name}" - gpu_pool.allocate_gpu(deployment_key, num_requested=1) - - # Verify GPU is allocated - assert gpu_pool.get_available_count() == 1 - allocated = gpu_pool.get_allocated_workloads() - assert deployment_key in allocated.values() - - # Setup mock container - mock_container = MagicMock() - docker_backend._backend_config.models_docker_networking_mode = "dond" - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.containers.list.return_value = [] - - # Create deployment and drive to completion (should release stale allocation first, then reallocate) - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=None) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update.status == "PENDING" - - # Verify GPU is still allocated (released then reallocated) - assert gpu_pool.get_available_count() == 1 - allocated = gpu_pool.get_allocated_workloads() - assert deployment_key in allocated.values() - - -@pytest.mark.asyncio -async def test_create_releases_stale_gpu_allocation_multi(docker_backend, sample_deployment, mock_docker_client): - """Test that create_model_deployment releases stale GPU allocation for multiple GPUs.""" - from nmp.common.docker.gpu_pool import DockerGPUPool - - gpu_pool = DockerGPUPool(reserved_gpu_device_ids=[0, 1, 2, 3]) - docker_backend._gpu_pool = gpu_pool - docker_backend._reconciler._gpu_pool = gpu_pool - - deployment_key = f"{sample_deployment.workspace}/{sample_deployment.name}" - gpu_pool.allocate_gpu(deployment_key, num_requested=2) - assert gpu_pool.get_available_count() == 2 - - config = MagicMock() - set_deployment_config( - config, - gpu=2, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - ) - - mock_container = MagicMock() - docker_backend._backend_config.models_docker_networking_mode = "dond" - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update.status == "PENDING" - assert gpu_pool.get_available_count() == 2 - allocated = gpu_pool.get_allocated_workloads() - assert list(allocated.values()).count(deployment_key) == 2 - - -@pytest.mark.asyncio -async def test_create_without_stale_allocation_succeeds( - docker_backend, sample_deployment, sample_config, mock_docker_client -): - """Test that create_model_deployment works when there's no stale allocation.""" - from nmp.common.docker.gpu_pool import DockerGPUPool - - gpu_pool = DockerGPUPool(reserved_gpu_device_ids=[0, 1]) - docker_backend._gpu_pool = gpu_pool - docker_backend._reconciler._gpu_pool = gpu_pool - assert gpu_pool.get_available_count() == 2 - - mock_container = MagicMock() - docker_backend._backend_config.models_docker_networking_mode = "dond" - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=None) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update.status == "PENDING" - - deployment_key = f"{sample_deployment.workspace}/{sample_deployment.name}" - assert gpu_pool.get_available_count() == 1 - allocated = gpu_pool.get_allocated_workloads() - assert deployment_key in allocated.values() - - -# ============================================================================ -# Tool Call Config & Chat Template — _compile_env_vars Tests -# ============================================================================ - - -@pytest.mark.asyncio -async def test_compile_env_vars_model_entity_chat_template(docker_backend, sample_deployment, sample_config): - """Test that chat_template from model entity spec sets NIM_CHAT_TEMPLATE.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = MagicMock() - model_entity.spec.chat_template = "{% for msg in messages %}{{ msg.role }}{% endfor %}" - model_entity.spec.tool_call_config = None - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=False, - ) - - assert env_vars["NIM_CHAT_TEMPLATE"] == "{% for msg in messages %}{{ msg.role }}{% endfor %}" - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - assert "NIM_ENABLE_AUTO_TOOL_CHOICE" not in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_model_entity_tool_call_config(docker_backend, sample_deployment, sample_config): - """Test that tool_call_config from model entity spec sets NIM env vars.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = MagicMock() - model_entity.spec.chat_template = None - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = "hermes" - model_entity.spec.tool_call_config.tool_call_plugin = None - model_entity.spec.tool_call_config.auto_tool_choice = True - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=False, - ) - - assert "NIM_CHAT_TEMPLATE" not in env_vars - assert env_vars["NIM_TOOL_CALL_PARSER"] == "hermes" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_auto_tool_choice_false(docker_backend, sample_deployment, sample_config): - """Test that auto_tool_choice=False does NOT set NIM_ENABLE_AUTO_TOOL_CHOICE from model entity.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = MagicMock() - model_entity.spec.chat_template = None - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = None - model_entity.spec.tool_call_config.tool_call_plugin = None - model_entity.spec.tool_call_config.auto_tool_choice = False - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=False, - ) - - # auto_tool_choice=False explicitly disables auto tool choice - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "0" - assert "NIM_TOOL_CALL_PARSER" not in env_vars - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - assert "NIM_CHAT_TEMPLATE" not in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_tool_call_plugin_path(docker_backend, sample_deployment, sample_config): - """Test that tool_call_plugin_path sets NIM_TOOL_PARSER_PLUGIN.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = MagicMock() - model_entity.spec.chat_template = None - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = "pythonic" - model_entity.spec.tool_call_config.tool_call_plugin = "ws/my-plugin" - model_entity.spec.tool_call_config.auto_tool_choice = None - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=False, - tool_call_plugin_path="/model-store/tool_call_plugin/custom.py", - ) - - assert env_vars["NIM_TOOL_CALL_PARSER"] == "pythonic" - assert env_vars["NIM_TOOL_PARSER_PLUGIN"] == "/model-store/tool_call_plugin/custom.py" - assert "NIM_CHAT_TEMPLATE" not in env_vars - assert "NIM_ENABLE_AUTO_TOOL_CHOICE" not in env_vars - - -@pytest.mark.asyncio -async def test_compile_env_vars_no_spec_no_tool_vars(docker_backend, sample_deployment, sample_config): - """When model entity has no spec, no tool call env vars should be set.""" - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = None - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=False, - ) - - assert "NIM_CHAT_TEMPLATE" not in env_vars - assert "NIM_TOOL_CALL_PARSER" not in env_vars - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - assert "NIM_ENABLE_AUTO_TOOL_CHOICE" not in env_vars - - -# ============================================================================ -# Deployment-level Overrides — _compile_env_vars Tests -# ============================================================================ - - -@pytest.mark.asyncio -async def test_compile_env_vars_deployment_overrides_model_entity(docker_backend, sample_deployment): - """Deployment-level chat_template/tool_call_config override model entity spec.""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - chat_template="deployment-template", - tool_call_config={ - "tool_call_parser": "openai", - "auto_tool_choice": True, - }, - ) - - # Model entity also has spec values (should be overridden) - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = MagicMock() - model_entity.spec.chat_template = "entity-template" - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = "hermes" - model_entity.spec.tool_call_config.tool_call_plugin = None - model_entity.spec.tool_call_config.auto_tool_choice = False - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - config, - model_entity=model_entity, - is_multi_llm=False, - ) - - # Deployment-level should win - assert env_vars["NIM_CHAT_TEMPLATE"] == "deployment-template" - assert env_vars["NIM_TOOL_CALL_PARSER"] == "openai" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - - -@pytest.mark.asyncio -async def test_compile_env_vars_deployment_auto_tool_choice_false_overrides(docker_backend, sample_deployment): - """Deployment auto_tool_choice=False overrides entity's True.""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - tool_call_config={"auto_tool_choice": False}, - ) - - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = MagicMock() - model_entity.spec.chat_template = None - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = None - model_entity.spec.tool_call_config.tool_call_plugin = None - model_entity.spec.tool_call_config.auto_tool_choice = True - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - config, - model_entity=model_entity, - is_multi_llm=False, - ) - - # Deployment False should override entity True → "0" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "0" - - -@pytest.mark.asyncio -async def test_compile_env_vars_deployment_only_no_entity(docker_backend, sample_deployment): - """Deployment-level config sets env vars even without a model entity.""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - chat_template="dep-tmpl", - tool_call_config={"tool_call_parser": "mistral", "auto_tool_choice": True}, - ) - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - config, - model_entity=None, - is_multi_llm=False, - ) - - assert env_vars["NIM_CHAT_TEMPLATE"] == "dep-tmpl" - assert env_vars["NIM_TOOL_CALL_PARSER"] == "mistral" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - - -# ============================================================================ -# Chat Template & Tool Call Config — Scenario-Based Tests -# -# These 7 scenarios mirror test_chat_template_tool_calling.py and verify -# that _compile_env_vars produces the correct NIM_* environment variables -# for every combination of model entity spec and deployment-level config. -# ============================================================================ - - -@pytest.mark.asyncio -async def test_scenario1_fileset_custom_fields_only(docker_backend, sample_deployment, sample_config): - """Scenario 1: chat_template + tool_call_config from model entity spec only. - - Simulates fileset custom_fields being merged into model spec by the - model spec task. NIMDeployment has no overrides. - """ - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "llama-3-2-1b-instruct-fileset-only" - model_entity.trust_remote_code = False - model_entity.spec = MagicMock() - model_entity.spec.chat_template = ( - "{%- set loop_messages = messages %}" - "{%- for message in loop_messages %}" - "{%- set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'" - " + message['content'] | trim + '<|eot_id|>' %}" - "{%- if loop.index0 == 0 %}{%- set content = '<|begin_of_text|>' + content %}{%- endif %}" - "{{ content }}{%- endfor %}" - "{%- if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}" - ) - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = "llama3_json" - model_entity.spec.tool_call_config.tool_call_plugin = None - model_entity.spec.tool_call_config.auto_tool_choice = True - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=True, - ) - - assert env_vars["NIM_CHAT_TEMPLATE"] == model_entity.spec.chat_template - assert env_vars["NIM_TOOL_CALL_PARSER"] == "llama3_json" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - - -@pytest.mark.asyncio -async def test_scenario2_deployment_config_only(docker_backend, sample_deployment): - """Scenario 2: chat_template + tool_call_config from NIMDeployment only. - - Fileset has no custom_fields, model entity spec has nothing. - Both values come entirely from the deployment config. - """ - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - chat_template="{%- for message in messages %}{{ message.role }}{% endfor %}", - tool_call_config={ - "tool_call_parser": "openai", - "auto_tool_choice": True, - }, - ) - - # Model entity has no spec values - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "llama-3-2-1b-instruct-deploy-only" - model_entity.trust_remote_code = False - model_entity.spec = None - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - config, - model_entity=model_entity, - is_multi_llm=True, - ) - - assert env_vars["NIM_CHAT_TEMPLATE"] == "{%- for message in messages %}{{ message.role }}{% endfor %}" - assert env_vars["NIM_TOOL_CALL_PARSER"] == "openai" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - - -@pytest.mark.asyncio -async def test_scenario3_both_deployment_wins(docker_backend, sample_deployment): - """Scenario 3: Both model entity spec and NIMDeployment set values. - - Deployment-level overrides should take precedence over model entity spec. - """ - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - chat_template=( - "{% for message in messages %}" - "{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' }}" - "{% endfor %}" - "{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}" - ), - tool_call_config={ - "tool_call_parser": "openai", - "auto_tool_choice": True, - }, - ) - - # Model entity has different values (should be overridden by deployment) - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "llama-3-2-1b-instruct-both-override" - model_entity.trust_remote_code = False - model_entity.spec = MagicMock() - model_entity.spec.chat_template = "entity-llama-template" - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = "llama3_json" - model_entity.spec.tool_call_config.tool_call_plugin = None - model_entity.spec.tool_call_config.auto_tool_choice = False - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - config, - model_entity=model_entity, - is_multi_llm=True, - ) - - # Deployment-level values should win - assert env_vars["NIM_CHAT_TEMPLATE"] == config.model_spec.chat_template - assert env_vars["NIM_TOOL_CALL_PARSER"] == "openai" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - - -@pytest.mark.asyncio -async def test_scenario4_baseline_nothing_set(docker_backend, sample_deployment, sample_config): - """Scenario 4: No chat_template or tool_call_config anywhere. - - NIM uses its built-in defaults from the tokenizer. - None of the tool-calling env vars should be present. - """ - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "llama-3-2-1b-instruct-baseline" - model_entity.trust_remote_code = False - model_entity.spec = MagicMock() - model_entity.spec.chat_template = None - model_entity.spec.tool_call_config = None - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=True, - ) - - assert "NIM_CHAT_TEMPLATE" not in env_vars - assert "NIM_TOOL_CALL_PARSER" not in env_vars - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - assert "NIM_ENABLE_AUTO_TOOL_CHOICE" not in env_vars - - -@pytest.mark.asyncio -async def test_scenario5_mixed_fileset_chat_template_deploy_tool_config(docker_backend, sample_deployment): - """Scenario 5: chat_template from model entity spec, tool_call_config from deployment. - - Tests that individual fields can come from different layers. - """ - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - # No chat_template at deployment level - tool_call_config={ - "tool_call_parser": "hermes", - "auto_tool_choice": True, - }, - ) - - # Model entity has chat_template but no tool_call_config - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "llama-3-2-1b-instruct-mixed" - model_entity.trust_remote_code = False - model_entity.spec = MagicMock() - model_entity.spec.chat_template = "llama-template-from-fileset" - model_entity.spec.tool_call_config = None - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - config, - model_entity=model_entity, - is_multi_llm=True, - ) - - # chat_template from model entity, tool config from deployment - assert env_vars["NIM_CHAT_TEMPLATE"] == "llama-template-from-fileset" - assert env_vars["NIM_TOOL_CALL_PARSER"] == "hermes" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - assert "NIM_TOOL_PARSER_PLUGIN" not in env_vars - - -@pytest.mark.asyncio -async def test_scenario6_plugin_from_model_entity_spec(docker_backend, sample_deployment, sample_config): - """Scenario 6: tool_call_plugin fileset reference in model entity spec. - - The plugin puller has already discovered the .py file and passed - its path via tool_call_plugin_path. - """ - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "llama-3-2-1b-instruct-plugin" - model_entity.trust_remote_code = False - model_entity.spec = MagicMock() - model_entity.spec.chat_template = "llama-chat-template" - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = "pythonic" - model_entity.spec.tool_call_config.tool_call_plugin = "default/my-tool-plugin" - model_entity.spec.tool_call_config.auto_tool_choice = True - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - sample_config, - model_entity=model_entity, - is_multi_llm=True, - tool_call_plugin_path="/model-store/tool_call_plugin/my_plugin.py", - ) - - assert env_vars["NIM_CHAT_TEMPLATE"] == "llama-chat-template" - assert env_vars["NIM_TOOL_CALL_PARSER"] == "pythonic" - assert env_vars["NIM_TOOL_PARSER_PLUGIN"] == "/model-store/tool_call_plugin/my_plugin.py" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - - -@pytest.mark.asyncio -async def test_scenario7_plugin_from_deployment_config(docker_backend, sample_deployment): - """Scenario 7: chat_template + tool_call_config (including tool_call_plugin) - set entirely from the deployment config, bypassing model entity spec. - - The plugin puller has already discovered the .py file path. - """ - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - chat_template="llama-deploy-template", - tool_call_config={ - "tool_call_parser": "pythonic", - "tool_call_plugin": "default/my-tool-plugin", - "auto_tool_choice": True, - }, - ) - - # Model entity has no spec (spec not yet populated, or no custom_fields) - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "llama-3-2-1b-instruct-plugin-deploy" - model_entity.trust_remote_code = False - model_entity.spec = None - - env_vars = await docker_backend._reconciler._compile_env_vars( - sample_deployment, - config, - model_entity=model_entity, - is_multi_llm=True, - tool_call_plugin_path="/model-store/tool_call_plugin/my_plugin.py", - ) - - assert env_vars["NIM_CHAT_TEMPLATE"] == "llama-deploy-template" - assert env_vars["NIM_TOOL_CALL_PARSER"] == "pythonic" - assert env_vars["NIM_TOOL_PARSER_PLUGIN"] == "/model-store/tool_call_plugin/my_plugin.py" - assert env_vars["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - - -# ============================================================================ -# Plugin Puller Tests -# ============================================================================ - - -@pytest.mark.asyncio -async def test_plugin_puller_success(docker_backend, sample_deployment, mock_docker_client, sample_resource_names): - """Test _run_plugin_puller discovers a single .py file and returns its path.""" - volume_name = sample_resource_names["volume"] - - # Mock puller container - mock_puller_container = MagicMock() - mock_puller_container.id = "pluginpuller1234" - mock_puller_container.wait.return_value = {"StatusCode": 0} - mock_puller_container.remove = MagicMock() - - # containers.run is called for: busybox mkdir, puller, busybox chown, busybox find - mock_docker_client.containers.run.side_effect = [ - None, # busybox mkdir - mock_puller_container, # puller - None, # busybox chown - b"/model-store/tool_call_plugin/my_plugin.py\n", # busybox find - ] - mock_docker_client.images.get.return_value = MagicMock() - - plugin_path, error = await docker_backend._reconciler._run_plugin_puller( - deployment=sample_deployment, - fileset_ref="workspace/my-tool-plugin", - volume_name=volume_name, - target_subdir="tool_call_plugin", - ) - - assert error is None - assert plugin_path == "/model-store/tool_call_plugin/my_plugin.py" - - # The plugin puller runs against the nmp-api image (entrypoint `nemo services - # run`), so the download command must run via the `hf` entrypoint override. - plugin_puller_calls = [ - c - for c in mock_docker_client.containers.run.call_args_list - if c.kwargs.get("labels", {}).get("nmp.nvidia.com/container-type") == "plugin-puller" - ] - assert len(plugin_puller_calls) == 1 - assert plugin_puller_calls[0].kwargs["name"] == sample_resource_names["plugin"] - assert plugin_puller_calls[0].kwargs["entrypoint"] == ["hf"] - assert plugin_puller_calls[0].kwargs["command"][0] == "download" - - -@pytest.mark.asyncio -async def test_plugin_puller_no_py_files(docker_backend, sample_deployment, mock_docker_client, sample_resource_names): - """Test _run_plugin_puller returns error when no .py files found.""" - volume_name = sample_resource_names["volume"] - - mock_puller_container = MagicMock() - mock_puller_container.id = "pluginpuller1234" - mock_puller_container.wait.return_value = {"StatusCode": 0} - mock_puller_container.remove = MagicMock() - - mock_docker_client.containers.run.side_effect = [ - None, # busybox mkdir - mock_puller_container, # puller - None, # busybox chown - b"\n", # busybox find (empty result) - ] - mock_docker_client.images.get.return_value = MagicMock() - - plugin_path, error = await docker_backend._reconciler._run_plugin_puller( - deployment=sample_deployment, - fileset_ref="workspace/empty-plugin", - volume_name=volume_name, - ) - - assert plugin_path is None - assert "no .py files" in error - - plugin_puller_calls = [ - c - for c in mock_docker_client.containers.run.call_args_list - if c.kwargs.get("labels", {}).get("nmp.nvidia.com/container-type") == "plugin-puller" - ] - assert len(plugin_puller_calls) == 1 - assert plugin_puller_calls[0].kwargs["name"] == sample_resource_names["plugin"] - - -@pytest.mark.asyncio -async def test_plugin_puller_multiple_py_files( - docker_backend, sample_deployment, mock_docker_client, sample_resource_names -): - """Test _run_plugin_puller returns error when multiple .py files found.""" - volume_name = sample_resource_names["volume"] - - mock_puller_container = MagicMock() - mock_puller_container.id = "pluginpuller1234" - mock_puller_container.wait.return_value = {"StatusCode": 0} - mock_puller_container.remove = MagicMock() - - mock_docker_client.containers.run.side_effect = [ - None, - mock_puller_container, - None, - b"/model-store/tool_call_plugin/a.py\n/model-store/tool_call_plugin/b.py\n", - ] - mock_docker_client.images.get.return_value = MagicMock() - - plugin_path, error = await docker_backend._reconciler._run_plugin_puller( - deployment=sample_deployment, - fileset_ref="workspace/multi-plugin", - volume_name=volume_name, - ) - - assert plugin_path is None - assert "2 .py files" in error - - plugin_puller_calls = [ - c - for c in mock_docker_client.containers.run.call_args_list - if c.kwargs.get("labels", {}).get("nmp.nvidia.com/container-type") == "plugin-puller" - ] - assert len(plugin_puller_calls) == 1 - assert plugin_puller_calls[0].kwargs["name"] == sample_resource_names["plugin"] - - -@pytest.mark.asyncio -async def test_plugin_puller_container_fails( - docker_backend, sample_deployment, mock_docker_client, sample_resource_names -): - """Test _run_plugin_puller returns error when puller container exits with non-zero.""" - volume_name = sample_resource_names["volume"] - - mock_puller_container = MagicMock() - mock_puller_container.id = "pluginpuller1234" - mock_puller_container.wait.return_value = {"StatusCode": 1} - mock_puller_container.logs.return_value = b"download failed" - mock_puller_container.remove = MagicMock() - - mock_docker_client.containers.run.side_effect = [ - None, - mock_puller_container, - ] - mock_docker_client.images.get.return_value = MagicMock() - - plugin_path, error = await docker_backend._reconciler._run_plugin_puller( - deployment=sample_deployment, - fileset_ref="workspace/bad-plugin", - volume_name=volume_name, - ) - - assert plugin_path is None - assert error is not None - assert "exit code 1" in error.lower() or "failed" in error.lower() - - plugin_puller_calls = [ - c - for c in mock_docker_client.containers.run.call_args_list - if c.kwargs.get("labels", {}).get("nmp.nvidia.com/container-type") == "plugin-puller" - ] - assert len(plugin_puller_calls) == 1 - assert plugin_puller_calls[0].kwargs["name"] == sample_resource_names["plugin"] - - -# ============================================================================ -# End-to-end: create_model_deployment with tool_call_config -# ============================================================================ - - -@pytest.mark.asyncio -async def test_create_deployment_with_tool_call_plugin_from_entity( - docker_backend, sample_deployment, mock_docker_client -): - """Test that create_model_deployment pulls tool_call_plugin fileset from model entity spec.""" - config = MagicMock() - set_deployment_config( - config, - gpu=1, - disk_size="50Gi", - image_name="nvcr.io/nim/meta/llama-3.2-1b-instruct", - image_tag="1.8.6", - model_name="llama-3.2-1b-instruct", - model_namespace="meta", - lora_enabled=False, - additional_envs=None, - ) - - model_entity = MagicMock() - model_entity.workspace = "default" - model_entity.name = "test-model" - model_entity.spec = MagicMock() - model_entity.spec.chat_template = None - model_entity.spec.tool_call_config = MagicMock() - model_entity.spec.tool_call_config.tool_call_parser = "hermes" - model_entity.spec.tool_call_config.tool_call_plugin = "ws/my-tool-fileset" - model_entity.spec.tool_call_config.auto_tool_choice = True - model_entity.peft = None - model_entity.fileset = None - model_entity.finetuning_type = None - - # Setup mock containers - mock_puller_container = MagicMock() - mock_puller_container.id = "pluginpuller1234" - mock_puller_container.wait.return_value = {"StatusCode": 0} - mock_puller_container.remove = MagicMock() - - mock_nim_container = MagicMock() - mock_nim_container.id = "nim1234567890ab" - mock_nim_container.start = MagicMock() - - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.run.side_effect = [ - None, # busybox mkdir - mock_puller_container, # plugin puller - None, # busybox chown - b"/model-store/tool_call_plugin/tool.py\n", # busybox find - ] - mock_docker_client.containers.create.return_value = mock_nim_container - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=model_entity) - ) - status_update = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status_update.status == "PENDING" - - # Verify the NIM container was created with the right env vars - create_call = mock_docker_client.containers.create.call_args - nim_env = create_call[1]["environment"] - assert nim_env["NIM_TOOL_CALL_PARSER"] == "hermes" - assert nim_env["NIM_TOOL_PARSER_PLUGIN"] == "/model-store/tool_call_plugin/tool.py" - assert nim_env["NIM_ENABLE_AUTO_TOOL_CHOICE"] == "1" - - -# ============================================================================ -# PENDING Timeout and Crash Loop Detection Tests -# ============================================================================ - - -@pytest.fixture -def make_mock_container(docker_backend, mock_docker_client): - """Factory to create a mock container wired into the docker backend for status tests. - - Handles the boilerplate of creating a MagicMock container, setting sensible - attrs for the given state, wiring it into ``mock_docker_client.containers.get``, - and switching the backend to "dond" networking mode. - """ - - def _make( - status="running", - restart_count=0, - logs=None, - exit_code=None, - ): - container = MagicMock() - container.status = status - container.id = "1234567890abcdef" - container.ports = {} - container.reload = MagicMock() - - if status == "running": - container.attrs = { - "State": {"StartedAt": "2024-01-01T00:00:00Z"}, - "RestartCount": restart_count, - } - elif status in ("exited", "dead"): - container.attrs = {"State": {"ExitCode": exit_code if exit_code is not None else 1}} - else: - container.attrs = {"RestartCount": restart_count} - - if logs is not None: - container.logs = MagicMock(return_value=logs) - - mock_docker_client.containers.get.side_effect = None - mock_docker_client.containers.get.return_value = container - docker_backend._backend_config.models_docker_networking_mode = "dond" - - return container - - return _make - - -class TestPendingTimeoutStatusTransition: - """Tests for PENDING -> ERROR transition after timeout.""" - - @pytest.mark.asyncio - async def test_running_not_healthy_within_timeout_returns_pending( - self, docker_backend, sample_deployment, make_mock_container - ): - """A running container that isn't healthy should return PENDING with timing info.""" - make_mock_container(status="running") - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "PENDING" - assert "still initializing" in status.status_message - # No elapsed/timeout in message (stable message to avoid new history entry every poll) - - @pytest.mark.asyncio - async def test_running_not_healthy_exceeds_timeout_returns_error( - self, docker_backend, sample_deployment, make_mock_container, sample_resource_names - ): - """A running container that isn't healthy and exceeds timeout should transition to ERROR.""" - make_mock_container(status="running", logs=b"NIM failed to start: model not supported") - docker_backend._backend_config.pending_timeout_seconds = 7200 - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=3) - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - assert "timed out" in status.status_message - assert "docker logs" in status.status_message - assert sample_resource_names["container"] in status.status_message - assert status.error_details["reason"] == "pending_timeout" - assert status.error_details["container_name"] == sample_resource_names["container"] - - @pytest.mark.asyncio - async def test_created_state_within_timeout_returns_pending( - self, docker_backend, sample_deployment, make_mock_container - ): - """A container in 'created' state should return PENDING with timing info.""" - make_mock_container(status="created") - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "PENDING" - assert "starting up" in status.status_message - # No elapsed/timeout in message (stable message to avoid new history entry every poll) - - @pytest.mark.asyncio - async def test_restarting_state_within_timeout_returns_pending_with_restart_count( - self, docker_backend, sample_deployment, make_mock_container - ): - """A container in 'restarting' state should include restart count.""" - make_mock_container(status="restarting", restart_count=3) - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "PENDING" - assert "restart count: 3" in status.status_message - # No elapsed/timeout in message (stable message to avoid new history entry every poll) - - @pytest.mark.asyncio - async def test_restarting_state_exceeds_timeout_returns_error( - self, docker_backend, sample_deployment, make_mock_container - ): - """A container in 'restarting' state that exceeds timeout should transition to ERROR.""" - make_mock_container(status="restarting", restart_count=3, logs=b"Segfault in model loading") - docker_backend._backend_config.pending_timeout_seconds = 3600 - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=2) - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - assert "timed out" in status.status_message - assert "docker logs" in status.status_message - assert status.error_details["reason"] == "pending_timeout" - assert status.error_details["container_state"] == "restarting" - - @pytest.mark.asyncio - async def test_ready_returns_ready_status(self, docker_backend, sample_deployment, make_mock_container): - """A healthy container should return READY.""" - make_mock_container(status="running") - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "READY" - - @pytest.mark.asyncio - async def test_exited_returns_error(self, docker_backend, sample_deployment, make_mock_container): - """A terminated container should return ERROR.""" - make_mock_container(status="exited", logs=b"Error occurred") - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - - @pytest.mark.asyncio - async def test_lost_returns_lost(self, docker_backend, sample_deployment, mock_docker_client): - """A missing container (LOST) should return LOST.""" - mock_docker_client.containers.get.side_effect = NotFound("Container not found") - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "LOST" - - -class TestPendingTimeoutErrorMessage: - """Tests for the content of the error message on PENDING timeout.""" - - @pytest.mark.asyncio - async def test_error_message_includes_docker_logs_command( - self, docker_backend, sample_deployment, make_mock_container, sample_resource_names - ): - """Timeout error message includes a runnable docker logs command.""" - make_mock_container(status="running", logs=b"Error: model architecture not supported") - docker_backend._backend_config.pending_timeout_seconds = 300 - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(seconds=400) - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - assert f"docker logs {sample_resource_names['container']}" in status.status_message - - @pytest.mark.asyncio - async def test_error_details_contain_container_logs(self, docker_backend, sample_deployment, make_mock_container): - """error_details should include the container log tail as error_stack.""" - make_mock_container( - status="running", - logs=b"ERROR: NemotronHForCausalLM is not supported\nFatal: exiting", - ) - docker_backend._backend_config.pending_timeout_seconds = 300 - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(seconds=400) - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.error_details["error_stack"] is not None - assert "NemotronHForCausalLM" in status.error_details["error_stack"] - - @pytest.mark.asyncio - async def test_running_not_healthy_shows_restart_count_in_pending_message( - self, docker_backend, sample_deployment, make_mock_container - ): - """PENDING message for running-not-healthy container includes restart count when > 0.""" - make_mock_container(status="running", restart_count=3) - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "PENDING" - assert "restarts: 3" in status.status_message - - -class TestCrashLoopDetection: - """Tests for crash loop detection: PENDING -> ERROR after too many restarts.""" - - @pytest.mark.asyncio - async def test_running_not_healthy_exceeds_max_restarts_returns_error( - self, docker_backend, sample_deployment, make_mock_container - ): - """A running container that isn't healthy and exceeds max restarts should transition to ERROR.""" - make_mock_container(status="running", restart_count=5, logs=b"CUDA error: out of memory") - docker_backend._backend_config.max_restart_count = 5 - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - assert "crash loop" in status.status_message - assert "5 container restarts" in status.status_message - assert "docker logs" in status.status_message - assert status.error_details["reason"] == "crash_loop" - assert status.error_details["restart_count"] == 5 - assert status.error_details["max_restart_count"] == 5 - - @pytest.mark.asyncio - async def test_running_not_healthy_below_max_restarts_returns_pending( - self, docker_backend, sample_deployment, make_mock_container - ): - """A running container below max restarts should remain PENDING.""" - make_mock_container(status="running", restart_count=3) - docker_backend._backend_config.max_restart_count = 5 - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "PENDING" - assert "restarts: 3" in status.status_message - - @pytest.mark.asyncio - async def test_restarting_state_exceeds_max_restarts_returns_error( - self, docker_backend, sample_deployment, make_mock_container - ): - """A container in 'restarting' state that exceeds max restarts should transition to ERROR.""" - make_mock_container(status="restarting", restart_count=7, logs=b"Segfault in model loading") - docker_backend._backend_config.max_restart_count = 5 - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - assert "crash loop" in status.status_message - assert "7 container restarts" in status.status_message - assert status.error_details["reason"] == "crash_loop" - assert status.error_details["restart_count"] == 7 - assert status.error_details["container_state"] == "restarting" - - @pytest.mark.asyncio - async def test_restarting_state_below_max_restarts_returns_pending( - self, docker_backend, sample_deployment, make_mock_container - ): - """A container in 'restarting' state below max restarts should remain PENDING.""" - make_mock_container(status="restarting", restart_count=2) - docker_backend._backend_config.max_restart_count = 5 - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "PENDING" - assert "restart count: 2" in status.status_message - - @pytest.mark.asyncio - async def test_crash_loop_takes_priority_over_pending_timeout( - self, docker_backend, sample_deployment, make_mock_container - ): - """Crash loop detection should fire even when pending timeout is also exceeded.""" - make_mock_container(status="running", restart_count=6, logs=b"OOM killed") - docker_backend._backend_config.pending_timeout_seconds = 300 - docker_backend._backend_config.max_restart_count = 5 - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(seconds=400) - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - assert status.error_details["reason"] == "crash_loop" - - @pytest.mark.asyncio - async def test_custom_max_restart_count(self, docker_backend, sample_deployment, make_mock_container): - """Custom max_restart_count should be respected.""" - make_mock_container(status="running", restart_count=10, logs=b"") - docker_backend._backend_config.max_restart_count = 15 - - with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: - mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "PENDING" - assert "restarts: 10" in status.status_message - - @pytest.mark.asyncio - async def test_crash_loop_error_includes_container_logs( - self, docker_backend, sample_deployment, make_mock_container - ): - """Crash loop error_details should include container logs as error_stack.""" - make_mock_container(status="restarting", restart_count=5, logs=b"RuntimeError: CUDA out of memory") - docker_backend._backend_config.max_restart_count = 5 - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - - assert status.status == "ERROR" - assert "CUDA out of memory" in status.error_details["error_stack"] - - -# ============================================================================= -# Orphan reconciliation: list_managed_deployment_names, delete_model_deployment (by workspace/name) -# ============================================================================= - - -@pytest.mark.asyncio -async def test_list_managed_deployment_names_returns_dedupe_workspace_name(backend_with_mock_client): - """list_managed_deployment_names returns sorted unique workspace/name from container labels.""" - backend, _ = backend_with_mock_client - c1 = MagicMock() - c1.labels = { - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - "nmp.nvidia.com/deployment-workspace": "ws-a", - "nmp.nvidia.com/deployment-name": "dep1", - } - c2 = MagicMock() - c2.labels = { - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - "nmp.nvidia.com/deployment-workspace": "ws-b", - "nmp.nvidia.com/deployment-name": "dep2", - } - c3 = MagicMock() - c3.labels = c1.labels # same deployment (e.g. puller) - should dedupe - with patch.object(backend._reconciler, "list_containers", return_value=[c1, c2, c3]): - names = await backend.list_managed_deployment_names() - assert names == ["ws-a/dep1", "ws-b/dep2"] - - -@pytest.mark.asyncio -async def test_list_managed_deployment_names_empty_when_no_containers(backend_with_mock_client): - """list_managed_deployment_names returns empty list when no managed containers.""" - backend, _ = backend_with_mock_client - with patch.object(backend._reconciler, "list_containers", return_value=[]): - names = await backend.list_managed_deployment_names() - assert names == [] - - -@pytest.mark.asyncio -async def test_list_managed_deployment_names_skips_missing_labels(backend_with_mock_client): - """list_managed_deployment_names skips containers missing workspace/name labels.""" - backend, _ = backend_with_mock_client - c1 = MagicMock() - c1.labels = {MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER} - with patch.object(backend._reconciler, "list_containers", return_value=[c1]): - names = await backend.list_managed_deployment_names() - assert names == [] - - -@pytest.mark.asyncio -async def test_list_managed_deployment_names_filters_by_owner_labels(mock_nmp_sdk, mock_docker_client): - """Owner labels keep orphan reconciliation scoped to this backend owner.""" - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - owner_labels = { - "nmp.nvidia.com/test-run": "run-1", - "nmp.nvidia.com/test-worker": "gw0", - } - with patch("nmp.core.models.controllers.backends.docker.backend.get_platform_config", return_value=platform_config): - backend = DockerServiceBackend( - nmp_sdk=mock_nmp_sdk, - config={"model_labels": owner_labels}, - ) - - matching_container = MagicMock() - matching_container.labels = { - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - "nmp.nvidia.com/deployment-workspace": "ws-a", - "nmp.nvidia.com/deployment-name": "dep1", - **owner_labels, - } - other_worker_container = MagicMock() - other_worker_container.labels = { - MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, - "nmp.nvidia.com/deployment-workspace": "ws-b", - "nmp.nvidia.com/deployment-name": "dep2", - "nmp.nvidia.com/test-run": "run-1", - "nmp.nvidia.com/test-worker": "gw1", - } - - with patch.object( - backend._reconciler, "list_containers", return_value=[matching_container, other_worker_container] - ): - names = await backend.list_managed_deployment_names() - - assert names == ["ws-a/dep1"] - - -@pytest.mark.asyncio -async def test_delete_model_deployment_by_id_calls_delete_by_model_deployment_id(backend_with_mock_client): - """delete_model_deployment(workspace, name) delegates to _delete_by_model_deployment_id.""" - backend, _ = backend_with_mock_client - with patch.object(backend, "_delete_by_model_deployment_id", new_callable=AsyncMock) as mock_delete: - mock_delete.return_value = DeploymentStatusUpdate(status="DELETED", status_message="") - result = await backend.delete_model_deployment("my-ws", "my-name") - mock_delete.assert_called_once_with("my-ws", "my-name") - assert result.status == "DELETED" - - -# ============================================================================ -# Stepped Creation Pipeline Tests -# ============================================================================ - - -class TestSteppedCreation: - """Tests for the non-blocking, staged deployment creation pipeline.""" - - @pytest.mark.asyncio - async def test_create_returns_pending_with_pulling_message( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """create_model_deployment returns PENDING immediately with pulling message.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - status = await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - assert status.status == "PENDING" - assert "pulling container image" in status.status_message.lower() - assert status.host_url is None - - @pytest.mark.asyncio - async def test_creation_state_stored_after_create( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """CreationState is stored in _creation_states after create_model_deployment.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - key = docker_backend._get_deployment_key(sample_deployment) - assert key in docker_backend._reconciler._creation_states - state = docker_backend._reconciler._creation_states[key] - assert state.stage == CreationStage.PULLING_NIM_IMAGE - assert state.task is not None - - @pytest.mark.asyncio - async def test_get_status_delegates_to_advance_creation( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """get_model_deployment_status delegates to _advance_creation when in creation pipeline.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - key = docker_backend._get_deployment_key(sample_deployment) - assert key in docker_backend._reconciler._creation_states - - # The image pull mock completes instantly, so status check should advance the stage - state = docker_backend._reconciler._creation_states[key] - if state.task and not state.task.done(): - await state.task - - status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) - assert status.status == "PENDING" - - @pytest.mark.asyncio - async def test_image_pull_failure_returns_error( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """If image pull fails, _advance_creation returns ERROR.""" - mock_docker_client.images.get.side_effect = ImageNotFound("not found") - mock_docker_client.images.pull.side_effect = ImageNotFound("Image not found in registry") - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - status = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status.status == "ERROR" - assert "image not found" in status.status_message.lower() - - @pytest.mark.asyncio - async def test_creation_state_removed_after_completion( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """CreationState is removed from _creation_states after creation completes.""" - docker_backend._backend_config.models_docker_networking_mode = "dond" - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend, sample_deployment) - - key = docker_backend._get_deployment_key(sample_deployment) - assert key not in docker_backend._reconciler._creation_states - - @pytest.mark.asyncio - async def test_delete_during_creation_cancels_task( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """Deleting a deployment during creation cancels the background task.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - mock_volume = MagicMock() - mock_docker_client.volumes.get.side_effect = None - mock_docker_client.volumes.get.return_value = mock_volume - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - key = docker_backend._get_deployment_key(sample_deployment) - assert key in docker_backend._reconciler._creation_states - - result = await docker_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) - - assert result.status == "DELETED" - assert key not in docker_backend._reconciler._creation_states - - @pytest.mark.asyncio - async def test_shutdown_cancels_all_creation_tasks( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """shutdown() cancels all in-flight creation tasks.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - key = docker_backend._get_deployment_key(sample_deployment) - assert key in docker_backend._reconciler._creation_states - - docker_backend.shutdown() - - assert key not in docker_backend._reconciler._creation_states - - @pytest.mark.asyncio - async def test_puller_stage_with_files_service_weights( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """When model needs Files service weights, creation goes through puller stages.""" - model_entity = MagicMock() - model_entity.workspace = "test" - model_entity.name = "sft-model" - model_entity.spec = None - model_entity.finetuning_type = "all_weights" - peft_mock = MagicMock() - peft_mock.finetuning_type = "all_weights" - model_entity.peft = peft_mock - model_entity.fileset = "hf://test/sft-model-weights" - - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity - ) - ) - - key = docker_backend._get_deployment_key(sample_deployment) - state = docker_backend._reconciler._creation_states[key] - assert state.stage == CreationStage.PULLING_NIM_IMAGE - - status = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status.status == "PENDING" - assert "container created" in status.status_message.lower() - assert key not in docker_backend._reconciler._creation_states - - @pytest.mark.asyncio - async def test_puller_running_stage_reports_downloading( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """While puller is running, status reports downloading weights.""" - model_entity = MagicMock() - model_entity.workspace = "test" - model_entity.name = "sft-model" - model_entity.spec = None - model_entity.finetuning_type = "all_weights" - peft_mock = MagicMock() - peft_mock.finetuning_type = "all_weights" - model_entity.peft = peft_mock - model_entity.fileset = "hf://test/sft-model-weights" - - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - mock_puller = _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - mock_puller.status = "running" - - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity - ) - ) - - key = docker_backend._get_deployment_key(sample_deployment) - # Advance through PULLING_NIM_IMAGE and PULLING_PULLER_IMAGE - for _ in range(5): - state = docker_backend._reconciler._creation_states.get(key) - if state is None: - break - if state.task and not state.task.done(): - try: - await state.task - except Exception: - pass - if state.stage == CreationStage.RUNNING_PULLER: - break - await docker_backend._reconciler.advance(key) - - assert key in docker_backend._reconciler._creation_states - state = docker_backend._reconciler._creation_states[key] - assert state.stage == CreationStage.RUNNING_PULLER - - status = await docker_backend._reconciler.advance(key) - assert status.status == "PENDING" - assert "downloading model weights" in status.status_message.lower() - - @pytest.mark.asyncio - async def test_concurrent_deployments_not_blocked(self, docker_backend, sample_config, mock_docker_client): - """Multiple deployments can have creation states concurrently.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - deployments = [] - for i in range(3): - d = MagicMock() - d.workspace = "test" - d.name = f"model-{i}" - d.hf_token_secret_name = None - deployments.append(d) - - for d in deployments: - await docker_backend.create_model_deployment( - ModelContext(model_deployment=d, model_deployment_config=sample_config) - ) - - # All three should have creation states - for d in deployments: - key = docker_backend._get_deployment_key(d) - assert key in docker_backend._reconciler._creation_states - - assert len(docker_backend._reconciler._creation_states) == 3 - - -class TestAsyncioToThreadOffloading: - """Verify blocking Docker operations are offloaded via asyncio.to_thread.""" - - @pytest.mark.asyncio - async def test_nim_image_pull_uses_to_thread( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """create_model_deployment offloads the NIM image pull to asyncio.to_thread.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - to_thread_calls: list[tuple] = [] - original_to_thread = asyncio.to_thread - - async def spy_to_thread(func, *args, **kwargs): - to_thread_calls.append((func, args, kwargs)) - return await original_to_thread(func, *args, **kwargs) - - with patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", - side_effect=spy_to_thread, - ): - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - key = docker_backend._get_deployment_key(sample_deployment) - state = docker_backend._reconciler._creation_states[key] - if state.task and not state.task.done(): - await state.task - - pull_calls = [c for c in to_thread_calls if getattr(c[0], "__name__", "") == "pull_image_if_not_local"] - assert len(pull_calls) >= 1, "NIM image pull should be offloaded to asyncio.to_thread" - - @pytest.mark.asyncio - async def test_container_creation_uses_to_thread( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """Container creation and start are offloaded to asyncio.to_thread.""" - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - to_thread_calls: list[tuple] = [] - original_to_thread = asyncio.to_thread - - async def spy_to_thread(func, *args, **kwargs): - to_thread_calls.append((func, args, kwargs)) - return await original_to_thread(func, *args, **kwargs) - - with patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", - side_effect=spy_to_thread, - ): - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - status = await drive_creation_to_completion(docker_backend, sample_deployment) - - assert status.status != "ERROR", f"Creation failed: {status.status_message}" - container_calls = [c for c in to_thread_calls if getattr(c[0], "__name__", "") == "create_and_start_container"] - assert len(container_calls) >= 1, "create_and_start_container should be offloaded to asyncio.to_thread" - - @pytest.mark.asyncio - async def test_puller_image_pull_uses_to_thread( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """When model puller is needed, its image pull is offloaded to asyncio.to_thread.""" - model_entity = MagicMock() - model_entity.workspace = "test" - model_entity.name = "sft-model" - model_entity.spec = None - model_entity.finetuning_type = "all_weights" - peft_mock = MagicMock() - peft_mock.finetuning_type = "all_weights" - model_entity.peft = peft_mock - model_entity.fileset = "hf://test/sft-model-weights" - - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - - to_thread_calls: list[tuple] = [] - original_to_thread = asyncio.to_thread - - async def spy_to_thread(func, *args, **kwargs): - to_thread_calls.append((func, args, kwargs)) - return await original_to_thread(func, *args, **kwargs) - - with patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", - side_effect=spy_to_thread, - ): - await docker_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity - ) - ) - await drive_creation_to_completion(docker_backend, sample_deployment) - - pull_calls = [c for c in to_thread_calls if getattr(c[0], "__name__", "") == "pull_image_if_not_local"] - assert len(pull_calls) >= 2, "Both NIM image and puller image pulls should be offloaded to asyncio.to_thread" - - @pytest.mark.asyncio - async def test_sidecar_creation_uses_to_thread( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """Sidecar container creation is offloaded to asyncio.to_thread.""" - sample_config.model_spec.lora_enabled = True - - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - to_thread_calls: list[tuple] = [] - original_to_thread = asyncio.to_thread - - async def spy_to_thread(func, *args, **kwargs): - to_thread_calls.append((func, args, kwargs)) - return await original_to_thread(func, *args, **kwargs) - - with patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", - side_effect=spy_to_thread, - ): - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend, sample_deployment) - - container_calls = [c for c in to_thread_calls if getattr(c[0], "__name__", "") == "create_and_start_container"] - assert len(container_calls) >= 2, ( - "Both NIM and sidecar container creation should be offloaded to asyncio.to_thread" - ) - - @pytest.mark.asyncio - async def test_no_blocking_calls_on_event_loop( - self, docker_backend, sample_deployment, sample_config, mock_docker_client - ): - """Verify blocking operations go through asyncio.to_thread, not directly on the loop.""" - mock_docker_client.images.get.return_value = MagicMock() - mock_docker_client.containers.list.return_value = [] - - mock_container = MagicMock() - mock_container.id = "1234567890abcdef" - mock_container.start = MagicMock() - mock_docker_client.containers.create.return_value = mock_container - - to_thread_funcs: list[str] = [] - original_to_thread = asyncio.to_thread - - async def spy_to_thread(func, *args, **kwargs): - name = getattr(func, "__name__", None) or getattr(func, "_mock_name", "unknown") - to_thread_funcs.append(str(name)) - return await original_to_thread(func, *args, **kwargs) - - with patch( - "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", - side_effect=spy_to_thread, - ): - await docker_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - await drive_creation_to_completion(docker_backend, sample_deployment) - - assert any("pull" in name.lower() for name in to_thread_funcs), ( - f"Image pull should go through asyncio.to_thread, got: {to_thread_funcs}" - ) - assert any("create" in name.lower() or "container" in name.lower() for name in to_thread_funcs), ( - f"Container creation should go through asyncio.to_thread, got: {to_thread_funcs}" - ) diff --git a/services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py b/services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py deleted file mode 100644 index 4dc47fad8c..0000000000 --- a/services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py +++ /dev/null @@ -1,2667 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for K8sNimOperatorServiceBackend.""" - -import contextlib -from datetime import datetime, timedelta, timezone -from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from kubernetes import client as k8s_client -from kubernetes.dynamic import exceptions as k8s_dynamic_exceptions -from nemo_platform.types.models.model_entity import ModelEntity -from nemo_platform.types.shared import LinearLayerSpec, MambaConfig, ModelSpec, MoEConfig, SlidingWindowConfig -from nmp.common.config import PlatformConfig -from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.common import deployment_elapsed_seconds, format_duration -from nmp.core.models.controllers.backends.k8s_nim_operator import K8sNimOperatorServiceBackend -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter -from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector -from nmp.core.models.controllers.context import ModelContext -from pydantic import ValidationError - -_K8S_BACKEND_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.backend" -_RECON_K8S_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.k8s" -_RECON_STATUS_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector" -_RECON_NIM_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator" - - -# --------------------------------------------------------------------------- -# Shared test helpers -# --------------------------------------------------------------------------- - - -def _nim_config(): - """A minimal NIM-routing ModelDeploymentConfig-like object. - - Engine is explicitly ``nim`` so the backend routes status/create/update to the - NIM reconciler (the backend rejects unknown engine strings rather than - defaulting them to NIM). The NIM status path only reads ``resource_name``, so - the resolved model fields are irrelevant here. - """ - config = MagicMock() - config.engine = "nim" - return config - - -def _make_nimservice_mock(state: str, conditions: list | None = None): - """Create a mock NIMService response dict.""" - mock_resource = MagicMock() - mock_resource.get.return_value = { - "status": {"state": state, "conditions": conditions or []}, - } - return mock_resource - - -def _make_pod( - name: str = "test-pod-abc123", - restart_count: int = 0, - waiting_reason: str | None = None, - phase: str = "Running", -) -> MagicMock: - """Create a mock V1Pod with configurable restart/waiting state.""" - pod = MagicMock() - pod.metadata.name = name - pod.metadata.creation_timestamp = datetime.now(timezone.utc) - pod.status.phase = phase - - cs = MagicMock() - cs.restart_count = restart_count - - if waiting_reason: - cs.state.waiting.reason = waiting_reason - cs.state.waiting.message = f"Back-off restarting failed container in pod {name}" - else: - cs.state.waiting = None - cs.state.running = MagicMock() - - pod.status.container_statuses = [cs] - return pod - - -def _wire_pod_lookup(mock_apps_v1, mock_core_v1, pod): - """Wire up AppsV1 + CoreV1 mocks so _get_pod_status_from_deployment finds *pod*.""" - mock_deployment = MagicMock() - mock_deployment.spec.selector.match_labels = {"app": "test"} - mock_apps_v1.read_namespaced_deployment.return_value = mock_deployment - - pods_list = MagicMock() - pods_list.items = [pod] - mock_core_v1.list_namespaced_pod.return_value = pods_list - - events_list = MagicMock() - events_list.items = [] - mock_core_v1.list_namespaced_event.return_value = events_list - - -@contextlib.contextmanager -def _mock_pod_backend(k8s_backend, pod=None, *, pod_logs=""): - """Context manager that wires up AppsV1Api / CoreV1Api mocks on *k8s_backend*. - - Yields ``(mock_apps_v1, mock_core_v1)`` for further customisation. - """ - mock_apps_v1 = MagicMock() - mock_core_v1 = MagicMock() - - if pod is not None: - _wire_pod_lookup(mock_apps_v1, mock_core_v1, pod) - - mock_core_v1.read_namespaced_pod_log.return_value = pod_logs - - with ( - patch(f"{_RECON_STATUS_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1), - patch(f"{_RECON_STATUS_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1), - ): - yield mock_apps_v1, mock_core_v1 - - -@pytest.fixture -def mock_nmp_sdk(): - """Create a mock AsyncNeMoPlatform SDK.""" - mock = AsyncMock() - return mock - - -@pytest.fixture -def mock_k8s_config(): - """Mock kubernetes config loading to avoid needing actual k8s config.""" - with ( - patch(f"{_K8S_BACKEND_MODULE}.k8s_config.load_incluster_config"), - patch(f"{_K8S_BACKEND_MODULE}.k8s_config.load_kube_config"), - patch(f"{_K8S_BACKEND_MODULE}.k8s_client.ApiClient"), - patch(f"{_K8S_BACKEND_MODULE}.DynamicClient"), - patch(f"{_K8S_BACKEND_MODULE}.os.path.exists", return_value=False), - ): - yield - - -def create_model_spec(): - """Create a sample ModelSpec for testing.""" - model_spec = ModelSpec( - base_num_parameters=7000000000, - context_size=4096, - num_virtual_tokens=0, - is_chat=True, - checkpoint_model_name="meta-llama/Llama-3.2-1b-instruct", - family="llama", - num_layers=32, - hidden_size=4096, - num_attention_heads=32, - num_kv_heads=32, - ffn_hidden_size=16384, - vocab_size=32000, - tied_embeddings=True, - gated_mlp=True, - precision="fp16", - moe_config=MoEConfig( - num_experts=128, - num_experts_per_tok=128, - num_expert_layers=128, - expert_ffn_size=16384, - num_shared_experts=128, - ), - mamba_config=MambaConfig( - num_layers=32, - hidden_size=4096, - num_attention_heads=32, - num_kv_heads=32, - ffn_hidden_size=16384, - vocab_size=32000, - is_hybrid=True, - num_mamba_layers=32, - ), - sliding_window_config=SlidingWindowConfig( - window_size=1024, - ), - minimum_gpus_all_weights=1, - minimum_gpus_lora=1, - linear_layers=[ - LinearLayerSpec( - name="linear-layer-1", - in_features=4096, - out_features=4096, - ) - ], - ) - return model_spec - - -@pytest.fixture -def k8s_backend(mock_nmp_sdk, mock_k8s_config): - """Create a K8sNimOperatorServiceBackend instance for testing.""" - return K8sNimOperatorServiceBackend( - nmp_sdk=mock_nmp_sdk, - config={}, - huggingface_model_puller="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10", - ) - - -def _sync_reconcilers(backend): - """Propagate the backend's (test-mocked) k8s state onto its reconcilers. - - Tests assign mock clients/config/namespace onto the backend *after* - construction (``backend._dynamic_client = MagicMock()`` etc.). Reconciliation - logic now lives on the two reconcilers, which captured their own clients at - ``init()`` time. This helper re-points the reconcilers at whatever the test - set on the backend so delegation exercises the test's mocks. Call it after - setting up the backend's ``_dynamic_client`` / ``_core_v1`` / ``_apps_v1`` / - ``_batch_v1`` / ``_backend_config`` / ``_k8s_namespace`` / ``_k8s_client``. - """ - nim = backend._nim_reconciler - k8s = backend._k8s_reconciler - status = getattr(backend, "_status_projector", None) - deleter = getattr(backend, "_resource_deleter", None) - namespace = backend._k8s_namespace - config = backend._backend_config - client = backend._k8s_client - - if nim is not None: - nim._k8s_namespace = namespace - nim._backend_config = config - if backend._dynamic_client is not None: - nim._dynamic_client = backend._dynamic_client - if k8s is not None: - k8s._k8s_namespace = namespace - k8s._backend_config = config - k8s._k8s_client = client - if getattr(backend, "_core_v1", None) is not None: - k8s._core_v1 = backend._core_v1 - if getattr(backend, "_apps_v1", None) is not None: - k8s._apps_v1 = backend._apps_v1 - if getattr(backend, "_batch_v1", None) is not None: - k8s._batch_v1 = backend._batch_v1 - if status is not None: - status._k8s_namespace = namespace - status._backend_config = config - status._k8s_client = client - if deleter is not None: - deleter._k8s_namespace = namespace - return backend - - -def _status_helper_reconciler(*, namespace="default", backend_config=None, k8s_client_=None): - """Build a StatusProjector exposing the shared status helpers for direct tests.""" - return StatusProjector( - k8s_client_=k8s_client_ if k8s_client_ is not None else MagicMock(), - backend_config=backend_config if backend_config is not None else K8sNimOperatorConfig(), - k8s_namespace=namespace, - ) - - -@pytest.fixture -def sample_deployment(): - """Create a sample ModelDeployment for testing. - - ``created_at`` defaults to 5 minutes ago so that PENDING timeout (2 h) - does NOT fire in the majority of tests. Override in individual tests - when timeout behaviour needs to be exercised. - """ - deployment = MagicMock() - deployment.workspace = "default" - deployment.name = "test-deployment" - deployment.entity_version = "v1" - deployment.status = "CREATED" - deployment.config = "test-config" - deployment.config_version = "v1" - deployment.created_at = datetime.now(timezone.utc) - timedelta(minutes=5) - return deployment - - -@pytest.fixture -def sample_config(): - """Create a sample ModelDeploymentConfig for testing. - - Engine is explicitly ``nim`` so the backend routes it to the NIM-operator - reconciler (the backend now rejects unknown engine strings rather than - defaulting them to NIM). - """ - config = MagicMock() - config.engine = "nim" - return config - - -@pytest.mark.asyncio -async def test_k8s_backend_create_model_deployment(k8s_backend, sample_deployment, sample_config): - """Test creating a model deployment with K8s NIM Operator backend.""" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - - # Mock the dynamic client resource operations - mock_resource = MagicMock() - mock_created = MagicMock() - mock_created.metadata.uid = "test-uid" - mock_resource.create.return_value = mock_created - k8s_backend._dynamic_client.resources.get.return_value = mock_resource - - # Mock the compile_nimservice function to avoid validation issues - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - # Verify compile_nimservice was called - mock_compile.assert_called_once() - - # Verify status update returned - assert status_update is not None - assert status_update.status == "PENDING" - assert "initiated successfully" in status_update.status_message - assert status_update.host_url is not None - - -@pytest.mark.asyncio -async def test_k8s_backend_update_model_deployment(k8s_backend, sample_deployment, sample_config): - """Test updating a model deployment with K8s NIM Operator backend.""" - # Mock the k8s clients - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - - # Mock the dynamic client resource operations - mock_resource = MagicMock() - mock_updated = MagicMock() - mock_updated.metadata.uid = "test-uid" - mock_resource.replace.return_value = mock_updated - k8s_backend._dynamic_client.resources.get.return_value = mock_resource - - # Mock the compile_nimservice function to avoid validation issues - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.update_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - # Verify compile_nimservice was called - mock_compile.assert_called_once() - - # Verify status update returned - assert status_update is not None - assert status_update.status == "PENDING" - assert "initiated successfully" in status_update.status_message - assert status_update.host_url is not None - - -@pytest.mark.asyncio -async def test_k8s_backend_get_model_deployment_status(k8s_backend, sample_deployment): - """Test getting model deployment status with K8s NIM Operator backend.""" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = K8sNimOperatorConfig() - - k8s_backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("Ready") - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - - assert status_update is not None - assert status_update.status == "READY" - assert status_update.status_message == "" - assert status_update.host_url is not None - - -@pytest.mark.asyncio -async def test_k8s_backend_get_status_without_config_is_unknown(k8s_backend, sample_deployment): - """No config -> backend cannot determine the engine/state, returns UNKNOWN. - - The controller retries on the next poll (which normally has a config) and - escalates to ERROR after its retry budget; the backend does not probe. - """ - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = K8sNimOperatorConfig() - _sync_reconcilers(k8s_backend) - - status_update = await k8s_backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=None) - ) - - assert status_update.status == "UNKNOWN" - # No reconciler/cluster lookups happen without a config. - k8s_backend._dynamic_client.resources.get.assert_not_called() - - -@pytest.mark.asyncio -async def test_k8s_backend_get_status_nimservice_not_found(k8s_backend, sample_deployment): - """Test getting status when NIMService doesn't exist yet - should keep current status.""" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = K8sNimOperatorConfig() - - mock_resource = MagicMock() - mock_resource.get.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock()) - k8s_backend._dynamic_client.resources.get.return_value = mock_resource - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - - assert status_update is not None - assert status_update.status == "LOST" - assert "not found" in status_update.status_message.lower() - assert status_update.host_url is None - - -@pytest.mark.asyncio -async def test_k8s_backend_get_status_nimservice_not_ready(k8s_backend, sample_deployment): - """Test getting status when NIMService is NotReady.""" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = K8sNimOperatorConfig() - - k8s_backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - with patch.object( - k8s_backend._status_projector, - "pod_status_from_deployment", - return_value=DeploymentStatusUpdate( - status="PENDING", status_message="Waiting for NIMService to become ready", host_url=None - ), - ): - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - - assert status_update is not None - assert status_update.status == "PENDING" - assert "ready" in status_update.status_message.lower() - # No elapsed/timeout in message (stable message to avoid new history entry every poll) - assert status_update.host_url is None - - -@pytest.mark.asyncio -async def test_k8s_backend_get_status_nimservice_crash_loop_backoff(k8s_backend, sample_deployment): - """Test getting status when NIMService pod is in CrashLoopBackoff with restarts >= max.""" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = K8sNimOperatorConfig() - k8s_backend._k8s_client = MagicMock() - - k8s_backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - pod = _make_pod(restart_count=5, waiting_reason="CrashLoopBackOff") - - with _mock_pod_backend(k8s_backend, pod=pod, pod_logs="ERROR: model failed to load"): - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - - assert status_update is not None - assert status_update.status == "ERROR" - assert "crash loop" in status_update.status_message - assert "container restarts" in status_update.status_message - assert "kubectl logs" in status_update.status_message - assert status_update.error_details["reason"] == "crash_loop" - assert status_update.error_details["restart_count"] == 5 - assert status_update.host_url is None - - -@pytest.mark.asyncio -async def test_k8s_backend_get_status_nimservice_pod_restarts_below_threshold(k8s_backend, sample_deployment): - """Test getting status when pod has restarts but below the max_restart_count threshold.""" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = K8sNimOperatorConfig() - k8s_backend._k8s_client = MagicMock() - - k8s_backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - pod = _make_pod(restart_count=2) - - with _mock_pod_backend(k8s_backend, pod=pod): - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - - assert status_update is not None - assert status_update.status == "PENDING" - assert "restarts: 2" in status_update.status_message - assert status_update.host_url is None - - -@pytest.mark.asyncio -async def test_k8s_backend_get_status_nimservice_pod_running_after_restarts(k8s_backend, sample_deployment): - """Test getting status when pod has restarts >= max but is now running (not in waiting state).""" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = K8sNimOperatorConfig() - k8s_backend._k8s_client = MagicMock() - - k8s_backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - pod = _make_pod(restart_count=5) # No waiting_reason → running - - with _mock_pod_backend(k8s_backend, pod=pod): - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - - assert status_update is not None - assert status_update.status == "PENDING" - assert "restarts: 5" in status_update.status_message - assert status_update.host_url is None - - -@pytest.mark.asyncio -async def test_k8s_backend_delete_model_deployment(k8s_backend, sample_deployment): - """Test deleting a model deployment with K8s NIM Operator backend.""" - # Mock the k8s clients - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - - # Mock the dynamic client resource operations - mock_resource = MagicMock() - k8s_backend._dynamic_client.resources.get.return_value = mock_resource - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) - - # Verify status update returned - assert status_update is not None - assert status_update.status == "DELETED" - assert "initiated successfully" in status_update.status_message - - -@pytest.mark.asyncio -async def test_k8s_backend_delete_model_deployment_with_secret(k8s_backend, sample_deployment): - """Test deleting a model deployment.""" - # Mock the k8s clients - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - k8s_backend._k8s_client = MagicMock() - - # Mock the dynamic client resource operations - mock_resource = MagicMock() - k8s_backend._dynamic_client.resources.get.return_value = mock_resource - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) - - # Verify status update returned - assert status_update is not None - assert status_update.status == "DELETED" - assert "initiated successfully" in status_update.status_message - - -@pytest.mark.asyncio -async def test_k8s_backend_delete_model_deployment_without_secret(k8s_backend, sample_deployment): - """Test deleting a model deployment (no HF secret).""" - # Mock the k8s clients - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - - # Mock the dynamic client resource operations - mock_resource = MagicMock() - k8s_backend._dynamic_client.resources.get.return_value = mock_resource - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) - - # Verify status update returned - assert status_update is not None - assert status_update.status == "DELETED" - - -@pytest.mark.asyncio -async def test_delete_attempts_all_resource_types_and_tolerates_404(k8s_backend, sample_deployment): - """Delete attempts CRs + raw vLLM objects by name; 404s are success -> DELETED.""" - k8s_backend._k8s_namespace = "default" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._core_v1 = MagicMock() - k8s_backend._apps_v1 = MagicMock() - k8s_backend._batch_v1 = MagicMock() - cr_api = MagicMock() - cr_api.delete.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock(status=404)) - k8s_backend._dynamic_client.resources.get.return_value = cr_api - notfound = k8s_client.exceptions.ApiException(status=404) - k8s_backend._apps_v1.delete_namespaced_deployment.side_effect = notfound - k8s_backend._core_v1.delete_namespaced_service.side_effect = notfound - k8s_backend._batch_v1.delete_namespaced_job.side_effect = notfound - k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.side_effect = notfound - - _sync_reconcilers(k8s_backend) - result = await k8s_backend.delete_model_deployment("default", "qwen") - - assert result.status == "DELETED" - assert k8s_backend._dynamic_client.resources.get.call_count == 2 - k8s_backend._apps_v1.delete_namespaced_deployment.assert_called_once() - k8s_backend._core_v1.delete_namespaced_service.assert_called_once() - k8s_backend._batch_v1.delete_namespaced_job.assert_called_once() - k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.assert_called_once() - - -@pytest.mark.asyncio -async def test_delete_real_failure_surfaces_error_but_attempts_all(k8s_backend, sample_deployment): - """A non-404 delete failure -> ERROR (not DELETED), and other deletes still run.""" - k8s_backend._k8s_namespace = "default" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._core_v1 = MagicMock() - k8s_backend._apps_v1 = MagicMock() - k8s_backend._batch_v1 = MagicMock() - cr_api = MagicMock() - cr_api.delete.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock(status=404)) - k8s_backend._dynamic_client.resources.get.return_value = cr_api - k8s_backend._apps_v1.delete_namespaced_deployment.side_effect = k8s_client.exceptions.ApiException(status=500) - notfound = k8s_client.exceptions.ApiException(status=404) - k8s_backend._core_v1.delete_namespaced_service.side_effect = notfound - k8s_backend._batch_v1.delete_namespaced_job.side_effect = notfound - k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.side_effect = notfound - - _sync_reconcilers(k8s_backend) - result = await k8s_backend.delete_model_deployment("default", "qwen") - - assert result.status == "ERROR" - k8s_backend._core_v1.delete_namespaced_service.assert_called_once() - k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.assert_called_once() - - -@pytest.mark.asyncio -async def test_delete_forbidden_cr_does_not_block_vllm_cleanup(k8s_backend, sample_deployment): - """A 403 deleting a NIMService still lets the raw vLLM objects be deleted (and surfaces ERROR).""" - k8s_backend._k8s_namespace = "default" - k8s_backend._dynamic_client = MagicMock() - k8s_backend._core_v1 = MagicMock() - k8s_backend._apps_v1 = MagicMock() - k8s_backend._batch_v1 = MagicMock() - cr_api = MagicMock() - cr_api.delete.side_effect = k8s_dynamic_exceptions.ForbiddenError(MagicMock(status=403)) - k8s_backend._dynamic_client.resources.get.return_value = cr_api - - _sync_reconcilers(k8s_backend) - result = await k8s_backend.delete_model_deployment("default", "qwen") - - assert result.status == "ERROR" - assert "forbidden" in result.status_message.lower() - k8s_backend._apps_v1.delete_namespaced_deployment.assert_called_once() - k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.assert_called_once() - - -def test_delete_one_404_is_success(): - """A typed 404 (object absent) is treated as success -> returns None.""" - deleter = ResourceDeleter(k8s_namespace="default") - delete_fn = MagicMock(side_effect=k8s_client.exceptions.ApiException(status=404)) - assert deleter.delete_one(delete_fn, "PVC", "obj") is None - - -def test_delete_one_dynamic_notfound_is_success(): - """A dynamic NotFoundError is treated as success -> returns None.""" - deleter = ResourceDeleter(k8s_namespace="default") - delete_fn = MagicMock(side_effect=k8s_dynamic_exceptions.NotFoundError(MagicMock(status=404))) - assert deleter.delete_one(delete_fn, "PVC", "obj") is None - - -def test_delete_one_forbidden_is_classified_not_raised(): - """A 403 is classified and returned as an error string, not raised.""" - deleter = ResourceDeleter(k8s_namespace="default") - delete_fn = MagicMock(side_effect=k8s_client.exceptions.ApiException(status=403)) - err = deleter.delete_one(delete_fn, "PVC", "obj") - assert err is not None - assert "forbidden" in err.lower() - - -def test_delete_one_unexpected_exception_is_classified_not_raised(): - """A non-API/transport error must be classified and returned, never raised. - - Guards the aggregation contract: the caller's per-resource delete loop must - continue (and surface the failure) rather than abort cleanup partway. - """ - deleter = ResourceDeleter(k8s_namespace="default") - delete_fn = MagicMock(side_effect=ConnectionError("connection reset")) - err = deleter.delete_one(delete_fn, "Deployment", "obj") - assert err is not None - assert "error deleting Deployment obj" in err - - -def test_k8s_backend_initialization(mock_nmp_sdk, mock_k8s_config): - """Test K8s NIM Operator backend initializes correctly with custom namespace config.""" - config = {"namespace": "nim-system"} - huggingface_model_puller = "nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10" - - backend = K8sNimOperatorServiceBackend( - nmp_sdk=mock_nmp_sdk, - config=config, - huggingface_model_puller=huggingface_model_puller, - ) - - # Verify backend initialized correctly with custom config - # Parent __init__ calls init() automatically, so everything should be set up - assert backend is not None - assert backend._nmp_sdk == mock_nmp_sdk - assert backend._config == config - assert backend._huggingface_model_puller == huggingface_model_puller - # After init(), these should be set (mocked by mock_k8s_config fixture) - assert backend._backend_config is not None - # The custom namespace from config should be used - assert backend._k8s_namespace == "nim-system" - - -def test_k8s_backend_initialization_with_empty_config(k8s_backend): - """Test K8s NIM Operator backend initializes correctly with empty config and defaults to 'default' namespace.""" - # The fixture creates backend with empty config - # Verify backend was fully initialized - assert k8s_backend is not None - assert k8s_backend._config == {} - assert k8s_backend._huggingface_model_puller == "nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10" - # After init(), these should be set (mocked by mock_k8s_config fixture) - assert k8s_backend._backend_config is not None - # With empty config and no service account file, should default to "default" - assert k8s_backend._k8s_namespace == "default" - - -@pytest.mark.asyncio -async def test_k8s_backend_create_when_nimservice_already_exists(k8s_backend, sample_deployment, sample_config): - """Test creating a deployment when NIMService already exists - should return PENDING without error.""" - # Mock the k8s clients - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - - # Mock the dynamic client to raise ConflictError on create - mock_resource = MagicMock() - mock_resource.create.side_effect = k8s_dynamic_exceptions.ConflictError(MagicMock()) - k8s_backend._dynamic_client.resources.get.return_value = mock_resource - - # Mock the compile_nimservice function - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) - ) - - # Verify status update returned PENDING (not ERROR) - assert status_update is not None - assert status_update.status == "PENDING" - assert "initiated successfully" in status_update.status_message - assert status_update.host_url is not None - - # Verify create was attempted but replace was NOT called - mock_resource.create.assert_called_once() - mock_resource.replace.assert_not_called() - - -# ============================================================================ -# SFT Model Deployment Integration Tests -# ============================================================================ - - -@pytest.mark.asyncio -async def test_create_model_deployment_with_sft_model(k8s_backend, sample_deployment, sample_config): - """Test that SFT models trigger NIMCache creation.""" - # Setup backend state - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - k8s_backend._backend_config.default_storage_class = "local-storage" - k8s_backend._backend_config.default_pvc_size = "200Gi" - k8s_backend._backend_config.files_auth_secret = "nemo-models-files-token" - k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - k8s_backend._backend_config.default_user_id = None - k8s_backend._backend_config.default_group_id = None - k8s_backend._backend_config.default_resources = None - k8s_backend._backend_config.default_tolerations = None - k8s_backend._backend_config.default_node_selector = None - - # Configure sample_config for SFT - sample_config.engine = "nim" - sample_config.model_spec = MagicMock() - sample_config.executor_config = MagicMock() - sample_config.model_spec.model_namespace = "test-ns" - sample_config.model_spec.model_name = "test-model" - sample_config.model_spec.model_revision = None - sample_config.executor_config.disk_size = "300Gi" - - # Create SFT model entity with fileset matching the model name - sft_model_entity = ModelEntity( - id="model-1", - entity_id="model-1", - created_at="2024-01-01T00:00:00Z", - updated_at="2024-01-01T00:00:00Z", - workspace="test-ns", - name="test-model", - parent="models", - db_version=1, - fileset="test-ns/test-model", - spec=create_model_spec(), - ) - - # Mock the dynamic client resource operations for both NIMCache and NIMService - mock_nimcache_resource = MagicMock() - mock_nimservice_resource = MagicMock() - mock_created_cache = MagicMock() - mock_created_cache.metadata.uid = "cache-uid" - mock_created_service = MagicMock() - mock_created_service.metadata.uid = "service-uid" - - mock_nimcache_resource.create.return_value = mock_created_cache - mock_nimservice_resource.create.return_value = mock_created_service - - def get_resource_side_effect(api_version, kind): - if kind == "NIMCache": - return mock_nimcache_resource - elif kind == "NIMService": - return mock_nimservice_resource - return MagicMock() - - k8s_backend._dynamic_client.resources.get.side_effect = get_resource_side_effect - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=sample_config, - model_entity=sft_model_entity, - ) - ) - - # Verify NIMCache was created - mock_nimcache_resource.create.assert_called_once() - nimcache_call_args = mock_nimcache_resource.create.call_args - created_nimcache = nimcache_call_args.kwargs["body"] - assert created_nimcache["kind"] == "NIMCache" - assert created_nimcache["spec"]["source"]["hf"]["namespace"] == "test-ns" - assert created_nimcache["spec"]["source"]["hf"]["modelName"] == "test-model" - - # Verify compile_nimservice was called with nimcache_name - mock_compile.assert_called_once() - compile_call_kwargs = mock_compile.call_args.kwargs - assert compile_call_kwargs["nimcache_name"] is not None - - # Verify status update returned - assert status_update is not None - assert status_update.status == "PENDING" - - -@pytest.mark.asyncio -async def test_create_model_deployment_with_files_service_model_triggers_nimcache( - k8s_backend, sample_deployment, sample_config -): - """Test that FILES_SERVICE (fileset-backed, non-SFT) models trigger NIMCache creation.""" - # Setup backend state - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - k8s_backend._backend_config.default_storage_class = "local-storage" - k8s_backend._backend_config.default_pvc_size = "200Gi" - k8s_backend._backend_config.files_auth_secret = "nemo-models-files-token" - k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - k8s_backend._backend_config.default_user_id = None - k8s_backend._backend_config.default_group_id = None - k8s_backend._backend_config.default_resources = None - k8s_backend._backend_config.default_tolerations = None - k8s_backend._backend_config.default_node_selector = None - - # Configure sample_config - sample_config.engine = "nim" - sample_config.model_spec = MagicMock() - sample_config.executor_config = MagicMock() - sample_config.model_spec.model_namespace = "test-ns" - sample_config.model_spec.model_name = "test-model" - sample_config.model_spec.model_revision = None - sample_config.executor_config.disk_size = "300Gi" - - # FILES_SERVICE: fileset set but NOT SFT (no spec, no finetuning_type LORA_MERGED/ALL_WEIGHTS) - files_service_model_entity = ModelEntity( - id="model-1", - entity_id="model-1", - created_at="2024-01-01T00:00:00Z", - updated_at="2024-01-01T00:00:00Z", - workspace="test-ns", - name="test-model", - parent="models", - db_version=1, - fileset="test-ns/test-model", - spec=None, - ) - - # Mock the dynamic client resource operations for both NIMCache and NIMService - mock_nimcache_resource = MagicMock() - mock_nimservice_resource = MagicMock() - mock_created_cache = MagicMock() - mock_created_cache.metadata.uid = "cache-uid" - mock_created_service = MagicMock() - mock_created_service.metadata.uid = "service-uid" - - mock_nimcache_resource.create.return_value = mock_created_cache - mock_nimservice_resource.create.return_value = mock_created_service - - def get_resource_side_effect(api_version, kind): - if kind == "NIMCache": - return mock_nimcache_resource - elif kind == "NIMService": - return mock_nimservice_resource - return MagicMock() - - k8s_backend._dynamic_client.resources.get.side_effect = get_resource_side_effect - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=sample_config, - model_entity=files_service_model_entity, - ) - ) - - # Verify NIMCache was created for FILES_SERVICE - mock_nimcache_resource.create.assert_called_once() - nimcache_call_args = mock_nimcache_resource.create.call_args - created_nimcache = nimcache_call_args.kwargs["body"] - assert created_nimcache["kind"] == "NIMCache" - assert created_nimcache["spec"]["source"]["hf"]["namespace"] == "test-ns" - assert created_nimcache["spec"]["source"]["hf"]["modelName"] == "test-model" - - # Verify compile_nimservice was called with nimcache_name - mock_compile.assert_called_once() - compile_call_kwargs = mock_compile.call_args.kwargs - assert compile_call_kwargs["nimcache_name"] is not None - - # Verify status update returned - assert status_update is not None - assert status_update.status == "PENDING" - - -@pytest.mark.asyncio -async def test_create_model_deployment_without_sft_model(k8s_backend, sample_deployment, sample_config): - """Test that non-SFT models do NOT trigger NIMCache creation.""" - # Setup backend state - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - - # Configure sample_config - sample_config.engine = "nim" - sample_config.model_spec = MagicMock() - sample_config.executor_config = MagicMock() - sample_config.model_spec.model_namespace = "test-ns" - sample_config.model_spec.model_name = "test-model" - - # Create non-SFT model entity - non_sft_model_entity = ModelEntity( - id="model-1", - entity_id="model-1", - created_at="2024-01-01T00:00:00Z", - updated_at="2024-01-01T00:00:00Z", - workspace="test-ns", - name="test-model", - parent="models", - db_version=1, - fileset=None, - ) - - # Mock the dynamic client resource operations for NIMService only - mock_nimservice_resource = MagicMock() - mock_created_service = MagicMock() - mock_created_service.metadata.uid = "service-uid" - mock_nimservice_resource.create.return_value = mock_created_service - k8s_backend._dynamic_client.resources.get.return_value = mock_nimservice_resource - - # Mock the compile_nimservice function - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=sample_config, - model_entity=non_sft_model_entity, - ) - ) - - # Verify compile_nimservice was called with nimcache_name=None - mock_compile.assert_called_once() - compile_call_kwargs = mock_compile.call_args.kwargs - assert compile_call_kwargs["nimcache_name"] is None - - # Verify status update returned - assert status_update is not None - assert status_update.status == "PENDING" - - -@pytest.mark.asyncio -async def test_create_model_deployment_with_sft_model_and_revision(k8s_backend, sample_deployment, sample_config): - """Test that SFT models with revision in model_name trigger NIMCache creation with parsed revision.""" - # Setup backend state - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - k8s_backend._backend_config.default_pvc_size = "100Gi" - k8s_backend._backend_config.default_storage_class = "standard" - k8s_backend._backend_config.default_user_id = 1000 - k8s_backend._backend_config.default_group_id = 1000 - k8s_backend._backend_config.files_auth_secret = "files-api-token" - k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcrimagepullsecret" - k8s_backend._backend_config.default_resources = None - k8s_backend._backend_config.default_tolerations = None - k8s_backend._backend_config.default_node_selector = None - - # Configure sample_config with model name containing revision - sample_config.engine = "nim" - sample_config.model_spec = MagicMock() - sample_config.executor_config = MagicMock() - sample_config.model_spec.model_namespace = "test-ns" - sample_config.model_spec.model_name = "test-model@v1.0" - sample_config.model_spec.model_revision = None - sample_config.executor_config.disk_size = "50Gi" - - # Create SFT model entity with fileset matching the model name - sft_model_entity = ModelEntity( - id="model-1", - entity_id="model-1", - created_at="2024-01-01T00:00:00Z", - updated_at="2024-01-01T00:00:00Z", - workspace="test-ns", - name="test-model", - parent="models", - db_version=1, - fileset="test-ns/test-model", - spec=create_model_spec(), - ) - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - # Mock the dynamic client resource operations - mock_nimcache_resource = MagicMock() - mock_nimservice_resource = MagicMock() - mock_created_cache = MagicMock() - mock_created_cache.metadata.name = "test-cache" - mock_nimcache_resource.create.return_value = mock_created_cache - mock_created_service = MagicMock() - mock_created_service.metadata.uid = "service-uid" - mock_nimservice_resource.create.return_value = mock_created_service - - def get_resource(api_version, kind): - if kind == "NIMCache": - return mock_nimcache_resource - return mock_nimservice_resource - - k8s_backend._dynamic_client.resources.get.side_effect = get_resource - - # Mock the compile_nimservice function - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = { - "apiVersion": "apps.nvidia.com/v1alpha1", - "kind": "NIMService", - } - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=sample_config, - model_entity=sft_model_entity, - ) - ) - - # Verify NIMCache was created - assert mock_nimcache_resource.create.called - - # Verify the NIMCache has the correct model name (without @revision) and revision field - nimcache_call = mock_nimcache_resource.create.call_args - nimcache_dict = nimcache_call.kwargs["body"] - # The body is a dict from model_dump() - assert nimcache_dict["spec"]["source"]["hf"]["modelName"] == "test-model" - assert nimcache_dict["spec"]["source"]["hf"]["revision"] == "v1.0" - - # Verify compile_nimservice was called with nimcache_name - mock_compile.assert_called_once() - compile_call_kwargs = mock_compile.call_args.kwargs - assert compile_call_kwargs["nimcache_name"] is not None - - # Verify status update returned - assert status_update is not None - assert status_update.status == "PENDING" - - -@pytest.mark.asyncio -async def test_create_model_deployment_nimcache_uses_fileset_not_entity_name( - k8s_backend, sample_deployment, sample_config -): - """Test that NIMCache uses the fileset name from model entity, not the model entity name. - - This is the critical scenario for LoRA deployments where the base model entity - name differs from the backing fileset name. The HF-compatible Files API resolves - by fileset name, so the NIMCache must use the fileset path. - """ - k8s_backend._dynamic_client = MagicMock() - k8s_backend._k8s_namespace = "default" - k8s_backend._backend_config = MagicMock() - k8s_backend._backend_config.default_storage_class = "local-storage" - k8s_backend._backend_config.default_pvc_size = "200Gi" - k8s_backend._backend_config.files_auth_secret = "nemo-models-files-token" - k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - k8s_backend._backend_config.default_user_id = None - k8s_backend._backend_config.default_group_id = None - k8s_backend._backend_config.default_resources = None - k8s_backend._backend_config.default_tolerations = None - k8s_backend._backend_config.default_node_selector = None - - sample_config.engine = "nim" - sample_config.model_spec = MagicMock() - sample_config.executor_config = MagicMock() - sample_config.model_spec.model_namespace = "my-workspace" - sample_config.model_spec.model_name = "my-model-entity" - sample_config.model_spec.model_revision = None - sample_config.executor_config.disk_size = "300Gi" - - mismatched_model_entity = ModelEntity( - id="model-1", - entity_id="model-1", - created_at="2024-01-01T00:00:00Z", - updated_at="2024-01-01T00:00:00Z", - workspace="my-workspace", - name="my-model-entity", - parent="models", - db_version=1, - fileset="my-workspace/my-actual-fileset", - spec=None, - ) - - mock_nimcache_resource = MagicMock() - mock_nimservice_resource = MagicMock() - mock_created_cache = MagicMock() - mock_created_cache.metadata.uid = "cache-uid" - mock_created_service = MagicMock() - mock_created_service.metadata.uid = "service-uid" - mock_nimcache_resource.create.return_value = mock_created_cache - mock_nimservice_resource.create.return_value = mock_created_service - - def get_resource_side_effect(api_version, kind): - if kind == "NIMCache": - return mock_nimcache_resource - elif kind == "NIMService": - return mock_nimservice_resource - return MagicMock() - - k8s_backend._dynamic_client.resources.get.side_effect = get_resource_side_effect - - platform_config = PlatformConfig( # type: ignore[abstract] - files_url="http://files-service:8000", - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" - ) as mock_compile: - mock_nimservice = MagicMock() - mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} - mock_compile.return_value = mock_nimservice - - _sync_reconcilers(k8s_backend) - status_update = await k8s_backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=sample_config, - model_entity=mismatched_model_entity, - ) - ) - - mock_nimcache_resource.create.assert_called_once() - nimcache_call_args = mock_nimcache_resource.create.call_args - created_nimcache = nimcache_call_args.kwargs["body"] - assert created_nimcache["kind"] == "NIMCache" - assert created_nimcache["spec"]["source"]["hf"]["namespace"] == "my-workspace" - assert created_nimcache["spec"]["source"]["hf"]["modelName"] == "my-actual-fileset" - - assert status_update is not None - assert status_update.status == "PENDING" - - -# ============================================================================ -# Config field tests -# ============================================================================ - - -class TestConfigFields: - """Tests for K8sNimOperatorConfig pending_timeout_seconds and max_restart_count.""" - - def test_defaults(self): - config = K8sNimOperatorConfig() - assert config.pending_timeout_seconds == 7200 - assert config.max_restart_count == 5 - - def test_explicit_values(self): - config = K8sNimOperatorConfig(pending_timeout_seconds=120, max_restart_count=3) - assert config.pending_timeout_seconds == 120 - assert config.max_restart_count == 3 - - def test_pending_timeout_minimum(self): - with pytest.raises(ValidationError): - K8sNimOperatorConfig(pending_timeout_seconds=30) - - def test_max_restart_count_minimum(self): - with pytest.raises(ValidationError): - K8sNimOperatorConfig(max_restart_count=0) - - -# ============================================================================ -# Static helper unit tests -# ============================================================================ - - -class TestFormatDuration: - """Tests for shared format_duration helper.""" - - @pytest.mark.parametrize( - ("seconds", "expected"), - [ - (0, "0s"), - (59, "59s"), - (60, "1m 0s"), - (90, "1m 30s"), - (3600, "1h 0s"), - (3661, "1h 1m 1s"), - (7200, "2h 0s"), - (7325, "2h 2m 5s"), - ], - ) - def test_format_duration(self, seconds, expected): - assert format_duration(seconds) == expected - - -class TestWithRestartInfo: - """Tests for StatusProjector._with_restart_info.""" - - def test_no_restarts(self): - assert StatusProjector._with_restart_info("some status", 0) == "some status" - - def test_with_restarts(self): - assert StatusProjector._with_restart_info("some status", 3) == "some status, restarts: 3" - - -class TestGetPodRestartCount: - """Tests for StatusProjector._get_pod_restart_count.""" - - def test_no_container_statuses(self): - pod = MagicMock() - pod.status.container_statuses = None - assert StatusProjector._get_pod_restart_count(pod) == 0 - - def test_multiple_containers_returns_max(self): - pod = MagicMock() - cs1 = MagicMock() - cs1.restart_count = 2 - cs2 = MagicMock() - cs2.restart_count = 7 - pod.status.container_statuses = [cs1, cs2] - assert StatusProjector._get_pod_restart_count(pod) == 7 - - -class TestDeploymentElapsedSeconds: - """Tests for shared deployment_elapsed_seconds helper.""" - - def test_returns_positive_elapsed(self): - deployment = MagicMock() - deployment.created_at = datetime.now(timezone.utc) - timedelta(minutes=10) - elapsed = deployment_elapsed_seconds(deployment) - assert 590 <= elapsed <= 620 - - def test_handles_naive_datetime(self): - deployment = MagicMock() - deployment.created_at = (datetime.now(timezone.utc) - timedelta(seconds=30)).replace(tzinfo=None) - elapsed = deployment_elapsed_seconds(deployment) - assert 25 <= elapsed <= 35 - - def test_returns_zero_when_created_at_is_none(self): - deployment = MagicMock() - deployment.created_at = None - assert deployment_elapsed_seconds(deployment) == 0.0 - - -# ============================================================================ -# PENDING timeout status transitions -# ============================================================================ - - -class TestPendingTimeoutStatusTransition: - """Tests for PENDING -> ERROR transition based on deployment.created_at.""" - - @pytest.fixture - def backend_with_short_timeout(self, mock_nmp_sdk, mock_k8s_config): - backend = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=60, max_restart_count=5) - backend._k8s_namespace = "default" - return backend - - @pytest.mark.asyncio - async def test_pending_within_timeout_stays_pending(self, backend_with_short_timeout, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(seconds=30) - - backend = backend_with_short_timeout - backend._dynamic_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - with patch.object( - backend._status_projector, - "pod_status_from_deployment", - return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), - ): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "PENDING" - # No elapsed/timeout in message (stable message to avoid new history entry every poll) - - @pytest.mark.asyncio - async def test_pending_beyond_timeout_transitions_to_error(self, backend_with_short_timeout, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(seconds=120) - - backend = backend_with_short_timeout - backend._dynamic_client = MagicMock() - backend._k8s_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - with patch.object( - backend._status_projector, - "pod_status_from_deployment", - return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), - ): - with _mock_pod_backend(backend, pod_logs="timeout error log"): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "ERROR" - assert "timed out" in result.status_message - assert "kubectl logs" in result.status_message - assert result.error_details["reason"] == "pending_timeout" - - @pytest.mark.asyncio - async def test_ready_not_affected_by_elapsed_time(self, backend_with_short_timeout, sample_deployment): - """READY status should be returned as-is regardless of elapsed time.""" - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=10) - - backend = backend_with_short_timeout - backend._dynamic_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("Ready") - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "READY" - - @pytest.mark.asyncio - @pytest.mark.parametrize("nim_state", ["Ready", "Failed"]) - async def test_terminal_states_not_affected_by_timeout( - self, backend_with_short_timeout, sample_deployment, nim_state - ): - """Terminal NIMService states should never be overridden by timeout logic.""" - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=10) - - backend = backend_with_short_timeout - backend._dynamic_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock(nim_state) - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status != "PENDING" - - @pytest.mark.asyncio - async def test_pending_timeout_at_exact_boundary(self, backend_with_short_timeout, sample_deployment): - """At exactly the timeout boundary, should transition to ERROR.""" - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(seconds=60) - - backend = backend_with_short_timeout - backend._dynamic_client = MagicMock() - backend._k8s_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - with patch.object( - backend._status_projector, - "pod_status_from_deployment", - return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), - ): - with _mock_pod_backend(backend): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "ERROR" - assert result.error_details["reason"] == "pending_timeout" - - -# ============================================================================ -# Crash loop detection -# ============================================================================ - - -class TestCrashLoopDetection: - """Tests for crash loop detection via _check_crash_loop.""" - - @pytest.fixture - def backend(self, mock_nmp_sdk, mock_k8s_config): - return _status_helper_reconciler( - namespace="test-ns", - backend_config=K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=5), - ) - - def test_no_container_statuses(self, backend): - pod = MagicMock() - pod.metadata.name = "pod-1" - pod.status.phase = "Pending" - pod.status.container_statuses = None - assert backend.check_crash_loop(pod, "res-1") is None - - def test_below_threshold_no_error(self, backend): - pod = _make_pod(restart_count=4, waiting_reason="CrashLoopBackOff") - assert backend.check_crash_loop(pod, "res-1") is None - - def test_at_threshold_with_waiting_returns_error(self, backend): - pod = _make_pod(restart_count=5, waiting_reason="CrashLoopBackOff") - - with _mock_pod_backend(backend, pod_logs="crash log"): - result = backend.check_crash_loop(pod, "res-1") - assert result is not None - assert result.status == "ERROR" - assert result.error_details["reason"] == "crash_loop" - assert result.error_details["restart_count"] == 5 - assert result.error_details["max_restart_count"] == 5 - - def test_above_threshold_without_waiting_returns_none(self, backend): - pod = _make_pod(restart_count=10) # running, not waiting - result = backend.check_crash_loop(pod, "res-1") - assert result is None - - @pytest.mark.parametrize("max_restarts", [1, 3, 10]) - def test_configurable_threshold(self, backend, max_restarts): - backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=max_restarts) - pod = _make_pod(restart_count=max_restarts, waiting_reason="CrashLoopBackOff") - with _mock_pod_backend(backend, pod_logs=""): - result = backend.check_crash_loop(pod, "res-1") - assert result is not None - assert result.status == "ERROR" - - def test_crash_loop_error_includes_kubectl_command(self, backend): - pod = _make_pod(name="my-pod-xyz", restart_count=5, waiting_reason="CrashLoopBackOff") - with _mock_pod_backend(backend, pod_logs="some logs"): - result = backend.check_crash_loop(pod, "res-1") - assert "kubectl logs -n test-ns my-pod-xyz" in result.status_message - - def test_crash_loop_error_includes_pod_logs_in_details(self, backend): - pod = _make_pod(restart_count=5, waiting_reason="CrashLoopBackOff") - with _mock_pod_backend(backend, pod_logs="RuntimeError: CUDA OOM"): - result = backend.check_crash_loop(pod, "res-1") - assert result.error_details["error_stack"] == "RuntimeError: CUDA OOM" - - -# ============================================================================ -# Error message content tests -# ============================================================================ - - -class TestPendingTimeoutErrorMessage: - """Tests for the error message content of pending timeout.""" - - @pytest.fixture - def backend(self, mock_nmp_sdk, mock_k8s_config): - return _status_helper_reconciler( - namespace="my-ns", - backend_config=K8sNimOperatorConfig(pending_timeout_seconds=120, max_restart_count=5), - ) - - def test_error_message_with_pod_name(self, backend): - with _mock_pod_backend(backend, pod_logs="error log tail"): - result = backend.build_pending_timeout_error("my-resource", 150.0, "my-pod-123") - assert result.status == "ERROR" - assert "timed out" in result.status_message - assert "kubectl logs -n my-ns my-pod-123" in result.status_message - assert result.error_details["pod_name"] == "my-pod-123" - assert result.error_details["namespace"] == "my-ns" - assert result.error_details["reason"] == "pending_timeout" - - def test_error_message_without_pod_name(self, backend): - result = backend.build_pending_timeout_error("my-resource", 150.0, None) - assert "kubectl logs -n my-ns deployment/my-resource" in result.status_message - assert "pod_name" not in result.error_details - - def test_error_details_contain_timing(self, backend): - result = backend.build_pending_timeout_error("res", 200.0, None) - assert result.error_details["elapsed_seconds"] == 200 - assert result.error_details["timeout_seconds"] == 120 - - def test_crash_loop_error_message(self, backend): - with _mock_pod_backend(backend, pod_logs="segfault"): - result = backend.build_crash_loop_error("res", "pod-abc", 7) - assert result.status == "ERROR" - assert "crash loop" in result.status_message - assert "7 container restarts" in result.status_message - assert "max: 5" in result.status_message - assert "kubectl logs -n my-ns pod-abc" in result.status_message - assert result.error_details["error_stack"] == "segfault" - - -# ============================================================================ -# Controller restart resilience -# ============================================================================ - - -class TestControllerRestartResilience: - """Verify that timeout detection works after a controller restart. - - Because we use deployment.created_at from the entity store (not in-memory - tracking), a new controller instance with no prior state should still be - able to detect that a deployment has been PENDING for too long. - """ - - @pytest.mark.asyncio - async def test_timeout_detected_after_restart(self, mock_nmp_sdk, mock_k8s_config, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=3) - - backend = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=5) - backend._k8s_namespace = "default" - backend._dynamic_client = MagicMock() - backend._k8s_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - with patch.object( - backend._status_projector, - "pod_status_from_deployment", - return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), - ): - with _mock_pod_backend(backend): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "ERROR" - assert result.error_details["reason"] == "pending_timeout" - - @pytest.mark.asyncio - async def test_no_false_positive_after_restart(self, mock_nmp_sdk, mock_k8s_config, sample_deployment): - """Fresh controller, deployment only 10 min old -> should stay PENDING.""" - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(minutes=10) - - backend = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=5) - backend._k8s_namespace = "default" - backend._dynamic_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - with patch.object( - backend._status_projector, - "pod_status_from_deployment", - return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), - ): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "PENDING" - # No elapsed/timeout in message (stable message to avoid new history entry every poll) - - -# ============================================================================ -# Pod log fetching edge cases -# ============================================================================ - - -class TestFetchPodLogs: - """Tests for _fetch_pod_logs edge cases.""" - - @pytest.fixture - def backend(self, mock_nmp_sdk, mock_k8s_config): - return _status_helper_reconciler(namespace="default") - - def test_returns_logs_normally(self, backend): - with _mock_pod_backend(backend, pod_logs="log line 1\nlog line 2"): - result = backend.fetch_pod_logs("pod-1") - assert result == "log line 1\nlog line 2" - - def test_truncates_long_logs(self, backend): - long_logs = "x" * 3000 - with _mock_pod_backend(backend, pod_logs=long_logs): - result = backend.fetch_pod_logs("pod-1") - assert len(result) == 2048 - - def test_returns_empty_on_exception(self, backend): - mock_core_v1 = MagicMock() - mock_core_v1.read_namespaced_pod_log.side_effect = Exception("API error") - with patch(f"{_RECON_STATUS_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1): - result = backend.fetch_pod_logs("pod-1") - assert result == "" - - -# ============================================================================ -# Augmented PENDING message tests -# ============================================================================ - - -class TestAugmentedPendingMessages: - """Verify that PENDING messages include elapsed/remaining timing and restart info.""" - - @pytest.fixture - def backend(self, mock_nmp_sdk, mock_k8s_config): - b = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - b._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=5) - b._k8s_namespace = "default" - b._k8s_client = MagicMock() - b._dynamic_client = MagicMock() - return b - - @pytest.mark.asyncio - async def test_timing_info_appended_to_pending(self, backend, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(minutes=30) - - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - with patch.object( - backend._status_projector, - "pod_status_from_deployment", - return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), - ): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "PENDING" - # No elapsed/timeout appended (stable message to avoid new history entry every poll) - assert result.status_message == "Waiting" - - @pytest.mark.asyncio - async def test_restart_info_in_pod_status(self, backend, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(minutes=5) - - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - pod = _make_pod(restart_count=3, phase="Running") - - with _mock_pod_backend(backend, pod=pod): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "PENDING" - assert "restarts: 3" in result.status_message - - @pytest.mark.asyncio - async def test_no_restart_info_when_zero(self, backend, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(minutes=5) - - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - pod = _make_pod(restart_count=0, phase="Running") - - with _mock_pod_backend(backend, pod=pod): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "PENDING" - assert "restarts" not in result.status_message - - -# ============================================================================ -# _find_pod_name edge cases -# ============================================================================ - - -class TestFindPodName: - """Tests for _find_pod_name best-effort lookup.""" - - @pytest.fixture - def backend(self, mock_nmp_sdk, mock_k8s_config): - return _status_helper_reconciler(namespace="default") - - def test_returns_pod_name(self, backend): - pod = _make_pod(name="found-pod-xyz") - with _mock_pod_backend(backend, pod=pod) as (mock_apps_v1, _): - result = backend.find_pod_name("resource-1") - assert result == "found-pod-xyz" - - def test_returns_none_when_no_pods(self, backend): - mock_apps_v1 = MagicMock() - mock_core_v1 = MagicMock() - mock_deployment = MagicMock() - mock_deployment.spec.selector.match_labels = {"app": "test"} - mock_apps_v1.read_namespaced_deployment.return_value = mock_deployment - pods_list = MagicMock() - pods_list.items = [] - mock_core_v1.list_namespaced_pod.return_value = pods_list - - with ( - patch(f"{_RECON_STATUS_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1), - patch(f"{_RECON_STATUS_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1), - ): - result = backend.find_pod_name("resource-1") - assert result is None - - def test_returns_none_on_api_exception(self, backend): - mock_apps_v1 = MagicMock() - mock_apps_v1.read_namespaced_deployment.side_effect = k8s_client.exceptions.ApiException(status=404) - - with patch(f"{_RECON_STATUS_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1): - result = backend.find_pod_name("resource-1") - assert result is None - - -# ============================================================================ -# Integration: crash loop takes precedence over pending timeout -# ============================================================================ - - -class TestCrashLoopPrecedence: - """Crash loop ERROR from pod status should be returned directly, not overridden by timeout.""" - - @pytest.mark.asyncio - async def test_crash_loop_overrides_timeout(self, mock_nmp_sdk, mock_k8s_config, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=5) - - backend = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=60, max_restart_count=3) - backend._k8s_namespace = "default" - backend._dynamic_client = MagicMock() - backend._k8s_client = MagicMock() - backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") - - pod = _make_pod(restart_count=3, waiting_reason="CrashLoopBackOff") - - with _mock_pod_backend(backend, pod=pod, pod_logs="crash"): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "ERROR" - assert result.error_details["reason"] == "crash_loop" - - -# ============================================================================ -# NIMService failed state -# ============================================================================ - - -class TestNIMServiceFailedState: - """Verify that a Failed NIMService goes to ERROR without timeout interference.""" - - @pytest.mark.asyncio - async def test_failed_nimservice_returns_error(self, mock_nmp_sdk, mock_k8s_config, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=5) - - backend = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=60) - backend._k8s_namespace = "default" - backend._dynamic_client = MagicMock() - - mock_resource = _make_nimservice_mock("Failed", [{"type": "Failed", "message": "out of GPU memory"}]) - backend._dynamic_client.resources.get.return_value = mock_resource - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "ERROR" - assert "NIMService failed" in result.status_message - - -# ============================================================================ -# LOST state -# ============================================================================ - - -class TestLostState: - """Verify that a LOST NIMService is not confused with PENDING timeout.""" - - @pytest.mark.asyncio - async def test_lost_nimservice_not_treated_as_pending(self, mock_nmp_sdk, mock_k8s_config, sample_deployment): - sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=5) - - backend = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=60) - backend._k8s_namespace = "default" - backend._dynamic_client = MagicMock() - - mock_resource = MagicMock() - mock_resource.get.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock()) - backend._dynamic_client.resources.get.return_value = mock_resource - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) - ) - assert result.status == "LOST" - - -# ============================================================================= -# Orphan reconciliation: list_managed_deployment_names, delete_model_deployment (by workspace/name) -# ============================================================================= - - -@pytest.mark.asyncio -async def test_list_managed_deployment_names_returns_workspace_name_from_labels(mock_nmp_sdk, mock_k8s_config): - """list_managed_deployment_names returns sorted workspace/name from NIMService metadata labels.""" - with patch(f"{_K8S_BACKEND_MODULE}.K8sNimOperatorServiceBackend._get_current_namespace", return_value="default"): - backend = K8sNimOperatorServiceBackend(mock_nmp_sdk, {}, "pull-image") - backend._k8s_namespace = "default" - backend._dynamic_client = MagicMock() - - item1 = MagicMock() - item1.metadata.labels = { - "nmp.nvidia.com/deployment-workspace": "ws-a", - "nmp.nvidia.com/deployment-name": "dep1", - } - item2 = MagicMock() - item2.metadata.labels = { - "nmp.nvidia.com/deployment-workspace": "ws-b", - "nmp.nvidia.com/deployment-name": "dep2", - } - list_result = MagicMock() - list_result.items = [item1, item2] - mock_nimservice_api = MagicMock() - mock_nimservice_api.get.return_value = list_result - backend._dynamic_client.resources.get.return_value = mock_nimservice_api - - _sync_reconcilers(backend) - names = await backend.list_managed_deployment_names() - - assert names == ["ws-a/dep1", "ws-b/dep2"] - - -@pytest.mark.asyncio -async def test_list_managed_deployment_names_empty_when_no_resources(mock_nmp_sdk, mock_k8s_config): - """list_managed_deployment_names returns empty list when no NIMServices match.""" - with patch(f"{_K8S_BACKEND_MODULE}.K8sNimOperatorServiceBackend._get_current_namespace", return_value="default"): - backend = K8sNimOperatorServiceBackend(mock_nmp_sdk, {}, "pull-image") - backend._k8s_namespace = "default" - backend._dynamic_client = MagicMock() - list_result = MagicMock() - list_result.items = [] - mock_nimservice_api = MagicMock() - mock_nimservice_api.get.return_value = list_result - backend._dynamic_client.resources.get.return_value = mock_nimservice_api - - _sync_reconcilers(backend) - names = await backend.list_managed_deployment_names() - - assert names == [] - - -@pytest.mark.asyncio -async def test_delete_model_deployment_by_id_calls_delete_resources(mock_nmp_sdk, mock_k8s_config): - """delete_model_deployment(workspace, name) calls _delete_resources_by_model_deployment_id.""" - with patch(f"{_K8S_BACKEND_MODULE}.K8sNimOperatorServiceBackend._get_current_namespace", return_value="default"): - backend = K8sNimOperatorServiceBackend(mock_nmp_sdk, {}, "pull-image") - backend._k8s_namespace = "default" - with patch.object(backend, "_delete_resources_by_model_deployment_id", new_callable=AsyncMock) as mock_delete: - mock_delete.return_value = DeploymentStatusUpdate(status="DELETED", status_message="") - result = await backend.delete_model_deployment("my-ws", "my-name") - mock_delete.assert_called_once_with("my-ws", "my-name") - assert result.status == "DELETED" - - -# =========================================================================== -# vLLM path (native Kubernetes objects, no operator) -# =========================================================================== - - -def _vllm_config( - *, gpu: int = 1, lora_enabled: bool = False, run_as_user: int | None = None, run_as_group: int | None = None -): - """A minimal vLLM ModelDeploymentConfig-like object for dispatch/compile.""" - return SimpleNamespace( - engine="vllm", - model_spec=SimpleNamespace( - model_type=None, - model_namespace="default", - model_name="qwen", - model_revision=None, - chat_template=None, - tool_call_config=None, - lora_enabled=lora_enabled, - ), - executor_config=SimpleNamespace( - gpu=gpu, - disk_size="50Gi", - image_name=None, - image_tag=None, - health_check_path=None, - run_as_user=run_as_user, - run_as_group=run_as_group, - additional_envs=None, - additional_args=[], - k8s_nim_operator_config=None, - override_config=None, - ), - ) - - -def _vllm_backend(k8s_backend): - """Wire a k8s_backend with mocked typed clients for the vLLM path.""" - k8s_backend._k8s_namespace = "nemo" - k8s_backend._backend_config = K8sNimOperatorConfig() - k8s_backend._k8s_client = MagicMock() - k8s_backend._core_v1 = MagicMock() - k8s_backend._apps_v1 = MagicMock() - k8s_backend._batch_v1 = MagicMock() - return k8s_backend - - -def _api_exception(status: int): - return k8s_client.exceptions.ApiException(status=status) - - -@pytest.mark.asyncio -async def test_vllm_create_emits_pvc_and_job_only(k8s_backend, sample_deployment): - """vLLM create (phase P0) emits the PVC + puller Job, not the Deployment/Service.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config(gpu=2) - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): - _sync_reconcilers(backend) - result = await backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - assert result.status == "PENDING" - backend._core_v1.create_namespaced_persistent_volume_claim.assert_called_once() - backend._batch_v1.create_namespaced_job.assert_called_once() - # Deployment + Service are NOT created at P0. - backend._apps_v1.create_namespaced_deployment.assert_not_called() - backend._core_v1.create_namespaced_service.assert_not_called() - - # The puller Job requests the same GPU as the server (topology pin). - job = backend._batch_v1.create_namespaced_job.call_args.kwargs["body"] - assert job.spec.template.spec.containers[0].resources.requests["nvidia.com/gpu"] == "2" - - -def _generic_config( - *, - gpu: int = 0, - image="nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", - tag="1.10.1", - run_as_user: int | None = None, - run_as_group: int | None = None, -): - """A minimal generic ModelDeploymentConfig-like object (no model weights).""" - return SimpleNamespace( - engine="generic", - model_spec=SimpleNamespace( - model_type=None, - model_namespace=None, - model_name=None, - model_revision=None, - chat_template=None, - tool_call_config=None, - lora_enabled=False, - ), - executor_config=SimpleNamespace( - gpu=gpu, - disk_size="50Gi", - image_name=image, - image_tag=tag, - health_check_path="/v1/health/ready", - run_as_user=run_as_user, - run_as_group=run_as_group, - additional_envs={"FOO": "bar"}, - additional_args=["--port", "8000"], - k8s_nim_operator_config=None, - override_config=None, - ), - ) - - -@pytest.mark.asyncio -async def test_generic_create_emits_deployment_and_service_no_pvc(k8s_backend, sample_deployment): - """Generic create emits the Deployment + Service immediately, with no PVC/puller Job.""" - backend = _vllm_backend(k8s_backend) - config = _generic_config() - - created_dep = MagicMock() - created_dep.metadata.name = backend._get_resource_name(sample_deployment) - created_dep.metadata.uid = "dep-uid" - backend._apps_v1.create_namespaced_deployment.return_value = created_dep - - _sync_reconcilers(backend) - result = await backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - assert result.status == "PENDING" - backend._apps_v1.create_namespaced_deployment.assert_called_once() - backend._core_v1.create_namespaced_service.assert_called_once() - # No model weights for generic: no PVC, no puller Job. - backend._core_v1.create_namespaced_persistent_volume_claim.assert_not_called() - backend._batch_v1.create_namespaced_job.assert_not_called() - - # The container runs the user's image + raw args + env verbatim, with no - # model-store volume mounted. - dep_obj = backend._apps_v1.create_namespaced_deployment.call_args.kwargs["body"] - container = dep_obj.spec.template.spec.containers[0] - assert container.image == "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect:1.10.1" - assert container.args == ["--port", "8000"] - assert {e.name: e.value for e in container.env} == {"FOO": "bar"} - volume_names = {v.name for v in dep_obj.spec.template.spec.volumes} - assert "model-store" not in volume_names - mount_names = {m.name for m in container.volume_mounts} - assert "model-store" not in mount_names - # Readiness/startup probes use the explicit health_check_path. - assert container.readiness_probe.http_get.path == "/v1/health/ready" - # No uid/gid override -> generic runs as the image's own user (no securityContext). - assert dep_obj.spec.template.spec.security_context is None - - -@pytest.mark.asyncio -async def test_generic_create_applies_run_as_user_override(k8s_backend, sample_deployment): - """executor_config.run_as_user/run_as_group set the pod securityContext for generic.""" - backend = _vllm_backend(k8s_backend) - config = _generic_config(run_as_user=1234, run_as_group=5678) - - created_dep = MagicMock() - created_dep.metadata.name = backend._get_resource_name(sample_deployment) - created_dep.metadata.uid = "dep-uid" - backend._apps_v1.create_namespaced_deployment.return_value = created_dep - - _sync_reconcilers(backend) - result = await backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - assert result.status == "PENDING" - dep_obj = backend._apps_v1.create_namespaced_deployment.call_args.kwargs["body"] - sec = dep_obj.spec.template.spec.security_context - assert sec is not None - assert sec.run_as_user == 1234 - assert sec.run_as_group == 5678 - - -@pytest.mark.asyncio -async def test_vllm_run_as_user_override_beats_engine_default(k8s_backend, sample_deployment): - """An explicit run_as_user overrides vLLM's default uid on the serving pod + puller.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config(gpu=1, run_as_user=4321) - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): - _sync_reconcilers(backend) - result = await backend.create_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - assert result.status == "PENDING" - # Puller Job runs as the overridden uid (so it writes the PVC as that user); - # group falls back to the vLLM default since run_as_group was not set. - job = backend._batch_v1.create_namespaced_job.call_args.kwargs["body"] - job_sec = job.spec.template.spec.security_context - assert job_sec.run_as_user == 4321 - assert job_sec.run_as_group == backend._backend_config.default_vllm_group_id - - -@pytest.mark.asyncio -async def test_generic_status_ready_when_deployment_ready(k8s_backend, sample_deployment): - """Generic status projects the serving Deployment's readiness directly (no Job/PVC).""" - backend = _vllm_backend(k8s_backend) - config = _generic_config() - - dep = MagicMock() - dep.status.ready_replicas = 1 - backend._apps_v1.read_namespaced_deployment.return_value = dep - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - - assert result.status == "READY" - # Generic status never consults the puller Job. - backend._batch_v1.read_namespaced_job.assert_not_called() - - -@pytest.mark.asyncio -async def test_generic_status_lost_when_deployment_missing(k8s_backend, sample_deployment): - """Generic status reports LOST when the serving Deployment was deleted externally.""" - backend = _vllm_backend(k8s_backend) - config = _generic_config() - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - assert result.status == "LOST" - - -def _generic_weighted_config(*, gpu: int = 1): - """A generic config that also references a model (fileset-backed weights).""" - config = _generic_config(gpu=gpu) - config.model_spec.model_namespace = "default" - config.model_spec.model_name = "qwen" - return config - - -def _fileset_model_entity(): - """A model entity with a fileset -> resolves to FILES_SERVICE weights.""" - return SimpleNamespace( - workspace="default", name="qwen", spec=None, trust_remote_code=False, fileset="hf://default/qwen" - ) - - -@pytest.mark.asyncio -async def test_generic_with_fileset_runs_staged_puller(k8s_backend, sample_deployment): - """A generic deployment whose config resolves to a fileset pulls weights (staged).""" - backend = _vllm_backend(k8s_backend) - config = _generic_weighted_config(gpu=1) - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): - _sync_reconcilers(backend) - result = await backend.create_model_deployment( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=config, - model_entity=_fileset_model_entity(), - ) - ) - - assert result.status == "PENDING" - # Weighted generic => staged rollout: PVC + puller Job, no Deployment yet. - backend._core_v1.create_namespaced_persistent_volume_claim.assert_called_once() - backend._batch_v1.create_namespaced_job.assert_called_once() - backend._apps_v1.create_namespaced_deployment.assert_not_called() - - -@pytest.mark.asyncio -async def test_generic_with_fileset_p3_mounts_model_store(k8s_backend, sample_deployment): - """At P3 the weighted generic serving Deployment mounts the model-store PVC + uses raw args.""" - backend = _vllm_backend(k8s_backend) - config = _generic_weighted_config(gpu=1) - - job = MagicMock() - job.status.failed = None - job.status.succeeded = 1 - backend._batch_v1.read_namespaced_job.return_value = job - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) - created_dep = MagicMock() - created_dep.metadata.name = backend._get_resource_name(sample_deployment) - created_dep.metadata.uid = "dep-uid" - backend._apps_v1.create_namespaced_deployment.return_value = created_dep - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext( - model_deployment=sample_deployment, - model_deployment_config=config, - model_entity=_fileset_model_entity(), - ) - ) - - assert result.status == "PENDING" - backend._apps_v1.create_namespaced_deployment.assert_called_once() - dep_obj = backend._apps_v1.create_namespaced_deployment.call_args.kwargs["body"] - container = dep_obj.spec.template.spec.containers[0] - # Weighted: model-store PVC is mounted so the pulled weights are available. - volume_names = {v.name for v in dep_obj.spec.template.spec.volumes} - assert "model-store" in volume_names - # Still a generic container: runs the user's raw args (no vLLM serve synthesis). - assert container.args == ["--port", "8000"] - - -@pytest.mark.asyncio -async def test_generic_update_patches_deployment(k8s_backend, sample_deployment): - """Updating a (weightless) generic deployment patches the serving objects in place.""" - backend = _vllm_backend(k8s_backend) - config = _generic_config() - # Serving Deployment already exists -> patch, don't recreate. - backend._apps_v1.read_namespaced_deployment.return_value = MagicMock() - - _sync_reconcilers(backend) - result = await backend.update_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - assert result.status == "PENDING" - backend._apps_v1.patch_namespaced_deployment.assert_called_once() - backend._core_v1.patch_namespaced_service.assert_called_once() - # Patched in place, not recreated. - backend._apps_v1.create_namespaced_deployment.assert_not_called() - - -@pytest.mark.asyncio -async def test_vllm_status_job_running_is_pending(k8s_backend, sample_deployment): - """While the puller Job is running, status is PENDING.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - job = MagicMock() - job.status.failed = None - job.status.succeeded = None - backend._batch_v1.read_namespaced_job.return_value = job - # No serving Deployment yet (still in pull phase). - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - assert result.status == "PENDING" - assert "weights" in result.status_message.lower() - backend._apps_v1.create_namespaced_deployment.assert_not_called() - - -@pytest.mark.asyncio -async def test_vllm_status_job_failed_is_error(k8s_backend, sample_deployment): - """A failed puller Job surfaces as ERROR.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - job = MagicMock() - job.status.failed = 5 - job.status.succeeded = None - backend._batch_v1.read_namespaced_job.return_value = job - backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) - # No serving Deployment yet (failed during pull phase). - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - assert result.status == "ERROR" - assert result.error_details["reason"] == "weight_pull_failed" - - -@pytest.mark.asyncio -async def test_vllm_status_job_complete_creates_deployment(k8s_backend, sample_deployment): - """When the Job completes (phase P3), the Deployment + Service are created.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config(gpu=1) - - job = MagicMock() - job.status.failed = None - job.status.succeeded = 1 - backend._batch_v1.read_namespaced_job.return_value = job - # Deployment does not exist yet -> triggers P3 creation. - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - # After the puller Job is deleted, no puller pod remains (volume released). - backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) - created_dep = MagicMock() - created_dep.metadata.name = backend._get_resource_name(sample_deployment) - created_dep.metadata.uid = "dep-uid" - backend._apps_v1.create_namespaced_deployment.return_value = created_dep - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - - assert result.status == "PENDING" - # Puller Job deleted (release RWO volume) before the Deployment is created. - backend._batch_v1.delete_namespaced_job.assert_called_once() - backend._apps_v1.create_namespaced_deployment.assert_called_once() - backend._core_v1.create_namespaced_service.assert_called_once() - # ownerRef patched onto the PVC so it cascades with the Deployment (Job is gone). - backend._core_v1.patch_namespaced_persistent_volume_claim.assert_called_once() - backend._batch_v1.patch_namespaced_job.assert_not_called() - - -@pytest.mark.asyncio -async def test_vllm_status_p3_waits_for_puller_pod_to_release_volume(k8s_backend, sample_deployment): - """At P3, if the puller pod is still present, defer Deployment creation (RWO release).""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config(gpu=1) - - job = MagicMock() - job.status.failed = None - job.status.succeeded = 1 - backend._batch_v1.read_namespaced_job.return_value = job - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - # Puller pod still terminating -> volume not yet released. - backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[MagicMock()]) - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - - assert result.status == "PENDING" - backend._batch_v1.delete_namespaced_job.assert_called_once() - # Deployment is NOT created until the puller pod is gone. - backend._apps_v1.create_namespaced_deployment.assert_not_called() - - -@pytest.mark.asyncio -async def test_vllm_status_job_complete_with_lora_wires_sidecar(k8s_backend, sample_deployment): - """At P3 with LoRA enabled, the Deployment gets the cache-init + adapter sidecar.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config(gpu=1, lora_enabled=True) - - job = MagicMock() - job.status.failed = None - job.status.succeeded = 1 - backend._batch_v1.read_namespaced_job.return_value = job - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) - created_dep = MagicMock() - created_dep.metadata.name = backend._get_resource_name(sample_deployment) - created_dep.metadata.uid = "dep-uid" - backend._apps_v1.create_namespaced_deployment.return_value = created_dep - - platform_cfg = MagicMock() - platform_cfg.image_pull_secrets = [] - platform_cfg.image_registry = "my-registry" - platform_cfg.image_tag = "local" - platform_cfg.to_shared_envvars.return_value = {"NMP_SHARED": "1"} - with patch(f"{_RECON_K8S_MODULE}.get_platform_config", return_value=platform_cfg): - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - - assert result.status == "PENDING" - dep_obj = backend._apps_v1.create_namespaced_deployment.call_args.kwargs["body"] - pod = dep_obj.spec.template.spec - assert pod.init_containers[0].name == "lora-cache-init" - sidecar = next(ctr for ctr in pod.containers if ctr.name == "lora-sidecar") - env = {e.name: e.value for e in sidecar.env} - assert env["NIM_PEFT_SOURCE"] == "/scratch/loras" - assert env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store" - assert env["NMP_SHARED"] == "1" - - -@pytest.mark.asyncio -async def test_vllm_status_job_absent_pvc_present_resumes_p3_not_lost(k8s_backend, sample_deployment): - """Job deleted (RWO release) + PVC present + no Deployment -> resume P3, not LOST.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config(gpu=1) - - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) # Job already deleted - # PVC still present -> we're mid-P3, not orphaned. - backend._core_v1.read_namespaced_persistent_volume_claim.return_value = MagicMock() - backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) - created_dep = MagicMock() - created_dep.metadata.name = backend._get_resource_name(sample_deployment) - created_dep.metadata.uid = "dep-uid" - backend._apps_v1.create_namespaced_deployment.return_value = created_dep - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - - assert result.status == "PENDING" - assert result.status != "LOST" - backend._apps_v1.create_namespaced_deployment.assert_called_once() - - -@pytest.mark.asyncio -async def test_vllm_status_job_and_pvc_absent_is_lost(k8s_backend, sample_deployment): - """Both Job and PVC gone + no Deployment -> genuine drift -> LOST.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config(gpu=1) - - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) - backend._core_v1.read_namespaced_persistent_volume_claim.side_effect = _api_exception(404) - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - assert result.status == "LOST" - - -@pytest.mark.asyncio -async def test_vllm_status_deployment_ready_is_ready(k8s_backend, sample_deployment): - """A ready serving Deployment maps to READY + host_url.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - - job = MagicMock() - job.status.failed = None - job.status.succeeded = 1 - backend._batch_v1.read_namespaced_job.return_value = job - - dep = MagicMock() - dep.status.ready_replicas = 1 - backend._apps_v1.read_namespaced_deployment.return_value = dep - - _sync_reconcilers(backend) - result = await backend.get_model_deployment_status( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config) - ) - assert result.status == "READY" - assert result.host_url is not None - - -@pytest.mark.asyncio -async def test_vllm_update_unchanged_source_does_not_repull(k8s_backend, sample_deployment): - """Unchanged model source: no resource deletion; serving objects patched in place.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - - existing_job = MagicMock() - existing_job.metadata.labels = {"nmp.nvidia.com/engine": "vllm"} - existing_job.metadata.annotations = {"nmp.nvidia.com/model-source": "default/qwen"} - backend._batch_v1.read_namespaced_job.return_value = existing_job - # Serving Deployment already exists -> update patches it in place. - backend._apps_v1.read_namespaced_deployment.return_value = MagicMock() - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): - with patch.object(backend._k8s_reconciler, "_delete_serving_resources") as mock_delete: - _sync_reconcilers(backend) - result = await backend.update_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - mock_delete.assert_not_called() - backend._batch_v1.create_namespaced_job.assert_not_called() - # Patched, not recreated. - backend._apps_v1.patch_namespaced_deployment.assert_called_once() - assert result.status == "PENDING" - - -@pytest.mark.asyncio -async def test_vllm_update_changed_source_repulls(k8s_backend, sample_deployment): - """Changed model source: delete resources and re-run the phased create.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - - existing_job = MagicMock() - existing_job.metadata.labels = {"nmp.nvidia.com/engine": "vllm"} - existing_job.metadata.annotations = {"nmp.nvidia.com/model-source": "default/old-model"} - backend._batch_v1.read_namespaced_job.return_value = existing_job - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", "v2")): - with patch.object(backend._k8s_reconciler, "_delete_serving_resources") as mock_delete: - _sync_reconcilers(backend) - result = await backend.update_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - mock_delete.assert_called_once() - # Re-pull: a new PVC + Job are created. - backend._core_v1.create_namespaced_persistent_volume_claim.assert_called_once() - backend._batch_v1.create_namespaced_job.assert_called_once() - assert result.status == "PENDING" - - -@pytest.mark.asyncio -async def test_vllm_update_changed_source_repulls_after_job_deleted(k8s_backend, sample_deployment): - """Changed source is detected via the PVC annotation once the puller Job is gone. - - At P3 the puller Job is deleted, so an already-serving deployment has no Job. - The model source must still be read (from the PVC) so a revision change - re-pulls instead of serving stale weights. - """ - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - - # Puller Job has been deleted (P3 completed). - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) - # PVC survives and carries the original model source. - existing_pvc = MagicMock() - existing_pvc.metadata.annotations = {"nmp.nvidia.com/model-source": "default/old-model"} - backend._core_v1.read_namespaced_persistent_volume_claim.return_value = existing_pvc - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", "v2")): - with patch.object(backend._k8s_reconciler, "_delete_serving_resources") as mock_delete: - _sync_reconcilers(backend) - result = await backend.update_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - # Source change detected from the PVC -> re-pull. - mock_delete.assert_called_once() - backend._core_v1.create_namespaced_persistent_volume_claim.assert_called_once() - backend._batch_v1.create_namespaced_job.assert_called_once() - assert result.status == "PENDING" - - -@pytest.mark.asyncio -async def test_vllm_update_unchanged_source_via_pvc_does_not_repull(k8s_backend, sample_deployment): - """Unchanged source read from the PVC (Job gone) patches in place, no re-pull.""" - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) - existing_pvc = MagicMock() - existing_pvc.metadata.annotations = {"nmp.nvidia.com/model-source": "default/qwen"} - backend._core_v1.read_namespaced_persistent_volume_claim.return_value = existing_pvc - # Serving Deployment exists -> patch path. - backend._apps_v1.read_namespaced_deployment.return_value = MagicMock() - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): - with patch.object(backend._k8s_reconciler, "_delete_serving_resources") as mock_delete: - _sync_reconcilers(backend) - result = await backend.update_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - mock_delete.assert_not_called() - backend._batch_v1.create_namespaced_job.assert_not_called() - backend._apps_v1.patch_namespaced_deployment.assert_called_once() - assert result.status == "PENDING" - - -@pytest.mark.asyncio -async def test_existing_model_source_reads_pvc_when_job_absent(k8s_backend): - """_existing_model_source falls back to the PVC annotation when the Job is gone.""" - backend = _vllm_backend(k8s_backend) - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) - pvc = MagicMock() - pvc.metadata.annotations = {"nmp.nvidia.com/model-source": "default/qwen@v3"} - backend._core_v1.read_namespaced_persistent_volume_claim.return_value = pvc - - _sync_reconcilers(backend) - assert backend._k8s_reconciler._existing_model_source("md-default-qwen") == "default/qwen@v3" - - -@pytest.mark.asyncio -async def test_existing_model_source_none_when_job_and_pvc_absent(k8s_backend): - """_existing_model_source returns None when neither Job nor PVC exists.""" - backend = _vllm_backend(k8s_backend) - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) - backend._core_v1.read_namespaced_persistent_volume_claim.side_effect = _api_exception(404) - - _sync_reconcilers(backend) - assert backend._k8s_reconciler._existing_model_source("md-default-qwen") is None - - -@pytest.mark.asyncio -async def test_existing_model_source_reraises_job_api_errors(k8s_backend): - """A non-404 error reading the Job propagates (not swallowed / no PVC fallback).""" - backend = _vllm_backend(k8s_backend) - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(500) - - _sync_reconcilers(backend) - with pytest.raises(k8s_client.exceptions.ApiException): - backend._k8s_reconciler._existing_model_source("md-default-qwen") - # The PVC fallback must not be consulted when the Job read fails hard. - backend._core_v1.read_namespaced_persistent_volume_claim.assert_not_called() - - -@pytest.mark.asyncio -async def test_existing_model_source_reraises_pvc_api_errors(k8s_backend): - """A non-404 error reading the PVC propagates (not swallowed / not None).""" - backend = _vllm_backend(k8s_backend) - backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) - backend._core_v1.read_namespaced_persistent_volume_claim.side_effect = _api_exception(500) - - _sync_reconcilers(backend) - with pytest.raises(k8s_client.exceptions.ApiException): - backend._k8s_reconciler._existing_model_source("md-default-qwen") - - -@pytest.mark.asyncio -async def test_vllm_update_during_pull_is_noop(k8s_backend, sample_deployment): - """Unchanged source, serving Deployment not yet created but puller Job present. - - Mid-pull updates must be a no-op (the status path emits the serving objects at - P3); we must not re-run create() or patch a non-existent Deployment. - """ - backend = _vllm_backend(k8s_backend) - config = _vllm_config() - - existing_job = MagicMock() - existing_job.metadata.labels = {"nmp.nvidia.com/engine": "vllm"} - existing_job.metadata.annotations = {"nmp.nvidia.com/model-source": "default/qwen"} - backend._batch_v1.read_namespaced_job.return_value = existing_job - # Serving Deployment does not exist yet (still pulling). - backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) - - with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): - _sync_reconcilers(backend) - result = await backend.update_model_deployment( - ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) - ) - - assert result.status == "PENDING" - # No-op: no re-create of PVC/Job, no Deployment patch. - backend._core_v1.create_namespaced_persistent_volume_claim.assert_not_called() - backend._batch_v1.create_namespaced_job.assert_not_called() - backend._apps_v1.patch_namespaced_deployment.assert_not_called() - - -@pytest.mark.asyncio -async def test_vllm_list_managed_unions_deployments(k8s_backend): - """list_managed_deployment_names unions NIMServices and raw vLLM Deployments.""" - backend = _vllm_backend(k8s_backend) - backend._dynamic_client = MagicMock() - backend._dynamic_client.resources.get.return_value.get.return_value = MagicMock(items=[]) - - dep = MagicMock() - dep.metadata.labels = { - "nmp.nvidia.com/deployment-workspace": "default", - "nmp.nvidia.com/deployment-name": "qwen", - } - backend._apps_v1.list_namespaced_deployment.return_value = MagicMock(items=[dep]) - - _sync_reconcilers(backend) - names = await backend.list_managed_deployment_names() - assert "default/qwen" in names diff --git a/services/core/models/tests/unit/controllers/test_nimservice_compiler.py b/services/core/models/tests/unit/controllers/test_nimservice_compiler.py deleted file mode 100644 index a1ff938078..0000000000 --- a/services/core/models/tests/unit/controllers/test_nimservice_compiler.py +++ /dev/null @@ -1,2088 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for NIMService compiler.""" - -import json -import subprocess -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import MagicMock, patch - -import pytest -import yaml -from nmp.common.config import PlatformConfig -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER -from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler import ( - TOOL_CALL_PLUGIN_FINALIZE_SCRIPT_TEMPLATE, - _apply_k8s_nim_operator_config, - compile_nimcache, - compile_nimservice, -) - - -@pytest.fixture -def backend_config(): - """Create a sample K8sNimOperatorConfig for testing.""" - return K8sNimOperatorConfig() - - -@pytest.fixture -def sample_deployment(): - """Create a sample ModelDeployment for testing.""" - deployment = MagicMock() - deployment.workspace = "test-ns" - deployment.name = "test-deployment" - deployment.entity_version = "v1" - return deployment - - -@pytest.fixture -def minimal_config(): - """Create a minimal ModelDeploymentConfig for testing.""" - config = MagicMock() - config.workspace = "test-ns" - config.name = "test-config" - config.entity_version = "v1" - - # Minimal model_spec configuration - config.engine = "nim" - config.model_spec = MagicMock() - config.model_spec.lora_enabled = False - config.model_spec.model_name = None - config.model_spec.model_namespace = None - config.model_spec.model_revision = None - config.model_spec.tool_call_config = None - - # Minimal executor_config configuration - config.executor_config = MagicMock() - config.executor_config.image_name = "nvcr.io/nim/meta/llama-3-8b-instruct" - config.executor_config.image_tag = "1.0.0" - config.executor_config.gpu = 1 - config.executor_config.disk_size = "50Gi" - config.executor_config.additional_envs = {} - config.executor_config.k8s_nim_operator_config = None - config.executor_config.override_config = {} - - return config - - -@pytest.fixture -def full_config(): - """Create a full ModelDeploymentConfig with all options enabled.""" - config = MagicMock() - config.workspace = "test-ns" - config.name = "test-config" - config.entity_version = "v1" - - # Full model_spec configuration - config.engine = "nim" - config.model_spec = MagicMock() - config.model_spec.lora_enabled = True - config.model_spec.model_name = "llama-3.2-3b-instruct" - config.model_spec.model_namespace = "meta" - config.model_spec.model_revision = None - config.model_spec.tool_call_config = None - - # Full executor_config configuration - config.executor_config = MagicMock() - config.executor_config.image_name = "nvcr.io/nim/meta/llama-3.2-3b-instruct" - config.executor_config.image_tag = "1.8.5" - config.executor_config.gpu = 2 - config.executor_config.disk_size = "200Gi" - config.executor_config.additional_envs = { - "CUSTOM_VAR": "custom_value", - "DEBUG": "true", - } - config.executor_config.k8s_nim_operator_config = None - config.executor_config.override_config = {} - - return config - - -def test_compile_nimservice_basic(backend_config, sample_deployment, minimal_config): - """Test basic NIMService compilation with minimal config.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify NIMService structure - assert nimservice.apiVersion == "apps.nvidia.com/v1alpha1" - assert nimservice.kind == "NIMService" - assert nimservice.metadata is not None - assert nimservice.spec is not None - - -def test_compile_nimservice_metadata(backend_config, sample_deployment, minimal_config): - """Test that metadata is correctly set.""" - resource_name = "md-test-ns-test-deployment" - k8s_namespace = "default" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace=k8s_namespace, - resource_name=resource_name, - ) - - # Verify metadata - assert nimservice.metadata["name"] == resource_name - assert nimservice.metadata["namespace"] == k8s_namespace - - # Verify labels - labels = nimservice.metadata["labels"] - assert labels["app.kubernetes.io/name"] == resource_name - assert labels[MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert labels["nmp.nvidia.com/deployment-workspace"] == sample_deployment.workspace - assert labels["nmp.nvidia.com/deployment-name"] == sample_deployment.name - - -def test_compile_nimservice_required_spec_fields(backend_config, sample_deployment, minimal_config): - """Test that all required spec fields are present.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - spec = nimservice.spec - - # Required fields from the NIMService CRD - assert spec.authSecret is not None - assert spec.authSecret == "ngc-api" - assert spec.image is not None - assert spec.resources is not None - assert spec.storage is not None - assert spec.expose is not None - assert spec.env is not None - - -def test_compile_nimservice_image_config(backend_config, sample_deployment, minimal_config): - """Test that image configuration is correctly set.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - image = nimservice.spec.image - assert image.repository == "nvcr.io/nim/meta/llama-3-8b-instruct" - assert image.tag == "1.0.0" - assert image.pullPolicy == "IfNotPresent" - - -def test_compile_nimservice_gpu_resources(backend_config, sample_deployment, minimal_config): - """Test that GPU resources are correctly configured.""" - minimal_config.executor_config.gpu = 4 - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - resources = nimservice.spec.resources - assert resources.limits is not None - assert "nvidia.com/gpu" in resources.limits - assert resources.limits["nvidia.com/gpu"].root == "4" - - # Verify CPU requests - assert resources.requests is not None - assert "cpu" in resources.requests - assert resources.requests["cpu"].root == "1000m" - - -def test_compile_nimservice_storage_pvc(backend_config, sample_deployment, minimal_config): - """Test that PVC storage is correctly configured.""" - resource_name = "md-test-ns-test-deployment" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name=resource_name, - ) - - storage = nimservice.spec.storage - assert storage.pvc is not None - - pvc = storage.pvc - assert pvc.create is True - assert pvc.name == resource_name - assert pvc.size == "50Gi" - assert pvc.storageClass is None # unset = use cluster default - assert pvc.volumeAccessMode == "ReadWriteOnce" - - -def test_compile_nimservice_storage_pvc_inherits_storage_class_from_backend_config(sample_deployment, minimal_config): - """Test that PVC storageClass is inherited from backend config when set.""" - backend_config = K8sNimOperatorConfig(default_storage_class="nfs") - resource_name = "md-test-ns-test-deployment" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name=resource_name, - ) - - storage = nimservice.spec.storage - assert storage.pvc is not None - assert storage.pvc.storageClass == "nfs" - - -def test_compile_nimservice_expose_service(backend_config, sample_deployment, minimal_config): - """Test that service exposure is correctly configured.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - expose = nimservice.spec.expose - assert expose.service is not None - assert expose.service.type == "ClusterIP" - assert expose.service.port == 8000 - - -def test_compile_nimservice_default_env_vars(backend_config, sample_deployment, minimal_config): - """Test that default environment variables are set.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_vars = {env.name: env.value for env in nimservice.spec.env} - - # Verify default env var - assert "NIM_GUIDED_DECODING_BACKEND" in env_vars - assert env_vars["NIM_GUIDED_DECODING_BACKEND"] == "outlines" - - -def test_compile_nimservice_model_env_vars(backend_config, sample_deployment, full_config): - """Test that model-specific environment variables are set.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=full_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_vars = {env.name: env.value for env in nimservice.spec.env} - - # Verify model env vars - expected_model_fqdn = "meta/llama-3.2-3b-instruct" - assert "NIM_SERVED_MODEL_NAME" in env_vars - assert env_vars["NIM_SERVED_MODEL_NAME"] == expected_model_fqdn - assert "NIM_MODEL_NAME" in env_vars - assert env_vars["NIM_MODEL_NAME"] == expected_model_fqdn - - -def test_compile_nimservice_lora_env_vars(backend_config, sample_deployment, full_config): - """Test that LoRA support environment variables are set when enabled.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=full_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_vars = {env.name: env.value for env in nimservice.spec.env} - - # Verify LoRA env vars - assert "NIM_PEFT_SOURCE" in env_vars - assert "NIM_PEFT_REFRESH_INTERVAL" in env_vars - # Default is now 30 seconds (matching DMS) - assert env_vars["NIM_PEFT_REFRESH_INTERVAL"] == "30" - - -def test_compile_nimservice_sidecar_container_name_truncated_for_long_resource_name( - backend_config, sample_deployment, full_config -): - """Sidecar container name must be ≤63 chars (K8s DNS label). Long resource names are truncated with hash.""" - long_resource_name = "md-e2e-d2ad136f-sft-model-deployment-qwen-lora-base" - assert len(long_resource_name) + len("-lora-sidecar") > 63 - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=full_config, - k8s_namespace="default", - resource_name=long_resource_name, - ) - - assert nimservice.spec.sidecarContainers is not None - assert len(nimservice.spec.sidecarContainers) == 1 - sidecar_name = nimservice.spec.sidecarContainers[0].name - assert len(sidecar_name) <= 63, f"Sidecar name {sidecar_name!r} exceeds 63 chars" - assert sidecar_name.endswith("-lora-sidecar") - - -def test_compile_nimservice_sidecar_command_includes_nemo_platform(backend_config, sample_deployment, full_config): - """Sidecar command must be full argv for K8s (container.command overrides image ENTRYPOINT).""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=full_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - assert nimservice.spec.sidecarContainers is not None - assert len(nimservice.spec.sidecarContainers) == 1 - sidecar = nimservice.spec.sidecarContainers[0] - assert sidecar.command == [ - "nemo", - "services", - "run", - "--sidecars", - "adapters", - ] - - -def test_compile_nimservice_sidecar_image_uses_platform_registry_and_tag( - backend_config, sample_deployment, full_config -): - """Sidecar image must use platform config registry and tag as separate fields (no split on ':').""" - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={ - "files": "http://nemo-files:8000", - "models": "http://nemo-models:8000", - }, - image_registry="localhost:5000", - image_tag="sidecar-tag", - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=full_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - assert nimservice.spec.sidecarContainers is not None - assert len(nimservice.spec.sidecarContainers) == 1 - sidecar = nimservice.spec.sidecarContainers[0] - assert sidecar.image.repository == "localhost:5000/nmp-api" - assert sidecar.image.tag == "sidecar-tag" - - -def test_compile_nimservice_additional_env_vars(backend_config, sample_deployment, full_config): - """Test that additional environment variables are added.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=full_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_vars = {env.name: env.value for env in nimservice.spec.env} - - # Verify additional env vars - assert "CUSTOM_VAR" in env_vars - assert env_vars["CUSTOM_VAR"] == "custom_value" - assert "DEBUG" in env_vars - assert env_vars["DEBUG"] == "true" - - -def test_compile_nimservice_additional_envs_override_defaults(backend_config, sample_deployment, minimal_config): - """Test that additional env vars can override defaults.""" - minimal_config.executor_config.additional_envs = { - "NIM_GUIDED_DECODING_BACKEND": "custom_backend", - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_vars = {env.name: env.value for env in nimservice.spec.env} - - # Verify override - assert env_vars["NIM_GUIDED_DECODING_BACKEND"] == "custom_backend" - - -def test_compile_nimservice_replicas(backend_config, sample_deployment, minimal_config): - """Test that replicas are set correctly.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Should default to 1 replica - assert nimservice.spec.replicas == 1 - - -def test_compile_nimservice_labels(backend_config, sample_deployment, minimal_config): - """Test that spec labels are set correctly.""" - resource_name = "md-test-ns-test-deployment" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name=resource_name, - ) - - labels = nimservice.spec.labels - assert labels["app.kubernetes.io/name"] == resource_name - assert labels[MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER - assert labels["nmp.nvidia.com/deployment-workspace"] == sample_deployment.workspace - assert labels["nmp.nvidia.com/deployment-name"] == sample_deployment.name - - -def test_compile_nimservice_override_config_env_vars(backend_config, sample_deployment, minimal_config): - """Test that override_config can add environment variables.""" - minimal_config.executor_config.override_config = { - "env": [ - {"name": "OVERRIDE_VAR", "value": "override_value"}, - {"name": "ANOTHER_VAR", "value": "another_value"}, - ] - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # The override should replace the entire env array - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert "OVERRIDE_VAR" in env_vars - assert env_vars["OVERRIDE_VAR"] == "override_value" - - -def test_compile_nimservice_override_config_tolerations(backend_config, sample_deployment, minimal_config): - """Test that override_config can add Kubernetes tolerations.""" - minimal_config.executor_config.override_config = { - "tolerations": [ - { - "key": "nvidia.com/gpu", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule", - }, - { - "key": "special-node", - "operator": "Exists", - "effect": "NoExecute", - }, - ] - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify tolerations were added - assert hasattr(nimservice.spec, "tolerations") - assert len(nimservice.spec.tolerations) == 2 - assert nimservice.spec.tolerations[0].key == "nvidia.com/gpu" - assert nimservice.spec.tolerations[1].key == "special-node" - - -def test_compile_nimservice_override_config_node_selector(backend_config, sample_deployment, minimal_config): - """Test that override_config can add node selectors.""" - minimal_config.executor_config.override_config = { - "nodeSelector": { - "node-type": "gpu-node", - "zone": "us-west1-a", - } - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify node selector was added - assert hasattr(nimservice.spec, "nodeSelector") - assert nimservice.spec.nodeSelector["node-type"] == "gpu-node" - assert nimservice.spec.nodeSelector["zone"] == "us-west1-a" - - -def test_compile_nimservice_override_config_resources(backend_config, sample_deployment, minimal_config): - """Test that override_config can override resource limits.""" - minimal_config.executor_config.override_config = { - "resources": { - "limits": { - "nvidia.com/gpu": "8", - "memory": "64Gi", - }, - "requests": { - "cpu": "4000m", - "memory": "32Gi", - }, - } - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - resources = nimservice.spec.resources - # Override should merge/replace resources - assert resources.limits["nvidia.com/gpu"].root == "8" - assert resources.limits["memory"].root == "64Gi" - assert resources.requests["cpu"].root == "4000m" - assert resources.requests["memory"].root == "32Gi" - - -def test_compile_nimservice_override_config_deep_merge(backend_config, sample_deployment, minimal_config): - """Test that override_config performs deep merge on nested structures.""" - # Set up a base config with some resources - minimal_config.executor_config.gpu = 2 - - # Override only the memory limit, GPU limit should remain - minimal_config.executor_config.override_config = { - "resources": { - "limits": { - "memory": "128Gi", - } - } - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - resources = nimservice.spec.resources - # GPU should still be there from base config - assert resources.limits["nvidia.com/gpu"].root == "2" - # Memory should be added from override - assert resources.limits["memory"].root == "128Gi" - # CPU requests should still be there from base config - assert resources.requests["cpu"].root == "1000m" - - -def test_compile_nimservice_override_config_serializes_correctly(backend_config, sample_deployment, minimal_config): - """Test that override_config values serialize correctly to dict for k8s API.""" - # Use override_config with multiple field types - minimal_config.executor_config.override_config = { - "env": [ - {"name": "OVERRIDE_VAR", "value": "override_value"}, - {"name": "ANOTHER_VAR", "value": "another_value"}, - ], - "tolerations": [ - { - "key": "nvidia.com/gpu", - "operator": "Equal", - "value": "true", - "effect": "NoSchedule", - }, - ], - "nodeSelector": { - "node-type": "gpu-node", - "zone": "us-west1-a", - }, - "resources": { - "limits": { - "nvidia.com/gpu": "4", - "memory": "64Gi", - }, - "requests": { - "cpu": "2000m", - "memory": "32Gi", - }, - }, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Serialize to dict as would be sent to k8s API - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - # Verify top-level structure - assert nimservice_dict["apiVersion"] == "apps.nvidia.com/v1alpha1" - assert nimservice_dict["kind"] == "NIMService" - assert "spec" in nimservice_dict - - spec = nimservice_dict["spec"] - - # Verify env vars are in correct k8s format (list of dicts with name/value) - assert "env" in spec - assert isinstance(spec["env"], list) - env_dict = {item["name"]: item["value"] for item in spec["env"]} - assert "OVERRIDE_VAR" in env_dict - assert env_dict["OVERRIDE_VAR"] == "override_value" - assert "ANOTHER_VAR" in env_dict - assert env_dict["ANOTHER_VAR"] == "another_value" - - # Verify tolerations are in correct k8s format - assert "tolerations" in spec - assert isinstance(spec["tolerations"], list) - assert len(spec["tolerations"]) == 1 - assert spec["tolerations"][0]["key"] == "nvidia.com/gpu" - assert spec["tolerations"][0]["operator"] == "Equal" - assert spec["tolerations"][0]["value"] == "true" - assert spec["tolerations"][0]["effect"] == "NoSchedule" - - # Verify nodeSelector is in correct k8s format (flat dict) - assert "nodeSelector" in spec - assert isinstance(spec["nodeSelector"], dict) - assert spec["nodeSelector"]["node-type"] == "gpu-node" - assert spec["nodeSelector"]["zone"] == "us-west1-a" - - # Verify resources are in correct k8s format (nested dicts with string values) - assert "resources" in spec - assert "limits" in spec["resources"] - assert "requests" in spec["resources"] - assert spec["resources"]["limits"]["nvidia.com/gpu"] == "4" - assert spec["resources"]["limits"]["memory"] == "64Gi" - assert spec["resources"]["requests"]["cpu"] == "2000m" - assert spec["resources"]["requests"]["memory"] == "32Gi" - - # Verify the dict can be serialized to JSON (final check for k8s compatibility) - json_str = json.dumps(nimservice_dict) - assert json_str # Should not raise an exception - parsed_back = json.loads(json_str) - assert parsed_back["spec"]["env"][0]["name"] == "OVERRIDE_VAR" - - -def test_compile_nimservice_override_config_serializes_to_yaml(backend_config, sample_deployment, minimal_config): - """Test that override_config values serialize correctly to YAML for kubectl apply.""" - # Use override_config with various field types - minimal_config.executor_config.override_config = { - "env": [ - {"name": "CUSTOM_ENV", "value": "custom_value"}, - ], - "tolerations": [ - { - "key": "special-hardware", - "operator": "Exists", - "effect": "NoSchedule", - }, - ], - "nodeSelector": { - "hardware": "gpu", - }, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Serialize to dict then to YAML (as kubectl would do) - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - yaml_output = yaml.dump(nimservice_dict, default_flow_style=False, sort_keys=False) - - # Parse back from YAML to verify roundtrip - parsed_yaml = yaml.safe_load(yaml_output) - - # Verify override values survived the roundtrip - assert "env" in parsed_yaml["spec"] - env_names = [e["name"] for e in parsed_yaml["spec"]["env"]] - assert "CUSTOM_ENV" in env_names - - assert "tolerations" in parsed_yaml["spec"] - assert parsed_yaml["spec"]["tolerations"][0]["key"] == "special-hardware" - assert parsed_yaml["spec"]["tolerations"][0]["operator"] == "Exists" - - assert "nodeSelector" in parsed_yaml["spec"] - assert parsed_yaml["spec"]["nodeSelector"]["hardware"] == "gpu" - - # Verify the YAML is in a format kubectl can apply - # (basic sanity checks for k8s YAML format) - assert parsed_yaml["apiVersion"] == "apps.nvidia.com/v1alpha1" - assert parsed_yaml["kind"] == "NIMService" - assert "metadata" in parsed_yaml - assert "name" in parsed_yaml["metadata"] - assert "namespace" in parsed_yaml["metadata"] - - -# ============================================================================ -# k8s_nim_operator_config Tests -# ============================================================================ - - -def test_compile_nimservice_k8s_nim_operator_config_resources(backend_config, sample_deployment, minimal_config): - """Test that k8s_nim_operator_config can add/override resource limits and requests.""" - # Add k8s_nim_operator_config with resources - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "resources": { - "limits": { - "memory": "32Gi", - }, - "requests": { - "cpu": "4", - "memory": "16Gi", - }, - } - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - resources = nimservice.spec.resources - # GPU should still be there from base config (gpu=1 from minimal_config) - assert "nvidia.com/gpu" in resources.limits - assert resources.limits["nvidia.com/gpu"].root == "1" - - # Memory limits from k8s_nim_operator_config should be merged - assert "memory" in resources.limits - assert resources.limits["memory"].root == "32Gi" - - # CPU requests should be overridden from k8s_nim_operator_config - assert "cpu" in resources.requests - assert resources.requests["cpu"].root == "4" - - # Memory requests from k8s_nim_operator_config - assert "memory" in resources.requests - assert resources.requests["memory"].root == "16Gi" - - -def test_compile_nimservice_k8s_nim_operator_config_tolerations(backend_config, sample_deployment, minimal_config): - """Test that k8s_nim_operator_config can add tolerations.""" - # Add k8s_nim_operator_config with tolerations - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "tolerations": [ - { - "key": "nvidia.com/gpu", - "operator": "Exists", - "effect": "NoSchedule", - }, - { - "key": "dedicated", - "operator": "Equal", - "value": "ml-workload", - "effect": "NoSchedule", - }, - ] - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify tolerations were added - assert nimservice.spec.tolerations is not None - assert len(nimservice.spec.tolerations) == 2 - - # Check first toleration - assert nimservice.spec.tolerations[0].key == "nvidia.com/gpu" - assert nimservice.spec.tolerations[0].operator == "Exists" - assert nimservice.spec.tolerations[0].effect == "NoSchedule" - - # Check second toleration - assert nimservice.spec.tolerations[1].key == "dedicated" - assert nimservice.spec.tolerations[1].operator == "Equal" - assert nimservice.spec.tolerations[1].value == "ml-workload" - assert nimservice.spec.tolerations[1].effect == "NoSchedule" - - -def test_compile_nimservice_k8s_nim_operator_config_node_selector(backend_config, sample_deployment, minimal_config): - """Test that k8s_nim_operator_config can add node selector with snake_case to camelCase conversion.""" - # Add k8s_nim_operator_config with node_selector (snake_case) - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "node_selector": { - "node-type": "gpu-node", - "zone": "us-west1-a", - "hardware": "a100", - } - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify nodeSelector was added (camelCase in K8s) - assert nimservice.spec.nodeSelector is not None - assert nimservice.spec.nodeSelector["node-type"] == "gpu-node" - assert nimservice.spec.nodeSelector["zone"] == "us-west1-a" - assert nimservice.spec.nodeSelector["hardware"] == "a100" - - -def test_compile_nimservice_k8s_nim_operator_config_startup_probe_grace_seconds( - backend_config, sample_deployment, minimal_config -): - """Test that k8s_nim_operator_config can set startup_probe_grace_seconds.""" - # Add k8s_nim_operator_config with startup_probe_grace_seconds - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.startup_probe_grace_seconds = 600 - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "startup_probe_grace_seconds": 600, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify startup probe uses the calculated failureThreshold - # 600 seconds / 10 = 60 failures - assert nimservice.spec.startupProbe is not None - assert nimservice.spec.startupProbe.enabled is True - assert nimservice.spec.startupProbe.probe is not None - assert nimservice.spec.startupProbe.probe.periodSeconds == 10 - assert nimservice.spec.startupProbe.probe.failureThreshold == 60 - - -def test_compile_nimservice_k8s_nim_operator_config_startup_probe_grace_seconds_rounds_up( - backend_config, sample_deployment, minimal_config -): - """Test that startup_probe_grace_seconds rounds up when dividing by 10.""" - # Add k8s_nim_operator_config with grace_seconds that doesn't divide evenly - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.startup_probe_grace_seconds = 605 - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "startup_probe_grace_seconds": 605, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify it rounds up: 605 / 10 = 60.5 → 61 - assert nimservice.spec.startupProbe.probe.failureThreshold == 61 - - -def test_compile_nimservice_k8s_nim_operator_config_startup_probe_grace_seconds_various_values( - backend_config, sample_deployment, minimal_config -): - """Test startup_probe_grace_seconds with various values to verify rounding.""" - test_cases = [ - (600, 60), # Exact division - (605, 61), # Rounds up - (1201, 121), # Large value rounds up - (10, 1), # Small value - (1, 1), # Minimum practical value - (999, 100), # Rounds up - (1000, 100), # Exact division - ] - - for grace_seconds, expected_threshold in test_cases: - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.startup_probe_grace_seconds = grace_seconds - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "startup_probe_grace_seconds": grace_seconds, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - assert nimservice.spec.startupProbe.probe.failureThreshold == expected_threshold, ( - f"For grace_seconds={grace_seconds}, expected failureThreshold={expected_threshold}, " - f"but got {nimservice.spec.startupProbe.probe.failureThreshold}" - ) - - -def test_compile_nimservice_startup_probe_default_when_no_grace_seconds( - backend_config, sample_deployment, minimal_config -): - """Test that startup probe uses default 600 seconds (60 failures) when grace_seconds is not provided.""" - # No k8s_nim_operator_config provided - minimal_config.executor_config.k8s_nim_operator_config = None - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify default startup probe: 600 seconds / 10 = 60 failures - assert nimservice.spec.startupProbe is not None - assert nimservice.spec.startupProbe.enabled is True - assert nimservice.spec.startupProbe.probe.periodSeconds == 10 - assert nimservice.spec.startupProbe.probe.failureThreshold == 60 - - -def test_compile_nimservice_k8s_nim_operator_config_multiple_fields(backend_config, sample_deployment, minimal_config): - """Test that k8s_nim_operator_config can set multiple fields at once.""" - # Add k8s_nim_operator_config with multiple fields - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.startup_probe_grace_seconds = 1200 - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "resources": { - "limits": {"memory": "64Gi"}, - "requests": {"cpu": "8", "memory": "32Gi"}, - }, - "tolerations": [ - {"key": "gpu-node", "operator": "Exists", "effect": "NoSchedule"}, - ], - "node_selector": { - "accelerator": "nvidia-a100", - }, - "startup_probe_grace_seconds": 1200, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Verify all fields were applied - assert nimservice.spec.resources.limits["memory"].root == "64Gi" - assert nimservice.spec.resources.requests["cpu"].root == "8" - assert len(nimservice.spec.tolerations) == 1 - assert nimservice.spec.tolerations[0].key == "gpu-node" - assert nimservice.spec.nodeSelector["accelerator"] == "nvidia-a100" - # Verify startup probe grace period: 1200 / 10 = 120 - assert nimservice.spec.startupProbe.probe.failureThreshold == 120 - - -def test_compile_nimservice_k8s_nim_operator_config_precedence_over_defaults( - backend_config, sample_deployment, minimal_config -): - """Test that k8s_nim_operator_config takes precedence over defaults.""" - # Base config has gpu=1 which sets nvidia.com/gpu limit to "1" - minimal_config.executor_config.gpu = 2 - - # k8s_nim_operator_config should override GPU limit - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "resources": { - "limits": { - "nvidia.com/gpu": "4", # Override from 2 to 4 - }, - } - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - resources = nimservice.spec.resources - # Should be 4, not 2 from the base config - assert resources.limits["nvidia.com/gpu"].root == "4" - - -def test_compile_nimservice_override_config_precedence_over_k8s_nim_operator_config( - backend_config, sample_deployment, minimal_config -): - """Test that override_config takes precedence over k8s_nim_operator_config.""" - # Set both k8s_nim_operator_config and override_config with conflicting values - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "node_selector": { - "zone": "us-west1-a", - "hardware": "gpu", - }, - "tolerations": [ - {"key": "from-k8s-config", "operator": "Exists"}, - ], - } - - minimal_config.executor_config.override_config = { - "nodeSelector": { - "zone": "us-east1-b", # Override zone - "environment": "production", # Add new field - }, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # override_config should win for nodeSelector - assert nimservice.spec.nodeSelector["zone"] == "us-east1-b" - assert nimservice.spec.nodeSelector["environment"] == "production" - # hardware should still be there from k8s_nim_operator_config (deep merge) - assert nimservice.spec.nodeSelector["hardware"] == "gpu" - - # tolerations from k8s_nim_operator_config should still be there - assert len(nimservice.spec.tolerations) == 1 - assert nimservice.spec.tolerations[0].key == "from-k8s-config" - - -def test_compile_nimservice_k8s_nim_operator_config_empty_does_nothing( - backend_config, sample_deployment, minimal_config -): - """Test that empty k8s_nim_operator_config doesn't affect the spec.""" - # Set k8s_nim_operator_config with no fields - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = {} - - nimservice_with_empty = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Remove k8s_nim_operator_config - minimal_config.executor_config.k8s_nim_operator_config = None - - nimservice_without = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Should be equivalent - assert nimservice_with_empty.spec.model_dump(exclude_none=True) == nimservice_without.spec.model_dump( - exclude_none=True - ) - - -def test_compile_nimservice_k8s_nim_operator_config_serializes_correctly( - backend_config, sample_deployment, minimal_config -): - """Test that k8s_nim_operator_config values serialize correctly to dict for k8s API.""" - # Set k8s_nim_operator_config with various field types - minimal_config.executor_config.k8s_nim_operator_config = MagicMock() - minimal_config.executor_config.k8s_nim_operator_config.model_dump.return_value = { - "resources": { - "limits": {"memory": "32Gi"}, - "requests": {"cpu": "4", "memory": "16Gi"}, - }, - "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, - ], - "node_selector": { - "hardware": "a100", - }, - } - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Serialize to dict as would be sent to k8s API - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - spec = nimservice_dict["spec"] - - # Verify resources are serialized correctly - assert "resources" in spec - assert spec["resources"]["limits"]["memory"] == "32Gi" - assert spec["resources"]["requests"]["cpu"] == "4" - assert spec["resources"]["requests"]["memory"] == "16Gi" - - # Verify tolerations are serialized correctly - assert "tolerations" in spec - assert len(spec["tolerations"]) == 1 - assert spec["tolerations"][0]["key"] == "nvidia.com/gpu" - - # Verify nodeSelector is serialized correctly (camelCase) - assert "nodeSelector" in spec - assert spec["nodeSelector"]["hardware"] == "a100" - - # Verify the dict can be serialized to JSON (final check for k8s compatibility) - json_str = json.dumps(nimservice_dict) - assert json_str - parsed_back = json.loads(json_str) - assert parsed_back["spec"]["nodeSelector"]["hardware"] == "a100" - - -def test_compile_nimservice_serializes_to_dict(backend_config, sample_deployment, minimal_config): - """Test that compiled NIMService can be serialized to dict for k8s API.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Should be able to serialize to dict - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - # Verify top-level structure - assert "apiVersion" in nimservice_dict - assert "kind" in nimservice_dict - assert "metadata" in nimservice_dict - assert "spec" in nimservice_dict - - # Verify it matches the expected API version - assert nimservice_dict["apiVersion"] == "apps.nvidia.com/v1alpha1" - assert nimservice_dict["kind"] == "NIMService" - - -def test_compile_nimservice_matches_example_structure(backend_config, sample_deployment): - """Test that compiled NIMService matches the structure from example YAML.""" - # Create a config that matches the example YAML - config = MagicMock() - config.workspace = "ben-test" - config.name = "llama-config" - config.entity_version = "v1" - - config.engine = "nim" - config.model_spec = MagicMock() - config.model_spec.lora_enabled = True - config.model_spec.model_name = "llama-3.2-3b-instruct" - config.model_spec.model_namespace = "ben-test" - config.model_spec.model_revision = None - config.model_spec.tool_call_config = None - config.executor_config = MagicMock() - config.executor_config.image_name = "nvcr.io/nim/meta/llama-3.2-3b-instruct" - config.executor_config.image_tag = "1.8.5" - config.executor_config.gpu = 1 - config.executor_config.disk_size = "200Gi" - config.executor_config.additional_envs = {} - config.executor_config.k8s_nim_operator_config = None - config.executor_config.override_config = {} - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=config, - k8s_namespace="aire-cicd", - resource_name="modeldeployment-ben-test-llama-3-2-3b-instruct-deployment", - ) - - # Verify structure matches example - assert nimservice.apiVersion == "apps.nvidia.com/v1alpha1" - assert nimservice.kind == "NIMService" - assert nimservice.spec.authSecret == "ngc-api" - assert nimservice.spec.image.repository == "nvcr.io/nim/meta/llama-3.2-3b-instruct" - assert nimservice.spec.image.tag == "1.8.5" - assert nimservice.spec.resources.limits["nvidia.com/gpu"].root == "1" - assert nimservice.spec.expose.service.type == "ClusterIP" - assert nimservice.spec.expose.service.port == 8000 - - # Verify env vars - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert "NIM_GUIDED_DECODING_BACKEND" in env_vars - assert "NIM_SERVED_MODEL_NAME" in env_vars - assert "NIM_PEFT_SOURCE" in env_vars - assert "NIM_PEFT_REFRESH_INTERVAL" in env_vars - - -def test_compile_nimservice_serializes_to_valid_yaml(backend_config, sample_deployment, full_config): - """Test that compiled NIMService can be serialized to valid YAML.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=full_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Serialize to dict (as would be sent to k8s API) - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - # Convert to YAML - yaml_output = yaml.dump(nimservice_dict, default_flow_style=False, sort_keys=False) - - # Verify YAML is parseable - parsed_yaml = yaml.safe_load(yaml_output) - - # Verify required top-level fields - assert parsed_yaml["apiVersion"] == "apps.nvidia.com/v1alpha1" - assert parsed_yaml["kind"] == "NIMService" - assert "metadata" in parsed_yaml - assert "spec" in parsed_yaml - - # Verify metadata structure - assert parsed_yaml["metadata"]["name"] == "md-test-ns-test-deployment" - assert parsed_yaml["metadata"]["namespace"] == "default" - assert "labels" in parsed_yaml["metadata"] - - # Verify spec has all required fields - spec = parsed_yaml["spec"] - assert "authSecret" in spec - assert "image" in spec - assert "resources" in spec - assert "storage" in spec - assert "expose" in spec - assert "env" in spec - - # Verify image structure - assert spec["image"]["repository"] == "nvcr.io/nim/meta/llama-3.2-3b-instruct" - assert spec["image"]["tag"] == "1.8.5" - assert spec["image"]["pullPolicy"] == "IfNotPresent" - - # Verify resources structure - assert "limits" in spec["resources"] - assert "nvidia.com/gpu" in spec["resources"]["limits"] - assert "requests" in spec["resources"] - assert "cpu" in spec["resources"]["requests"] - - # Verify storage structure - assert "pvc" in spec["storage"] - assert spec["storage"]["pvc"]["create"] is True - assert spec["storage"]["pvc"]["size"] == "200Gi" - - # Verify expose structure - assert "service" in spec["expose"] - assert spec["expose"]["service"]["type"] == "ClusterIP" - assert spec["expose"]["service"]["port"] == 8000 - - # Verify env vars are present as a list - assert isinstance(spec["env"], list) - assert len(spec["env"]) > 0 - - # Verify env vars have correct structure - env_names = [env["name"] for env in spec["env"]] - assert "NIM_GUIDED_DECODING_BACKEND" in env_names - assert "NIM_SERVED_MODEL_NAME" in env_names - - -def test_compile_nimservice_yaml_structure_matches_example(backend_config, sample_deployment): - """Test that compiled NIMService YAML has same structure as example.""" - # Create config matching the example - config = MagicMock() - config.workspace = "ben-test" - config.name = "llama-config" - config.entity_version = "v1" - - config.engine = "nim" - config.model_spec = MagicMock() - config.model_spec.lora_enabled = True - config.model_spec.model_name = "llama-3.2-3b-instruct" - config.model_spec.model_namespace = "ben-test" - config.model_spec.model_revision = None - config.model_spec.tool_call_config = None - config.executor_config = MagicMock() - config.executor_config.image_name = "nvcr.io/nim/meta/llama-3.2-3b-instruct" - config.executor_config.image_tag = "1.8.5" - config.executor_config.gpu = 1 - config.executor_config.disk_size = "200Gi" - config.executor_config.additional_envs = {} - config.executor_config.k8s_nim_operator_config = None - config.executor_config.override_config = {} - - # Compile NIMService - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=config, - k8s_namespace="aire-cicd", - resource_name="modeldeployment-ben-test-llama-3-2-3b-instruct-deployment", - ) - - # Serialize to dict - compiled_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - # Load example YAML - example_path = Path(__file__).parent / "data" / "example-nimservice.yaml" - with open(example_path) as f: - example_dict = yaml.safe_load(f) - - # Compare key structural elements (not exact values, since our config differs slightly) - assert compiled_dict["apiVersion"] == example_dict["apiVersion"] - assert compiled_dict["kind"] == example_dict["kind"] - - # Verify same top-level spec fields exist - compiled_spec_keys = set(compiled_dict["spec"].keys()) - - # Our compiled version should have all the same major sections - # (though we may not implement every field yet) - assert "authSecret" in compiled_spec_keys - assert "image" in compiled_spec_keys - assert "env" in compiled_spec_keys - assert "expose" in compiled_spec_keys - assert "resources" in compiled_spec_keys - assert "storage" in compiled_spec_keys - - # Verify image structure matches (we may have a subset of fields) - compiled_image_keys = set(compiled_dict["spec"]["image"].keys()) - # Our compiled version should have the core fields - assert "repository" in compiled_image_keys - assert "tag" in compiled_image_keys - assert "pullPolicy" in compiled_image_keys - - # Verify expose structure matches - assert "service" in compiled_dict["spec"]["expose"] - assert "service" in example_dict["spec"]["expose"] - assert set(compiled_dict["spec"]["expose"]["service"].keys()) <= set( - example_dict["spec"]["expose"]["service"].keys() - ) - - # Verify resources structure matches - assert "limits" in compiled_dict["spec"]["resources"] - assert "requests" in compiled_dict["spec"]["resources"] - assert "limits" in example_dict["spec"]["resources"] - assert "requests" in example_dict["spec"]["resources"] - - # Verify storage structure matches - assert "pvc" in compiled_dict["spec"]["storage"] - assert "pvc" in example_dict["spec"]["storage"] - - -def test_compile_nimservice_can_roundtrip_through_yaml(backend_config, sample_deployment, minimal_config): - """Test that NIMService can be serialized to YAML and parsed back.""" - # Compile NIMService - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - # Serialize to dict - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - # Convert to YAML string - yaml_string = yaml.dump(nimservice_dict, default_flow_style=False) - - # Parse back from YAML - parsed_dict = yaml.safe_load(yaml_string) - - # Verify key fields survived the roundtrip - assert parsed_dict["apiVersion"] == "apps.nvidia.com/v1alpha1" - assert parsed_dict["kind"] == "NIMService" - assert parsed_dict["metadata"]["name"] == "md-test-ns-test-deployment" - assert parsed_dict["spec"]["authSecret"] == "ngc-api" - assert parsed_dict["spec"]["image"]["repository"] == "nvcr.io/nim/meta/llama-3-8b-instruct" - assert parsed_dict["spec"]["resources"]["limits"]["nvidia.com/gpu"] == "1" - assert parsed_dict["spec"]["expose"]["service"]["port"] == 8000 - - # Verify the parsed dict can be used to create a Kubernetes resource - # (In real usage, this would be sent to kubectl or k8s Python client) - assert "metadata" in parsed_dict - assert "spec" in parsed_dict - assert parsed_dict["kind"] == "NIMService" - - -# ============================================================================ -# Multi-LLM Configuration Tests -# ============================================================================ - - -def test_compile_nimservice_multi_llm_with_files_service(backend_config, sample_deployment, minimal_config): - """Test multi-LLM configuration for Files service (no HF_TOKEN/HF_ENDPOINT in env).""" - # Set up multi-LLM image (using default) - minimal_config.executor_config.image_name = None # Will use default - minimal_config.model_spec.model_namespace = "nvidia" - minimal_config.model_spec.model_name = "Llama-3.1-Nemotron-Nano-4B-v1.1" - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={ - "files": "http://nemo-files:8000", - "models": "http://nemo-models:8000", - }, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - env_names = [env.name for env in nimservice.spec.env] - - # NIMCache pulls from Files; NIMService does not need HF_TOKEN or HF_ENDPOINT in env - assert "HF_TOKEN" not in env_names - assert "HF_ENDPOINT" not in env_dict - - assert "NIM_MODEL_NAME" in env_dict - assert env_dict["NIM_MODEL_NAME"] == "nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1" - assert "NIM_SERVED_MODEL_NAME" in env_dict - assert env_dict["NIM_SERVED_MODEL_NAME"] == "nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1" - assert "NIM_GUIDED_DECODING_BACKEND" in env_dict - assert env_dict["NIM_GUIDED_DECODING_BACKEND"] == "outlines" - - -def test_compile_nimservice_multi_llm_user_overrides_decoding_backend( - backend_config, sample_deployment, minimal_config -): - """Test that user can override NIM_GUIDED_DECODING_BACKEND for multi-LLM.""" - # Set up multi-LLM with user override - minimal_config.executor_config.image_name = None # Will use default - minimal_config.model_spec.model_namespace = "nvidia" - minimal_config.model_spec.model_name = "Llama-3.1-Nemotron-Nano-4B-v1.1" - minimal_config.executor_config.additional_envs = { - "NIM_GUIDED_DECODING_BACKEND": "custom_backend", - } - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={ - "files": "http://files-service:8000", - "models": "http://models-service:8000", - }, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - - # User override should take precedence - assert env_dict["NIM_GUIDED_DECODING_BACKEND"] == "custom_backend" - - -def test_compile_nimservice_llm_specific_nim_traditional_behavior(backend_config, sample_deployment, minimal_config): - """Test that LLM-specific NIMs use traditional configuration (not multi-LLM behavior).""" - # Set up LLM-specific image (NOT default) - minimal_config.executor_config.image_name = "nvcr.io/nim/meta/llama-3-8b-instruct" - minimal_config.model_spec.model_namespace = "meta" - minimal_config.model_spec.model_name = "llama-3-8b-instruct" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - - # Should NOT have hf:// prefix for LLM-specific NIMs - assert "NIM_MODEL_NAME" in env_dict - assert env_dict["NIM_MODEL_NAME"] == "meta/llama-3-8b-instruct" - assert not env_dict["NIM_MODEL_NAME"].startswith("hf://") - - # Should NOT have HF_ENDPOINT - assert "HF_ENDPOINT" not in env_dict - - # Should use default backend (outlines) for LLM-specific - assert "NIM_GUIDED_DECODING_BACKEND" in env_dict - assert env_dict["NIM_GUIDED_DECODING_BACKEND"] == "outlines" - - -# ============================================================================ -# NIMCache Compilation Tests -# ============================================================================ - - -def test_compile_nimcache_basic(backend_config): - """Test NIMCache CR generation for SFT models.""" - # Configure backend config - backend_config.default_storage_class = "local-storage" - backend_config.files_auth_secret = "nemo-models-files-token" - backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - backend_config.default_user_id = 1000 - backend_config.default_group_id = 1000 - - # Mock platform config - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={ - "files": "http://files-service:8000", - "models": "http://models-service:8000", - }, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - # Generate NIMCache - nimcache = compile_nimcache( - backend_config=backend_config, - k8s_namespace="default", - resource_name="test-deployment", - model_namespace="test-ns", - model_name="test-model", - pvc_size="200Gi", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - model_revision="v1", - ) - - # Verify NIMCache structure - assert nimcache.apiVersion == "apps.nvidia.com/v1alpha1" - assert nimcache.kind == "NIMCache" - assert nimcache.metadata["name"] == "test-deployment" - assert nimcache.metadata["namespace"] == "default" - - # Verify storage configuration - assert nimcache.spec.storage.pvc.create is True - assert nimcache.spec.storage.pvc.size == "200Gi" - assert nimcache.spec.storage.pvc.storageClass == "local-storage" - - # Verify Files service source configuration (uses Hf CRD type for HF-compatible API) - assert nimcache.spec.source.hf is not None - assert nimcache.spec.source.hf.endpoint == "http://files-service:8000/apis/files/v2/hf" - assert nimcache.spec.source.hf.namespace == "test-ns" - assert nimcache.spec.source.hf.modelName == "test-model" - assert nimcache.spec.source.hf.revision == "v1" - assert nimcache.spec.source.hf.authSecret == "nemo-models-files-token" - assert nimcache.spec.source.hf.modelPuller == "nvcr.io/nvidia/model-puller:latest" - assert nimcache.spec.source.hf.pullSecret == "nvcr-secret" - - -def test_compile_nimcache_without_v2hf_suffix(backend_config): - """Test NIMCache CR generation when files service URL does NOT include /v2/hf suffix.""" - # Configure backend config - backend_config.default_storage_class = "local-storage" - backend_config.files_auth_secret = "nemo-models-files-token" - backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - backend_config.default_user_id = None - backend_config.default_group_id = None - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - # Generate NIMCache - nimcache = compile_nimcache( - backend_config=backend_config, - k8s_namespace="default", - resource_name="test-deployment", - model_namespace="test-ns", - model_name="test-model", - pvc_size="200Gi", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - model_revision=None, - ) - - # Verify endpoint HAS /apis/files/v2/hf appended - assert nimcache.spec.source.hf is not None - assert nimcache.spec.source.hf.endpoint == "http://files-service:8000/apis/files/v2/hf" - - -def test_compile_nimcache_with_v2hf_suffix(backend_config): - """Test NIMCache CR generation when files service URL already includes /v2/hf suffix.""" - # Configure backend config - backend_config.default_storage_class = "local-storage" - backend_config.files_auth_secret = "nemo-models-files-token" - backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - backend_config.default_user_id = None - backend_config.default_group_id = None - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000/v2/hf"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - # Generate NIMCache - nimcache = compile_nimcache( - backend_config=backend_config, - k8s_namespace="default", - resource_name="test-deployment", - model_namespace="test-ns", - model_name="test-model", - pvc_size="200Gi", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - model_revision=None, - ) - - # When base URL has /v2/hf, urljoin appends apis/files/v2/hf to it - assert nimcache.spec.source.hf is not None - assert nimcache.spec.source.hf.endpoint == "http://files-service:8000/v2/apis/files/v2/hf" - - -def test_compile_nimcache_with_custom_auth_secret(backend_config): - """Test NIMCache CR generation with custom files_auth_secret config.""" - # Configure backend config with CUSTOM auth secret name - backend_config.default_storage_class = "local-storage" - backend_config.files_auth_secret = "my-custom-files-secret" - backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - backend_config.default_user_id = None - backend_config.default_group_id = None - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - # Generate NIMCache - nimcache = compile_nimcache( - backend_config=backend_config, - k8s_namespace="default", - resource_name="test-deployment", - model_namespace="test-ns", - model_name="test-model", - pvc_size="200Gi", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - model_revision=None, - ) - - # Verify custom auth secret is used in NIMCache - assert nimcache.spec.source.hf is not None - assert nimcache.spec.source.hf.authSecret == "my-custom-files-secret" - # Ensure it's not using the default value - assert nimcache.spec.source.hf.authSecret != "nemo-models-files-token" - - -def test_compile_nimcache_default_resources_tolerations_node_selector(backend_config): - """Test NIMCache honors default_resources, default_tolerations, default_node_selector like NIMService.""" - backend_config.default_storage_class = "local-storage" - backend_config.files_auth_secret = "nemo-models-files-token" - backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - backend_config.default_user_id = 1000 - backend_config.default_group_id = 1000 - backend_config.default_resources = { - "requests": {"cpu": "2", "memory": "8Gi"}, - "limits": {"memory": "16Gi"}, - } - backend_config.default_tolerations = [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, - ] - backend_config.default_node_selector = {"node-type": "gpu-node", "zone": "us-west1-a"} - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimcache = compile_nimcache( - backend_config=backend_config, - k8s_namespace="default", - resource_name="test-deployment", - model_namespace="test-ns", - model_name="test-model", - pvc_size="200Gi", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - model_revision=None, - ) - - # Resources: mapped from requests (then limits) to NIMCache cpu/memory - assert nimcache.spec.resources is not None - assert nimcache.spec.resources.cpu is not None - assert nimcache.spec.resources.memory is not None - # RootModel types may wrap int/str - cpu_val = getattr(nimcache.spec.resources.cpu, "root", nimcache.spec.resources.cpu) - mem_val = getattr(nimcache.spec.resources.memory, "root", nimcache.spec.resources.memory) - assert cpu_val == "2" - assert mem_val == "8Gi" - - # Tolerations - assert nimcache.spec.tolerations is not None - assert len(nimcache.spec.tolerations) == 1 - assert nimcache.spec.tolerations[0].key == "nvidia.com/gpu" - assert nimcache.spec.tolerations[0].operator == "Exists" - assert nimcache.spec.tolerations[0].effect == "NoSchedule" - - # Node selector - assert nimcache.spec.nodeSelector is not None - assert nimcache.spec.nodeSelector == {"node-type": "gpu-node", "zone": "us-west1-a"} - - -def test_compile_nimcache_no_defaults_when_unset(backend_config): - """Test NIMCache has no resources/tolerations/nodeSelector when backend defaults are unset.""" - backend_config.default_storage_class = "local-storage" - backend_config.files_auth_secret = "nemo-models-files-token" - backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - backend_config.default_user_id = None - backend_config.default_group_id = None - # Explicitly leave default_resources, default_tolerations, default_node_selector as None - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimcache = compile_nimcache( - backend_config=backend_config, - k8s_namespace="default", - resource_name="test-deployment", - model_namespace="test-ns", - model_name="test-model", - pvc_size="200Gi", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - model_revision=None, - ) - - assert nimcache.spec.resources is None - assert nimcache.spec.tolerations is None - assert nimcache.spec.nodeSelector is None - - -def test_compile_nimcache_default_labels_and_annotations(backend_config): - """Test that default_labels and default_annotations from backend config are applied to NIMCache CR and PVC.""" - backend_config.default_storage_class = "local-storage" - backend_config.files_auth_secret = "nemo-models-files-token" - backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret" - backend_config.default_labels = {"team": "ml-platform", "env": "prod"} - backend_config.default_annotations = {"prometheus.io/scrape": "true"} - - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files-service:8000"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimcache = compile_nimcache( - backend_config=backend_config, - k8s_namespace="default", - resource_name="test-deployment", - model_namespace="test-ns", - model_name="test-model", - pvc_size="200Gi", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - model_revision=None, - ) - - assert nimcache.metadata["labels"]["team"] == "ml-platform" - assert nimcache.metadata["labels"]["env"] == "prod" - assert nimcache.metadata["labels"]["app.kubernetes.io/name"] == "test-deployment" - assert nimcache.metadata["annotations"] == {"prometheus.io/scrape": "true"} - assert nimcache.spec.storage.pvc.annotations == {"prometheus.io/scrape": "true"} - - -def test_compile_nimservice_nimcache_files_service_no_ft_env(backend_config, sample_deployment, minimal_config): - """With NIMCache + multi-LLM image, do not set NIM_FT_MODEL (match docker).""" - minimal_config.model_spec.model_namespace = "e2e-workspace" - minimal_config.model_spec.model_name = "qwen-2-5-1-5b" - minimal_config.executor_config.image_name = "nvcr.io/nim/nvidia/llm-nim" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - nimcache_name="md-test-ns-test-deployment", - ) - - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - assert env_dict["NIM_MODEL_NAME"] == "/model-store" - assert env_dict["NIM_SERVED_MODEL_NAME"] == "e2e-workspace/qwen-2-5-1-5b" - # NIM_CACHE_PATH not set by compiler (reverted for isolation test) - assert "NIM_FT_MODEL" not in env_dict - # FILES_SERVICE (multi-LLM) does not get NIM_FT_MODEL or NIM_CUSTOM_MODEL - assert "NIM_CUSTOM_MODEL" not in env_dict - # NGC_API_KEY from auth secret for parity with Docker - ngc_env = next(e for e in nimservice.spec.env if e.name == "NGC_API_KEY") - assert ngc_env.valueFrom is not None - assert ngc_env.valueFrom.secretKeyRef is not None - assert ngc_env.valueFrom.secretKeyRef.key == "NGC_API_KEY" - assert ngc_env.valueFrom.secretKeyRef.name == backend_config.auth_secret - - -def test_compile_nimservice_nimcache_files_service_sft_has_ft_env(backend_config, sample_deployment, minimal_config): - """With NIMCache and model-specific image, set NIM_FT_MODEL and NGC_API_KEY.""" - minimal_config.model_spec.model_namespace = "e2e-workspace" - minimal_config.model_spec.model_name = "sft-model" - # Model-specific image (not multi-LLM) so NIM_FT_MODEL is set. - minimal_config.executor_config.image_name = "nvcr.io/nim/meta/llama-3_1-8b-instruct" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - nimcache_name="md-test-ns-test-deployment", - ) - - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - assert env_dict["NIM_MODEL_NAME"] == "/model-store" - assert env_dict["NIM_SERVED_MODEL_NAME"] == "e2e-workspace/sft-model" - assert env_dict["NIM_FT_MODEL"] == "/model-store" - assert env_dict["NIM_CUSTOM_MODEL"] == "/model-store" - # NIM_CACHE_PATH not set by compiler (reverted for isolation test) - # NGC_API_KEY from auth secret for parity with Docker - ngc_env = next(e for e in nimservice.spec.env if e.name == "NGC_API_KEY") - assert ngc_env.valueFrom is not None - assert ngc_env.valueFrom.secretKeyRef is not None - assert ngc_env.valueFrom.secretKeyRef.key == "NGC_API_KEY" - assert ngc_env.valueFrom.secretKeyRef.name == backend_config.auth_secret - - -def test_compile_nimservice_nimcache_files_service_sft_multi_llm_no_ft_env( - backend_config, sample_deployment, minimal_config -): - """With NIMCache but multi-LLM image, do NOT set NIM_FT_MODEL (breaks LoRA).""" - minimal_config.model_spec.model_namespace = "e2e-workspace" - minimal_config.model_spec.model_name = "sft-model" - # Explicit multi-LLM image so NIM_FT_MODEL is omitted (minimal_config fixture uses model-specific image). - minimal_config.executor_config.image_name = "nvcr.io/nim/nvidia/llm-nim" - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - nimcache_name="md-test-ns-test-deployment", - ) - - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - assert env_dict["NIM_MODEL_NAME"] == "/model-store" - assert "NIM_FT_MODEL" not in env_dict - - -def test_compile_nimservice_tool_call_plugin_init_containers(backend_config, sample_deployment, minimal_config): - """tool_call_plugin compiles three init containers and sets deterministic plugin path.""" - minimal_config.model_spec.tool_call_config = SimpleNamespace( - tool_call_plugin="test-ws/my-plugin-fileset", - tool_call_parser=None, - auto_tool_choice=None, - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - huggingface_model_puller="nvcr.io/nvidia/model-puller:latest", - ) - - assert nimservice.spec.initContainers is not None - assert len(nimservice.spec.initContainers) == 3 - - prepare_container, pull_container, finalize_container = nimservice.spec.initContainers - - assert prepare_container.command is not None - assert prepare_container.command[0:2] == ["sh", "-c"] - assert "/model-store/plugin" in prepare_container.command[2] - - assert pull_container.command == ["download", "test-ws/my-plugin-fileset", "--local-dir", "/scratch/plugin"] - pull_env = {env.name: env.value for env in (pull_container.env or [])} - assert pull_env["HF_TOKEN"] == "service:models" - assert pull_env["HF_ENDPOINT"].endswith("/apis/files/v2/hf") - - assert finalize_container.command is not None - assert finalize_container.command[0:2] == ["sh", "-c"] - assert "/model-store/plugin/plugin.py" in finalize_container.command[2] - - env_dict = {env.name: env.value for env in nimservice.spec.env if env.value} - assert env_dict["NIM_TOOL_PARSER_PLUGIN"] == "/model-store/plugin/plugin.py" - - -def test_tool_call_plugin_finalize_script_moves_single_py(tmp_path): - """Finalize script moves exactly one discovered .py to plugin_path.""" - scratch_dir = tmp_path / "scratch" / "plugin" - plugin_path = tmp_path / "model-store" / "plugin" / "plugin.py" - scratch_dir.mkdir(parents=True, exist_ok=True) - plugin_path.parent.mkdir(parents=True, exist_ok=True) - source = scratch_dir / "custom_plugin.py" - source.write_text("print('ok')\n") - - script = TOOL_CALL_PLUGIN_FINALIZE_SCRIPT_TEMPLATE.format( - scratch_dir=str(scratch_dir), plugin_path=str(plugin_path) - ) - result = subprocess.run(["bash", "-c", script], capture_output=True, text=True) - - assert result.returncode == 0, f"stdout={result.stdout}\nstderr={result.stderr}" - assert plugin_path.exists() - assert plugin_path.read_text() == "print('ok')\n" - assert not source.exists() - - -def test_tool_call_plugin_finalize_script_fails_with_multiple_py_files(tmp_path): - """Finalize script fails when more than one .py file exists.""" - scratch_dir = tmp_path / "scratch" / "plugin" - plugin_path = tmp_path / "model-store" / "plugin" / "plugin.py" - scratch_dir.mkdir(parents=True, exist_ok=True) - plugin_path.parent.mkdir(parents=True, exist_ok=True) - (scratch_dir / "a.py").write_text("print('a')\n") - (scratch_dir / "b.py").write_text("print('b')\n") - - script = TOOL_CALL_PLUGIN_FINALIZE_SCRIPT_TEMPLATE.format( - scratch_dir=str(scratch_dir), plugin_path=str(plugin_path) - ) - result = subprocess.run(["bash", "-c", script], capture_output=True, text=True) - - assert result.returncode != 0 - combined_output = (result.stdout or "") + (result.stderr or "") - assert "must contain exactly one .py file" in combined_output - assert not plugin_path.exists() - - -def test_apply_k8s_nim_operator_config_tolerations_replaced(backend_config, sample_deployment, minimal_config): - """Tolerations from config replace spec tolerations when present (full list).""" - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files:8000", "models": "http://models:8000"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test", - ) - spec = nimservice.spec - assert spec.tolerations is None - k8s_config = { - "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, - ], - } - result = _apply_k8s_nim_operator_config(spec, k8s_config) - assert len(result.tolerations) == 1 - t = result.tolerations[0] - assert t.key == "nvidia.com/gpu" - assert t.operator == "Exists" - assert t.effect == "NoSchedule" - - -def test_apply_k8s_nim_operator_config_empty_config_returns_unchanged( - backend_config, sample_deployment, minimal_config -): - """Empty k8s_config leaves spec unchanged.""" - platform_config = PlatformConfig( # type: ignore[abstract] - service_discovery={"files": "http://files:8000", "models": "http://models:8000"}, - ) - with patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", - return_value=platform_config, - ): - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test", - ) - spec = nimservice.spec - result = _apply_k8s_nim_operator_config(spec, {}) - assert result.resources == spec.resources - assert result.nodeSelector == spec.nodeSelector - - -def test_compile_nimservice_trust_remote_code_env_var(backend_config, sample_deployment, minimal_config): - """Test that NIM_FORCE_TRUST_REMOTE_CODE=1 is set when model_entity.trust_remote_code is True.""" - from nemo_platform.types.models.model_entity import ModelEntity - - model_entity = ModelEntity( - id="model-1", - entity_id="model-1", - created_at="2024-01-01T00:00:00Z", - updated_at="2024-01-01T00:00:00Z", - workspace="test-ns", - name="test-model", - parent="models", - db_version=1, - fileset="test-ns/test-model", - spec=None, - trust_remote_code=True, - ) - - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - model_entity=model_entity, - ) - - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert env_vars.get("NIM_FORCE_TRUST_REMOTE_CODE") == "1" - - -def test_compile_nimservice_no_trust_remote_code_by_default(backend_config, sample_deployment, minimal_config): - """Test that NIM_FORCE_TRUST_REMOTE_CODE is NOT set when model_entity is None or trust_remote_code is False.""" - nimservice = compile_nimservice( - backend_config=backend_config, - deployment=sample_deployment, - config=minimal_config, - k8s_namespace="default", - resource_name="md-test-ns-test-deployment", - ) - - env_vars = {env.name: env.value for env in nimservice.spec.env} - assert "NIM_FORCE_TRUST_REMOTE_CODE" not in env_vars diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/nimservice.py b/services/core/models/tests/unit/nimservice_spec_types.py similarity index 100% rename from services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/types/nimservice.py rename to services/core/models/tests/unit/nimservice_spec_types.py diff --git a/services/core/models/tests/unit/test_config.py b/services/core/models/tests/unit/test_config.py index 162ce8d078..385a469b43 100644 --- a/services/core/models/tests/unit/test_config.py +++ b/services/core/models/tests/unit/test_config.py @@ -3,6 +3,8 @@ """Tests for Models service configuration.""" +from unittest.mock import MagicMock, patch + import pytest from nmp.common.config import Runtime from nmp.core.models.config import ( @@ -12,11 +14,7 @@ get_default_backends_for_runtime, merge_backends, ) -from nmp.core.models.controllers.backends.registry import ( - DockerBackendConfigModel, - K8sNimOperatorBackendConfigModel, - NoneBackendConfigModel, -) +from nmp.core.models.controllers.backends.deployments_plugin.config import DeploymentsPluginBackendConfigModel from pydantic import ValidationError @@ -54,137 +52,136 @@ def test_config_structure(): def test_get_default_backends_for_docker_runtime(): - """Test that docker backend is selected and enabled for DOCKER runtime.""" + """Test that deployments_plugin backend is selected and enabled for DOCKER runtime.""" backends = get_default_backends_for_runtime(Runtime.DOCKER) - assert "docker" in backends - assert isinstance(backends["docker"], DockerBackendConfigModel) - assert backends["docker"].enabled is True + assert "deployments_plugin" in backends + assert isinstance(backends["deployments_plugin"], DeploymentsPluginBackendConfigModel) + assert backends["deployments_plugin"].enabled is True + assert backends["deployments_plugin"].docker_executor == "local-docker" + assert backends["deployments_plugin"].default_executor == "local-docker" + assert "docker" not in backends assert "nim_operator" not in backends def test_get_default_backends_for_kubernetes_runtime(): - """Test that nim_operator backend is selected and enabled for KUBERNETES runtime.""" + """Test that deployments_plugin backend is selected and enabled for KUBERNETES runtime.""" backends = get_default_backends_for_runtime(Runtime.KUBERNETES) - assert "nim_operator" in backends - assert isinstance(backends["nim_operator"], K8sNimOperatorBackendConfigModel) - assert backends["nim_operator"].enabled is True + assert "deployments_plugin" in backends + assert isinstance(backends["deployments_plugin"], DeploymentsPluginBackendConfigModel) + assert backends["deployments_plugin"].enabled is True + assert backends["deployments_plugin"].k8s_executor == "local-k8s" + assert backends["deployments_plugin"].default_executor == "local-k8s" + assert "nim_operator" not in backends assert "docker" not in backends +def test_get_default_backends_for_none_runtime(): + """Test that deployments_plugin backend is selected for NONE runtime.""" + backends = get_default_backends_for_runtime(Runtime.NONE) + assert "deployments_plugin" in backends + assert isinstance(backends["deployments_plugin"], DeploymentsPluginBackendConfigModel) + assert backends["deployments_plugin"].enabled is True + assert backends["deployments_plugin"].default_executor is None + + def test_merge_backends_with_no_custom_backends(): """Test that merge returns only default backends when no custom backends provided.""" - default_backends = {"docker": DockerBackendConfigModel()} + default_backends = {"deployments_plugin": DeploymentsPluginBackendConfigModel()} custom_backends = {} merged = merge_backends(custom_backends, default_backends) - assert "docker" in merged + assert "deployments_plugin" in merged assert len(merged) == 1 def test_merge_backends_with_no_default_backends(): """Test that merge returns only custom backends when no defaults provided.""" default_backends = {} - custom_backends = {"docker": DockerBackendConfigModel()} + custom_backends = {"deployments_plugin": DeploymentsPluginBackendConfigModel()} merged = merge_backends(custom_backends, default_backends) - assert "docker" in merged + assert "deployments_plugin" in merged assert len(merged) == 1 def test_merge_backends_custom_overrides_default(): """Test that custom backend config overrides default backend config.""" default_backends = { - "docker": DockerBackendConfigModel(enabled=False), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=False), } custom_backends = { - "docker": DockerBackendConfigModel(enabled=True), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True), } merged = merge_backends(custom_backends, default_backends) - assert "docker" in merged - assert merged["docker"].enabled is True - - -def test_merge_backends_combines_different_backends(): - """Test that merge combines different backend types.""" - default_backends = {"docker": DockerBackendConfigModel()} - custom_backends = {"nim_operator": K8sNimOperatorBackendConfigModel()} - - merged = merge_backends(custom_backends, default_backends) - - assert "docker" in merged - assert "nim_operator" in merged - assert len(merged) == 2 + assert merged["deployments_plugin"].enabled is True def test_merge_backends_preserves_enabled_flag(): """Test that merge correctly handles enabled flag overrides.""" default_backends = { - "docker": DockerBackendConfigModel(enabled=False), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=False), } custom_backends = { - "docker": DockerBackendConfigModel(enabled=True), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True), } merged = merge_backends(custom_backends, default_backends) - assert merged["docker"].enabled is True + assert merged["deployments_plugin"].enabled is True -def test_merge_backends_disables_conflicting_when_runtime_demoted_to_none(): - """When the runtime auto-demotes to NONE (docker unavailable), any - user-enabled non-`none` backend must be disabled so the registry can - pick the `none` fallback cleanly instead of crashing with - 'Multiple backends are enabled'.""" - # Default reflects post-demotion state: NemoPlatformConfig.validate_runtime - # flipped DOCKER → NONE because the docker socket wasn't reachable. - default_backends = {"none": NoneBackendConfigModel(enabled=True)} - # User config (e.g. local.yaml or a downstream override) still asks for docker. - custom_backends = {"docker": DockerBackendConfigModel(enabled=True)} +@patch("nmp.core.models.config.get_platform_config") +def test_merge_backends_disables_conflicting_when_runtime_demoted_to_none(mock_platform_config): + """When runtime is NONE, force-enable deployments_plugin and disable other enabled backends.""" + mock_platform_config.return_value = MagicMock(runtime=Runtime.NONE) + default_backends = {"deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True)} + custom_backends = { + "deployments_plugin": DeploymentsPluginBackendConfigModel( + enabled=True, + docker_executor="local-docker", + ), + } merged = merge_backends(custom_backends, default_backends) - # `none` survives as the runtime fallback; the conflicting docker entry - # gets force-disabled. - assert merged["none"].enabled is True - assert merged["docker"].enabled is False + assert merged["deployments_plugin"].enabled is True -def test_merge_backends_leaves_docker_enabled_when_runtime_is_docker(): - """Sanity-check that the demotion-handling logic doesn't disable a - legitimately-enabled custom backend when the runtime is still DOCKER.""" - default_backends = {"docker": DockerBackendConfigModel(enabled=True)} - custom_backends = {"docker": DockerBackendConfigModel(enabled=True)} +@patch("nmp.core.models.config.get_platform_config") +def test_merge_backends_leaves_deployments_plugin_enabled_when_runtime_is_docker(mock_platform_config): + """Sanity-check demotion logic does not disable a valid docker-runtime backend.""" + mock_platform_config.return_value = MagicMock(runtime=Runtime.DOCKER) + default_backends = {"deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True)} + custom_backends = { + "deployments_plugin": DeploymentsPluginBackendConfigModel( + enabled=True, + docker_executor="local-docker", + ), + } merged = merge_backends(custom_backends, default_backends) - assert merged["docker"].enabled is True - assert "none" not in merged + assert merged["deployments_plugin"].enabled is True -def test_merge_backends_force_enables_none_when_user_disabled_it_during_demotion(): - """If the user explicitly disabled `none` AND another backend is enabled, - runtime demotion to NONE must still leave exactly one enabled backend. - - Without the force-enable, the user's `none: enabled=False` would win on - merge, my non-`none` disable loop would drop docker, and the registry - would crash with "No backends are enabled" (the zero-enabled case). - """ - default_backends = {"none": NoneBackendConfigModel(enabled=True)} +@patch("nmp.core.models.config.get_platform_config") +def test_merge_backends_force_enables_deployments_plugin_when_user_disabled_it_during_demotion( + mock_platform_config, +): + """Runtime NONE must still leave exactly one enabled backend.""" + mock_platform_config.return_value = MagicMock(runtime=Runtime.NONE) + default_backends = {"deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=True)} custom_backends = { - "none": NoneBackendConfigModel(enabled=False), - "docker": DockerBackendConfigModel(enabled=True), + "deployments_plugin": DeploymentsPluginBackendConfigModel(enabled=False), } merged = merge_backends(custom_backends, default_backends) - # `none` is force-enabled (overrides user's explicit disable) so the - # registry has exactly one enabled backend in the demotion path. - assert merged["none"].enabled is True - assert merged["docker"].enabled is False + assert merged["deployments_plugin"].enabled is True def test_module_level_backends_variable_exists(): @@ -201,26 +198,23 @@ def test_merge_backends_with_flat_config_partial_override(): """ # Default backend with flat config default_backends = { - "nim_operator": K8sNimOperatorBackendConfigModel( + "deployments_plugin": DeploymentsPluginBackendConfigModel( enabled=True, default_storage_class="standard", default_pvc_size="100Gi", ), } - # Custom backend that only overrides one field custom_backends = { - "nim_operator": K8sNimOperatorBackendConfigModel( + "deployments_plugin": DeploymentsPluginBackendConfigModel( default_storage_class="fast-ssd", ), } merged = merge_backends(custom_backends, default_backends) - # The custom storage class should override - assert merged["nim_operator"].default_storage_class == "fast-ssd" - # But the PVC size should be preserved from default - assert merged["nim_operator"].default_pvc_size == "100Gi" + assert merged["deployments_plugin"].default_storage_class == "fast-ssd" + assert merged["deployments_plugin"].default_pvc_size == "100Gi" # ============================================================================ diff --git a/services/core/models/tests/unit/test_schemas_override_config.py b/services/core/models/tests/unit/test_schemas_override_config.py index a5658dcd2c..a46987fa0a 100644 --- a/services/core/models/tests/unit/test_schemas_override_config.py +++ b/services/core/models/tests/unit/test_schemas_override_config.py @@ -11,7 +11,7 @@ from datetime import datetime import pytest -from nmp.core.models.controllers.backends.k8s_nim_operator.types.nimservice import Spec +from nimservice_spec_types import Spec from nmp.core.models.schemas import ( ContainerExecutorConfig, CreateModelDeploymentConfigRequest, diff --git a/tests/auth_idp/static/test_authentik_kubernetes_demo.py b/tests/auth_idp/static/test_authentik_kubernetes_demo.py index b1d0502e30..20b90bff1f 100644 --- a/tests/auth_idp/static/test_authentik_kubernetes_demo.py +++ b/tests/auth_idp/static/test_authentik_kubernetes_demo.py @@ -796,8 +796,7 @@ def test_authentik_umbrella_values_configure_nemo_envoy_as_the_only_edge_proxy() assert values["integration"]["nemoPlatform"]["envoyServiceName"] == "nemo-platform-envoy" assert nemo_values["rbac"]["volcanoEnabled"] is False assert nemo_values["platformConfig"]["models"]["controller"]["backends"] == { - "none": {"enabled": True}, - "nim_operator": {"enabled": False}, + "deployments_plugin": {"enabled": True}, } diff --git a/uv.lock b/uv.lock index 84431b5eff..1de277123c 100644 --- a/uv.lock +++ b/uv.lock @@ -2289,6 +2289,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2", marker = "(python_full_version >= '3.12' and platform_machine == 'arm64' and sys_platform == 'darwin') or (python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] + [[package]] name = "httpx-aiohttp" version = "0.1.12" @@ -4732,9 +4737,7 @@ jobs-service = [ { name = "uvicorn", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] models-service = [ - { name = "docker", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "fastapi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "kubernetes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -5130,7 +5133,6 @@ requires-dist = [ { name = "docker", marker = "extra == 'all'", specifier = ">=7.1.0" }, { name = "docker", marker = "extra == 'core-service'", specifier = ">=7.1.0" }, { name = "docker", marker = "extra == 'jobs-service'", specifier = ">=7.1.0" }, - { name = "docker", marker = "extra == 'models-service'", specifier = ">=7.1.0" }, { name = "docker", marker = "extra == 'nemo-platform-sdk'", specifier = ">=7.0.0" }, { name = "docker", marker = "extra == 'services'", specifier = ">=7.1.0" }, { name = "duckdb", marker = "extra == 'all'", specifier = ">=1.1.3" }, @@ -5227,14 +5229,10 @@ requires-dist = [ { name = "jsonschema", marker = "extra == 'nemo-evaluator-sdk'", specifier = ">=4.23.0" }, { name = "jsonschema", marker = "extra == 'nemo-platform-plugin'", specifier = ">=4.0.0" }, { name = "kubernetes", marker = "extra == 'all'", specifier = ">=30.1.0" }, - { name = "kubernetes", marker = "extra == 'all'", specifier = ">=31.0.0" }, { name = "kubernetes", marker = "extra == 'core-service'", specifier = ">=30.1.0" }, - { name = "kubernetes", marker = "extra == 'core-service'", specifier = ">=31.0.0" }, { name = "kubernetes", marker = "extra == 'jobs-service'", specifier = ">=30.1.0" }, - { name = "kubernetes", marker = "extra == 'models-service'", specifier = ">=31.0.0" }, { name = "kubernetes", marker = "extra == 'nmp-common'", specifier = ">=30.1.0" }, { name = "kubernetes", marker = "extra == 'services'", specifier = ">=30.1.0" }, - { name = "kubernetes", marker = "extra == 'services'", specifier = ">=31.0.0" }, { name = "langchain", marker = "extra == 'all'", specifier = ">=1.3.14" }, { name = "langchain", marker = "extra == 'nemo-guardrails-plugin'", specifier = ">=1.3.14" }, { name = "langchain", marker = "extra == 'plugins'", specifier = ">=1.3.14" }, @@ -7306,9 +7304,7 @@ name = "nmp-models" version = "0.0.0" source = { editable = "services/core/models" } dependencies = [ - { name = "docker", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "fastapi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "kubernetes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -7331,14 +7327,11 @@ dev = [ { name = "pytest-xdist", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "ruff", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "ty", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "types-docker", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.metadata] requires-dist = [ - { name = "docker", specifier = ">=7.1.0" }, { name = "fastapi", specifier = ">=0.115.8" }, - { name = "kubernetes", specifier = ">=31.0.0" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, { name = "nmp-common", editable = "packages/nmp_common" }, { name = "pydantic", specifier = ">=2.10.6" }, @@ -7361,7 +7354,6 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "ruff", specifier = ">=0.11.8" }, { name = "ty", specifier = ">=0.0.1a14" }, - { name = "types-docker", specifier = ">=7.1.0" }, ] [[package]] @@ -11175,32 +11167,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/e5/47a573bbbd0a790f8f9fe452f7188ea72b212d21c9be57d5fc0cbc442075/types_awscrt-0.31.3-py3-none-any.whl", hash = "sha256:e5ce65a00a2ab4f35eacc1e3d700d792338d56e4823ee7b4dbe017f94cfc4458", size = 43340, upload-time = "2026-03-08T02:31:13.38Z" }, ] -[[package]] -name = "types-docker" -version = "7.1.0.20260328" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "types-paramiko", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "types-requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "urllib3", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4d/de/5f13b22eca604e58af908fbef60f4704a28bcc87ca1d9241d5b8bd433c85/types_docker-7.1.0.20260328.tar.gz", hash = "sha256:6e1a614685bc494580226891da1d01e0cfca6bd00b2761d8239ef09806b2154b", size = 32930, upload-time = "2026-03-28T04:08:02.78Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/64/cb764366d6d76495a0ffbda4bdedc5663d585215d0dd8ac808ed6484b48e/types_docker-7.1.0.20260328-py3-none-any.whl", hash = "sha256:3fb95d3ad63fae06d9d5cb18a0394252453b7c06d843bed567d634aa8ede3ebc", size = 47466, upload-time = "2026-03-28T04:08:01.519Z" }, -] - -[[package]] -name = "types-paramiko" -version = "4.0.0.20260322" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/cc/b83f1c085cc2c4d85f4ba2f799d1b18840b768d7b1a7dfb7d5cc5470fbc9/types_paramiko-4.0.0.20260322.tar.gz", hash = "sha256:dfcb13d8cf52499a198ced552b78fa685369a376b143abfb90cd49f465e383a0", size = 29040, upload-time = "2026-03-22T04:08:47.815Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d3/92/10415430e8035fe0155582757d9829784202bb198ed84fe9afa5e59d5263/types_paramiko-4.0.0.20260322-py3-none-any.whl", hash = "sha256:c585bcf81b5d2fc722279763d50eca8095777ee949af8706900e9f8411af979b", size = 38809, upload-time = "2026-03-22T04:08:46.622Z" }, -] - [[package]] name = "types-requests" version = "2.33.0.20260327"