feat(models): deployments_plugin cutover, drop legacy backends and k8s-nim-operator (AIRCORE-759)

.github/actions/setup-kind-cluster/action.yaml

@@ -144,10 +144,14 @@ runs:
         KIND_IMAGE_PULL_USER: ${{ inputs['kind-image-pull-user'] }}
         NMP_E2E_REGISTRY: ${{ inputs['image-registry'] }}
         NMP_E2E_TAG: ${{ inputs['image-tag'] }}
+        # CPU generic HTTP smoke image; tag must match e2e/test_models_deployments_plugin.py.
+        GENERIC_HTTP_IMAGE: docker.io/library/python
+        GENERIC_HTTP_TAG: 3.12-alpine
       run: |
         e2e/k8s/scripts/prepull_kind_images.sh \
           "${NMP_E2E_REGISTRY}/nmp-api:${NMP_E2E_TAG}" \
-          "${NMP_E2E_REGISTRY}/nmp-cpu-tasks:${NMP_E2E_TAG}"
+          "${NMP_E2E_REGISTRY}/nmp-cpu-tasks:${NMP_E2E_TAG}" \
+          "${GENERIC_HTTP_IMAGE}:${GENERIC_HTTP_TAG}"
 
     - name: Install NeMo Platform
       if: ${{ inputs['install-nemo-platform'] == 'true' }}

.github/workflows/ci.yaml

@@ -447,6 +447,7 @@ jobs:
           export NMP_BASE_URL="${NMP_E2E_CLUSTER_URL}"
           uv run --frozen pytest \
             e2e/test_jobs.py::test_job_using_secret_environment_variable \
+            e2e/test_models_deployments_plugin.py::test_generic_model_deployment_lifecycle \
             -v \
             --run-e2e \
             --no-cov \

contrib/auth/authentik/helm/values.yaml

@@ -255,10 +255,8 @@ nemo-platform:
     models:
       controller:
         backends:
-          none:
+          deployments_plugin:
             enabled: true
-          nim_operator:
-            enabled: false
 
 integration:
   nemoPlatform:

docs/set-up/config-reference.mdx

@@ -504,149 +504,6 @@ models:
     interval_seconds: 5
     # Dict of custom backend configurations for the Models Controller
     backends:
-      docker:
-        # Default NIM image when none is specified (multi-LLM image) | default: 'nvcr.io/nim/nvidia/llm-nim'
-        default_nimservice_image: nvcr.io/nim/nvidia/llm-nim
-        # Default NIM image tag when none is specified | default: '1.13.1'
-        default_nimservice_image_tag: 1.13.1
-        # Default vLLM image when none is specified for engine='vllm' | default: 'vllm/vllm-openai'
-        default_vllm_image: vllm/vllm-openai
-        # Default vLLM image tag when none is specified for engine='vllm' | default: 'v0.22.1'
-        default_vllm_image_tag: v0.22.1
-        # NIM guided decoding backend | default: 'outlines'
-        nim_guided_decoding_backend: outlines
-        # PEFT/LoRA source URL for models service | default: ''
-        peft_source: ''
-        # PEFT/LoRA refresh interval in seconds | default: 30
-        peft_refresh_interval: 30
-        # Secret name for Files service authentication | default: 'files-hf-token'
-        files_auth_secret: files-hf-token
-        # Docker client timeout in seconds for long-running operations (default: 10 minutes) | default: 600
-        docker_timeout: 600
-        # Networking mode for NIM containers: 'local' (port forwarding to localhost for local dev), 'dond' (container names on shared network for quickstart), 'dind' (port forwarding to docker service for DinD setups). | default: 'local' | values: 'local' | 'dond' | 'dind'
-        models_docker_networking_mode: local
-        # Docker network name for 'dond' mode. NIMs will join this network to communicate with the NeMo Platform container. Required when MODELS_DOCKER_NETWORKING_MODE='dond'. Quickstart sets this automatically via MODELS_DOCKER_NETWORK env var. | default: ''
-        models_docker_network: ''
-        # Container name for 'dond' mode. Used to replace localhost in URLs passed to NIMs so they can reach services via the Docker network. Quickstart sets this automatically via MODELS_DOCKER_CONTAINER_NAME env var. | default: ''
-        models_docker_container_name: ''
-        # Hostname for port forwarding in 'dind' mode. Typically 'localhost' or the DinD service name. Used to construct URLs like http://{hostname}:{port}. | default: 'localhost'
-        models_docker_host_service_name: localhost
-        # Start of port range for port forwarding (inclusive). Defaults to start of IANA dynamic/ephemeral port range. | default: 49152
-        models_docker_port_range_start: 49152
-        # End of port range for port forwarding (inclusive). Defaults to 500-port range within IANA dynamic/ephemeral range. | default: 49652
-        models_docker_port_range_end: 49652
-        # Image used to pull model weights. Its entrypoint is overridden to the Hugging Face CLI ('hf download ...'), so any image with the 'hf' CLI on PATH works. Defaults to the platform's nmp-api image (registry/tag from platform config); override to use a different puller image.
-        huggingface_model_puller: my-registry/nmp-api:local
-        # Extra environment variables for the model puller container. Values override the reconciler defaults (HF_ENDPOINT, HF_TOKEN). E.g. set HF_HUB_ENABLE_HF_TRANSFER='0' to disable the hf_transfer download path.
-        huggingface_model_puller_env: {}
-        # Timeout in seconds for the model puller container to complete (default: 30 minutes) | default: 1800
-        model_puller_timeout: 1800
-        # Per-file HTTP download timeout in seconds for the model puller (HF_HUB_DOWNLOAD_TIMEOUT). Increase if large model files fail with IncompleteRead/ChunkedEncodingError (default: 2 hours). | default: 7200
-        model_puller_download_timeout: 7200
-        # Max concurrent file downloads in the model puller (hf download --max-workers). Default 1 (sequential) reduces IncompleteRead/ChunkedEncodingError on slow or flaky links; increase for speed. | default: 1
-        model_puller_max_workers: 1
-        # Number of retries when the puller fails with a transient error (IncompleteRead, ChunkedEncodingError, connection broken). Same volume is reused so partial downloads can be completed. | default: 3
-        model_puller_retries: 3
-        # Additional labels copied onto model resources managed by the models controller.
-        model_labels: {}
-        # BusyBox image repository used for helper containers (permissions/find/chown). | default: 'busybox'
-        busybox_image: busybox
-        # BusyBox image tag used for helper containers. | default: 'latest'
-        busybox_image_tag: latest
-        # Image name (without registry/tag) used for the LoRA adapters sidecar container. Registry and tag are taken from NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG. The sidecar is invoked via the lora_sidecar_command. | default: 'nmp-api'
-        lora_sidecar_image_name: nmp-api
-        # Command passed to the LoRA sidecar container. Default uses the nmp-platform-runner entrypoint present in nmp-api.
-        lora_sidecar_command:
-        - --sidecars
-        - adapters
-        - --port
-        - '60830'
-        # Optional entrypoint override for the LoRA sidecar container. Leave empty to use the image's default entrypoint (correct for nmp-api). | default: ''
-        lora_sidecar_entrypoint: ''
-        # Maximum time (in seconds) a deployment may stay in PENDING before being transitioned to ERROR. Default: 7200 (2 hours). | default: 7200
-        pending_timeout_seconds: 7200
-        # Maximum number of container restarts before a PENDING deployment is transitioned to ERROR (crash loop detection). Default: 5. | default: 5
-        max_restart_count: 5
-        # Optional fixed shared memory size (/dev/shm) for multi-GPU NIM containers. If set, overrides the per-GPU calculation. Leave empty to use shm_size_per_gpu x GPU count. Override via MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE env. Format: e.g. '2g', '4g'. | default: ''
-        nim_multi_gpu_shm_size: ''
-        # Shared memory size (/dev/shm) per GPU in megabytes for multi-GPU NIM containers. Total shm = this value x GPU count (e.g. 1024 x 2 GPUs = 2048m). Override via MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU env (integer string). | default: 1024
-        nim_multi_gpu_shm_size_per_gpu: 1024
-        # Whether this backend is enabled | default: False
-        enabled: false
-      nim_operator:
-        # Default storage class for PVCs. If not set, the cluster's default StorageClass is used.
-        default_storage_class:
-        # Default PVC size for model storage (used if not specified in deployment config) | default: '200Gi'
-        default_pvc_size: 200Gi
-        # LoRA/PEFT source endpoint (only used when lora_enabled is true) | default: 'http://nemo-entity-store:8000'
-        peft_source: http://nemo-entity-store:8000
-        # PEFT refresh interval in seconds (only used when lora_enabled is true) | default: 30
-        peft_refresh_interval: 30
-        # Image name (without registry/tag) for the LoRA adapters sidecar container. Registry and tag are taken from the platform config (NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG). Override to 'nmp-customizer-tasks' for local dev when that image is already available but nmp-api is not. | default: 'nmp-api'
-        lora_sidecar_image_name: nmp-api
-        # Kubernetes container command (entrypoint) for the LoRA sidecar. Default uses the nmp-platform-runner entrypoint present in nmp-api. When using nmp-customizer-tasks set to ['python'] and set lora_sidecar_args to ['-m', 'nmp.core.models.sidecars.adapters.main'].
-        lora_sidecar_command:
-        - nemo
-        - services
-        - run
-        - --sidecars
-        - adapters
-        # Kubernetes container args for the LoRA sidecar (appended after lora_sidecar_command). Leave empty for nmp-api. Set to ['-m', 'nmp.core.models.sidecars.adapters.main'] when using nmp-customizer-tasks. | default: []
-        lora_sidecar_args: []
-        # Default user ID for NIM containers (security context)
-        default_user_id:
-        # Default group ID for NIM containers (security context)
-        default_group_id:
-        # Default user ID for vLLM puller + server pods (security context). Defaults to 2000 to match the upstream vLLM image's 'vllm' user, which has an /etc/passwd entry (avoids torch getpwuid crashes from an unknown uid). | default: 2000
-        default_vllm_user_id: 2000
-        # Default group ID / fsGroup for vLLM puller + server pods. Defaults to 0 (root group) to match the upstream vLLM image and keep weights readable across the puller and server pods. | default: 0
-        default_vllm_group_id: 0
-        # Kubernetes secret name for Files service authentication (HF_TOKEN) | default: 'nemo-models-files-token'
-        files_auth_secret: nemo-models-files-token
-        # The name of the image pull secret for the modelPuller image | default: 'nvcrimagepullsecret'
-        huggingface_model_puller_image_pull_secret: nvcrimagepullsecret
-        # BusyBox image repository used by plugin init containers. Fully qualified (docker.io/library/...) so it resolves on container runtimes that enforce fully-qualified image names (short names like 'busybox' fail there). | default: 'docker.io/library/busybox'
-        busybox_image: docker.io/library/busybox
-        # BusyBox image tag used by plugin init containers. | default: 'latest'
-        busybox_image_tag: latest
-        # NGC API key secret name for pulling NIM images | default: 'ngc-api'
-        auth_secret: ngc-api
-        # Default NIMService image repository (used if not specified in deployment config) | default: 'nvcr.io/nim/nvidia/llm-nim'
-        default_nimservice_image: nvcr.io/nim/nvidia/llm-nim
-        # Default NIMService image tag (used if not specified in deployment config) | default: '1.13.1'
-        default_nimservice_image_tag: 1.13.1
-        # Default vLLM server image repository (used if not specified in deployment config) | default: 'vllm/vllm-openai'
-        default_vllm_image: vllm/vllm-openai
-        # Default vLLM server image tag (used if not specified in deployment config) | default: 'v0.22.1'
-        default_vllm_image_tag: v0.22.1
-        # Default guided decoding backend for NIM (e.g., 'outlines', 'auto', 'lm-format-enforcer') | default: 'outlines'
-        nim_guided_decoding_backend: outlines
-        # Kubernetes namespace for NIM deployments (defaults to controller's namespace if not set)
-        namespace:
-        # ServiceAccount name for directly-emitted vLLM Deployment pods and the weight-puller Job. If not set, the namespace default ServiceAccount is used.
-        service_account_name:
-        # Shared memory (/dev/shm) size limit for vLLM Deployment pods (e.g. '8Gi'). If not set, the emptyDir uses the node default size.
-        default_shared_memory_size_limit:
-        # Default Kubernetes resource requirements for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: {'requests': {'cpu': '2', 'memory': '8Gi'}, 'limits': {'memory': '16Gi'}}
-        default_resources:
-        # Default Kubernetes tolerations for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: [{'key': 'nvidia.com/gpu', 'operator': 'Exists', 'effect': 'NoSchedule'}]
-        default_tolerations:
-        # Default Kubernetes node selector for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: {'node-type': 'gpu-node', 'zone': 'us-west1-a'}
-        default_node_selector:
-        # Additional labels copied onto model resources managed by the models controller.
-        model_labels:
-        # Default Kubernetes labels applied to NIMService and NIMCache resources and their child resources (e.g. pods). Merged with controller-managed labels; controller labels take precedence on conflict. Example: {'team': 'ml-platform', 'environment': 'prod'}
-        default_labels:
-        # Default Kubernetes annotations applied to NIMService and NIMCache resources and their child resources (e.g. pods, PVCs). Merged with controller-managed annotations; controller annotations take precedence on conflict. Example: {'prometheus.io/scrape': 'true'}
-        default_annotations:
-        # Default grace period in seconds for NIM startup. Can be overridden per-deployment via k8s_nim_operator_config. Determines how long Kubernetes will wait for the NIM to become ready before restarting it. If not set, defaults to 600 seconds (10 minutes). Example: 600 (10 minutes)
-        default_startup_probe_grace_period_seconds:
-        # Maximum time in seconds a deployment may stay in PENDING before being transitioned to ERROR. Default: 7200 (2 hours). | default: 7200
-        pending_timeout_seconds: 7200
-        # Maximum number of pod container restarts before a PENDING deployment is transitioned to ERROR (crash loop detection). Default: 5. | default: 5
-        max_restart_count: 5
-        # Whether this backend is enabled | default: False
-        enabled: false
       deployments_plugin:
         default_executor:
         docker_executor:
@@ -662,10 +519,10 @@ models:
         default_nimservice_image: nvcr.io/nim/meta/llama-3.1-8b-instruct
         # default: '1.8.5'
         default_nimservice_image_tag: 1.8.5
-        # default: 'vllm/vllm-openai'
-        default_vllm_image: vllm/vllm-openai
-        # default: 'v0.8.5'
-        default_vllm_image_tag: v0.8.5
+        # Default vLLM image repository. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/vllm/vllm-openai'
+        default_vllm_image: docker.io/vllm/vllm-openai
+        # default: 'v0.22.1'
+        default_vllm_image_tag: v0.22.1
         # default: 1000
         default_user_id: 1000
         # default: 2000
@@ -689,15 +546,10 @@ models:
         busybox_image: docker.io/library/busybox
         # BusyBox image tag for LoRA cache init containers. | default: 'latest'
         busybox_image_tag: latest
-        # default: 5.0
-        delete_wait_seconds: 5.0
-        # default: 0.5
-        delete_poll_seconds: 0.5
+        # Maximum seconds a deployment may stay DELETING before ERROR. 0 disables timeout escalation. | default: 60
+        deleting_timeout_seconds: 60
         # default: False
         enabled: false
-      none:
-        # Whether this backend is enabled | default: False
-        enabled: false
     # Time-to-live in seconds for DELETED deployments before they are permanently removed from the database | default: 30
     model_deployment_garbage_collection_ttl_seconds: 30
     # Time-to-live in seconds for ERROR deployments before backend resources are garbage collected | default: 10800

docs/set-up/helm/persistent-volumes.mdx

@@ -31,11 +31,11 @@ As an alternative to PVC-based file storage, you can configure the Files service
 
  ```yaml
  core:
- storage:
- storageClass: "nfs"
- accessModes:
- - ReadWriteMany
- size: 200Gi
+   storage:
+     storageClass: "nfs"
+     accessModes:
+       - ReadWriteMany
+     size: 200Gi
  ```
 
  If `storageClass` is empty (default), the cluster’s default StorageClass is used.
@@ -46,8 +46,8 @@ To use a pre-created PersistentVolume instead of having the chart create a PVC,
 
 ```yaml
 core:
- storage:
- existingPersistentVolumeName: "my-existing-pv-name"
+  storage:
+    existingPersistentVolumeName: "my-existing-pv-name"
 ```
 
 When set, the chart does not create a new PVC; pods mount the named volume.
@@ -58,12 +58,11 @@ For NIM deployments launched via the NeMo Platform, you can set the default Stor
 
 ```yaml
 platformConfig:
- models:
- controller:
- backends:
- nim_operator:
- config:
- default_storage_class: "nfs"
+  models:
+    controller:
+      backends:
+        deployments_plugin:
+          default_storage_class: "nfs"
 ```
 
 Replace `"nfs"` with your StorageClass name (e.g. `oci-nfs`, `gp3`). For NIM scaling and multi-node deployments, use a ReadWriteMany-capable StorageClass.

e2e/k8s/scripts/prepull_kind_images.sh

@@ -37,6 +37,11 @@ pull_image_on_node() {
             return 1
         fi
         auth_args=(--user "\$oauthtoken:${NGC_API_KEY}")
+    elif [[ "${image}" == docker.io/* || "${image}" == registry-1.docker.io/* ]]; then
+        # Public Docker Hub images are pulled anonymously. Passing the GHCR
+        # KIND_IMAGE_PULL_* credentials here makes auth.docker.io reject the
+        # request with a 401, so leave auth_args empty.
+        auth_args=()
     elif [ -n "${KIND_IMAGE_PULL_USER:-}" ] && [ -n "${KIND_IMAGE_PULL_TOKEN:-}" ]; then
         auth_args=(--user "${KIND_IMAGE_PULL_USER}:${KIND_IMAGE_PULL_TOKEN}")
     fi

e2e/k8s/values/default.yaml

@@ -1,9 +1,5 @@
 # Default E2E values for running tests in a Kubernetes cluster, for ephemeral testing
 
-# Always disable nim operator for E2E tests
-k8s-nim-operator:
-  enabled: false
-
 postgresql:
   image:
     repository: docker.io/library/postgres

e2e/k8s/values/kind.yaml

@@ -1,10 +1,6 @@
 # kind values for CPU-only GitHub Actions Kubernetes E2E runs.
 
-k8s-nim-operator:
-  enabled: false
-
 rbac:
-  k8sNimOperatorEnabled: false
   volcanoEnabled: false
 
 postgresql:
@@ -31,8 +27,10 @@ platformConfig:
   models:
     controller:
       backends:
-        nim_operator:
-          enabled: false
+        deployments_plugin:
+          enabled: true
+          k8s_executor: kind-k8s
+          default_executor: kind-k8s
   # Wire a nemo-deployments k8s executor so deployment_mode=k8s agent
   # deployments (test_nemo_agents_k8s.py) run as real Deployment+Service
   # workloads. default_namespace is omitted so the executor uses its own pod

e2e/k8s/values/minikube-auth-portforward.yaml

@@ -8,9 +8,6 @@
 # is blocked and the ingress addon cannot bootstrap. The chart still keeps split
 # pods on the API service URL while the API pod itself loops back to localhost.
 
-k8s-nim-operator:
-  enabled: false
-
 postgresql:
   persistence:
     storageClass: standard
@@ -38,5 +35,14 @@ platformConfig:
   models:
     controller:
       backends:
-        nim_operator:
-          enabled: false
+        deployments_plugin:
+          enabled: true
+          k8s_executor: local-k8s
+          default_executor: local-k8s
+  deployments:
+    executors:
+      - name: local-k8s
+        backend: k8s
+        config:
+          default_namespace: default
+    default_executor: local-k8s

e2e/k8s/values/minikube-auth.yaml

@@ -10,9 +10,6 @@
 #   while the API pod itself loops back to localhost
 # - keeps ingress enabled for browser/curl access through minikube ingress
 
-k8s-nim-operator:
-  enabled: false
-
 postgresql:
   persistence:
     storageClass: standard
@@ -44,5 +41,14 @@ platformConfig:
   models:
     controller:
       backends:
-        nim_operator:
-          enabled: false
+        deployments_plugin:
+          enabled: true
+          k8s_executor: local-k8s
+          default_executor: local-k8s
+  deployments:
+    executors:
+      - name: local-k8s
+        backend: k8s
+        config:
+          default_namespace: default
+    default_executor: local-k8s