feat(models): vllm k8s support

docs/set-up/config-reference.mdx

@@ -529,12 +529,16 @@ models:
         default_user_id:
         # Default group ID for NIM containers (security context)
         default_group_id:
+        # Default user ID for vLLM puller + server pods (security context). Defaults to 2000 to match the upstream vLLM image's 'vllm' user, which has an /etc/passwd entry (avoids torch getpwuid crashes from an unknown uid). | default: 2000
+        default_vllm_user_id: 2000
+        # Default group ID / fsGroup for vLLM puller + server pods. Defaults to 0 (root group) to match the upstream vLLM image and keep weights readable across the puller and server pods. | default: 0
+        default_vllm_group_id: 0
         # Kubernetes secret name for Files service authentication (HF_TOKEN) | default: 'nemo-models-files-token'
         files_auth_secret: nemo-models-files-token
         # The name of the image pull secret for the modelPuller image | default: 'nvcrimagepullsecret'
         huggingface_model_puller_image_pull_secret: nvcrimagepullsecret
-        # BusyBox image repository used by plugin init containers. | default: 'busybox'
-        busybox_image: busybox
+        # BusyBox image repository used by plugin init containers. Fully qualified (docker.io/library/...) so it resolves on container runtimes that enforce fully-qualified image names (short names like 'busybox' fail there). | default: 'docker.io/library/busybox'
+        busybox_image: docker.io/library/busybox
         # BusyBox image tag used by plugin init containers. | default: 'latest'
         busybox_image_tag: latest
         # NGC API key secret name for pulling NIM images | default: 'ngc-api'
@@ -543,10 +547,18 @@ models:
         default_nimservice_image: nvcr.io/nim/nvidia/llm-nim
         # Default NIMService image tag (used if not specified in deployment config) | default: '1.13.1'
         default_nimservice_image_tag: 1.13.1
+        # Default vLLM server image repository (used if not specified in deployment config) | default: 'vllm/vllm-openai'
+        default_vllm_image: vllm/vllm-openai
+        # Default vLLM server image tag (used if not specified in deployment config) | default: 'v0.22.1'
+        default_vllm_image_tag: v0.22.1
         # Default guided decoding backend for NIM (e.g., 'outlines', 'auto', 'lm-format-enforcer') | default: 'outlines'
         nim_guided_decoding_backend: outlines
         # Kubernetes namespace for NIM deployments (defaults to controller's namespace if not set)
         namespace:
+        # ServiceAccount name for directly-emitted vLLM Deployment pods and the weight-puller Job. If not set, the namespace default ServiceAccount is used.
+        service_account_name:
+        # Shared memory (/dev/shm) size limit for vLLM Deployment pods (e.g. '8Gi'). If not set, the emptyDir uses the node default size.
+        default_shared_memory_size_limit:
         # Default Kubernetes resource requirements for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: {'requests': {'cpu': '2', 'memory': '8Gi'}, 'limits': {'memory': '16Gi'}}
         default_resources:
         # Default Kubernetes tolerations for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: [{'key': 'nvidia.com/gpu', 'operator': 'Exists', 'effect': 'NoSchedule'}]

services/core/inference-gateway/tests/integration/conftest.py

@@ -87,29 +87,19 @@ def init(self) -> None:
         """No-op init for mock backend."""
         pass
 
-    async def create_model_deployment(
-        self,
-        deployment: Any,
-        config: Any,
-        model_entity: Any = None,
-    ) -> DeploymentStatusUpdate:
+    async def create_model_deployment(self, ctx: Any) -> DeploymentStatusUpdate:
         """Record call and return configured response."""
-        self.create_calls.append((deployment, config, model_entity))
+        self.create_calls.append((ctx.model_deployment, ctx.model_deployment_config, ctx.model_entity))
         return self.create_response
 
-    async def update_model_deployment(
-        self,
-        deployment: Any,
-        config: Any,
-        model_entity: Any = None,
-    ) -> DeploymentStatusUpdate:
+    async def update_model_deployment(self, ctx: Any) -> DeploymentStatusUpdate:
         """Record call and return configured response."""
-        self.update_calls.append((deployment, config, model_entity))
+        self.update_calls.append((ctx.model_deployment, ctx.model_deployment_config, ctx.model_entity))
         return self.create_response
 
-    async def get_model_deployment_status(self, deployment: Any) -> DeploymentStatusUpdate:
+    async def get_model_deployment_status(self, ctx: Any) -> DeploymentStatusUpdate:
         """Record call and return configured response."""
-        self.status_calls.append(deployment)
+        self.status_calls.append(ctx.model_deployment)
         return self.status_response
 
     async def delete_model_deployment(self, deployment: Any) -> DeploymentStatusUpdate:

services/core/models/src/nmp/core/models/controllers/backends/backends.py

@@ -4,13 +4,11 @@
 """Base backend interface for Models Controller service."""
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Optional
+from typing import Any, Dict
 
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform.types.inference import ModelDeploymentStatus
-from nemo_platform.types.inference.model_deployment import ModelDeployment
-from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig
-from nemo_platform.types.models.model_entity import ModelEntity
+from nmp.core.models.controllers.context import ModelContext
 from pydantic import BaseModel
 
 
@@ -67,15 +65,12 @@ def shutdown(self) -> None:
         ...
 
     @abstractmethod
-    async def create_model_deployment(
-        self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None
-    ) -> DeploymentStatusUpdate:
+    async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate:
         """Create a new model deployment.
 
         Args:
-            deployment: The ModelDeployment object to create
-            config: The ModelDeploymentConfig for this deployment
-            model_entity: Optional Model entity from Entity Store (contains peft, artifact, etc.)
+            ctx: The reconciliation context bundling the ModelDeployment, its
+                ModelDeploymentConfig, and the optional Model entity.
 
         Returns:
             DeploymentStatusUpdate with the current status after creation attempt
@@ -86,15 +81,13 @@ async def create_model_deployment(
         ...
 
     @abstractmethod
-    async def update_model_deployment(
-        self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None
-    ) -> DeploymentStatusUpdate:
+    async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate:
         """Update an existing model deployment.
 
         Args:
-            deployment: The ModelDeployment object with updated configuration
-            config: The ModelDeploymentConfig for this deployment (may be a new version)
-            model_entity: Optional Model entity from Entity Store (contains peft, artifact, etc.)
+            ctx: The reconciliation context bundling the ModelDeployment, its
+                (possibly new-version) ModelDeploymentConfig, and the optional
+                Model entity.
 
         Returns:
             DeploymentStatusUpdate with the current status after update attempt
@@ -105,11 +98,14 @@ async def update_model_deployment(
         ...
 
     @abstractmethod
-    async def get_model_deployment_status(self, deployment: ModelDeployment) -> DeploymentStatusUpdate:
+    async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate:
         """Get the current status of a model deployment.
 
         Args:
-            deployment: The ModelDeployment object to check
+            ctx: The reconciliation context bundling the ModelDeployment, its
+                ModelDeploymentConfig, and the optional Model entity. Some backends
+                need the config to advance creation (e.g. the k8s vLLM path emits
+                the serving Deployment once the weight-puller Job completes).
 
         Returns:
             DeploymentStatusUpdate with the current deployment status

services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py

@@ -12,14 +12,12 @@
 import asyncio
 import os
 from logging import getLogger
-from typing import Any, Optional
+from typing import Any
 
 import httpx
 from docker.errors import APIError, NotFound
 from nemo_platform import NotFoundError
 from nemo_platform.types.inference.model_deployment import ModelDeployment
-from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig
-from nemo_platform.types.models.model_entity import ModelEntity
 from nmp.common.config import get_platform_config
 from nmp.common.docker.gpu_pool import DockerGPUPool
 from nmp.common.resources import SharedResourceManager
@@ -37,6 +35,7 @@
     NGC_IMAGE_REGISTRY_USER_NAME,
     DockerDeploymentCreationReconciler,
 )
+from nmp.core.models.controllers.context import ModelContext
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import ReadTimeout
 from urllib3.exceptions import ReadTimeoutError as Urllib3ReadTimeoutError
@@ -184,17 +183,15 @@ async def _ensure_ngc_login(self, ngc_api_key: str | None) -> None:
     # ServiceBackend CRUD interface
     # ==================================================================
 
-    async def create_model_deployment(
-        self,
-        deployment: ModelDeployment,
-        config: ModelDeploymentConfig,
-        model_entity: Optional[ModelEntity] = None,
-    ) -> DeploymentStatusUpdate:
+    async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate:
         """Create a new model deployment as a Docker container.
 
         Resolves NGC credentials and delegates the multi-stage creation
         pipeline to :class:`DockerDeploymentCreationReconciler`.
         """
+        deployment = ctx.model_deployment
+        config = ctx.model_deployment_config
+        model_entity = ctx.model_entity
         resolved_ngc_key = await self._resolve_ngc_api_key()
         await self._ensure_ngc_login(resolved_ngc_key)
 
@@ -205,25 +202,22 @@ async def create_model_deployment(
             resolved_ngc_key,
         )
 
-    async def update_model_deployment(
-        self,
-        deployment: ModelDeployment,
-        config: ModelDeploymentConfig,
-        model_entity: Optional[ModelEntity] = None,
-    ) -> DeploymentStatusUpdate:
+    async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate:
         """Update a model deployment by recreating the container."""
+        deployment = ctx.model_deployment
         logger.info(f"Updating Docker deployment: {deployment.workspace}/{deployment.name}")
         delete_result = await self.delete_model_deployment(deployment.workspace, deployment.name)
         if delete_result.status == "ERROR":
             return delete_result
-        return await self.create_model_deployment(deployment, config, model_entity)
+        return await self.create_model_deployment(ctx)
 
-    async def get_model_deployment_status(self, deployment: ModelDeployment) -> DeploymentStatusUpdate:
+    async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate:
         """Get the status of a Docker model deployment.
 
         While the deployment is still progressing through the creation
         pipeline this delegates to the reconciler's ``advance`` method.
         """
+        deployment = ctx.model_deployment
         if self._reconciler.is_deploying(deployment.workspace, deployment.name):
             deployment_key = self._reconciler.get_deployment_key(deployment.workspace, deployment.name)
             return await self._reconciler.advance(deployment_key)

services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py

@@ -35,14 +35,27 @@
 from nmp.core.models.app import ModelWeightsType, get_model_weights_type, is_multi_llm_image, parse_model_name_revision
 from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER
 from nmp.core.models.app.utils import _get_k8s_safe_name
+from nmp.core.models.controllers.backends import vllm_compiler
 from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate
-from nmp.core.models.controllers.backends.common import DeploymentConfigView, deployment_config_view
-from nmp.core.models.controllers.backends.docker import vllm_compiler
+from nmp.core.models.controllers.backends.common import deployment_config_view
 from nmp.core.models.controllers.backends.docker.config import (
     MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE,
     MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU,
     DockerBackendConfig,
 )
+from nmp.core.models.controllers.backends.engine import (
+    ENGINE_HEALTH_PATHS,
+    ENGINE_LABEL,
+    ENGINE_NIM,
+    ENGINE_VLLM,
+    HEALTH_PATH_LABEL,
+)
+from nmp.core.models.controllers.backends.engine import (
+    config_engine as _config_engine,
+)
+from nmp.core.models.controllers.backends.engine import (
+    resolve_health_path as _resolve_health_path,
+)
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import ReadTimeout
 from tenacity import before_sleep_log, retry, stop_after_attempt, wait_exponential
@@ -65,44 +78,6 @@
 NGC_IMAGE_REGISTRY = os.getenv("NGC_IMAGE_REGISTRY", "nvcr.io")
 NGC_IMAGE_REGISTRY_USER_NAME = os.getenv("NGC_IMAGE_REGISTRY_USER_NAME", "$oauthtoken")
 
-ENGINE_NIM = "nim"
-ENGINE_VLLM = "vllm"
-ENGINE_GENERIC = "generic"
-
-# Docker label recording the engine, read back at status time to pick the health probe.
-ENGINE_LABEL = "nmp.nvidia.com/engine"
-
-# Docker label recording the resolved readiness-probe path, read back at status
-# time. Stamped at create so status polling doesn't need the deployment config.
-HEALTH_PATH_LABEL = "nmp.nvidia.com/health-path"
-
-# Per-engine readiness probe paths (relative to the container host URL).
-ENGINE_HEALTH_PATHS: dict[str, str] = {
-    ENGINE_NIM: "/v1/health/ready",
-    ENGINE_VLLM: "/health",
-}
-
-
-def _config_engine(config: Any) -> str:
-    """Return the engine discriminant as a lowercase string (defaults to nim)."""
-    engine = getattr(config, "engine", None)
-    if engine is None:
-        return ENGINE_NIM
-    # engine may be an enum or a plain string depending on the SDK model.
-    return str(getattr(engine, "value", engine)).lower()
-
-
-def _resolve_health_path(engine: str, view: DeploymentConfigView) -> str:
-    """Resolve the readiness-probe path for a deployment.
-
-    Precedence: an explicit ``executor_config.health_check_path`` wins; otherwise
-    fall back to the engine's standard endpoint. ``generic`` containers have no
-    engine default, so they fall back to the NIM path unless they set their own.
-    """
-    if getattr(view, "health_check_path", None):
-        return view.health_check_path
-    return ENGINE_HEALTH_PATHS.get(engine, ENGINE_HEALTH_PATHS[ENGINE_NIM])
-
 
 def _should_retry_docker_error(exception: BaseException) -> bool:
     """Determine if a Docker exception should be retried."""

services/core/models/src/nmp/core/models/controllers/backends/engine.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Backend-agnostic engine dispatch + readiness-probe helpers.
+
+The ``engine`` discriminant on a ``ModelDeploymentConfig`` selects the compiler
+path (nim / vllm / generic). These constants and helpers are shared by every
+service backend (docker container labels, k8s object labels) so engine selection
+and readiness-probe resolution behave identically regardless of where the
+deployment runs.
+"""
+
+from typing import Any
+
+from nmp.core.models.controllers.backends.common import DeploymentConfigView
+
+ENGINE_NIM = "nim"
+ENGINE_VLLM = "vllm"
+ENGINE_GENERIC = "generic"
+
+# Label recording the engine, read back at status time to pick the health probe.
+# Used as a docker container label and a k8s object/pod label.
+ENGINE_LABEL = "nmp.nvidia.com/engine"
+
+# Label recording the resolved readiness-probe path, read back at status time.
+# Stamped at create so status polling doesn't need the deployment config.
+HEALTH_PATH_LABEL = "nmp.nvidia.com/health-path"
+
+# Per-engine readiness probe paths (relative to the container/pod host URL).
+ENGINE_HEALTH_PATHS: dict[str, str] = {
+    ENGINE_NIM: "/v1/health/ready",
+    ENGINE_VLLM: "/health",
+}
+
+
+def config_engine(config: Any) -> str:
+    """Return the engine discriminant as a lowercase string (defaults to nim)."""
+    engine = getattr(config, "engine", None)
+    if engine is None:
+        return ENGINE_NIM
+    # engine may be an enum or a plain string depending on the SDK model.
+    return str(getattr(engine, "value", engine)).lower()
+
+
+def resolve_health_path(engine: str, view: DeploymentConfigView) -> str:
+    """Resolve the readiness-probe path for a deployment.
+
+    Precedence: an explicit ``executor_config.health_check_path`` wins; otherwise
+    fall back to the engine's standard endpoint. ``generic`` containers have no
+    engine default, so they fall back to the NIM path unless they set their own.
+    """
+    explicit_path = getattr(view, "health_check_path", None)
+    if explicit_path:
+        return explicit_path
+    return ENGINE_HEALTH_PATHS.get(engine, ENGINE_HEALTH_PATHS[ENGINE_NIM])