diff --git a/.github/workflows/post-merge-ci.yml b/.github/workflows/post-merge-ci.yml index 7e48980a5504..6c63f4bb1390 100644 --- a/.github/workflows/post-merge-ci.yml +++ b/.github/workflows/post-merge-ci.yml @@ -738,7 +738,7 @@ jobs: with: vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} vcluster_namespace: ${{ needs.deploy-operator.outputs.namespace }} - - name: Install GAIE prerequisites (Gateway API, kgateway, Inference Extension CRDs) + - name: Install GAIE prerequisites (Gateway API, agentgateway, Inference Extension CRDs) shell: bash env: KUBECONFIG_B64: ${{ steps.connect-vcluster.outputs.kubeconfig_base64 }} @@ -746,7 +746,7 @@ jobs: echo "${KUBECONFIG_B64}" | base64 -d > ${{ github.workspace }}/.kubeconfig_gaie export KUBECONFIG=${{ github.workspace }}/.kubeconfig_gaie export NAMESPACE=default - bash deploy/inference-gateway/scripts/install_gaie_crd_kgateway.sh + bash deploy/inference-gateway/scripts/install_gaie_crd_agentgateway.sh rm -f ${{ github.workspace }}/.kubeconfig_gaie - name: Run GAIE Deploy Test id: deploy-test diff --git a/deploy/inference-gateway/scripts/install_gaie_crd_kgateway.sh b/deploy/inference-gateway/scripts/install_gaie_crd_kgateway.sh deleted file mode 100755 index c8d2e5ae7d8c..000000000000 --- a/deploy/inference-gateway/scripts/install_gaie_crd_kgateway.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) - -echo "install_gaie_crd_kgateway.sh is deprecated; installing agentgateway instead." >&2 -exec "${SCRIPT_DIR}/install_gaie_crd_agentgateway.sh" "$@" diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/.helmignore b/deploy/inference-gateway/standalone/helm/dynamo-gaie/.helmignore deleted file mode 100644 index 0e8a0eb36f4c..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/Chart.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/Chart.yaml deleted file mode 100644 index b632dcd2c66e..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/Chart.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: dynamo-gaie -description: A Helm chart for installing Inference Gateway Inference Extension with Dynamo - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/epp-config-dynamo.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/epp-config-dynamo.yaml deleted file mode 100644 index 83da5d28d6e9..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/epp-config-dynamo.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Dynamo EPP Configuration - -apiVersion: inference.networking.x-k8s.io/v1alpha1 -kind: EndpointPickerConfig -# This config uses the same disagg-profile-handler as disaggregated deployments. -# With no "prefill" profile defined, the handler runs only the "decode" profile -# and the decode scorer receives isDisaggregated=false, enabling full KV cache -# overlap scoring for aggregated mode. Adding prefill workers and a prefill -# profile would automatically switch to disaggregated routing. -plugins: - - type: disagg-profile-handler - - # Label filter: restricts pods to decode workers - - name: decode-filter - type: label-filter - parameters: - label: "nvidia.com/dynamo-sub-component-type" - validValues: - - "decode" - allowsNoLabel: true - - # Picker: chooses the final endpoint after scoring - - name: picker - type: max-score-picker - - # Dynamo decode scorer: calls Dynamo FFI decode router, handles request lifecycle - - name: dyn-decode - type: dyn-decode-scorer - -schedulingProfiles: - - name: decode - plugins: - - pluginRef: decode-filter - weight: 1 - - pluginRef: dyn-decode - weight: 1 - - pluginRef: picker - weight: 1 diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/NOTES.txt b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/NOTES.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/_helpers.tpl b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/_helpers.tpl deleted file mode 100644 index 30f4ba59349c..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/_helpers.tpl +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{/* -Expand the name of the chart. -*/}} -{{- define "dynamo-gaie.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "dynamo-gaie.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "dynamo-gaie.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "dynamo-gaie.labels" -}} -helm.sh/chart: {{ include "dynamo-gaie.chart" . }} -{{ include "dynamo-gaie.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "dynamo-gaie.selectorLabels" -}} -app.kubernetes.io/name: {{ include "dynamo-gaie.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Resolve the fully qualified Dynamo namespace as "-" -*/}} -{{- define "dynamo-gaie.dynamoNamespace" -}} -{{- $dgdName := (.Values.dynamoGraphDeploymentName | default "") | trim -}} -{{- if not $dgdName }} -{{- fail "set dynamoGraphDeploymentName to derive the Dynamo namespace" }} -{{- end }} -{{- $releaseNamespace := (.Release.Namespace | default "") | trim -}} -{{- if not $releaseNamespace }} -{{- fail "Release.Namespace must be set to derive the Dynamo namespace" }} -{{- end }} -{{- printf "%s-%s" $releaseNamespace $dgdName }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "dynamo-gaie.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "dynamo-gaie.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/cluster-role-binding.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/cluster-role-binding.yaml deleted file mode 100644 index 4b2345a26f6f..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/cluster-role-binding.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: pod-read-binding -subjects: -- kind: ServiceAccount - name: default - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: pod-read - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/cluster-role.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/cluster-role.yaml deleted file mode 100644 index b9336231b9c0..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/cluster-role.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: pod-read -rules: -# Gateway API inference resources (experimental API) -- apiGroups: ["inference.networking.x-k8s.io"] - resources: ["inferencepools", "inferenceobjectives", "inferencemodelrewrites"] - verbs: ["get", "watch", "list"] -# Gateway API inference resources (stable API) -- apiGroups: ["inference.networking.k8s.io"] - resources: ["inferencepools"] - verbs: ["get", "watch", "list"] -# Core resources for pod discovery -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list"] -# Dynamo k8s service discovery - endpointslices -- apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["get", "list", "watch"] -# Dynamo k8s service discovery - worker metadata CRs -- apiGroups: ["nvidia.com"] - resources: ["dynamoworkermetadatas"] - verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] -# Authentication/authorization -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/dynamo-epp.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/dynamo-epp.yaml deleted file mode 100644 index 4e8d8503e0cb..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/dynamo-epp.yaml +++ /dev/null @@ -1,166 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- /* ------------ file-scope vars (no output) ------------ */ -}} -{{- $platformNs := default .Release.Namespace .Values.platformNamespace -}} -{{- $platformName := default "dynamo-platform" .Values.platformReleaseName -}} -{{- $useDynamo := default false .Values.epp.useDynamo -}} -{{- $resolvedDynNs := (include "dynamo-gaie.dynamoNamespace" .) | trim -}} -{{- $ns := ternary (required "set dynamoGraphDeploymentName when epp.useDynamo=true" $resolvedDynNs) "" $useDynamo -}} -{{- $useEtcd := default false .Values.epp.dynamo.useEtcd -}} -{{- $eppImage := required "extension.image is required - set via --set-string extension.image=$EPP_IMAGE or in values file" .Values.extension.image }} - ---- -# Deployment for the EPP (Endpoint Picker Plugin) -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.model.shortName }}-epp - namespace: {{ .Release.Namespace }} - labels: - app: {{ .Values.model.shortName }}-epp -spec: - replicas: 1 - selector: - matchLabels: - app: {{ .Values.model.shortName }}-epp - template: - metadata: - labels: - app: {{ .Values.model.shortName }}-epp - annotations: - # Exclude Istio sidecar injection: EPP serves TLS on port 9002 - # (--secure-serving true); an Istio sidecar in STRICT mTLS mode - # causes a double-TLS handshake failure on that port. - sidecar.istio.io/inject: "false" - spec: - terminationGracePeriodSeconds: 130 - - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . | quote }} - {{- end }} - {{- end }} - - containers: - - name: epp - image: "{{ $eppImage }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.epp.imagePullPolicy }} - args: - {{- if .Values.epp.argsOverride }} - {{- toYaml .Values.epp.argsOverride | nindent 8 }} - {{- else }} - - --pool-name - - "{{ .Values.model.shortName }}-pool" - - --pool-namespace - - "{{ .Release.Namespace }}" - - --pool-group - - "inference.networking.k8s.io" - - -v - - "4" - - --zap-encoder - - "json" - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - {{- if $useDynamo }} - - --config-file - - "{{ .Values.epp.configFile }}" - {{- end }} - {{- end }} - - volumeMounts: - - name: hf-cache - mountPath: /home/nonroot/.cache - {{- if $useDynamo }} - - name: epp-config - mountPath: /etc/epp - readOnly: true - {{- end }} - - env: - {{- if $useDynamo }} - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - {{- if $useEtcd }} - - name: ETCD_ENDPOINTS - value: "{{ $platformName }}-etcd.{{ $platformNs }}:2379" - {{- else }} - - name: DYN_DISCOVERY_BACKEND - value: "kubernetes" - {{- end }} - - name: NATS_SERVER - value: "nats://{{ $platformName }}-nats.{{ $platformNs }}:4222" - - name: DYN_NAMESPACE - value: "{{ $ns }}" - - name: USE_STREAMING - value: "true" - # HuggingFace token for downloading model config files - # Without this, HuggingFace rate-limits requests (429 Too Many Requests) - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: HF_TOKEN - optional: true - {{- end }} - {{- range .Values.epp.extraEnv }} - - name: {{ .name }} - value: {{ .value | quote }} - {{- end }} - - name: RUST_LOG - value: "debug,dynamo_llm::kv_router=trace" - - ports: - - containerPort: 9002 - - containerPort: 9003 - - name: metrics - containerPort: 9090 - livenessProbe: - grpc: - port: 9003 - service: inference-extension - initialDelaySeconds: 5 - periodSeconds: 10 - readinessProbe: - grpc: - port: 9003 - service: inference-extension - initialDelaySeconds: 5 - periodSeconds: 10 - - volumes: - - name: hf-cache - emptyDir: {} - {{- if $useDynamo }} - - name: epp-config - configMap: - name: {{ include "dynamo-gaie.fullname" . }}-epp-config - items: - - key: epp-config-dynamo.yaml - path: epp-config-dynamo.yaml - {{- end }} diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/epp-configmap.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/epp-configmap.yaml deleted file mode 100644 index bb53f0f9b137..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/epp-configmap.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "dynamo-gaie.fullname" . }}-epp-config - labels: - app.kubernetes.io/name: {{ include "dynamo-gaie.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} -data: - epp-config-dynamo.yaml: | -{{ (.Files.Get "epp-config-dynamo.yaml") | indent 4 }} diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/http-router.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/http-router.yaml deleted file mode 100644 index 4f65abe54a69..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/http-router.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.httpRoute.enabled }} -{{- /* Default gatewayNamespace to the release namespace if not specified */ -}} -{{- $gatewayNs := default .Release.Namespace .Values.httpRoute.gatewayNamespace }} -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: {{ .Values.model.shortName }}-route - namespace: {{ .Release.Namespace }} -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: {{ .Values.httpRoute.gatewayName }} - namespace: {{ $gatewayNs }} - rules: - - backendRefs: - - group: inference.networking.k8s.io - kind: InferencePool - name: {{ .Values.model.shortName }}-pool - namespace: {{ .Release.Namespace }} - port: {{ .Values.inferencePool.port }} - weight: 1 - matches: - - path: - type: PathPrefix - value: {{ .Values.httpRoute.path.prefix }} - timeouts: - request: {{ .Values.httpRoute.timeout.request }} -{{- end }} \ No newline at end of file diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/inference-pool.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/inference-pool.yaml deleted file mode 100644 index ce41905b6661..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/inference-pool.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# InferencePool (stable API - inference.networking.k8s.io/v1) -# Used by both kGateway (to resolve HTTPRoute backends) and EPP (to discover backend pods) -apiVersion: inference.networking.k8s.io/v1 -kind: InferencePool -metadata: - name: {{ .Values.model.shortName }}-pool - namespace: {{ .Release.Namespace }} -spec: - targetPorts: - - number: {{ .Values.inferencePool.port }} - selector: - matchLabels: - nvidia.com/dynamo-component: Frontend - nvidia.com/dynamo-namespace: {{ include "dynamo-gaie.dynamoNamespace" . }} - endpointPickerRef: - group: "" - kind: Service - name: {{ .Values.model.shortName }}-epp - port: - number: 9002 \ No newline at end of file diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/service.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/service.yaml deleted file mode 100644 index 389546c9e3aa..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/templates/service.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.model.shortName }}-epp - namespace: {{ .Release.Namespace }} -spec: - selector: - app: {{ .Values.model.shortName }}-epp - ports: - - protocol: TCP - port: 9002 - targetPort: 9002 - appProtocol: http2 - type: ClusterIP \ No newline at end of file diff --git a/deploy/inference-gateway/standalone/helm/dynamo-gaie/values.yaml b/deploy/inference-gateway/standalone/helm/dynamo-gaie/values.yaml deleted file mode 100644 index 5a0e805b9e0b..000000000000 --- a/deploy/inference-gateway/standalone/helm/dynamo-gaie/values.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for dynamo-gaie. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# Name of the DynamoGraphDeployment (DGD) that backs this gateway integration -dynamoGraphDeploymentName: "vllm-agg" - -# This is the port on which the model is exposed -model: - # This is the model name that will be used to route traffic to the dynamo model - # for example, if the model name is Qwen/Qwen3-0.6B, then the modelShortName should be qwen - identifier: "Qwen/Qwen3-0.6B" - # This is the short name of the model that will be used to generate the resource names - shortName: "qwen" - -# InferencePool configuration -inferencePool: - # Target port number for the inference pool - port: 8000 - -# HTTPRoute configuration -httpRoute: - # Enable the HTTPRoute - enabled: true - # Gateway parent reference configuration - gatewayName: "inference-gateway" - # Namespace where the Gateway resource is deployed - # Leave empty ("") to use the release namespace, or set explicitly (e.g., "default") - gatewayNamespace: "" - # Path matching configuration - path: - prefix: "/" - # Timeout configuration - timeout: - request: "300s" - -extension: - # EPP image for the GAIE extension (required - no default) - # Set via --set-string extension.image=$EPP_IMAGE or in your values file - image: "" - -# generic knobs you may want in both modes -imagePullSecrets: - - docker-imagepullsecret - -epp: - imagePullPolicy: IfNotPresent - # Add env in name/value pairs - extraEnv: [] - # If you ever want to completely override args, supply a list here. - # When empty, chart will render sane defaults - argsOverride: [] - - # Dynamo routing mode - set to true to enable KV-aware routing via Dynamo EPP image - useDynamo: true - - # Dynamo-specific settings (only used when useDynamo: true) - configFile: "/etc/epp/epp-config-dynamo.yaml" - dynamo: - # Use ETCD for discovery instead of Kubernetes (default: false) - # Set to true via --set epp.dynamo.useEtcd=true to enable ETCD discovery - useEtcd: false - -# Platform configuration (for Dynamo mode) -platformReleaseName: dynamo-platform -platformNamespace: "my-model" diff --git a/deploy/inference-gateway/standalone/vllm_agg_qwen.yaml b/deploy/inference-gateway/standalone/vllm_agg_qwen.yaml deleted file mode 100644 index 481a94533f58..000000000000 --- a/deploy/inference-gateway/standalone/vllm_agg_qwen.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Default values for dynamo-gaie. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# Name of the DynamoGraphDeployment (DGD) backing this model deployment. -dynamoGraphDeploymentName: "vllm-agg" - -# This is the port on which the model is exposed -model: - # This is the model name that will be used to route traffic to the dynamo model - # for example, if the model name is Qwen/Qwen3-0.6B, then the modelShortName should be qwen - identifier: "Qwen/Qwen3-0.6B" - # This is the short name of the model that will be used to generate the resource names - shortName: "qwen" diff --git a/docs/kubernetes/inference-gateway.md b/docs/kubernetes/inference-gateway.md index 6bd9a905101d..8440ba690f58 100644 --- a/docs/kubernetes/inference-gateway.md +++ b/docs/kubernetes/inference-gateway.md @@ -107,12 +107,6 @@ make info # Check image tag ### 5. Deploy -We recommend deploying Inference Gateway's Endpoint Picker as a Dynamo operator's managed component. Alternatively, -you could deploy it as a standalone pod. -Note that when deploying Dynamo with the Inference Gateway Extension each worker must have the FrontEnd as a sidecar. - -#### 5.a. Deploy as a DGD component (recommended) - We provide an example for the Qwen vLLM below. You have to deploy the Dynamo Graph and the `HTTPRoute`. The example `http-route.yaml` resolves the `Gateway` in the same namespace as @@ -200,44 +194,6 @@ The example `http-route.yaml` resolves the `Gateway` in the same namespace as the route. If you install the `Gateway` in one namespace and apply the route in another, add `parentRefs[].namespace: ` to `http-route.yaml`. -#### 5.b. Deploy as a standalone pod - -We do not recommend this method but there are hints on how to do this here. - -##### 5.b.1 Deploy Your Model ### - -##### 5.b.2 Install Dynamo GIE helm chart ### - -```bash -cd deploy/inference-gateway/standalone - -# Export the EPP image - use the Dynamo FrontEnd image or build your own EPP image (see section 4) -export EPP_IMAGE= -``` -Create a model configuration file similar to the vllm_agg_qwen.yaml for your model. - -```bash -helm upgrade --install dynamo-gaie ./helm/dynamo-gaie -n my-model -f ./vllm_agg_qwen.yaml --set-string extension.image=$EPP_IMAGE -``` - -By default, the Kubernetes discovery mechanism is used. If you prefer etcd, please use the `--set epp.dynamo.useEtcd=true` flag below. - -```bash -helm upgrade --install dynamo-gaie ./helm/dynamo-gaie -n my-model -f ./vllm_agg_qwen.yaml --set-string extension.image=$EPP_IMAGE --set epp.dynamo.useEtcd=true -``` - -Key configurations include: - -- An InferenceModel resource for the Qwen model -- A service for the inference gateway -- Required RBAC roles and bindings -- RBAC permissions -- dynamoGraphDeploymentName - the name of the Dynamo Graph where your model is deployed. - - -**Configuration** -You can configure the plugin by setting environment variables in the EPP component of your DGD in case of the operator-managed installation or in your [values.yaml](https://github.com/ai-dynamo/dynamo/blob/main/deploy/inference-gateway/standalone/helm/dynamo-gaie/values.yaml). - Common Vars for Routing Configuration: **Enabling KV-Aware Routing (most precise)** @@ -271,8 +227,6 @@ To disable the EPP from listening for KV events (e.g., when prefix caching is of - `DYN_ROUTER_TRACK_OUTPUT_BLOCKS` — Track output blocks during generation (default: false) - See the [KV cache routing design](../design-docs/router-design.md) for details. -Stand-Alone installation only: -- Overwrite the `DYN_NAMESPACE` env var if needed to match your model's dynamo namespace. **Service Mesh Integration (Istio)**