diff --git a/deploy/helm/admin-token-issuer-proxy/chart/templates/poddisruptionbudget.yaml b/deploy/helm/admin-token-issuer-proxy/chart/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..f63e53289 --- /dev/null +++ b/deploy/helm/admin-token-issuer-proxy/chart/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.adminIssuerProxy.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.adminIssuerProxy.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.adminIssuerProxy.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "admin-issuer-proxy.fullname" . }} + namespace: {{ include "admin-issuer-proxy.namespace" . }} + labels: {{- include "admin-issuer-proxy.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.adminIssuerProxy.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.adminIssuerProxy.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "admin-issuer-proxy.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/admin-token-issuer-proxy/chart/values.yaml b/deploy/helm/admin-token-issuer-proxy/chart/values.yaml index e1c691a7a..da5d0b4da 100644 --- a/deploy/helm/admin-token-issuer-proxy/chart/values.yaml +++ b/deploy/helm/admin-token-issuer-proxy/chart/values.yaml @@ -21,6 +21,13 @@ adminIssuerProxy: # and configuring appropriate affinity rules. replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: # Container image repository path. This field is required and must be set during installation. repository: "" diff --git a/deploy/helm/api-keys-colocated/api-keys/templates/poddisruptionbudget.yaml b/deploy/helm/api-keys-colocated/api-keys/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..9970e807b --- /dev/null +++ b/deploy/helm/api-keys-colocated/api-keys/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.apikeys.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.apikeys.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.apikeys.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "nv_api_keys.fullname" . }} + namespace: {{ include "nv_api_keys.namespace" . }} + labels: {{- include "nv_api_keys.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.apikeys.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.apikeys.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "nv_api_keys.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/api-keys-colocated/api-keys/values.yaml b/deploy/helm/api-keys-colocated/api-keys/values.yaml index 157c73141..9b5e247e4 100644 --- a/deploy/helm/api-keys-colocated/api-keys/values.yaml +++ b/deploy/helm/api-keys-colocated/api-keys/values.yaml @@ -17,6 +17,13 @@ apikeys: replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: registry: "" repository: "" diff --git a/deploy/helm/cassandra/helm/templates/poddisruptionbudget.yaml b/deploy/helm/cassandra/helm/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..6fd98021b --- /dev/null +++ b/deploy/helm/cassandra/helm/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.cassandra.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.cassandra.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.cassandra.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "cassandra.fullname" . }} + namespace: {{ include "cassandra.namespace" . }} + labels: {{- include "cassandra.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.cassandra.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.cassandra.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "cassandra.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/cassandra/helm/values.yaml b/deploy/helm/cassandra/helm/values.yaml index 95e650aa2..bf80f4348 100644 --- a/deploy/helm/cassandra/helm/values.yaml +++ b/deploy/helm/cassandra/helm/values.yaml @@ -36,6 +36,13 @@ cassandra: replicaCount: 3 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + cluster: name: cassandra seed: 2 diff --git a/deploy/helm/container-cache/deploy/values.yaml b/deploy/helm/container-cache/deploy/values.yaml index b89312cb1..877730801 100644 --- a/deploy/helm/container-cache/deploy/values.yaml +++ b/deploy/helm/container-cache/deploy/values.yaml @@ -18,6 +18,13 @@ nodeSelector: {} # Number of replicas to create of container-cache. replicaCount: 1 +podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + # Domain(s) of target hosts where the proxy passes requests (comma-separated). targetHost: nvcr.io,stg.nvcr.io diff --git a/deploy/helm/encrypted-secret-store/ess-api/templates/poddisruptionbudget.yaml b/deploy/helm/encrypted-secret-store/ess-api/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..2feaf31fa --- /dev/null +++ b/deploy/helm/encrypted-secret-store/ess-api/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.ess.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.ess.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.ess.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "ess-api.fullname" . }} + namespace: {{ include "ess-api.namespace" . }} + labels: {{- include "ess-api.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.ess.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.ess.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "ess-api.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/encrypted-secret-store/ess-api/values.yaml b/deploy/helm/encrypted-secret-store/ess-api/values.yaml index 7ed4d6ecc..bd91f7887 100644 --- a/deploy/helm/encrypted-secret-store/ess-api/values.yaml +++ b/deploy/helm/encrypted-secret-store/ess-api/values.yaml @@ -16,6 +16,13 @@ ess: replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: registry: "" # must be supplied in additional values diff --git a/deploy/helm/function-autoscaler/templates/poddisruptionbudget.yaml b/deploy/helm/function-autoscaler/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..3ec7a3a3c --- /dev/null +++ b/deploy/helm/function-autoscaler/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.functionautoscaler.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.functionautoscaler.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.functionautoscaler.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "function-autoscaler.fullname" . }} + namespace: {{ include "function-autoscaler.namespace" . }} + labels: {{- include "function-autoscaler.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.functionautoscaler.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.functionautoscaler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "function-autoscaler.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/function-autoscaler/values.yaml b/deploy/helm/function-autoscaler/values.yaml index 3da023d4e..93b6a5d1a 100644 --- a/deploy/helm/function-autoscaler/values.yaml +++ b/deploy/helm/function-autoscaler/values.yaml @@ -27,6 +27,13 @@ functionautoscaler: replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: registry: "" # must be supplied repository: "" # must be supplied diff --git a/deploy/helm/grpc-proxy/grpc-proxy/templates/poddisruptionbudget.yaml b/deploy/helm/grpc-proxy/grpc-proxy/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..40e8aa9cb --- /dev/null +++ b/deploy/helm/grpc-proxy/grpc-proxy/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.grpcproxy.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.grpcproxy.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.grpcproxy.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "grpc-proxy.fullname" . }} + namespace: {{ include "grpc-proxy.namespace" . }} + labels: {{- include "grpc-proxy.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.grpcproxy.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.grpcproxy.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "grpc-proxy.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/grpc-proxy/grpc-proxy/values.yaml b/deploy/helm/grpc-proxy/grpc-proxy/values.yaml index 2dc3de4dc..19e0af183 100644 --- a/deploy/helm/grpc-proxy/grpc-proxy/values.yaml +++ b/deploy/helm/grpc-proxy/grpc-proxy/values.yaml @@ -23,6 +23,13 @@ grpcproxy: replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: registry: "" repository: "" diff --git a/deploy/helm/helm-reval/templates/poddisruptionbudget.yaml b/deploy/helm/helm-reval/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..40b071ced --- /dev/null +++ b/deploy/helm/helm-reval/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.reval.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.reval.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.reval.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "reval.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "reval.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.reval.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.reval.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "reval.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/helm-reval/values.yaml b/deploy/helm/helm-reval/values.yaml index 09a10baed..d287406de 100644 --- a/deploy/helm/helm-reval/values.yaml +++ b/deploy/helm/helm-reval/values.yaml @@ -26,6 +26,13 @@ reval: imagePullSecrets: [] replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + serviceAccount: create: false # automount: false — the reval service does not talk to the Kubernetes API; diff --git a/deploy/helm/http-invocation/nvcf-invocation-service/templates/poddisruptionbudget.yaml b/deploy/helm/http-invocation/nvcf-invocation-service/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..9de68d2d5 --- /dev/null +++ b/deploy/helm/http-invocation/nvcf-invocation-service/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.invocation.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.invocation.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.invocation.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "nvcf-invocation-service.fullname" . }} + namespace: {{ include "nvcf-invocation-service.namespace" . }} + labels: {{- include "nvcf-invocation-service.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.invocation.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.invocation.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "nvcf-invocation-service.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/http-invocation/nvcf-invocation-service/values.yaml b/deploy/helm/http-invocation/nvcf-invocation-service/values.yaml index d71c8280b..b0b5d008e 100644 --- a/deploy/helm/http-invocation/nvcf-invocation-service/values.yaml +++ b/deploy/helm/http-invocation/nvcf-invocation-service/values.yaml @@ -26,6 +26,13 @@ invocation: replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: registry: "" # must be supplied repository: "" # must be supplied diff --git a/deploy/helm/llm-api-gateway/llm-api-gateway/templates/poddisruptionbudget.yaml b/deploy/helm/llm-api-gateway/llm-api-gateway/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..10306495a --- /dev/null +++ b/deploy/helm/llm-api-gateway/llm-api-gateway/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.llmApiGateway.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.llmApiGateway.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.llmApiGateway.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "llm-api-gateway.fullname" . }} + namespace: {{ include "llm-api-gateway.namespace" . }} + labels: {{- include "llm-api-gateway.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.llmApiGateway.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.llmApiGateway.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "llm-api-gateway.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml b/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml index a2515de6c..b0f8aaa56 100644 --- a/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml +++ b/deploy/helm/llm-api-gateway/llm-api-gateway/values.yaml @@ -16,6 +16,13 @@ llmApiGateway: replicaCount: 3 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: registry: "" repository: "" diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/poddisruptionbudget.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..3b6c839b1 --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.llmRequestRouter.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.llmRequestRouter.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.llmRequestRouter.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "llm-request-router.fullname" . }} + namespace: {{ include "llm-request-router.namespace" . }} + labels: {{- include "llm-request-router.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.llmRequestRouter.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.llmRequestRouter.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "llm-request-router.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/values.yaml b/deploy/helm/llm-request-router/llm-request-router/values.yaml index 77a7f57d8..e834ec5d6 100644 --- a/deploy/helm/llm-request-router/llm-request-router/values.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/values.yaml @@ -19,6 +19,13 @@ llmRequestRouter: # the StatefulSet and headless service. replicaCount: 3 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: registry: "" repository: "" diff --git a/deploy/helm/nats-auth-callout/templates/poddisruptionbudget.yaml b/deploy/helm/nats-auth-callout/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..5d3dcbe9a --- /dev/null +++ b/deploy/helm/nats-auth-callout/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "app.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "app.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "app.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/nats-auth-callout/values.yaml b/deploy/helm/nats-auth-callout/values.yaml index 44c987d59..dccba01a3 100644 --- a/deploy/helm/nats-auth-callout/values.yaml +++ b/deploy/helm/nats-auth-callout/values.yaml @@ -19,6 +19,13 @@ fullnameOverride: "" replicaCount: 1 +podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + # Container registry credentials for pulling images registry: # Use existing secret (leave empty to create one) diff --git a/deploy/helm/nats/values.yaml b/deploy/helm/nats/values.yaml index 0acc30f78..978c1f3cd 100644 --- a/deploy/helm/nats/values.yaml +++ b/deploy/helm/nats/values.yaml @@ -16,6 +16,14 @@ # NATS Configuration (reference: https://github.com/nats-io/k8s/tree/main/helm/charts/nats) nats: enabled: true + + # Pod disruption budget. The upstream chart enables this by default. + # Set enabled: false to disable, or configure merge/patch for custom spec fields. + podDisruptionBudget: + enabled: true + merge: {} + patch: [] + global: image: pullSecretNames: [] diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/poddisruptionbudget.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..6662875aa --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "nvcaop.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "nvcaop.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "nvcaop.baseSelectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/nvca-operator/nvca-operator/values.yaml b/deploy/helm/nvca-operator/nvca-operator/values.yaml index 61d246354..7075e97cb 100644 --- a/deploy/helm/nvca-operator/nvca-operator/values.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/values.yaml @@ -65,6 +65,13 @@ serviceAccount: name: "" ## @param replicaCount Replica count for the operator deployment replicaCount: 1 +## @param podDisruptionBudget PodDisruptionBudget configuration +podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" ## @param systemNamespace Namespace in which NVCFBackend objects are created. systemNamespace: nvca-operator ## @param logLevel Logging level for the module diff --git a/deploy/helm/nvcf-unbound/values.yaml b/deploy/helm/nvcf-unbound/values.yaml index f61c7d865..2236cac06 100644 --- a/deploy/helm/nvcf-unbound/values.yaml +++ b/deploy/helm/nvcf-unbound/values.yaml @@ -22,6 +22,13 @@ nodeSelector: replicaCount: 3 +podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + images: server: nvcr.io/nv-ngc-devops/unbound:1.22.0-nv1.3.1 logger: nvcr.io/nv-ngc-devops/dns-logger:v2.2.1 diff --git a/deploy/helm/openbao/helm/values.yaml b/deploy/helm/openbao/helm/values.yaml index 0dd537f07..8852fa2ca 100644 --- a/deploy/helm/openbao/helm/values.yaml +++ b/deploy/helm/openbao/helm/values.yaml @@ -83,6 +83,14 @@ openbao: ha: enabled: true replicas: 3 + + # Pod disruption budget for the HA server StatefulSet. + # The upstream chart enables this by default; maxUnavailable defaults to + # (replicas/2)-1 when null. + disruptionBudget: + enabled: true + maxUnavailable: null + raft: enabled: true config: | diff --git a/deploy/helm/ratelimiter/nvcf-ratelimiter/templates/poddisruptionbudget.yaml b/deploy/helm/ratelimiter/nvcf-ratelimiter/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..634e44932 --- /dev/null +++ b/deploy/helm/ratelimiter/nvcf-ratelimiter/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.rateLimiter.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.rateLimiter.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.rateLimiter.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "nvcf-ratelimiter.fullname" . }} + namespace: {{ include "nvcf-ratelimiter.namespace" . }} + labels: {{- include "nvcf-ratelimiter.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.rateLimiter.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.rateLimiter.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "nvcf-ratelimiter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/ratelimiter/nvcf-ratelimiter/values.yaml b/deploy/helm/ratelimiter/nvcf-ratelimiter/values.yaml index 8dd161a32..0baf68b5d 100644 --- a/deploy/helm/ratelimiter/nvcf-ratelimiter/values.yaml +++ b/deploy/helm/ratelimiter/nvcf-ratelimiter/values.yaml @@ -32,6 +32,13 @@ rateLimiter: replicaCount: 1 + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + image: registry: "" repository: "" diff --git a/deploy/stacks/self-managed/Makefile b/deploy/stacks/self-managed/Makefile index 64f4850a0..d88d75ea4 100644 --- a/deploy/stacks/self-managed/Makefile +++ b/deploy/stacks/self-managed/Makefile @@ -4,3 +4,4 @@ include Makefile.dist .PHONY: test test: @tests/llm-router-worker-address.sh + @tests/pdb-value-wiring.sh diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index a9e9b927b..12dd22c8f 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -168,6 +168,13 @@ accounts: # These static global values are processed in the values template nats: enabled: true + # PDB: the upstream NATS chart enables a disruption budget by default. + # Override here to disable it or adjust the minAvailable threshold. + podDisruptionBudget: + enabled: false + merge: + spec: + minAvailable: 2 cassandra: enabled: true @@ -184,6 +191,11 @@ cassandra: # Do not use small for default cloud installs. Cassandra can OOM during # first boot and leave migrations in a failed or dirty state. resourcesPreset: "xlarge" + # PDB: disabled by default. Recommended for production 3-node clusters. + # Set exactly one of minAvailable or maxUnavailable. + podDisruptionBudget: + enabled: false + minAvailable: 2 # keep at least 2 of 3 nodes up during any disruption certManager: # cert-manager controller required to reconcile Certificate resources for @@ -220,6 +232,16 @@ openbao: # - nats-system # - nvcf # - sis + # PDB for the injector Deployment. minAvailable: 1 by default (always active). + podDisruptionBudget: + minAvailable: 1 + server: + ha: + # PDB for the HA Raft StatefulSet. The upstream chart enables this by + # default with maxUnavailable derived from replica count. + disruptionBudget: + enabled: false + maxUnavailable: 1 addons: # LLS/TURN Low-Latency Streaming addon @@ -314,6 +336,54 @@ stateMetrics: rateLimiter: enabled: true replicaCount: 1 + # PDB: enable when replicaCount > 1. + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +# PDB knobs for control-plane services. Disabled by default. +# Set enabled: true for any service running more than one replica. + +apikeys: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +ess: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +invocation: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +adminIssuerProxy: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +functionautoscaler: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +reval: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +# LLM addon services (active only when addons.llm.enabled: true): +llmApiGateway: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +llmRequestRouter: + podDisruptionBudget: + enabled: false + maxUnavailable: 1 # Ingress Gateway Configuration ingress: @@ -373,6 +443,10 @@ ingress: listenerName: nats grpcproxy: + # PDB: disabled by default. Enable when running multiple grpc-proxy replicas. + podDisruptionBudget: + enabled: false + maxUnavailable: 1 # Optional NVCF API gRPC endpoint used only by grpc-proxy. # Leave empty when grpc-proxy can use the same endpoint advertised to workers # through global.workerEndpoints.nvcfGrpcServiceURL. Set this when workers diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index b3e0b560b..b09aa5527 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -39,6 +39,11 @@ cassandra: replicaCount: {{ dig "cassandra" "replicaCount" 3 .Values }} resourcesPreset: {{ dig "cassandra" "resourcesPreset" "xlarge" .Values }} + {{- with dig "cassandra" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with include "nvcf.nodeSelector" (dict "type" "cassandra" "selectors" .Values.global.nodeSelectors) }} {{- . | nindent 2 }} {{- end }} @@ -154,7 +159,7 @@ openbao: {{- toYaml $webhook | nindent 6 }} {{- end }} podDisruptionBudget: - minAvailable: 1 + minAvailable: {{ dig "openbao" "injector" "podDisruptionBudget" "minAvailable" 1 .Values }} server: image: registry: {{ .Values.global.image.registry }} @@ -172,6 +177,12 @@ openbao: {{- end }} size: {{ .Values.global.storageSize | default "10Gi" }} + ha: + {{- with dig "openbao" "server" "ha" "disruptionBudget" dict .Values }} + disruptionBudget: + {{- toYaml . | nindent 8 }} + {{- end }} + nats: {{- if .Values.global.imagePullSecrets }} global: @@ -223,6 +234,11 @@ nats: storageClassName: {{ .Values.global.storageClass }} {{- end }} + {{- with dig "nats" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + apikeys: fullnameOverride: api-keys {{- if .Values.global.imagePullSecrets }} @@ -238,6 +254,10 @@ apikeys: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with dig "apikeys" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} natsAuthCalloutService: {{- if .Values.global.imagePullSecrets }} @@ -459,7 +479,11 @@ invocation: {{- with $invocationWorkerBaseURL }} WORKER_STREAM_PROPERTIES__SELF_ADDRESS: {{ . | quote }} {{- end }} - + {{- with dig "invocation" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + nvctApi: fullnameOverride: nvct-api {{- if .Values.global.imagePullSecrets }} @@ -534,6 +558,10 @@ grpcproxy: RATE_LIMIT_ENABLED: "true" RATE_LIMIT_ADDR: "http://ratelimiter.nvcf.svc.cluster.local:7777" {{- end }} + {{- with dig "grpcproxy" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} rateLimiter: {{- if .Values.global.imagePullSecrets }} @@ -552,6 +580,10 @@ rateLimiter: env: OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}" {{- end }} + {{- with dig "rateLimiter" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} ess: fullnameOverride: ess-api @@ -575,6 +607,10 @@ ess: {{- if .Values.global.observability.tracing.enabled }} MANAGEMENT_OTLP_TRACING_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}/v1/traces" {{- end }} + {{- with dig "ess" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} notary: fullnameOverride: notary-service @@ -660,6 +696,10 @@ adminIssuerProxy: name: {{ required "ingress.gatewayApi.gateways.shared.name is required" .Values.ingress.gatewayApi.gateways.shared.name }} hostname: "api-keys.{{ .Values.global.domain }}" path: "/v1/admin/keys" + {{- with dig "adminIssuerProxy" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} stateMetrics: enabled: false @@ -728,6 +768,10 @@ functionautoscaler: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with dig "functionautoscaler" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} reval: @@ -748,6 +792,10 @@ reval: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with dig "reval" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} llmApiGateway: enabled: {{ dig "addons" "llm" "enabled" false .Values }} @@ -786,6 +834,10 @@ llmApiGateway: {{- if .Values.global.observability.tracing.enabled }} endpoint: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}" {{- end }} + {{- with dig "llmApiGateway" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} llmRequestRouter: enabled: {{ dig "addons" "llm" "enabled" false .Values }} @@ -867,6 +919,10 @@ llmRequestRouter: {{- toYaml .Values.global.imagePullSecrets | nindent 6 }} {{- end }} {{- end }} + {{- with dig "llmRequestRouter" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} vanityGateway: fullnameOverride: vanity-gateway diff --git a/deploy/stacks/self-managed/tests/pdb-value-wiring.sh b/deploy/stacks/self-managed/tests/pdb-value-wiring.sh new file mode 100755 index 000000000..fbb7cbb0d --- /dev/null +++ b/deploy/stacks/self-managed/tests/pdb-value-wiring.sh @@ -0,0 +1,283 @@ +#!/usr/bin/env bash +# Test that podDisruptionBudget values thread correctly from environment files +# through global.yaml.gotmpl into each chart's rendered output, and that the +# chart-level fail validation fires when both or neither availability field is set. +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +environment_name="pdb-value-wiring-test" +environment_file="$test_stack_dir/environments/$environment_name.yaml" +secrets_file="$test_stack_dir/secrets/$environment_name-secrets.yaml" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "pdb-value-wiring: $*" >&2 + exit 1 +} + +mkdir -p "$test_stack_dir" +cp -R "$stack_dir"/. "$test_stack_dir" +printf '{}\n' >"$secrets_file" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +render_chart_values() { + local release="$1" + local output_file="$2" + shift 2 + + HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default \ + --selector "name=$release" \ + "$@" \ + write-values \ + --output-file-template "$output_file" 2>/dev/null || + HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" \ + --environment default \ + --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ + --selector "name=$release" \ + "$@" \ + write-values \ + --output-file-template "$output_file" +} + +pdb_enabled_in_values() { + local values_file="$1" + local key_prefix="$2" # e.g. "cassandra" or "" for root-level + grep -q "enabled: true" "$values_file" +} + +pdb_key_present() { + local values_file="$1" + grep -q "podDisruptionBudget:" "$values_file" +} + +pdb_min_available() { + local values_file="$1" + grep -q "minAvailable:" "$values_file" +} + +pdb_max_unavailable() { + local values_file="$1" + grep -q "maxUnavailable:" "$values_file" +} + +write_env() { + cat >"$environment_file" +} + +# --------------------------------------------------------------------------- +# 1. Cassandra PDB — omitted (default off) +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +EOF + +render_chart_values cassandra "$work_dir/cassandra-off-values.yaml" >/dev/null +if grep -q "podDisruptionBudget:" "$work_dir/cassandra-off-values.yaml" && + grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-off-values.yaml" | grep -q "enabled: true"; then + fail "cassandra: PDB should be disabled by default but rendered enabled: true" +fi + +# --------------------------------------------------------------------------- +# 2. Cassandra PDB — minAvailable passthrough +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +cassandra: + podDisruptionBudget: + enabled: true + minAvailable: 2 +EOF + +render_chart_values cassandra "$work_dir/cassandra-min-values.yaml" >/dev/null +grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-min-values.yaml" | grep -q "enabled: true" || + fail "cassandra: minAvailable PDB did not set enabled: true" +grep -q "minAvailable: 2" "$work_dir/cassandra-min-values.yaml" || + fail "cassandra: minAvailable: 2 did not reach the chart values" + +# --------------------------------------------------------------------------- +# 3. Cassandra PDB — maxUnavailable passthrough +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +cassandra: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +EOF + +render_chart_values cassandra "$work_dir/cassandra-max-values.yaml" >/dev/null +grep -q "maxUnavailable: 1" "$work_dir/cassandra-max-values.yaml" || + fail "cassandra: maxUnavailable: 1 did not reach the chart values" + +# --------------------------------------------------------------------------- +# 4. Cassandra PDB — chart fail: both fields set +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +cassandra: + podDisruptionBudget: + enabled: true + minAvailable: 2 + maxUnavailable: 1 +EOF + +# helm template should fail when both fields are set +if HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default \ + --selector name=cassandra \ + template >"$work_dir/cassandra-both.log" 2>&1; then + fail "cassandra: setting both minAvailable and maxUnavailable should have failed" +fi +grep -qi "exactly one" "$work_dir/cassandra-both.log" || + fail "cassandra: both-fields error did not contain expected message" + +# --------------------------------------------------------------------------- +# 4b. Cassandra PDB — chart fail: neither field set +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +cassandra: + podDisruptionBudget: + enabled: true +EOF + +# helm template should fail when enabled but neither field is set +if HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default \ + --selector name=cassandra \ + template >"$work_dir/cassandra-neither.log" 2>&1; then + fail "cassandra: enabled with neither minAvailable nor maxUnavailable should have failed" +fi +grep -qi "exactly one" "$work_dir/cassandra-neither.log" || + fail "cassandra: neither-field error did not contain expected message" + +# --------------------------------------------------------------------------- +# 5. NATS PDB — passthrough (upstream chart; enabled by default) +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +nats: + podDisruptionBudget: + enabled: false +EOF + +render_chart_values nats "$work_dir/nats-disabled-values.yaml" >/dev/null +# The nats wrapper values should reflect the override +grep -A2 "podDisruptionBudget:" "$work_dir/nats-disabled-values.yaml" | grep -q "enabled: false" || + fail "nats: podDisruptionBudget.enabled: false did not reach chart values" + +# --------------------------------------------------------------------------- +# 6. OpenBao HA disruptionBudget passthrough +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +openbao: + server: + ha: + disruptionBudget: + enabled: true + maxUnavailable: 1 +EOF + +render_chart_values openbao-server "$work_dir/openbao-ha-values.yaml" >/dev/null +grep -q "maxUnavailable: 1" "$work_dir/openbao-ha-values.yaml" || + fail "openbao: server.ha.disruptionBudget.maxUnavailable: 1 did not reach chart values" + +# --------------------------------------------------------------------------- +# 7. OpenBao injector PDB minAvailable passthrough +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +openbao: + injector: + podDisruptionBudget: + minAvailable: 2 +EOF + +render_chart_values openbao-server "$work_dir/openbao-injector-values.yaml" >/dev/null +grep -q "minAvailable: 2" "$work_dir/openbao-injector-values.yaml" || + fail "openbao: injector.podDisruptionBudget.minAvailable: 2 did not reach chart values" + +# --------------------------------------------------------------------------- +# 8. rateLimiter PDB passthrough +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +rateLimiter: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +EOF + +render_chart_values ratelimiter "$work_dir/ratelimiter-values.yaml" >/dev/null +grep -A2 "podDisruptionBudget:" "$work_dir/ratelimiter-values.yaml" | grep -q "enabled: true" || + fail "rateLimiter: podDisruptionBudget.enabled: true did not reach chart values" + +# --------------------------------------------------------------------------- +# 9. ess PDB passthrough +# --------------------------------------------------------------------------- +write_env <<'EOF' +global: + image: + registry: nvcr.io + repository: test/nvcf +ess: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +EOF + +render_chart_values ess-api "$work_dir/ess-values.yaml" >/dev/null +grep -A2 "podDisruptionBudget:" "$work_dir/ess-values.yaml" | grep -q "enabled: true" || + fail "ess: podDisruptionBudget.enabled: true did not reach chart values" + +echo "pdb-value-wiring: all checks passed" diff --git a/docs/user/optional-enhancements.md b/docs/user/optional-enhancements.md index 1becae946..f88b1d6a1 100644 --- a/docs/user/optional-enhancements.md +++ b/docs/user/optional-enhancements.md @@ -4,6 +4,10 @@ NVCF supports several optional components that can enhance your deployment's performance, routing, and GPU cluster capabilities. Each component has its own installation and configuration guide. +## Pod Disruption Budgets + +- [Pod Disruption Budgets](pod-disruption-budgets.md) - Protect stateful infrastructure and control-plane services from simultaneous voluntary evictions during node drains and cluster upgrades + ## Low-Latency Streaming - [LLS Installation](lls-installation.md) - Required for streaming Cloud Functions using WebRTC diff --git a/docs/user/pod-disruption-budgets.md b/docs/user/pod-disruption-budgets.md new file mode 100644 index 000000000..6ca8a21a9 --- /dev/null +++ b/docs/user/pod-disruption-budgets.md @@ -0,0 +1,128 @@ +# Pod Disruption Budgets + +PodDisruptionBudgets (PDBs) limit the number of pods that can be voluntarily +evicted at once during node drains, cluster upgrades, or autoscaler scale-down +events. Enabling PDBs prevents entire stateful tiers from going offline during +maintenance and is recommended for production deployments. + +See the [Kubernetes PDB documentation](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) +for a full explanation of how disruption budgets work. + +## How it works + +When a PDB is active, the Kubernetes eviction API blocks any voluntary +disruption that would reduce the number of running pods below `minAvailable` (or +above `maxUnavailable`). Node drain operations will pause and wait for displaced +pods to reschedule and become ready before proceeding. + +PDBs only protect against voluntary disruptions (drains, upgrades, +autoscaler). They do not prevent evictions caused by node failure or +out-of-memory pressure. + +## Configuration + +All supported PDB knobs are pre-declared with `enabled: false` in +`deploy/stacks/self-managed/environments/base.yaml`. To enable a budget, +set `enabled: true` and choose a value for the relevant block. For +environment-specific overrides, copy the block into your environment file +(e.g. `deploy/stacks/self-managed/environments/.yaml`) and adjust there. + +### Infrastructure components + +These are the stateful dependencies that underpin the NVCF control plane. PDBs +are most critical here. + +The custom Cassandra chart has PDB disabled by default. NATS and OpenBao use +upstream charts that enable a disruption budget by default. + +Cassandra (3-node cluster, namespace `cassandra-system`): + +```yaml +cassandra: + podDisruptionBudget: + enabled: true + minAvailable: 2 # keep at least 2 of 3 nodes up during any disruption +``` + +NATS (3-node JetStream cluster, namespace `nats-system`): + +The upstream NATS chart enables a PDB by default. Override to disable or +customise. These values are set in the Helmfile environment file +(`environments/.yaml`); the `merge:` key is interpreted by the upstream +NATS chart's values schema and is not a Helmfile directive: + +```yaml +nats: + podDisruptionBudget: + enabled: true + merge: + spec: + minAvailable: 2 +``` + +OpenBao server (3-node HA Raft cluster, namespace `vault-system`): + +The upstream OpenBao chart enables an HA disruption budget by default. +Override `maxUnavailable` when needed: + +```yaml +openbao: + server: + ha: + disruptionBudget: + enabled: true + maxUnavailable: 1 +``` + +The OpenBao injector PDB (`minAvailable: 1`) is always active and can be +adjusted: + +```yaml +openbao: + injector: + podDisruptionBudget: + minAvailable: 1 +``` + +### Control-plane services + +These services run as Deployments and default to a single replica. Enable PDBs +only when you increase `replicaCount` above 1. + +| Environment key | Default replicas | +|---|---| +| `ess.podDisruptionBudget` | 1 | +| `grpcproxy.podDisruptionBudget` | 1 | +| `invocation.podDisruptionBudget` | 1 | +| `rateLimiter.podDisruptionBudget` | 1 | +| `llmApiGateway.podDisruptionBudget` | 3 | +| `llmRequestRouter.podDisruptionBudget` | 3 | +| `adminIssuerProxy.podDisruptionBudget` | 1 | +| `apikeys.podDisruptionBudget` | 1 | +| `natsAuthCalloutService.podDisruptionBudget` | 1 | +| `functionautoscaler.podDisruptionBudget` | 1 | +| `reval.podDisruptionBudget` | 1 | +| `podDisruptionBudget` (nvca-operator) | 1 | + +Example for a scaled-up LLM gateway: + +```yaml +llmApiGateway: + replicaCount: 5 + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +``` + +### Value reference + +Each PDB block accepts the same fields: + +| Field | Type | Description | +|---|---|---| +| `enabled` | bool | Set `true` to create the PDB resource. Default: `false` for custom charts. | +| `minAvailable` | int or string | Minimum pods that must remain available. Accepts an integer (`2`) or a percentage (`"50%"`). Mutually exclusive with `maxUnavailable`. | +| `maxUnavailable` | int or string | Maximum pods that may be unavailable at once. Accepts an integer (`1`) or a percentage (`"33%"`). Mutually exclusive with `minAvailable`. | + +Set exactly one of `minAvailable` or `maxUnavailable` when `enabled: true`. +The chart will fail at render time if both or neither are set. diff --git a/fern/versions/dev.yml b/fern/versions/dev.yml index f3ec7ca54..6eb3b5140 100644 --- a/fern/versions/dev.yml +++ b/fern/versions/dev.yml @@ -51,6 +51,8 @@ navigation: contents: - page: Optional Enhancements path: ../../docs/user/optional-enhancements.md + - page: Pod Disruption Budgets + path: ../../docs/user/pod-disruption-budgets.md - page: LLM Function Enablement path: ../../docs/user/llm-function-enablement.md - page: LLM Request Router Load Balancing