ci(deploy-test): align vCluster Kubernetes with host and remove readiness workaround

.github/actions/setup-dynamo-operator/action.yml

@@ -46,10 +46,6 @@ inputs:
     description: 'Docker Hub password for helm registry login'
     required: false
     default: ''
-  vcluster_k8s_version:
-    description: 'Kubernetes version for the vCluster control plane (must be supported by kr8s)'
-    required: false
-    default: 'v1.32.13'
   vcluster_host_priority_class:
     description: 'Host PriorityClass to set on pods synced from the vCluster. Use auto to set ci-nightly-high-priority only when the active kube context targets aws-dev-01 or aws-dev-02; use an empty value to disable.'
     required: false
@@ -250,14 +246,22 @@ runs:
           VCLUSTER_HELM_ARGS+=(--values "${VCLUSTER_CHECKPOINT_VALUES}")
         fi
 
-        # The K8s version must be supported by the kr8s Python library used in deploy tests.
-        # This is independent of the host cluster version.
+        HOST_K8S_GIT_VERSION=$(kubectl version --output=json | jq -r '.serverVersion.gitVersion // empty')
+        if [[ ! "${HOST_K8S_GIT_VERSION}" =~ ^(v[0-9]+\.[0-9]+\.[0-9]+) ]]; then
+          echo "::error::Unable to resolve the host Kubernetes version from '${HOST_K8S_GIT_VERSION}'"
+          exit 1
+        fi
+        VCLUSTER_K8S_VERSION="${BASH_REMATCH[1]}"
+
+        # vCluster distro tags use upstream versions, so strip provider suffixes.
+        # Guest/host skew can break Pod status sync (loft-sh/vcluster#3578).
+        echo "Using host Kubernetes ${HOST_K8S_GIT_VERSION}; pinning vCluster guest to ${VCLUSTER_K8S_VERSION}"
         vcluster create "${VCLUSTER_NAME}" \
           --namespace "${NAMESPACE}" \
           --connect=false \
           --upgrade \
           --set controlPlane.distro.k8s.enabled=true \
-          --set controlPlane.distro.k8s.version=${{ inputs.vcluster_k8s_version }} \
+          --set controlPlane.distro.k8s.version="${VCLUSTER_K8S_VERSION}" \
           "${VCLUSTER_HELM_ARGS[@]}"
         echo "::endgroup::"
 
@@ -438,7 +442,6 @@ runs:
         helm upgrade --install dynamo-platform . --namespace default \
           "${IMAGE_ARGS[@]}" \
           --set dynamo-operator.gpuDiscovery.enabled=false \
-          --set dynamo-operator.webhook.service.publishNotReadyAddresses=true \
           "${CHECKPOINT_HELM_ARGS[@]}" \
           --set global.grove.install=true \
           --set global.kai-scheduler.install=false \
@@ -451,18 +454,6 @@ runs:
           --set 'nats.reloader.merge.resources.limits.cpu=100m' \
           --set 'nats.reloader.merge.resources.limits.memory=64Mi' \
           --debug
-
-        # Temporary vCluster CI workaround for #12266. The host/guest Kubernetes
-        # version skew can leave the translated host Pod's Ready condition stale
-        # after Grove is serving its webhook. Grove's pinned chart does not
-        # expose publishNotReadyAddresses, so patch only the short-lived CI
-        # Service. The readiness wait and admission smoke test below still prove
-        # that the actual webhook process is healthy before deploy tests start.
-        kubectl --kubeconfig=${{ github.workspace }}/.kubeconfig-vcluster \
-          patch service grove-operator \
-          --namespace default \
-          --type merge \
-          --patch '{"spec":{"publishNotReadyAddresses":true}}'
         echo "::endgroup::"
 
     - name: Wait for operator CRDs to be established
@@ -529,7 +520,6 @@ runs:
       shell: bash
       env:
         NAMESPACE: ${{ steps.resolve-names.outputs.namespace }}
-        VCLUSTER_NAME: ${{ steps.resolve-names.outputs.vcluster_name }}
       run: |
         set -euo pipefail
         VKUBECONFIG=${{ github.workspace }}/.kubeconfig-vcluster
@@ -542,64 +532,6 @@ runs:
           exit 1
         fi
 
-        echo "::group::Webhook Service routing override"
-        VIRTUAL_PUBLISH_NOT_READY=$(kubectl --kubeconfig="${VKUBECONFIG}" \
-          get services -n default \
-          -l app.kubernetes.io/component=webhook \
-          -o jsonpath='{.items[0].spec.publishNotReadyAddresses}')
-        if [ "${VIRTUAL_PUBLISH_NOT_READY}" != "true" ]; then
-          echo "::error::vCluster webhook Service does not publish not-ready addresses"
-          exit 1
-        fi
-
-        HOST_WEBHOOK_SERVICE=$(kubectl get services -n "${NAMESPACE}" \
-          -l app.kubernetes.io/component=webhook \
-          -o jsonpath='{.items[0].metadata.name}')
-        HOST_PUBLISH_NOT_READY=$(kubectl get service "${HOST_WEBHOOK_SERVICE}" \
-          -n "${NAMESPACE}" \
-          -o jsonpath='{.spec.publishNotReadyAddresses}')
-        if [ "${HOST_PUBLISH_NOT_READY}" != "true" ]; then
-          echo "::error::vCluster did not preserve publishNotReadyAddresses on the host webhook Service"
-          exit 1
-        fi
-        echo "Webhook Service publishes not-ready addresses in both virtual and host clusters"
-
-        GROVE_VIRTUAL_PUBLISH_NOT_READY=$(kubectl --kubeconfig="${VKUBECONFIG}" \
-          get service grove-operator \
-          -n default \
-          -o jsonpath='{.spec.publishNotReadyAddresses}')
-        if [ "${GROVE_VIRTUAL_PUBLISH_NOT_READY}" != "true" ]; then
-          echo "::error::Grove webhook Service does not publish not-ready addresses in the vCluster"
-          exit 1
-        fi
-
-        # The injected vCluster labels identify the translated Service without
-        # depending solely on application labels being preserved by the syncer.
-        GROVE_HOST_SELECTOR="vcluster.loft.sh/managed-by=${VCLUSTER_NAME},vcluster.loft.sh/namespace=default,app.kubernetes.io/name=grove-operator"
-        HOST_GROVE_SERVICE=
-        HOST_GROVE_PUBLISH_NOT_READY=
-        end=$((SECONDS + 120))
-        while true; do
-          HOST_GROVE_SERVICE=$(kubectl get services -n "${NAMESPACE}" \
-            -l "${GROVE_HOST_SELECTOR}" \
-            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
-          if [ -n "${HOST_GROVE_SERVICE}" ]; then
-            HOST_GROVE_PUBLISH_NOT_READY=$(kubectl get service "${HOST_GROVE_SERVICE}" \
-              -n "${NAMESPACE}" \
-              -o jsonpath='{.spec.publishNotReadyAddresses}' 2>/dev/null || true)
-            if [ "${HOST_GROVE_PUBLISH_NOT_READY}" = "true" ]; then
-              break
-            fi
-          fi
-          if [ "${SECONDS}" -ge "${end}" ]; then
-            echo "::error::vCluster did not preserve publishNotReadyAddresses on the host Grove webhook Service"
-            exit 1
-          fi
-          sleep 2
-        done
-        echo "Grove webhook Service publishes not-ready addresses in both virtual and host clusters"
-        echo "::endgroup::"
-
         echo "::group::Resolved operator image identity"
         kubectl get "${OPERATOR_POD}" -n "${NAMESPACE}" \
           -o jsonpath='{range .status.initContainerStatuses[*]}initContainer={.name} image={.image} imageID={.imageID}{"\n"}{end}{range .status.containerStatuses[*]}container={.name} image={.image} imageID={.imageID}{"\n"}{end}'

deploy/helm/charts/platform/README.md

@@ -168,7 +168,6 @@ Kubernetes: `>=1.30.0-0`
 | dynamo-operator.dynamo.serviceMesh.istio.caCertificates | string | `""` | Optional path (in the istio-proxy sidecar's filesystem) to CA certificates used to verify the EPP server certificate. Used only when tlsMode is "MUTUAL". |
 | dynamo-operator.dynamo.metrics.prometheusEndpoint | string | `""` | Endpoint that services can use to retrieve metrics. If set, dynamo operator will automatically inject the PROMETHEUS_ENDPOINT environment variable into services it manages. Users can override the value of the PROMETHEUS_ENDPOINT environment variable by modifying the corresponding deployment's environment variables |
 | dynamo-operator.dynamo.mpiRun.secretName | string | `"mpi-run-ssh-secret"` | Name of the secret containing the SSH key for MPI Run |
-| dynamo-operator.webhook.service.publishNotReadyAddresses | bool | `false` | Publish the webhook Service endpoint before the Pod Ready condition is true. Keep false for production; short-lived vCluster CI may enable this to tolerate stale host Pod readiness while admission smoke tests verify the webhook itself. |
 | dynamo-operator.webhook.certificateSecret.name | string | `"webhook-server-cert"` | Name of the Kubernetes secret containing webhook TLS certificates. The secret must contain three keys: tls.crt (server certificate), tls.key (server private key), and ca.crt (Certificate Authority certificate). |
 | dynamo-operator.webhook.certificateSecret.external | bool | `false` | Whether to manage the certificate secret externally. When false (default), the operator's built-in cert-controller generates and rotates certificates automatically. When true, you must create the secret manually before installing the chart. |
 | dynamo-operator.webhook.caBundle | string | `""` | CA bundle (base64 encoded) for webhook validation. Only used when certificateSecret.external=true. For automatic certificate generation or cert-manager integration, leave this empty as it will be injected automatically. |
@@ -186,7 +185,7 @@ Kubernetes: `>=1.30.0-0`
 | dynamo-operator.checkpoint.storage | object | `{}` | Optional PVC storage used when the snapshot-agent is installed outside workload namespaces with snapshot.storage.accessMode=podMount. Set create=true for operator-managed namespace PVCs, or omit/create=false to require an already-present PVC with the configured name. ReadWriteOnce can be used with podMount for sequential checkpoint/restore on suitable storage backends; use ReadWriteMany for concurrent multi-node access. |
 | grove.tolerations | list | `[]` | Node tolerations for Grove pods |
 | grove.affinity | object | `{}` | Affinity for Grove pods |
-| grove.config.server.healthProbes.enable | bool | `true` | Enable Grove's webhook-aware liveness and readiness probes. The readiness endpoint stays false until certificates and the webhook server are ready. vCluster CI temporarily bypasses this signal only for Service routing: its setup gate still waits for this container probe, which will also resume gating Service endpoints after that workaround is removed. |
+| grove.config.server.healthProbes.enable | bool | `true` | Enable Grove's webhook-aware liveness and readiness probes. The readiness endpoint stays false until certificates and the webhook server are ready. |
 | kai-scheduler.global.tolerations | list | `[]` | Node tolerations for kai-scheduler pods |
 | kai-scheduler.global.affinity | object | `{}` | Affinity for kai-scheduler pods |
 | etcd.image.repository | string | `"bitnamilegacy/etcd"` | following bitnami announcement for brownout - https://github.com/bitnami/charts/tree/main/bitnami/etcd#%EF%B8%8F-important-notice-upcoming-changes-to-the-bitnami-catalog, we need to use the legacy repository until we migrate to the new "secure" repository |

deploy/helm/charts/platform/components/operator/templates/webhook-service.yaml

@@ -25,7 +25,6 @@ metadata:
     app.kubernetes.io/part-of: dynamo-operator
   {{- include "dynamo-operator.labels" . | nindent 4 }}
 spec:
-  publishNotReadyAddresses: {{ .Values.webhook.service.publishNotReadyAddresses }}
   ports:
   - port: 443
     protocol: TCP

deploy/helm/charts/platform/components/operator/values.yaml

@@ -189,12 +189,6 @@ checkpoint:
 
 # Webhook configuration
 webhook:
-  service:
-    # -- Publish the webhook Service endpoint before the Pod Ready condition is true.
-    # Keep false for production; short-lived vCluster CI may enable this to tolerate
-    # stale host Pod readiness while admission smoke tests verify the webhook itself.
-    publishNotReadyAddresses: false
-
   # Certificate configuration
   certificateSecret:
     # Name of the Kubernetes Secret holding webhook TLS certificates.

deploy/helm/charts/platform/values.yaml

@@ -218,12 +218,6 @@ dynamo-operator:
 
   # Webhook configuration for admission control and validation
   webhook:
-    service:
-      # -- Publish the webhook Service endpoint before the Pod Ready condition is true.
-      # Keep false for production; short-lived vCluster CI may enable this to tolerate
-      # stale host Pod readiness while admission smoke tests verify the webhook itself.
-      publishNotReadyAddresses: false
-
     # Certificate configuration for webhook TLS
     certificateSecret:
       # -- Name of the Kubernetes secret containing webhook TLS certificates. The secret must contain three keys: tls.crt (server certificate), tls.key (server private key), and ca.crt (Certificate Authority certificate).
@@ -297,9 +291,6 @@ grove:
       healthProbes:
         # -- Enable Grove's webhook-aware liveness and readiness probes.
         # The readiness endpoint stays false until certificates and the webhook server are ready.
-        # vCluster CI temporarily bypasses this signal only for Service routing:
-        # its setup gate still waits for this container probe, which will also
-        # resume gating Service endpoints after that workaround is removed.
         enable: true
 
 # Kai Scheduler component - advanced workload scheduling