fix(ci): recover deploy tests from vcluster tunnel drops

.dockerignore

@@ -25,13 +25,15 @@
 **/*.libtorch_model_store/*
 **/.git
 **/.github
-# Planner test images exercise the version-stamping script without copying the
-# rest of the GitHub metadata into every Docker build context.
+# Test images exercise these scripts without copying the rest of the GitHub
+# metadata into every Docker build context.
 !.github
 .github/*
 !.github/scripts
 .github/scripts/*
 !.github/scripts/apply_dev_version.py
+!.github/scripts/retry_kubectl.sh
+!.github/scripts/collect_vcluster_diagnostics.sh
 **/*backup*/
 .dockerignore
 **/target/*

.github/actions/connect-vcluster/action.yml

@@ -49,7 +49,8 @@ runs:
         KUBECONFIG_TMP="${KUBECONFIG_PATH}.tmp"
 
         start_port_forward() {
-          kubectl port-forward -n "${NAMESPACE}" "svc/${VCLUSTER_NAME}" 8443:443 > "${PF_LOG}" 2>&1 &
+          printf '\n[%s] Starting vCluster API port-forward\n' "$(date -u +%FT%TZ)" >> "${PF_LOG}"
+          kubectl port-forward -n "${NAMESPACE}" "svc/${VCLUSTER_NAME}" 8443:443 >> "${PF_LOG}" 2>&1 &
           PF_PID=$!
           echo "${PF_PID}" > "${PF_PID_FILE}"
           echo "Started vCluster API port-forward pid ${PF_PID}; log: ${PF_LOG}"
@@ -143,7 +144,8 @@ runs:
             vcluster_name="$4"
 
             start_pf() {
-              kubectl port-forward -n "${namespace}" "svc/${vcluster_name}" 8443:443 > "${pf_log}" 2>&1 &
+              printf "\n[%s] Watchdog restarting vCluster API port-forward\n" "$(date -u +%FT%TZ)" >> "${pf_log}"
+              kubectl port-forward -n "${namespace}" "svc/${vcluster_name}" 8443:443 >> "${pf_log}" 2>&1 &
               echo "$!" > "${pf_pid_file}"
             }
 
@@ -159,7 +161,7 @@ runs:
               fi
               sleep 5
             done
-          ' _ "${PF_PID_FILE}" "${PF_LOG}" "${NAMESPACE}" "${VCLUSTER_NAME}" > "${WATCHDOG_LOG}" 2>&1 &
+          ' _ "${PF_PID_FILE}" "${WATCHDOG_LOG}" "${NAMESPACE}" "${VCLUSTER_NAME}" > "${WATCHDOG_LOG}" 2>&1 &
           echo "$!" > "${WATCHDOG_PID_FILE}"
           echo "Started vCluster API port-forward watchdog; log: ${WATCHDOG_LOG}"
         fi

.github/actions/dynamo-deploy-test/action.yml

@@ -157,7 +157,8 @@ runs:
           namespaces="${NAMESPACE},default"
         fi
 
-        mkdir -p "${DYN_TEST_OUTPUT_PATH}"
+        bash "${{ github.workspace }}/.github/scripts/collect_vcluster_diagnostics.sh" "${DYN_TEST_OUTPUT_PATH}"
+
         kubectl cluster-info dump \
           --namespaces="${namespaces}" \
           --output-directory="${DYN_TEST_OUTPUT_PATH}/cluster-info" || true

.github/scripts/collect_vcluster_diagnostics.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+output_dir="$1"
+mkdir -p "$output_dir"
+for log_name in vcluster-port-forward.log vcluster-port-forward-watchdog.log; do
+    if [ -f "${GITHUB_WORKSPACE}/.${log_name}" ]; then
+        # Non-hidden names ensure upload-artifact includes the tunnel logs.
+        cp "${GITHUB_WORKSPACE}/.${log_name}" "${output_dir}/${log_name}"
+    fi
+done
+{
+    echo "captured_at=$(date -u +%FT%TZ)"
+    curl -skS --connect-timeout 1 --max-time 2 \
+        -o /dev/null -w 'healthz_http_code=%{http_code}\n' \
+        https://127.0.0.1:8443/healthz || echo "healthz_unreachable=true"
+    for pid_name in vcluster-port-forward.pid vcluster-port-forward-watchdog.pid; do
+        if [ -f "${GITHUB_WORKSPACE}/.${pid_name}" ]; then
+            pid=$(cat "${GITHUB_WORKSPACE}/.${pid_name}")
+            echo "${pid_name}=${pid}"
+            ps -p "$pid" -o pid=,ppid=,stat=,etime=,command= || true
+        else
+            echo "${pid_name}=missing"
+        fi
+    done
+} > "${output_dir}/vcluster-port-forward-status.log" 2>&1

.github/scripts/retry_kubectl.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Retry idempotent kubectl operations while the vCluster watchdog restores its
+# tunnel. Use -f <file>, not stdin, so every attempt receives the same manifest.
+retry_kubectl() (
+    local error_log attempt status
+    error_log=$(mktemp)
+    trap 'rm -f "$error_log"' EXIT
+    for attempt in 1 2 3 4; do
+        if kubectl --request-timeout=30s "$@" 2>"$error_log"; then
+            cat "$error_log" >&2
+            return 0
+        else
+            status=$?
+        fi
+        cat "$error_log" >&2
+        if (( attempt == 4 )) || ! grep -Eiq \
+            'Unable to connect to the server:.*(EOF|connection refused|connection reset|i/o timeout|TLS handshake timeout)|The connection to the server .* was refused|read: connection reset by peer' \
+            "$error_log"; then
+            return "$status"
+        fi
+        echo "vCluster connection lost; retrying in 5s ($attempt/3)" >&2
+        sleep 5
+    done
+)

.github/workflows/shared-dgdr-deploy-test.yml

@@ -121,25 +121,32 @@ jobs:
           KUBECONFIG: ${{ github.workspace }}/.kubeconfig-vcluster
         run: |
           set -euo pipefail
-          kubectl delete namespace "${DGDR_NAMESPACE}" \
+          source .github/scripts/retry_kubectl.sh
+          retry_kubectl delete namespace "${DGDR_NAMESPACE}" \
             --ignore-not-found --wait=true --timeout=5m
+          manifest=$(mktemp)
+          chmod 600 "$manifest"
+          trap 'rm -f "$manifest"' EXIT
           kubectl create namespace "${DGDR_NAMESPACE}" \
-            --dry-run=client -o yaml | kubectl apply -f -
+            --dry-run=client -o yaml > "$manifest"
+          retry_kubectl apply -f "$manifest"
           kubectl create secret generic hf-token-secret \
             --namespace="${DGDR_NAMESPACE}" \
             --from-literal=HF_TOKEN="${HF_TOKEN}" \
-            --dry-run=client -o yaml | kubectl apply -f -
+            --dry-run=client -o yaml > "$manifest"
+          retry_kubectl apply -f "$manifest"
 
       - name: Verify DGDR test prerequisites
         shell: bash
         env:
           KUBECONFIG: ${{ github.workspace }}/.kubeconfig-vcluster
         run: |
           set -euo pipefail
-          kubectl wait --for=condition=established \
+          source .github/scripts/retry_kubectl.sh
+          retry_kubectl wait --for=condition=established \
             crd/dynamographdeploymentrequests.nvidia.com \
             --timeout=120s
-          kubectl get secret hf-token-secret -n "${DGDR_NAMESPACE}" >/dev/null
+          retry_kubectl get secret hf-token-secret -n "${DGDR_NAMESPACE}" >/dev/null
 
       - name: Run CPU DGDR tests
         uses: $/.github/actions/dynamo-deploy-test
@@ -162,5 +169,21 @@ jobs:
         env:
           KUBECONFIG: ${{ github.workspace }}/.kubeconfig-vcluster
         run: |
-          kubectl delete namespace "${DGDR_NAMESPACE}" \
+          set -euo pipefail
+          source .github/scripts/retry_kubectl.sh
+          retry_kubectl delete namespace "${DGDR_NAMESPACE}" \
             --ignore-not-found --wait=true --timeout=5m
+
+      - name: Collect vCluster diagnostics on failure
+        if: failure()
+        shell: bash
+        run: bash .github/scripts/collect_vcluster_diagnostics.sh "${RUNNER_TEMP}/dgdr-vcluster-diagnostics"
+
+      - name: Upload vCluster diagnostics
+        if: failure()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
+        with:
+          name: vcluster-diagnostics-dgdr-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/dgdr-vcluster-diagnostics/
+          retention-days: 7
+          if-no-files-found: warn

container/Dockerfile.test

@@ -37,10 +37,12 @@ RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/
 COPY --chmod=664 --chown=dynamo:0 pyproject.toml /workspace/pyproject.toml
 COPY --chmod=664 --chown=dynamo:0 tests/__init__.py /workspace/tests/__init__.py
 COPY --chmod=664 --chown=dynamo:0 tests/marker_categories.py /workspace/tests/marker_categories.py
-# The nightly version-stamping tests execute this script against temporary
-# fixture repositories. Keep the production image unchanged and expose only
-# this script in the test image.
-COPY --chmod=664 --chown=dynamo:0 .github/scripts/apply_dev_version.py /workspace/.github/scripts/apply_dev_version.py
+# Version-stamping and shell regression tests execute these workflow helpers.
+# COPY also creates parent directories: keep them traversable by ARC's uid 1001.
+COPY --chmod=755 --chown=dynamo:0 \
+    .github/scripts/apply_dev_version.py \
+    .github/scripts/retry_kubectl.sh \
+    .github/scripts/collect_vcluster_diagnostics.sh /workspace/.github/scripts/
 # Required: blocks dynamo.vllm/sglang from shadowing the installed packages.
 COPY --chmod=664 --chown=dynamo:0 conftest.py /workspace/conftest.py
 COPY --chmod=775 --chown=dynamo:0 benchmarks/ /workspace/benchmarks/