fix(nvsnap): repair install path - chart drift, disabled L2 fan-out, unbuildable server

src/compute-plane-services/nvsnap/.gitignore

@@ -22,8 +22,13 @@ dist/
 # Output of the go coverage tool
 coverage/
 *.out
-*.html
-!ui/dist/**.html
+# Scoped to the coverage artifact rather than a bare *.html: the broad
+# pattern silently swallowed ui/index.html, Vite's entry point, so
+# nvsnap-server could not be built from a clean checkout. The
+# !ui/dist/**.html negation that used to sit here was a workaround for
+# the same over-broad rule and is no longer needed.
+coverage.html
+*.cover.html
 
 # Go workspace
 go.work

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/post-install-smoke.yaml

@@ -88,6 +88,17 @@ rules:
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list"]
+  # StorageClasses are cluster-scoped; step 7 reads one to confirm the L2
+  # class an operator named actually exists. Without this the check would
+  # fail on RBAC rather than on the thing it is testing.
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list"]
+  # Same reasoning for the VolumeSnapshotClass step 7 checks when
+  # agent.l2.snapshotClass is set.
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -203,6 +214,111 @@ spec:
                 echo "  OK ($READY/$DESIRED agents ready)"
               fi
 
+              # Steps 1-5 above prove the components are alive. Everything
+              # below proves they are wired to each other and configured the
+              # way the operator asked. Liveness checks pass happily through
+              # a stale agent image, a silently disabled L2 tier, and a
+              # server that cannot authenticate to its own agents — all three
+              # shipped undetected and are what steps 6-9 exist to catch.
+
+              echo "--- [6/9] agent image matches this release ---"
+              # nvsnap#731: the chart pinned an agent 31 versions behind
+              # versions.sh and every liveness probe stayed green, because a
+              # stale agent is a perfectly healthy agent.
+              WANT="{{ include "nvsnap.agent.image" . }}"
+              GOT=$(kubectl get ds nvsnap-agent -n {{ .Release.Namespace }} -o jsonpath='{.spec.template.spec.containers[0].image}')
+              if [ "$GOT" != "$WANT" ]; then
+                echo "  FAIL: deployed $GOT, chart expects $WANT"
+                FAIL=1
+              else
+                echo "  OK ($GOT)"
+              fi
+
+              echo "--- [7/9] L2 fan-out configuration ---"
+              # An empty storageClass disables L2 and degrades restore to the
+              # L3 peer cascade. That is a legitimate configuration, but it
+              # must be a decision rather than an accident, so say which it is.
+              {{- if .Values.agent.l2.storageClass }}
+              if kubectl get storageclass {{ .Values.agent.l2.storageClass | quote }} >/dev/null 2>&1; then
+                echo "  OK (L2 enabled on StorageClass {{ .Values.agent.l2.storageClass }})"
+                {{- if .Values.agent.l2.snapshotClass }}
+                # A named snapshot class that does not exist is worse than none:
+                # the snapshot-clone promote silently has nowhere to go. An unset
+                # one is NOT an error — internal/checkpointstore/storage_profile.go
+                # only needs it for StrategySnapshotClone (GKE PD, EBS).
+                # SharedVolumePromoter provisioners (nvmesh, EFS) promote
+                # zero-copy and never take a snapshot.
+                if ! kubectl get volumesnapshotclass {{ .Values.agent.l2.snapshotClass | quote }} >/dev/null 2>&1; then
+                  echo "  FAIL: agent.l2.snapshotClass={{ .Values.agent.l2.snapshotClass }} does not exist"
+                  FAIL=1
+                else
+                  echo "  OK (snapshot class {{ .Values.agent.l2.snapshotClass }} present)"
+                fi
+                {{- else }}
+                PROV=$(kubectl get storageclass {{ .Values.agent.l2.storageClass | quote }} -o jsonpath='{.provisioner}' 2>/dev/null)
+                echo "  NOTE: no snapshotClass set; provisioner $PROV must be one that promotes without snapshots"
+                {{- end }}
+              else
+                echo "  FAIL: agent.l2.storageClass={{ .Values.agent.l2.storageClass }} does not exist"
+                FAIL=1
+              fi
+              {{- else }}
+              echo "  WARN: L2 disabled (agent.l2.storageClass unset) — restore falls back to the L3 peer cascade"
+              {{- end }}
+
+              {{- if (.Values.agent.auth).enabled }}
+              echo "--- [8/9] agent token reaches every client ---"
+              # nvsnap#736: nvsnap-server calls the agent API but had no
+              # notion of the token. Under --auth-mode=required its cascade
+              # delete 401'd, returned 204 anyway, dropped the catalog row and
+              # orphaned the dump. Nothing was unhealthy; the wiring was just
+              # incomplete. Assert every component that talks to the agent
+              # actually carries the credential.
+              if ! kubectl get secret nvsnap-agent-token -n {{ .Release.Namespace }} >/dev/null 2>&1; then
+                echo "  FAIL: Secret nvsnap-agent-token missing while auth is enabled"
+                FAIL=1
+              else
+                for pair in "ds/nvsnap-agent:agent"{{ if .Values.server.enabled }}" deploy/nvsnap-server:server"{{ end }}; do
+                  obj="${pair%%:*}"; who="${pair##*:}"
+                  # Check where the value comes from, not just that the name is
+                  # present. A literal, or a secretKeyRef aimed at the wrong
+                  # Secret or key, produces a token that does not match the one
+                  # the agent verifies against — which fails exactly like having
+                  # no token at all, but looks correct in a name-only check.
+                  SRC=$(kubectl get "$obj" -n {{ .Release.Namespace }} -o jsonpath='
+                    {range .spec.template.spec.containers[0].env[?(@.name=="NVSNAP_AGENT_TOKEN")]}
+                    {.valueFrom.secretKeyRef.name}/{.valueFrom.secretKeyRef.key}{end}' 2>/dev/null | tr -d ' \n')
+                  if [ "$SRC" = "nvsnap-agent-token/token" ]; then
+                    echo "  OK ($who reads NVSNAP_AGENT_TOKEN from nvsnap-agent-token/token)"
+                  elif [ -z "$SRC" ] || [ "$SRC" = "/" ]; then
+                    echo "  FAIL: $who has no NVSNAP_AGENT_TOKEN secretKeyRef — its agent calls will 401"
+                    FAIL=1
+                  else
+                    echo "  FAIL: $who reads NVSNAP_AGENT_TOKEN from $SRC, want nvsnap-agent-token/token"
+                    FAIL=1
+                  fi
+                done
+              fi
+
+              echo "--- [9/9] auth is enforced, not merely configured ---"
+              # Prove the guard actually runs: a token-less API call must be
+              # rejected in required mode, and a token-bearing one accepted.
+              # Configuration that is present but not enforced looks identical
+              # to configuration that works.
+              AGENT_URL="http://nvsnap-agent.{{ .Release.Namespace }}.svc.cluster.local:8081"
+              # `|| true` so a missing Secret reports a FAIL below rather than
+              # killing the script under `set -e` and losing the remaining checks.
+              TOK=$(kubectl get secret nvsnap-agent-token -n {{ .Release.Namespace }} -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null || true)
+              AUTHED=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 -H "Authorization: Bearer $TOK" "$AGENT_URL/v1/checkpoints" || echo 000)
+              [ "$AUTHED" = "200" ] && echo "  OK (authenticated call accepted)" \
+                || { echo "  FAIL: authenticated call returned $AUTHED, want 200"; FAIL=1; }
+              {{- if eq ((.Values.agent.auth).mode | default "") "required" }}
+              ANON=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$AGENT_URL/v1/checkpoints" || echo 000)
+              [ "$ANON" = "401" ] && echo "  OK (anonymous call rejected)" \
+                || { echo "  FAIL: anonymous call returned $ANON, want 401 — required mode is not enforcing"; FAIL=1; }
+              {{- end }}
+              {{- end }}
+
               echo
               if [ "$FAIL" -ne 0 ]; then
                 echo "=== SMOKE TEST FAILED ==="

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml

@@ -137,7 +137,7 @@ agent:
     # is no AppVersion fallback (see _helpers.tpl) — empty tag fails
     # the chart render with a clear error rather than templating a
     # known-broken image ref.
-    tag: "v0.1.3"
+    tag: "v0.2.32"
     pullPolicy: Always
 
   # nodeSelector is intentionally empty — GPU detection runs through

src/compute-plane-services/nvsnap/scripts/install-nvsnap.sh

@@ -227,6 +227,48 @@ else
     info "skipping cert-manager + webhook (--without-webhook)"
 fi
 
+# L2 per-capture PVC fan-out is off unless agent.l2.storageClass names an
+# RWX-capable class. Left empty the install still succeeds, but restore
+# fan-out silently degrades to the L3 peer cascade — a large throughput
+# difference that only shows up under multi-node fan-out, long after the
+# installer has printed a clean banner. Say so at install time.
+#
+# Reported, not auto-selected: picking the wrong class yields PVCs that
+# never bind, and the right choice depends on cluster topology. RWX
+# capability isn't exposed on the StorageClass API, so candidates are
+# matched on known RWX provisioners.
+#
+# Only --set is inspected here. An operator supplying the value through
+# -f/--values is passing a file this script does not parse, so stay quiet
+# rather than claim L2 is off on evidence we do not have.
+l2_configured=false
+printf '%s\n' "${EXTRA_HELM_ARGS[@]}" | grep -q "agent.l2.storageClass=" && l2_configured=true
+printf '%s\n' "${EXTRA_HELM_ARGS[@]}" | grep -qE '^(-f|--values)$' && l2_configured=true
+
+if [ "$l2_configured" = false ]; then
+    rwx_re='smb\.csi|nfs\.csi|efs\.csi|filestore\.csi|azurefile|excelero|nvmesh|cephfs'
+    # Keep the query's exit status: a denied or unreachable API returns
+    # nothing, which is indistinguishable from "no RWX classes exist" unless
+    # the failure is recorded separately. Reporting the wrong one of those two
+    # sends the operator to provision storage they may already have.
+    sc_ok=true
+    sc_list=$(kubectl get storageclass -o jsonpath='{range .items[*]}{.metadata.name}{" ("}{.provisioner}{")"}{"\n"}{end}' 2>/dev/null) || sc_ok=false
+    candidates=$(printf '%s\n' "$sc_list" | grep -iE "$rwx_re" || true)
+
+    echo "     WARNING: agent.l2.storageClass is unset — L2 per-capture PVC fan-out is DISABLED." >&2
+    echo "              Restore falls back to the L3 peer cascade (slower multi-node fan-out)." >&2
+    if [ "$sc_ok" = false ]; then
+        echo "              Unable to inspect StorageClasses (query failed); cannot list candidates." >&2
+    elif [ -n "$candidates" ]; then
+        echo "              RWX-capable StorageClasses on this cluster:" >&2
+        echo "$candidates" | sed 's/^/                /' >&2
+        echo "              Enable with: --set agent.l2.storageClass=<name>" >&2
+    else
+        echo "              No RWX-capable StorageClass detected; L2 needs one provisioned first." >&2
+        echo "              Reference: deploy/k8s/nvcf-cluster-prep/storage-classes.yaml" >&2
+    fi
+fi
+
 # ─── 6. Helm install ───────────────────────────────────────────────────
 
 step "[6/7] helm install / upgrade nvsnap"

src/compute-plane-services/nvsnap/scripts/sync-versions.sh

@@ -27,6 +27,38 @@ source "$(dirname "${BASH_SOURCE[0]}")/versions.sh"
 
 DIRS=(deploy/k8s deploy)
 
+# Helm values files spell an image as split `repository:` / `tag:` fields
+# rather than one registry/name:tag token, so the substitution below can
+# never see them and the grep-based check below can never flag them. They
+# are handled separately by chart_tag()/set_chart_tag().
+CHART_VALUES=(deploy/helm/nvsnap/values.yaml)
+
+# Print the tag a chart values file pins for <image-name>, or nothing when
+# that repository isn't present. \042 and \047 are " and ' — spelled in
+# octal so this awk program survives shell quoting intact.
+chart_tag() {
+    awk -v name="$2" '
+        $1 == "repository:" { pending = ($2 == name) }
+        pending && $1 == "tag:" { gsub(/[\042\047]/, "", $2); print $2; exit }
+    ' "$1"
+}
+
+# Rewrite the tag a chart values file pins for <image-name>, preserving the
+# original indentation.
+set_chart_tag() {
+    local file="$1"
+    awk -v name="$2" -v ver="$3" '
+        $1 == "repository:" { pending = ($2 == name) }
+        pending && $1 == "tag:" {
+            match($0, /^[ \t]*/)
+            print substr($0, 1, RLENGTH) "tag: \"" ver "\""
+            pending = 0
+            next
+        }
+        { print }
+    ' "$file" > "$file.tmp" && mv "$file.tmp" "$file"
+}
+
 # image-name → version-var
 declare -A IMAGES=(
     [nvsnap-agent]="$NVSNAP_APP_VERSION"
@@ -47,7 +79,19 @@ for name in "${!IMAGES[@]}"; do
     # nvsnap-agent-base (different prefix) and nvsnap-init from nvsnap-init-config.
     sed_re="s|[^[:space:]\"']*/${name}:[^[:space:]\"']*|${new}|g"
     for dir in "${DIRS[@]}"; do
-        find "$dir" -name "*.yaml" -exec sed -i -E "${sed_re}" {} \;
+        # Skip chart templates. They never carry a literal image reference --
+        # the chart reads images from values.yaml, handled by set_chart_tag
+        # below -- so there is nothing here to sync, and the pattern is loose
+        # enough to corrupt ordinary text. It rewrote the shell literal
+        # "ds/nvsnap-agent:agent" in post-install-smoke.yaml into a full image
+        # ref, which broke the smoke test and, because the smoke test gates
+        # the release, made every subsequent install and upgrade fail.
+        find "$dir" -name "*.yaml" -not -path "*/templates/*" \
+            -exec sed -i -E "${sed_re}" {} \;
+    done
+    for f in "${CHART_VALUES[@]}"; do
+        [ -f "$f" ] && [ -n "$(chart_tag "$f" "$name")" ] || continue
+        set_chart_tag "$f" "$name" "$ver"
     done
     echo "Synced ${name} -> ${new}"
 done
@@ -58,7 +102,10 @@ fail=0
 for name in "${!IMAGES[@]}"; do
     ver="${IMAGES[$name]}"
     expected="${NVSNAP_REGISTRY}/${name}:${ver}"
-    if remaining=$(grep -rn "/${name}:" "${DIRS[@]}" 2>/dev/null | grep -v "${expected}" || true); then
+    # Same templates exclusion as the substitution above, or this reports the
+    # shell literals it deliberately no longer rewrites as stale references.
+    if remaining=$(grep -rn "/${name}:" "${DIRS[@]}" 2>/dev/null \
+                   | grep -v '/templates/' | grep -v "${expected}" || true); then
         if [ -n "$remaining" ]; then
             echo "WARNING: stale ${name} references found:" >&2
             echo "$remaining" >&2
@@ -67,4 +114,28 @@ for name in "${!IMAGES[@]}"; do
     fi
 done
 
+# Same check for chart values. The grep above matches a combined
+# registry/name:tag token, which split repository:/tag: fields never form —
+# so without this loop a drifting chart tag passes verification silently.
+# That is exactly how the chart shipped nvsnap-agent v0.1.3 against an
+# NVSNAP_APP_VERSION of v0.2.32 (nvsnap#731).
+for f in "${CHART_VALUES[@]}"; do
+    [ -f "$f" ] || continue
+    for name in "${!IMAGES[@]}"; do
+        actual=$(chart_tag "$f" "$name")
+        [ -n "$actual" ] || continue
+        if [ "$actual" != "${IMAGES[$name]}" ]; then
+            echo "WARNING: ${f} pins ${name} tag ${actual}, expected ${IMAGES[$name]}" >&2
+            fail=1
+        fi
+    done
+    # The chart builds refs as <imageRegistry>/<repository>:<tag>, so a
+    # drifting registry breaks every image at once.
+    registry=$(awk '$1 == "imageRegistry:" { print $2; exit }' "$f")
+    if [ -n "$registry" ] && [ "$registry" != "$NVSNAP_REGISTRY" ]; then
+        echo "WARNING: ${f} imageRegistry is ${registry}, expected ${NVSNAP_REGISTRY}" >&2
+        fail=1
+    fi
+done
+
 exit $fail

src/compute-plane-services/nvsnap/ui/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en" class="dark">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>NvSnap · GPU Capture &amp; Restore</title>
+    <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
+  </head>
+  <body class="bg-[#0a0a0f] text-[#e4e4e7]">
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>