feat(nvsnap): shared-token authentication for the agent API

src/compute-plane-services/nvsnap/cmd/agent/main.go

@@ -71,6 +71,8 @@ func main() {
 		"NvSnap-server base URL for peer-fanout catalog lookups (e.g. http://nvsnap-server.nvsnap-system.svc.cluster.local:8080). Empty disables cross-node cascade.")
 	flag.StringVar(&config.NodeIP, "node-ip", os.Getenv("HOST_IP"),
 		"This agent's reachable address from peers (downward API status.hostIP when hostNetwork:true). Empty disables peer registration.")
+	flag.StringVar(&config.AdvertiseIP, "advertise-ip", os.Getenv("POD_IP"),
+		"Address peers dial to reach this agent (downward API status.podIP; equals the node IP under hostNetwork). Falls back to --node-ip when empty. See GH #490.")
 	flag.StringVar(&config.BlobStoreURL, "blob-store-url", os.Getenv("NVSNAP_BLOB_STORE_URL"),
 		"NvSnap-blobstore base URL for Phase 5d.2 durable backstop (e.g. http://nvsnap-blobstore.nvsnap-system.svc.cluster.local:9000). Empty disables capture-side upload AND cascade tier-3 fallback.")
 	// Cross-cluster replication (docs/design/cross-cluster-replication.md).
@@ -90,6 +92,11 @@ func main() {
 	flag.StringVar(&config.FSStorePath, "fsstore-path", os.Getenv("NVSNAP_FSSTORE_PATH"),
 		"Path to a shared filesystem mounted on every node (Lustre/Weka/EFS/Filestore/NFS). When set, captures are published here and the restore cascade copies from this path before peer fanout. Empty disables.")
 	flag.StringVar(&config.ListenAddr, "listen", ":8081", "Listen address")
+	// The token itself is env-only, never a flag: flag values show up in the
+	// pod spec and in `ps`, and this is a credential.
+	var authMode string
+	flag.StringVar(&authMode, "auth-mode", os.Getenv("NVSNAP_AGENT_AUTH_MODE"),
+		"Agent API authentication: disabled (default), permissive (check, log failures, still serve), or required (401). Token comes from NVSNAP_AGENT_TOKEN. See GH #486.")
 	flag.StringVar(&config.CheckpointDir, "checkpoint-dir", "/var/lib/nvsnap/checkpoints", "Checkpoint storage directory (in-agent-container path)")
 	flag.StringVar(&config.CheckpointHostDir, "checkpoint-host-dir", "/var/lib/containerd/nvsnap-checkpoints", "Host path that backs --checkpoint-dir (must match the DaemonSet hostPath mount; used to translate paths for the capture-write writer Job)")
 	flag.StringVar(&config.CRIUPath, "criu-path", "/usr/local/sbin/criu", "Path to CRIU binary (on host filesystem)")
@@ -183,6 +190,8 @@ func main() {
 		"Strategy for restore-side overlay mount prep: inline (do mounts during admission, default) or init-container (delegate to nvsnap-mount-prep init container on the restored pod)")
 	flag.StringVar(&config.Webhook.MountPrepInitImage, "webhook-mount-prep-init-image", "",
 		"Image ref for the nvsnap-mount-prep init container injected when --webhook-restore-prep-strategy=init-container. Must contain /nvsnap-mount-prep (the agent image satisfies this).")
+	flag.StringVar(&config.Webhook.AgentBaseURL, "webhook-agent-base-url", os.Getenv("NVSNAP_WEBHOOK_AGENT_BASE_URL"),
+		"Base URL the injected nvsnap-mount-prep init container uses to reach its node-local agent. Empty uses http://$(NVSNAP_HOST_IP):<port>, which requires hostPort. Set to the internalTrafficPolicy:Local Service under pod networking. See GH #490.")
 	flag.IntVar(&config.Webhook.AgentHostPort, "webhook-agent-host-port", 8081,
 		"Port the nvsnap-mount-prep init container reaches the agent on (matches --listen and the agent DaemonSet's hostPort).")
 
@@ -204,6 +213,19 @@ func main() {
 		"imagePullSecret name for the mount-holder pod (created by operators in the workload namespace). Defaults to nvsnap-agent-pull; set to '-' to disable.")
 
 	flag.Parse()
+
+	// Fail startup on a bad mode rather than falling back to disabled: an
+	// operator who typo'd --auth-mode should hear about it now, not discover
+	// months later that the API was open the whole time.
+	var authErr error
+	if config.AuthMode, authErr = agent.ParseAuthMode(authMode); authErr != nil {
+		logrus.WithError(authErr).Fatal("invalid --auth-mode")
+	}
+	config.AuthToken = os.Getenv("NVSNAP_AGENT_TOKEN")
+	if config.AuthMode != agent.AuthDisabled && config.AuthToken == "" {
+		logrus.Fatalf("--auth-mode=%s requires NVSNAP_AGENT_TOKEN to be set", config.AuthMode)
+	}
+
 	config.RootfsCapture.WarmupDelay = time.Duration(rootfsWarmupSec) * time.Second
 	for _, b := range strings.Split(replicationPeerBuckets, ",") {
 		if b = strings.TrimSpace(b); b != "" {

src/compute-plane-services/nvsnap/cmd/nvsnap-mount-prep/main.go

@@ -40,6 +40,9 @@ limitations under the License.
 //	NVSNAP_POD_UID         (required) downward API: metadata.uid
 //	NVSNAP_RESTORE_HASH    (required) full sha256 of the capture
 //	NVSNAP_AGENT_URL       (required) e.g. http://$(HOST_IP):8081
+//	NVSNAP_AGENT_TOKEN     (optional) bearer token for the agent API (GH #486);
+//	                       empty sends no header, which is correct while the
+//	                       agent still runs with auth disabled
 //	NVSNAP_CAPTURE_NODE    (optional) where capture data lives; empty=this node
 //	NVSNAP_PREP_MOUNTS     (required) JSON-encoded []VolumeMeta from the manifest
 //	NVSNAP_PREP_DEADLINE   (optional) duration; default 15m
@@ -196,6 +199,7 @@ func startWithRetry(agentURL string, req prepRequest) error {
 			return err
 		}
 		httpReq.Header.Set("Content-Type", "application/json")
+		setAgentAuth(httpReq)
 		resp, err := http.DefaultClient.Do(httpReq)
 		if err != nil {
 			lastErr = err
@@ -220,6 +224,7 @@ func startWithRetry(agentURL string, req prepRequest) error {
 
 func getStatus(agentURL, podUID string) (*prepStatus, error) {
 	httpReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, agentURL+"/v1/restore/prep/"+podUID, http.NoBody)
+	setAgentAuth(httpReq)
 	if err != nil {
 		return nil, err
 	}
@@ -244,3 +249,15 @@ func getStatus(agentURL, podUID string) (*prepStatus, error) {
 	}
 	return &s, nil
 }
+
+// setAgentAuth attaches the agent API bearer token when one is configured.
+// Empty is the normal state until the operator turns auth on, and sending no
+// header is exactly what a disabled or permissive agent expects. See GH #486.
+func setAgentAuth(r *http.Request) {
+	if r == nil {
+		return
+	}
+	if tok := os.Getenv("NVSNAP_AGENT_TOKEN"); tok != "" {
+		r.Header.Set("Authorization", "Bearer "+tok)
+	}
+}

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml

@@ -42,9 +42,9 @@ spec:
         {{- toYaml .Values.agent.tolerations | nindent 8 }}
       hostPID: {{ .Values.agent.hostPID }}
       hostNetwork: {{ .Values.agent.hostNetwork }}
-      # ClusterFirstWithHostNet so Service DNS still resolves under
-      # hostNetwork (nvsnap-server.<ns>.svc, nvsnap-blobstore.<ns>.svc).
-      dnsPolicy: ClusterFirstWithHostNet
+      # ClusterFirstWithHostNet is only correct under hostNetwork; with pod
+      # networking it is wrong (it points resolution at the node's resolv.conf).
+      dnsPolicy: {{ if .Values.agent.hostNetwork }}ClusterFirstWithHostNet{{ else }}ClusterFirst{{ end }}
       serviceAccountName: nvsnap-agent
       {{- include "nvsnap.imagePullSecrets" . | nindent 6 }}
       initContainers:
@@ -82,6 +82,18 @@ spec:
           image: {{ include "nvsnap.agent.image" . }}
           imagePullPolicy: {{ .Values.agent.image.pullPolicy }}
           args:
+            {{- if .Values.agent.auth.enabled }}
+            # permissive counts and logs unauthenticated callers but still
+            # serves them; required returns 401. Roll out on permissive until
+            # nvsnap_agent_auth_total{result="missing"} is zero.
+            - --auth-mode={{ .Values.agent.auth.mode }}
+            {{- end }}
+            {{- if not .Values.agent.hostNetwork }}
+            # Pod networking: the init container reaches its node-local agent
+            # through the internalTrafficPolicy:Local Service instead of the
+            # node IP, so no hostPort is needed (GH #490).
+            - --webhook-agent-base-url=http://nvsnap-agent-local.{{ .Release.Namespace }}.svc.cluster.local:8081
+            {{- end }}
             - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint
             - --criu-path=/criu-bundle/criu
             # Translate in-container --checkpoint-dir to the host path
@@ -168,6 +180,24 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.hostIP
+            # POD_IP is what peers dial (--advertise-ip). Under hostNetwork
+            # kubelet reports status.podIP as the node IP, so this is correct
+            # in both network modes and needs no conditional (GH #490).
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            {{- if .Values.agent.auth.enabled }}
+            # Shared bearer token for the agent API (GH #486). Env rather than
+            # a flag: flag values are visible in the pod spec and in `ps`.
+            # The agent uses it both to verify inbound requests and to sign
+            # its own peer calls.
+            - name: NVSNAP_AGENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: nvsnap-agent-token
+                  key: token
+            {{- end }}
             {{- if .Values.server.enabled }}
             - name: NVSNAP_CATALOG_URL
               value: "http://nvsnap-server.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}"
@@ -225,7 +255,13 @@ spec:
             {{- end }}
           ports:
             - containerPort: 8081
+              {{- if .Values.agent.hostNetwork }}
+              # Binds the API to every node's IP. Only declared under
+              # hostNetwork; with pod networking peers dial the pod IP and
+              # same-node callers use the internalTrafficPolicy:Local
+              # Service, so no node-wide listener is needed (GH #490).
               hostPort: 8081
+              {{- end }}
               name: http-api
             {{- if .Values.webhook.enabled }}
             - containerPort: 8443
@@ -386,3 +422,33 @@ spec:
       name: http-api
   clusterIP: None
 {{- end }}
+
+{{- if not .Values.agent.hostNetwork }}
+---
+# Node-local Service: the pod-network replacement for hostPort (GH #490).
+#
+# Callers that must reach the agent on THEIR OWN node -- the nvsnap-mount-prep
+# init container is the one that matters -- used to do it via
+# http://$(status.hostIP):8081, which requires the API to be bound to every
+# node's IP. internalTrafficPolicy:Local is the Kubernetes-native way to say
+# the same thing: this ClusterIP only ever routes to the endpoint on the
+# calling node, and has no node-IP listener at all.
+#
+# The headless nvsnap-agent Service above stays for tools that want to address
+# a specific agent; this one is for "whichever agent is on my node".
+apiVersion: v1
+kind: Service
+metadata:
+  name: nvsnap-agent-local
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nvsnap.agent.labels" . | nindent 4 }}
+spec:
+  selector:
+    {{- include "nvsnap.agent.selectorLabels" . | nindent 4 }}
+  internalTrafficPolicy: Local
+  ports:
+    - port: 8081
+      targetPort: 8081
+      name: http-api
+{{- end }}

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-token-secret.yaml

@@ -0,0 +1,45 @@
+{{- if .Values.agent.auth.enabled }}
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Shared bearer token for the agent HTTP API (GH #486).
+#
+# The agent API is the control surface of a privileged process and the
+# DaemonSet binds it to every node's IP, so it needs authentication in the
+# request path. This Secret holds the token both the agent (to verify) and its
+# callers (to present) read.
+#
+# Generated once and preserved across upgrades: `helm upgrade` re-renders every
+# template, so a freshly random token on each upgrade would rotate the
+# credential out from under running callers and cause a self-inflicted outage
+# mid-rollout. The lookup below reuses the existing value when the Secret is
+# already present. Set agent.auth.token explicitly to manage it yourself (or to
+# rotate deliberately).
+{{- $ns := .Release.Namespace }}
+{{- $name := "nvsnap-agent-token" }}
+{{- $existing := lookup "v1" "Secret" $ns $name }}
+{{- $token := "" }}
+{{- if .Values.agent.auth.token }}
+{{-   $token = .Values.agent.auth.token | b64enc }}
+{{- else if and $existing $existing.data $existing.data.token }}
+{{-   $token = $existing.data.token }}
+{{- else }}
+{{-   $token = randAlphaNum 48 | b64enc }}
+{{- end }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $name }}
+  namespace: {{ $ns }}
+  labels:
+    app.kubernetes.io/name: nvsnap
+    app.kubernetes.io/part-of: nvsnap
+  annotations:
+    # helm.sh/resource-policy keeps the Secret if the release is removed with
+    # --keep-history style workflows; without it a delete/reinstall cycle
+    # silently rotates the token.
+    helm.sh/resource-policy: keep
+type: Opaque
+data:
+  token: {{ $token }}
+{{- end }}

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/network-policy-restore-pods.yaml

@@ -107,7 +107,7 @@ Only rendered when the init-container strategy is selected AND
 agentHostCIDR is set. The default inline strategy does the mount inside
 the webhook and needs no pod->agent egress.
 */ -}}
-{{- if and .Values.webhook.enabled (eq (.Values.webhook.restorePrepStrategy | default "inline") "init-container") .Values.webhook.agentHostCIDR .Values.agent.l2.restoreNamespaces -}}
+{{- if and .Values.webhook.enabled (eq (.Values.webhook.restorePrepStrategy | default "inline") "init-container") (or (not .Values.agent.hostNetwork) .Values.webhook.agentHostCIDR) .Values.agent.l2.restoreNamespaces -}}
 {{- range $ns := .Values.agent.l2.restoreNamespaces }}
 ---
 apiVersion: networking.k8s.io/v1
@@ -129,8 +129,25 @@ spec:
     - Egress
   egress:
     - to:
+        {{- if $.Values.agent.hostNetwork }}
+        # hostNetwork: the agent carries NODE identity, so a podSelector never
+        # matches it (verified on GKE Dataplane V2 / Cilium) and the rule has
+        # to name the whole node CIDR -- every node, on this port, for every
+        # pod in the namespace.
         - ipBlock:
             cidr: {{ $.Values.webhook.agentHostCIDR }}
+        {{- else }}
+        # Pod networking: the agent has a pod identity again, so the rule can
+        # name exactly the agent pods and nothing else. This is the concrete
+        # payoff of GH #490 -- no operator-supplied CIDR, and the grant shrinks
+        # from "the node network" to "these pods".
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ $.Release.Namespace }}
+          podSelector:
+            matchLabels:
+              {{- include "nvsnap.agent.selectorLabels" $ | nindent 14 }}
+        {{- end }}
       ports:
         - protocol: TCP
           port: {{ $.Values.webhook.agentHostPort | default 8081 }}

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml

@@ -176,13 +176,54 @@ agent:
       effect: NoSchedule
 
   # The agent runs privileged with hostPID/hostNetwork — required to
-  # see host processes (for CRIU) and to expose hostPort 8081 reliably.
-  # Don't disable unless you know what you're trading away.
+  # hostPID is required: CRIU and cuda-checkpoint address target processes by
+  # host PID, and the pre-checkpoint socket sweep opens /proc/<pid>/ns/net.
   hostPID: true
+
+  # hostNetwork is NOT required by any agent capability (GH #490). Everything
+  # that looked like it needed the host netns actually enters the TARGET pod's
+  # namespace: external_tcp.go setns's via /proc/<pid>/ns/net, and CRIU
+  # dump/restore nsenter into the container's netns. The apiserver reaches the
+  # webhook through a Service, not the node IP.
+  #
+  # What it does carry is hostPort 8081 -- which is what binds the agent's
+  # privileged API to every node's IP and makes NetworkPolicy unable to fence
+  # it, since a hostNetwork pod has node identity rather than pod identity.
+  #
+  # Setting this false switches to: peers dial the pod IP (--advertise-ip from
+  # status.podIP), same-node callers use the internalTrafficPolicy:Local
+  # Service, no hostPort is declared, and the restore-pod egress policy
+  # tightens from a node CIDR to a podSelector.
+  #
+  # Still true by default because the flip is a network topology change that
+  # has not been validated on a cluster yet. Do that before flipping.
   hostNetwork: true
 
   # Host paths the agent bind-mounts. Override if your nodes use
   # non-standard layouts (e.g. K3s on a single laptop).
+
+  # Authentication for the agent HTTP API (GH #486).
+  #
+  # The agent API restores and deletes checkpoints, serves any file inside a
+  # checkpoint, and exposes pprof, on a process running privileged with
+  # /var/lib and the containerd root bind-mounted. The DaemonSet binds it to
+  # every node's IP (hostNetwork + hostPort), and NetworkPolicy cannot fence a
+  # hostNetwork pod, so access control has to live in the request path.
+  #
+  # Rollout: enable with mode=permissive first. The agent then counts and logs
+  # unauthenticated callers via nvsnap_agent_auth_total{result="missing"} but
+  # still serves them, so nothing breaks while callers pick up the token.
+  # Switch to required once that series is flat at zero.
+  auth:
+    # Off by default so an upgrade does not lock out callers that have not
+    # been given the token yet. The agent logs a warning while it is off.
+    enabled: false
+    # permissive | required. Ignored when enabled=false.
+    mode: permissive
+    # Leave empty to have the chart generate one and preserve it across
+    # upgrades. Set explicitly to manage or rotate the credential yourself.
+    token: ""
+
   hostPaths:
     checkpoints: /var/lib/containerd/nvsnap-checkpoints
     containerdSock: /run/containerd/containerd.sock

src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "agent.go",
         "blob_uploader.go",
+        "auth.go",
         "capture_cascade.go",
         "capture_peer.go",
         "cascade_fetch.go",
@@ -90,6 +91,8 @@ go_test(
         "l2_promote_async_test.go",
         "l2_writer_test.go",
         "nim_backend_test.go",
+        "advertise_test.go",
+        "auth_test.go",
         "pathsafe_test.go",
         "peer_fanout_test.go",
         "peer_load_test.go",