feat: add durable identity-validated microVM cleanup

.github/workflows/test-cloud-hypervisor.yml

@@ -252,15 +252,16 @@ jobs:
         if: always()
         run: |
           set -euo pipefail
-          while read -r namespace _; do
-            case "$namespace" in
-              awfvm-*) sudo ip netns delete "$namespace" ;;
-            esac
-          done < <(sudo ip netns list)
           if sudo ip netns list | grep -q '^awfvm-'; then
+            sudo ip netns list >&2
             echo "::error::Cloud Hypervisor namespace residue remains after cleanup"
             exit 1
           fi
+          if sudo iptables -S DOCKER-USER | grep -q -- '--comment awf:awf_vm_'; then
+            sudo iptables -S DOCKER-USER >&2
+            echo "::error::Cloud Hypervisor bridge-rule residue remains after cleanup"
+            exit 1
+          fi
           # /sys/fs/cgroup/awf-cloud-hypervisor is a parent cgroup that
           # persists across the whole job; only per-run sub-cgroups are
           # created one level inside it (see cgroupPath in

docs/cloud-hypervisor-foundation.md

@@ -86,7 +86,30 @@ AWF performs these steps for each run:
     and remove network, cgroup, and run-directory resources.
 
 Cleanup is idempotent and aggregates errors so one cleanup failure does not
-skip later cleanup steps.
+skip later cleanup steps. Before the first privileged per-run resource is
+created, AWF atomically writes a root-owned mode-`0600` recovery record under
+`/run/awf-cloud-hypervisor/pending-cleanup/`, itself a root-owned mode-`0700`
+directory. The record contains the owning AWF PID, `/proc` start time,
+executable identity, exact resource names, and immutable inode/ifindex
+identities captured immediately after each namespace, interface, run directory,
+cgroup, VMM, and `virtiofsd` process becomes live. The host bridge-forwarding
+rule is tagged with a per-run iptables comment and recorded by its exact tuple,
+so concurrent runs do not share an anonymously owned rule. Any staged
+virtio-fs bind mounts are recorded by mount ID, device, root, target, filesystem
+type, and source; stale recovery revalidates and unmounts them deepest-first
+before removing their inode-validated share directory.
+
+Every subsequent Cloud Hypervisor startup reaps stale records before creating
+its own resources. A record whose owner still has the same PID, start time,
+executable inode, credentials, and network namespace is active and is skipped,
+so concurrent sibling runs cannot reap one another. For an abandoned record,
+AWF revalidates every existing resource and process immediately before acting.
+It never treats a name or PID alone as ownership evidence: PID reuse, a changed
+namespace/interface inode or ifindex, an uncommitted launch identity, malformed
+state, or an unsafe record mode stops cleanup, reports an error, and preserves
+the record and resources for diagnosis. The record is removed only after normal
+teardown succeeds. `--keep-containers` is an explicit diagnostic opt-out: its
+record is removed while the requested resources remain preserved.
 
 ## Security boundaries
 

scripts/ci/cloud-hypervisor-live-smoke.sh

@@ -7,8 +7,9 @@ set -euo pipefail
 # This covers allowed/blocked domains, direct
 # egress, arbitrary TCP, DNS, metadata IP, mandatory API-proxy reflect with
 # secret-sentinel absence, live workspace sharing incl. symlinks/permissions,
-# exit-code propagation, timeout, SIGTERM cancellation, partial-start
-# rollback, keep/preserve diagnostics, plus backend-specific live checks:
+# exit-code propagation, timeout, SIGTERM cancellation, abrupt process-death
+# recovery, partial-start rollback, keep/preserve diagnostics, plus
+# backend-specific live checks:
 #
 #   - device-assumptions: confirms eth0, the sole /dev/vda block disk, and
 #     virtio-fs workspace layout documented in Part 6.
@@ -95,6 +96,11 @@ assert_no_residue() {
     echo "Cloud Hypervisor veth/TAP residue detected" >&2
     return 1
   fi
+  if sudo iptables -S DOCKER-USER | grep -q -- '--comment awf:awf_vm_'; then
+    sudo iptables -S DOCKER-USER >&2
+    echo "Cloud Hypervisor per-run bridge rule residue detected" >&2
+    return 1
+  fi
   # $CGROUP_ROOT (.../awf-cloud-hypervisor) is a *parent* cgroup that
   # persists across runs; only per-run sub-cgroups live one level
   # inside it (see cgroupPath in src/cloud-hypervisor/manager.ts). Any
@@ -427,6 +433,65 @@ if [ "$cleanup_ms" -gt "$CLEANUP_CEILING_MS" ]; then
   exit 1
 fi
 
+# SIGKILL cannot run in-process finally/signal cleanup. Kill the root AWF Node
+# process itself, prove its VMM/netns/cgroup survive, then let the next ordinary
+# invocation recover them through the durable identity-validated registry.
+crash_work="$RUN_ROOT/process-death/work"
+crash_workspace="$RUN_ROOT/process-death/workspace"
+crash_audit="$RUN_ROOT/process-death/audit"
+mkdir -p "$crash_work" "$crash_workspace" "$crash_audit"
+(
+  export GITHUB_WORKSPACE="$crash_workspace"
+  export OPENAI_API_KEY="$SECRET_SENTINEL"
+  exec sudo -E node "$ROOT/dist/cli.js" \
+    "${COMMON[@]}" \
+    --work-dir "$crash_work" \
+    --audit-dir "$crash_audit" \
+    -- 'sleep 300'
+) >"$RUN_ROOT/process-death/stdout.log" 2>"$RUN_ROOT/process-death/stderr.log" &
+crash_wrapper_pid=$!
+crash_node_pid=
+for _ in $(seq 1 90); do
+  for candidate in $(pgrep -f "node $ROOT/dist/cli.js.*--work-dir $crash_work" || true); do
+    [ "$candidate" = "$crash_wrapper_pid" ] && continue
+    candidate_exe=$(sudo readlink "/proc/$candidate/exe" 2>/dev/null || true)
+    case "$candidate_exe" in
+      */node) crash_node_pid=$candidate; break ;;
+    esac
+  done
+  if [ -n "$crash_node_pid" ] &&
+     sudo ip netns list | grep -q '^awfvm-' &&
+     sudo find /run/awf-cloud-hypervisor/pending-cleanup -maxdepth 1 -name '*.json' | grep -q . &&
+     sudo find "$CGROUP_ROOT" -mindepth 1 -maxdepth 1 -type d | grep -q . &&
+     pgrep -f "$ARTIFACT_DIR/cloud-hypervisor --api-socket" >/dev/null; then
+    break
+  fi
+  sleep 1
+done
+[ -n "$crash_node_pid" ] || {
+  echo "process-death: AWF process did not become live" >&2
+  exit 1
+}
+sudo kill -KILL "$crash_node_pid"
+set +e
+wait "$crash_wrapper_pid"
+crash_status=$?
+set -e
+[ "$crash_status" -ne 0 ] || {
+  echo "process-death: SIGKILL unexpectedly returned success" >&2
+  exit 1
+}
+sudo ip netns list | grep -q '^awfvm-' || {
+  echo "process-death: abrupt exit did not leave the expected recovery fixture" >&2
+  exit 1
+}
+pgrep -f "$ARTIFACT_DIR/cloud-hypervisor --api-socket" >/dev/null || {
+  echo "process-death: VMM did not survive abrupt owner death" >&2
+  exit 1
+}
+run_case process-death-reaper 0 'true'
+assert_no_residue
+
 keep_work="$RUN_ROOT/keep/work"
 keep_workspace="$RUN_ROOT/keep/workspace"
 keep_audit="$RUN_ROOT/keep/audit"
@@ -480,6 +545,16 @@ sudo find "$keep_audit/cloud-hypervisor" -type f -size +1048576c -print -quit \
     exit 1
   }
 
+readarray -t keep_forward_rule < <(
+  sudo node -e '
+    const plan = require(process.argv[1]);
+    console.log(plan.infrastructureBridge);
+    console.log(plan.hostForwardRuleComment);
+  ' "$keep_audit/cloud-hypervisor/network-plan.json"
+)
+sudo iptables -t filter -D DOCKER-USER \
+  -i "${keep_forward_rule[0]}" -o "${keep_forward_rule[0]}" \
+  -m comment --comment "${keep_forward_rule[1]}" -j ACCEPT
 while read -r namespace _; do
   case "$namespace" in
     awfvm-*) sudo ip netns delete "$namespace" ;;