ci: add Cloud Hypervisor live-KVM integration

.github/workflows/test-cloud-hypervisor.yml

@@ -0,0 +1,277 @@
+name: Cloud Hypervisor Preview Integration
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_live_kvm:
+        description: Run the live KVM job on GitHub-hosted x64 Ubuntu 24.04
+        required: true
+        type: boolean
+        default: true
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - '.github/workflows/test-cloud-hypervisor.yml'
+      - 'guest/cloud-hypervisor/**'
+      - 'guest/firecracker-supervisor/**'
+      - 'src/cloud-hypervisor/**'
+      - 'src/cloud-hypervisor-runtime-backend.ts'
+      - 'src/cloud-hypervisor-runtime-backend.test.ts'
+      - 'src/microvm/**'
+      - 'src/types/runtime-options.ts'
+      - 'scripts/ci/cloud-hypervisor-*.sh'
+      - 'docs/cloud-hypervisor-foundation.md'
+
+permissions:
+  contents: read
+  id-token: write
+  attestations: write
+
+concurrency:
+  group: cloud-hypervisor-preview-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build-test-artifacts:
+    name: Build deterministic test guest
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version: '1.25.0'
+          cache-dependency-path: guest/firecracker-supervisor/go.mod
+
+      - name: Run guest supervisor unit tests
+        working-directory: guest/firecracker-supervisor
+        # guest/firecracker-supervisor is shared, unmodified, between the
+        # Firecracker and Cloud Hypervisor backends (see build.sh above).
+        # Running its unit tests here (not just building it) catches
+        # defects like an incorrect syscall.Mount() fstype before they
+        # only surface as a guest kernel panic during the live-KVM job
+        # below, which is much slower to diagnose.
+        run: go test ./...
+
+      - name: Install deterministic guest build prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends \
+            bc \
+            binutils \
+            bison \
+            build-essential \
+            ca-certificates \
+            cpio \
+            e2fsprogs \
+            file \
+            flex \
+            libelf-dev \
+            libssl-dev \
+            rsync \
+            xz-utils
+
+      - name: Build and verify pinned artifacts
+        run: |
+          ./guest/cloud-hypervisor/build-test-artifacts.sh
+          ./guest/cloud-hypervisor/verify-test-artifacts.sh \
+            release/cloud-hypervisor-test-x86_64
+
+      - name: Attest guest artifact provenance
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-path: release/cloud-hypervisor-test-x86_64/awf-cloud-hypervisor-test-x86_64.tar.gz
+
+      - name: Upload guest artifacts
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: cloud-hypervisor-test-x86_64
+          path: release/cloud-hypervisor-test-x86_64/
+          if-no-files-found: error
+          retention-days: 7
+
+  live-kvm:
+    name: Live Cloud Hypervisor KVM smoke/security
+    needs: build-test-artifacts
+    if: >-
+      (github.event_name == 'workflow_dispatch' && inputs.run_live_kvm) ||
+      (github.event_name == 'pull_request' &&
+       contains(github.event.pull_request.labels.*.name, 'cloud-hypervisor-kvm'))
+    runs-on: ubuntu-24.04
+    # 15 cases, each with up to a 90s boot budget and (since the guest
+    # connectivity probe raised its own timeout for the same nested-KVM
+    # scheduling reasons) up to a further 90s probe budget in the worst
+    # case; 60 minutes leaves headroom without masking a genuine hang.
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
+        with:
+          node-version: '22'
+          cache: npm
+
+      - name: Download verified guest artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: cloud-hypervisor-test-x86_64
+          path: ${{ runner.temp }}/cloud-hypervisor-test-x86_64
+
+      - name: Restore artifact executable permissions
+        run: |
+          # actions/upload-artifact + actions/download-artifact do not
+          # reliably preserve the executable bit on binary files (a known
+          # GitHub Actions artifact limitation). Digest verification below
+          # still proves file integrity; this only restores the mode bits
+          # needed to exec the pinned, already-verified binaries.
+          chmod 0755 \
+            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64/cloud-hypervisor" \
+            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64/awf-supervisor"
+
+      - name: Grant workflow user access to KVM
+        run: |
+          if [ -e /dev/kvm ]; then
+            sudo chmod 666 /dev/kvm
+          fi
+
+      - name: Verify capable host and artifact digests
+        run: |
+          ./scripts/ci/cloud-hypervisor-host-preflight.sh \
+            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64"
+
+      - name: Install, build, and prepare infrastructure images
+        run: |
+          npm ci
+          npm run build
+          docker build -t ghcr.io/github/gh-aw-firewall/squid:latest containers/squid
+          docker build -t ghcr.io/github/gh-aw-firewall/api-proxy:latest containers/api-proxy
+
+      - name: Run live fail-closed smoke/security coverage
+        run: |
+          ./scripts/ci/cloud-hypervisor-live-smoke.sh \
+            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64"
+
+      - name: Print guest boot diagnostics on failure
+        if: failure()
+        run: |
+          set -uo pipefail
+          # Cloud Hypervisor's own collectDiagnostics() runs before the run
+          # directory is torn down on a startup failure (see
+          # src/cloud-hypervisor-runtime-backend.ts), so the guest serial
+          # console and Cloud Hypervisor log land under each case's
+          # audit/cloud-hypervisor/ directory. Print them directly in the
+          # job log for fast triage, in addition to the uploaded artifact.
+          # Redact the test-only secret sentinel defensively, matching the
+          # scan the next step performs on the copied artifact.
+          for f in "$RUNNER_TEMP"/awf-cloud-hypervisor-live/*/audit/cloud-hypervisor/{serial.log,cloud-hypervisor.log,network-diagnostics.txt,vm-info.json,counters.json}; do
+            [ -f "$f" ] || continue
+            echo "--- $f ---"
+            sed 's/awf-cloud-hypervisor-real-secret-do-not-expose/[REDACTED]/g' "$f"
+          done
+
+      - name: Collect redacted diagnostics
+        if: always()
+        run: |
+          set -euo pipefail
+          source_root="$RUNNER_TEMP/awf-cloud-hypervisor-live"
+          destination="$RUNNER_TEMP/cloud-hypervisor-diagnostics-safe"
+          rm -rf "$destination"
+          mkdir -p "$destination"
+          if [ -d "$source_root" ]; then
+            # --keep-containers (the keep-containers/preserve-diagnostics
+            # live case) intentionally leaves its own work/audit files
+            # root-owned (they are written by the CLI process, itself run
+            # via sudo, and deliberately never cleaned up so the case's own
+            # assertions can inspect them afterward). Every other case's
+            # files are owned by the runner user and readable without
+            # sudo, but find/cp must run as root here to reach the
+            # keep-containers case's own preserved files too -- reading a
+            # root-owned file as the runner user is a permission error,
+            # not evidence of a missing file.
+            while IFS= read -r -d '' file; do
+              relative=${file#"$source_root/"}
+              mkdir -p "$destination/$(dirname "$relative")"
+              sudo cp "$file" "$destination/$relative"
+            done < <(
+              sudo find "$source_root" -type f \
+                \( -path '*/audit/*' \
+                -o -path '*/proxy-logs/*' \
+                -o -name 'stdout.log' \
+                -o -name 'stderr.log' \) \
+                -print0
+            )
+          fi
+          # Hand ownership of the copied destination tree back to the
+          # runner user: sudo cp above creates new destination files as
+          # root, which the secret-sentinel scan below and the later
+          # upload-artifact step (both running as the runner user, not
+          # root) need to be able to read.
+          sudo chown -R "$(id -u):$(id -g)" "$destination"
+          # awf-resolved-config.json's agentCommand field always contains the
+          # smoke test's own shell command verbatim -- and this specific
+          # command (the api-proxy-reflect case) intentionally references
+          # the sentinel string itself as the pattern it greps for, to
+          # assert the sentinel is absent from `env`. That is expected,
+          # self-referential test source text, not a leak of the sentinel
+          # *value* into somewhere it shouldn't be (guest console output,
+          # network captures, proxy logs, etc. are all still fully scanned
+          # below) -- so this one, known, always-matching file is excluded
+          # from the scan rather than silently disabling it everywhere.
+          if grep -R --binary-files=without-match \
+            -F 'awf-cloud-hypervisor-real-secret-do-not-expose' \
+            --exclude='awf-resolved-config.json' \
+            "$destination"; then
+            echo "::error::Secret sentinel found in diagnostic artifacts"
+            rm -rf "$destination"
+            exit 1
+          fi
+
+      - name: Upload actionable diagnostics
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: cloud-hypervisor-live-diagnostics
+          path: ${{ runner.temp }}/cloud-hypervisor-diagnostics-safe/
+          if-no-files-found: warn
+          retention-days: 7
+
+      - name: Enforce final residue cleanup
+        if: always()
+        run: |
+          set -euo pipefail
+          while read -r namespace _; do
+            case "$namespace" in
+              awffc-*) sudo ip netns delete "$namespace" ;;
+            esac
+          done < <(sudo ip netns list)
+          if sudo ip netns list | grep -q '^awffc-'; then
+            echo "::error::Cloud Hypervisor namespace residue remains after cleanup"
+            exit 1
+          fi
+          # /sys/fs/cgroup/awf-cloud-hypervisor is a parent cgroup that
+          # persists across the whole job; only per-run sub-cgroups are
+          # created one level inside it (see cgroupPath in
+          # src/cloud-hypervisor/manager.ts). Any cgroup v2 directory --
+          # including this parent itself -- always contains standard
+          # controller interface files (cpu.max, memory.max,
+          # cgroup.controllers, ...) simply by virtue of existing;
+          # matching all entries here (not just directories) made this
+          # check a guaranteed false positive the moment the live-KVM job
+          # ever actually completed successfully. See the identical fix
+          # in scripts/ci/cloud-hypervisor-live-smoke.sh's own
+          # assert_no_residue for the same root cause.
+          if [ -d /sys/fs/cgroup/awf-cloud-hypervisor ] && \
+             [ -n "$(sudo find /sys/fs/cgroup/awf-cloud-hypervisor -mindepth 1 -maxdepth 1 -type d 2>/dev/null)" ]; then
+            echo "::error::Cloud Hypervisor cgroup residue remains after cleanup"
+            exit 1
+          fi
+          if pgrep -f 'cloud-hypervisor --api-socket' >/dev/null 2>&1; then
+            echo "::error::Cloud Hypervisor process residue remains after cleanup"
+            exit 1
+          fi

.github/workflows/test-firecracker.yml

@@ -98,6 +98,19 @@ jobs:
           name: firecracker-test-x86_64
           path: ${{ runner.temp }}/firecracker-test-x86_64
 
+      - name: Restore artifact executable permissions
+        run: |
+          # actions/upload-artifact + actions/download-artifact do not
+          # reliably preserve the executable bit on binary files (a known
+          # GitHub Actions artifact limitation). Digest verification in the
+          # preflight step below still proves file integrity; this only
+          # restores the mode bits needed to exec the pinned, already-
+          # verified binaries.
+          chmod 0755 \
+            "$RUNNER_TEMP/firecracker-test-x86_64/firecracker" \
+            "$RUNNER_TEMP/firecracker-test-x86_64/jailer" \
+            "$RUNNER_TEMP/firecracker-test-x86_64/awf-firecracker-supervisor"
+
       - name: Grant workflow user access to KVM
         run: |
           if [ -e /dev/kvm ]; then
@@ -130,21 +143,40 @@ jobs:
           rm -rf "$destination"
           mkdir -p "$destination"
           if [ -d "$source_root" ]; then
+            # See the identical comment in test-cloud-hypervisor.yml:
+            # --keep-containers intentionally leaves its own work/audit
+            # files root-owned (written by the CLI process via sudo, never
+            # cleaned up so the case's own assertions can inspect them
+            # afterward), so find/cp must run as root here too.
             while IFS= read -r -d '' file; do
               relative=${file#"$source_root/"}
               mkdir -p "$destination/$(dirname "$relative")"
-              cp "$file" "$destination/$relative"
+              sudo cp "$file" "$destination/$relative"
             done < <(
-              find "$source_root" -type f \
+              sudo find "$source_root" -type f \
                 \( -path '*/audit/*' \
                 -o -path '*/proxy-logs/*' \
                 -o -name 'stdout.log' \
                 -o -name 'stderr.log' \) \
                 -print0
             )
           fi
+          # Hand ownership of the copied destination tree back to the
+          # runner user: sudo cp above creates new destination files as
+          # root, which the secret-sentinel scan below and the later
+          # upload-artifact step (both running as the runner user) need to
+          # be able to read.
+          sudo chown -R "$(id -u):$(id -g)" "$destination"
+          # See the identical comment in test-cloud-hypervisor.yml:
+          # awf-resolved-config.json's agentCommand field always contains
+          # the smoke test's own shell command verbatim, which for the
+          # api-proxy-reflect case intentionally references the sentinel
+          # string itself as the pattern it greps for -- expected,
+          # self-referential test source text, not a leak of the sentinel
+          # value elsewhere (still fully scanned below).
           if grep -R --binary-files=without-match \
             -F 'awf-firecracker-real-secret-do-not-expose' \
+            --exclude='awf-resolved-config.json' \
             "$destination"; then
             echo "::error::Secret sentinel found in diagnostic artifacts"
             rm -rf "$destination"

.gitignore

@@ -34,3 +34,8 @@ paper-data/.gh-aw-skip-cache.json
 
 # Python bytecode from local enclave-bootstrap syntax checks
 __pycache__/
+
+# Local `go build`/`go vet` output for the guest supervisor module
+# (built binary shares its directory's module name with no extension,
+# easy to accidentally leave behind after a local build/test cycle)
+/guest/firecracker-supervisor/firecracker-supervisor