From dead69d854fb3e0649747a3997b4b2e677497865 Mon Sep 17 00:00:00 2001 From: Landon Cox Date: Fri, 14 Aug 2026 10:34:59 -0700 Subject: [PATCH 1/2] feat!: remove Firecracker support Remove the runtime backend, configuration surface, tests, CI scripts, and documentation. Rename shared microVM components and vendor the Cloud Hypervisor kernel configuration. BREAKING CHANGE: Firecracker runtime options and configuration are no longer supported. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1895b94b-be33-4f6f-a6b2-5b3ae2ec2c6f --- .github/workflows/release.yml | 2 +- .github/workflows/test-cloud-hypervisor.yml | 14 +- .gitignore | 2 +- README.md | 1 - docs/INTEGRATION-TESTS.md | 56 +- docs/architecture.md | 43 +- docs/awf-config-spec.md | 48 +- docs/awf-config.schema.json | 77 +- docs/cloud-hypervisor-foundation.md | 1194 ++---- docs/compatibility.md | 20 - docs/firecracker-integration.md | 1089 ----- docs/gvisor-integration.md | 6 +- docs/releasing.md | 7 - docs/sandbox-design.md | 13 +- docs/sbx-integration.md | 55 +- .../cloud-hypervisor/build-test-artifacts.sh | 30 +- guest/cloud-hypervisor/kernel.config | 3556 +++++++++++++++++ guest/firecracker-supervisor/go.mod | 5 - guest/firecracker/build-test-artifacts.sh | 309 -- guest/firecracker/verify-test-artifacts.sh | 34 - .../build.sh | 2 +- .../config.go | 0 .../config_test.go | 0 guest/microvm-supervisor/go.mod | 5 + .../main.go | 4 +- .../protocol.go | 8 +- .../protocol_test.go | 0 .../runtime_linux.go | 3 +- .../runtime_linux_test.go | 3 +- .../runtime_other.go | 2 +- .../ci/cloud-hypervisor-ci-scripts.test.ts | 31 +- scripts/ci/cloud-hypervisor-host-preflight.sh | 9 +- scripts/ci/cloud-hypervisor-live-smoke.sh | 34 +- scripts/ci/firecracker-host-preflight.sh | 43 - scripts/ci/firecracker-live-smoke.sh | 221 - .../ci/test-cloud-hypervisor-workflow.test.ts | 2 +- src/awf-config-schema.json | 77 +- src/cli-options.ts | 28 +- src/cloud-hypervisor-runtime-backend.test.ts | 4 +- src/cloud-hypervisor-runtime-backend.ts | 2 +- src/cloud-hypervisor/launcher.test.ts | 10 +- src/cloud-hypervisor/launcher.ts | 12 +- src/cloud-hypervisor/manager.test.ts | 2 +- src/cloud-hypervisor/manager.ts | 8 +- src/cloud-hypervisor/preflight.ts | 10 +- src/cloud-hypervisor/runtime-validation.ts | 9 +- src/cloud-hypervisor/vm-config-builder.ts | 3 +- src/commands/build-config.ts | 72 +- src/commands/main-action.test.ts | 8 +- src/commands/validate-options.test.ts | 69 - src/commands/validators/config-assembly.ts | 22 - src/commands/validators/security-mode.ts | 6 +- src/compose-generator.ts | 2 +- src/config-file.ts | 14 +- src/config-mapper.ts | 14 - src/container-runtime.test.ts | 16 - src/container-runtime.ts | 6 - src/enclave/agent-runner-spec.test.ts | 2 +- src/enclave/runtime-preflight.test.ts | 13 - src/enclave/runtime-preflight.ts | 9 +- src/external-runtime-backend-resolver.ts | 8 - src/external-runtime-backend.test.ts | 19 - src/firecracker-runtime-backend.test.ts | 440 -- src/firecracker-runtime-backend.ts | 430 -- src/firecracker/api-client.test.ts | 176 - src/firecracker/api-client.ts | 247 -- src/firecracker/config.test.ts | 127 - src/firecracker/manager.test.ts | 782 ---- src/firecracker/manager.ts | 699 ---- src/firecracker/preflight.test.ts | 401 -- src/firecracker/preflight.ts | 405 -- src/firecracker/runtime-validation.test.ts | 106 - src/firecracker/runtime-validation.ts | 92 - src/microvm/guest-protocol.ts | 2 +- src/microvm/network-plan.ts | 8 +- src/microvm/network-types.ts | 8 +- src/microvm/network.test.ts | 25 +- src/microvm/vsock-client.ts | 4 +- src/microvm/workspace.ts | 7 +- src/services/agent-service-build.test.ts | 10 - src/types/index.ts | 8 - src/types/runtime-options.ts | 37 - 82 files changed, 3985 insertions(+), 7402 deletions(-) delete mode 100644 docs/firecracker-integration.md create mode 100644 guest/cloud-hypervisor/kernel.config delete mode 100644 guest/firecracker-supervisor/go.mod delete mode 100755 guest/firecracker/build-test-artifacts.sh delete mode 100755 guest/firecracker/verify-test-artifacts.sh rename guest/{firecracker-supervisor => microvm-supervisor}/build.sh (91%) rename guest/{firecracker-supervisor => microvm-supervisor}/config.go (100%) rename guest/{firecracker-supervisor => microvm-supervisor}/config_test.go (100%) create mode 100644 guest/microvm-supervisor/go.mod rename guest/{firecracker-supervisor => microvm-supervisor}/main.go (67%) rename guest/{firecracker-supervisor => microvm-supervisor}/protocol.go (97%) rename guest/{firecracker-supervisor => microvm-supervisor}/protocol_test.go (100%) rename guest/{firecracker-supervisor => microvm-supervisor}/runtime_linux.go (99%) rename guest/{firecracker-supervisor => microvm-supervisor}/runtime_linux_test.go (98%) rename guest/{firecracker-supervisor => microvm-supervisor}/runtime_other.go (53%) delete mode 100755 scripts/ci/firecracker-host-preflight.sh delete mode 100755 scripts/ci/firecracker-live-smoke.sh delete mode 100644 src/firecracker-runtime-backend.test.ts delete mode 100644 src/firecracker-runtime-backend.ts delete mode 100644 src/firecracker/api-client.test.ts delete mode 100644 src/firecracker/api-client.ts delete mode 100644 src/firecracker/config.test.ts delete mode 100644 src/firecracker/manager.test.ts delete mode 100644 src/firecracker/manager.ts delete mode 100644 src/firecracker/preflight.test.ts delete mode 100644 src/firecracker/preflight.ts delete mode 100644 src/firecracker/runtime-validation.test.ts delete mode 100644 src/firecracker/runtime-validation.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e60d0defe..53670d528 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1020,7 +1020,7 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.25.0' - cache-dependency-path: guest/firecracker-supervisor/go.mod + cache-dependency-path: guest/microvm-supervisor/go.mod - name: Install guest build prerequisites run: | diff --git a/.github/workflows/test-cloud-hypervisor.yml b/.github/workflows/test-cloud-hypervisor.yml index 6510cf92b..56c690d00 100644 --- a/.github/workflows/test-cloud-hypervisor.yml +++ b/.github/workflows/test-cloud-hypervisor.yml @@ -14,7 +14,7 @@ on: - '.github/workflows/test-cloud-hypervisor.yml' - 'guest/cloud-hypervisor/**' - 'containers/build-tools/**' - - 'guest/firecracker-supervisor/**' + - 'guest/microvm-supervisor/**' - 'src/cloud-hypervisor/**' - 'src/cloud-hypervisor-runtime-backend.ts' - 'src/cloud-hypervisor-runtime-backend.test.ts' @@ -46,13 +46,11 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.25.0' - cache-dependency-path: guest/firecracker-supervisor/go.mod + cache-dependency-path: guest/microvm-supervisor/go.mod - name: Run guest supervisor unit tests - working-directory: guest/firecracker-supervisor - # guest/firecracker-supervisor is shared, unmodified, between the - # Firecracker and Cloud Hypervisor backends (see build.sh above). - # Running its unit tests here (not just building it) catches + working-directory: guest/microvm-supervisor + # Running unit tests here (not just building it) catches # defects like an incorrect syscall.Mount() fstype before they # only surface as a guest kernel panic during the live-KVM job # below, which is much slower to diagnose. @@ -255,10 +253,10 @@ jobs: set -euo pipefail while read -r namespace _; do case "$namespace" in - awffc-*) sudo ip netns delete "$namespace" ;; + awfvm-*) sudo ip netns delete "$namespace" ;; esac done < <(sudo ip netns list) - if sudo ip netns list | grep -q '^awffc-'; then + if sudo ip netns list | grep -q '^awfvm-'; then echo "::error::Cloud Hypervisor namespace residue remains after cleanup" exit 1 fi diff --git a/.gitignore b/.gitignore index cfd07bc22..0693f1916 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,4 @@ __pycache__/ # Local `go build`/`go vet` output for the guest supervisor module # (built binary shares its directory's module name with no extension, # easy to accidentally leave behind after a local build/test cycle) -/guest/firecracker-supervisor/firecracker-supervisor +/guest/microvm-supervisor/microvm-supervisor diff --git a/README.md b/README.md index 27699ae76..0d651ea1b 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,6 @@ See [GitHub Actions](docs/github_actions.md) for advanced setup and `awf logs su - [Diagnosing AWF failures](docs/diagnosing-awf-failures.md) — use the Self-Hosted Runner Doctor agent to triage self-hosted/ARC/GHES/GHEC failures - [Auth Doctor Updater workflow](.github/workflows/auth-doctor-updater.md) — daily/manual audit that opens bounded PRs with evidence-backed authentication and API-proxy documentation corrections - [Image verification](docs/image-verification.md) — cosign signature verification -- [Firecracker integration (preview)](docs/firecracker-integration.md) — Firecracker v1.16.1 microVM backend: explicit opt-in, Linux/KVM only, macOS/Windows unsupported, operator-managed artifacts with mandatory SHA-256 digests, fail-closed egress, mandatory API proxy credential isolation - [Cloud Hypervisor integration (preview)](docs/cloud-hypervisor-foundation.md) — Cloud Hypervisor v53.0 microVM backend: explicit opt-in, GitHub-hosted Ubuntu x86_64 KVM runners only, operator-managed artifacts with mandatory SHA-256 digests, Landlock/seccomp-confined launcher in place of a jailer, fail-closed egress, mandatory API proxy credential isolation ## Development diff --git a/docs/INTEGRATION-TESTS.md b/docs/INTEGRATION-TESTS.md index 667c00917..b02c5ba69 100644 --- a/docs/INTEGRATION-TESTS.md +++ b/docs/INTEGRATION-TESTS.md @@ -193,44 +193,12 @@ Each document provides per-test-case analysis with plain-language descriptions, - **[CI & Smoke Tests](test-analysis/ci-smoke.md)** — All 27 CI/smoke/build-test workflows analyzed - **[Test Infrastructure](test-analysis/test-infra.md)** — Runner architecture, batch pattern, cleanup strategy, limitations -## Firecracker preview integration tests - -The dedicated Firecracker CI workflow is disabled. The deterministic artifact -build and live KVM smoke scripts remain available for explicit local validation, -but they are not run by pull request, push, schedule, or manual Actions events. - -The live smoke/security suite verifies all five SHA-256 digests before running. -Its preflight requires usable KVM and fails closed if `/dev/kvm` or another -required host capability is unavailable. - -Live assertions (see `scripts/ci/firecracker-live-smoke.sh`): - -| Case | What it proves | -|------|---------------| -| `allowed-https` | Allowed domains reach the internet through Squid | -| `blocked-domain` | Non-allowlisted domains are blocked | -| `direct-egress` | Bypassing proxy env vars does not enable direct egress | -| `arbitrary-tcp` | Raw TCP to arbitrary IPs is blocked | -| `dns-denial` | Direct DNS (8.8.8.8:53) is blocked from the guest | -| `metadata-denial` | EC2/GCP/Azure instance metadata IP (`169.254.169.254`) is unreachable | -| `api-proxy-reflect` | API proxy `/reflect` reachable; secret sentinel not present in output | -| `workspace-copyback` | Guest file writes, permission changes, and symlinks survive copy-back | -| `exit-code` | Agent exit code propagates faithfully (37 → 37) | -| `timeout-124` | Timed-out agent exits 124 | -| `partial-start-cleanup` | Corrupt rootfs causes clean failure; no namespace residue | -| `cancellation` | `SIGTERM` cleans up network namespace; exits 143 | -| `keep` | `--keep-containers` preserves jail/namespace/images; all diagnostics ≤1 MiB | - -After every case, the suite asserts no `awffc-*` namespaces or Firecracker -interface residue remain. See [Firecracker integration (preview)](../docs/firecracker-integration.md#part-14--ci-workflow) -for the current automation status and local validation details. - ## Cloud Hypervisor preview integration tests The Cloud Hypervisor backend has its own separate CI workflow -(`test-cloud-hypervisor.yml`), based on the retained Firecracker test -conventions but scoped to Cloud Hypervisor paths and **GitHub-hosted Ubuntu -x86_64 runners only** (self-hosted runners are explicitly rejected). +(`test-cloud-hypervisor.yml`), scoped to Cloud Hypervisor paths and +**GitHub-hosted Ubuntu x86_64 runners only**. Self-hosted runners are explicitly +rejected. **Trigger:** `workflow_dispatch`, or pull request open/synchronize/reopen/label scoped to `guest/cloud-hypervisor/**`, `src/cloud-hypervisor/**`, @@ -239,8 +207,8 @@ scoped to `guest/cloud-hypervisor/**`, `src/cloud-hypervisor/**`, schedule. **Build job** (`ubuntu-24.04`): Builds deterministic guest artifacts — Cloud -Hypervisor v53.0 binary, the same pinned Linux 6.1.141 kernel config -Firecracker uses, BusyBox 1.36.1 rootfs, and the shared AWF guest supervisor — +Hypervisor v53.0 binary, the pinned Linux 6.1.141 kernel config, BusyBox 1.36.1 +rootfs, and the shared AWF guest supervisor — from pinned, SHA-256 verified sources. Attests provenance. Uploads as a 7-day workflow artifact (`cloud-hypervisor-test-x86_64`). @@ -250,9 +218,8 @@ SHA-256 digests plus GitHub-hosted-only host eligibility (`GITHUB_ACTIONS`, live smoke/security suite. The preflight requires usable KVM and fails closed if `/dev/kvm` or another required host capability is unavailable. -Live assertions (see `scripts/ci/cloud-hypervisor-live-smoke.sh`) reproduce -Firecracker's full 13-case contract verbatim, plus two Cloud Hypervisor-only -cases: +Live assertions (see `scripts/ci/cloud-hypervisor-live-smoke.sh`) cover the +following behavior: | Case | What it proves | |------|---------------| @@ -272,10 +239,9 @@ cases: | `keep` | `--keep-containers` preserves namespace/run-directory; diagnostics ≤1 MiB | | `security-assertions` **(CH-only)** | Live jailer-replacement boundary: non-root uid, `CapEff` limited to `CAP_NET_ADMIN` alone, `no_new_privs`, active seccomp filter, per-run cgroup membership/bounded memory, `landlock_enable` + exactly-minimal disk/net/vsock topology via `vm.info` | -After every case, the suite asserts no `awffc-*` namespaces, `fch*`/`fcn*`/`fct*` -interfaces (shared naming with Firecracker), `awf-cloud-hypervisor` cgroup -entries, or `cloud-hypervisor` processes remain. The secret sentinel -(`awf-cloud-hypervisor-real-secret-do-not-expose`, distinct from -Firecracker's) is scanned for in the same way. See +After every case, the suite asserts no `awfvm-*` namespaces, +`vmh*`/`vmn*`/`vmt*` interfaces, `awf-cloud-hypervisor` cgroup entries, or +`cloud-hypervisor` processes remain. The suite also scans output for the secret +sentinel (`awf-cloud-hypervisor-real-secret-do-not-expose`). See [Cloud Hypervisor integration (preview)](../docs/cloud-hypervisor-foundation.md#part-14--ci-workflow) for the full CI workflow specification. diff --git a/docs/architecture.md b/docs/architecture.md index d9a63b2b9..88dbe6459 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -240,27 +240,22 @@ Use `--keep-containers` to preserve containers and files after execution for deb - `js-yaml`: YAML generation for Docker Compose config - TypeScript 5.x, compiled to ES2020 CommonJS -## Firecracker microVM runtime (preview) - -When `--container-runtime firecracker --firecracker-preview` is supplied, AWF -uses a substantially different execution architecture: - -- The **agent container is replaced** by a Firecracker microVM — a hardware-isolated - virtual machine with its own Linux kernel. -- The Squid proxy and API proxy **remain as Docker Compose containers** on the host. -- The workspace is **not bind-mounted**; it is copied into a bounded ext4 image - (`workspace.ext4`) before boot and copied back after the agent exits. - There is no live filesystem passthrough (no virtiofs). -- A **dedicated network namespace** (`awffc-`) isolates the VM's network. - nftables rules inside the namespace allow only Squid and the API proxy; all - other guest outbound connections are denied. -- The **API proxy is mandatory**; provider credentials are never passed as guest - environment variables and an explicit assertion enforces this. -- **TTY, Docker-in-Docker, topology peers, enclaves, extra volume mounts, and - remote Docker hosts all fail closed** in this preview. -- **Linux/KVM only** — macOS and Windows are permanently unsupported. - CI specifically supports GitHub-hosted x64 `ubuntu-24.04`; KVM remains - mandatory, and hosts without usable `/dev/kvm` access fail closed. - -See [Firecracker integration (preview)](./firecracker-integration.md) for the -full architecture, trust model, and operator guide. +## Cloud Hypervisor microVM runtime (preview) + +With `--container-runtime cloud-hypervisor --cloud-hypervisor-preview`, AWF +runs the agent in a hardware-isolated microVM: + +- Squid and the API proxy remain Docker Compose services on the host. +- A sandboxed `virtiofsd` exports the workspace read-write to `/workspace`. +- A dedicated `awfvm-` network namespace contains `vmh*`, `vmn*`, and + `vmt*` veth/TAP interfaces. nftables permits access only to Squid and the API + proxy. +- The mandatory API proxy keeps provider credentials out of the guest + environment. +- The VMM runs as a non-root identity with `no_new_privs`, a minimal capability + set, Landlock filesystem rules, seccomp, and explicit cgroup v2 limits. +- The preview supports only GitHub-hosted Ubuntu x86_64 KVM runners and fails + closed on unsupported hosts or missing artifacts. + +See [Cloud Hypervisor integration](./cloud-hypervisor-foundation.md) for the +complete architecture, trust model, and operator guide. diff --git a/docs/awf-config-spec.md b/docs/awf-config-spec.md index 3c0179997..6437c6030 100644 --- a/docs/awf-config-spec.md +++ b/docs/awf-config-spec.md @@ -75,7 +75,6 @@ following top-level properties. All are OPTIONAL: | `apiProxy` | object | API proxy sidecar configuration | | `security` | object | Security and isolation settings | | `container` | object | Container and Docker settings | -| `firecracker` | object | Firecracker v1.16.1 control-plane preview settings | | `cloudHypervisor` | object | Cloud Hypervisor v53.0 microVM preview settings (see §4.1) | | `chroot` | object | Chroot execution overrides for split-filesystem ARC/DinD runners | | `dind` | object | Bootstrap helpers for ARC/DinD split runner/daemon filesystems | @@ -93,18 +92,18 @@ normatively by `docs/awf-config.schema.json`. The `cloudHypervisor` surface pins Cloud Hypervisor v53.0 artifacts and digests (binary, PCI-capable guest kernel, rootfs, and the shared AWF guest -supervisor) and, like Firecracker, requires explicit -`--cloud-hypervisor-preview` opt-in plus `container.containerRuntime: +supervisor) and requires explicit `--cloud-hypervisor-preview` opt-in plus +`container.containerRuntime: "cloud-hypervisor"` to execute a workload. Supported host target is GitHub-hosted Ubuntu `x86_64` runners with KVM only — self-hosted and non-Ubuntu/non-x86_64 hosts are rejected explicitly by [`src/cloud-hypervisor/host-eligibility.ts`](../src/cloud-hypervisor/host-eligibility.ts), -unlike Firecracker's preview which permits self-hosted hosts. See +with no fallback to another runtime. See [`src/cloud-hypervisor/preflight.ts`](../src/cloud-hypervisor/preflight.ts) for the artifact/host trust-check module, [`src/cloud-hypervisor/launcher.ts`](../src/cloud-hypervisor/launcher.ts) -for the secure host launcher (network-namespace join, privilege drop, and -Landlock-based filesystem confinement in place of Firecracker's jailer), +for the secure host launcher (network-namespace join, privilege drop, Landlock +filesystem confinement, and seccomp), [`src/cloud-hypervisor/manager.ts`](../src/cloud-hypervisor/manager.ts) for the VM lifecycle, and [`guest/cloud-hypervisor/`](../guest/cloud-hypervisor/) for the guest artifact build/verification pipeline. See @@ -223,21 +222,7 @@ AWF settings MAY be supplied via config files, including stdin (`--config -`). - `container.dockerHostPathPrefix` → `--docker-host-path-prefix` - `container.runnerToolCachePath` → *(config-only; checked first for optional read-only runner tool cache mount, before `RUNNER_TOOL_CACHE` and `/home/runner/work/_tool` auto-detection)* - `container.mounts[]` → `-v, --mount` *(repeatable; each array entry maps to one Docker volume mount in `/host_path:/container_path[:ro|rw]` format (both paths must be absolute; host path must exist); in chroot mode, container paths are automatically prefixed with `/host`)* -- `container.containerRuntime` → `--container-runtime` *(user-facing runtime name: `"gvisor"` for OCI runtime in compose, `"sbx"` for Docker sbx microVM, `"firecracker"` for the explicit Firecracker v1.16.1 workload preview, or `"cloud-hypervisor"` for the explicit Cloud Hypervisor v53.0 workload preview (GitHub-hosted Ubuntu x86_64 KVM runners only; see §4.1). For gvisor: translates to `"runsc"`, injects `extra_hosts` for DNS workaround. For sbx, Firecracker, and Cloud Hypervisor: infrastructure stays in Compose while the primary agent runs in a microVM.)* -- `firecracker.previewEnabled` → `--firecracker-preview` -- `firecracker.firecrackerBinary` → `--firecracker-binary` -- `firecracker.jailerBinary` → `--firecracker-jailer-binary` -- `firecracker.kernelPath` → `--firecracker-kernel` -- `firecracker.rootfsPath` → `--firecracker-rootfs` -- `firecracker.supervisorPath` → `--firecracker-supervisor` -- `firecracker.vcpuCount` → `--firecracker-vcpus` -- `firecracker.memoryMib` → `--firecracker-memory-mib` -- `firecracker.apiTimeoutMs` → `--firecracker-api-timeout-ms` -- `firecracker.sha256.firecracker` → `--firecracker-binary-sha256` -- `firecracker.sha256.jailer` → `--firecracker-jailer-sha256` -- `firecracker.sha256.kernel` → `--firecracker-kernel-sha256` -- `firecracker.sha256.rootfs` → `--firecracker-rootfs-sha256` -- `firecracker.sha256.supervisor` → `--firecracker-supervisor-sha256` +- `container.containerRuntime` → `--container-runtime` *(user-facing runtime name: `"gvisor"` for an OCI runtime in Compose, `"sbx"` for a Docker sbx microVM, or `"cloud-hypervisor"` for the explicit Cloud Hypervisor v53.0 workload preview (GitHub-hosted Ubuntu x86_64 KVM runners only; see §4.1). gVisor translates to `"runsc"` and injects `extra_hosts` for its DNS workaround. For sbx and Cloud Hypervisor, infrastructure stays in Compose while the primary agent runs in a microVM.)* - `cloudHypervisor.previewEnabled` → `--cloud-hypervisor-preview` *(requires `container.containerRuntime: "cloud-hypervisor"` and a GitHub-hosted Ubuntu x86_64 KVM runner to execute a workload)* - `cloudHypervisor.cloudHypervisorBinary` → `--cloud-hypervisor-binary` - `cloudHypervisor.kernelPath` → `--cloud-hypervisor-kernel` @@ -300,27 +285,6 @@ AWF settings MAY be supplied via config files, including stdin (`--config -`). When `container.dockerHostPathPrefix` points at a daemon-visible shared `/tmp` path, the implementation stages the invoking CLI binary together with `/etc/passwd`, `/etc/group`, and the generated chroot `/etc/hosts` under that shared path so chroot mode can bootstrap on split-filesystem ARC/DinD hosts. -The `firecracker` surface is an explicit workload preview pinned to Firecracker -v1.16.1 on Linux/KVM (`x86_64` or `aarch64`). It requires strict network -isolation, a local Unix-socket Docker daemon, the matching jailer, and explicit -SHA-256 digests for Firecracker, jailer, kernel, rootfs, and the AWF guest -supervisor. AWF starts Compose infrastructure only, attaches the jailed -microVM to the proven internal bridge, and executes through vsock. Host access, -DinD, extra mounts, TTY, topology peers, and enclaves fail closed in this -preview. Selecting `firecracker` never falls back to another runtime. - -**macOS and Windows are permanently unsupported.** CI specifically supports -GitHub-hosted x64 `ubuntu-24.04`; KVM remains mandatory, and hosts without usable -`/dev/kvm` access fail closed. The API proxy is mandatory; provider credentials -are never passed as guest environment variables. No auto-download of artifacts; -all five artifact paths and their SHA-256 digests are required on every invocation. -The `firecracker-test-x86_64` x86_64 test/preview artifacts can be built -explicitly from the repository, but are not built or published by GitHub -Actions. They are not production defaults and are never auto-downloaded. See -[Firecracker integration (preview)](./firecracker-integration.md) for the -complete operator guide, trust model, workspace semantics, local validation, -and troubleshooting reference. - When DinD is detected, AWF preserves the detected `DOCKER_HOST` value for the agent environment (including MCP servers) so DinD-aware tooling can reach the correct daemon without manual workflow env overrides. `security.allowHostPorts` (`--allow-host-ports`) is accepted together with diff --git a/docs/awf-config.schema.json b/docs/awf-config.schema.json index 58b4dfea9..60d95db24 100644 --- a/docs/awf-config.schema.json +++ b/docs/awf-config.schema.json @@ -650,82 +650,9 @@ "enum": [ "gvisor", "sbx", - "firecracker", "cloud-hypervisor" ], - "description": "Runtime for the primary agent. \"gvisor\" uses runsc in Docker Compose. \"sbx\" uses a Docker sbx microVM. \"firecracker\" selects the explicit Linux/KVM Firecracker v1.16.1 workload preview. \"cloud-hypervisor\" selects the explicit Linux/KVM Cloud Hypervisor v53.0 workload preview (GitHub-hosted Ubuntu x86_64 KVM runners only). Infrastructure containers always use the default runc runtime." - } - } - }, - "firecracker": { - "type": "object", - "description": "Firecracker v1.16.1 workload preview configuration. Requires strict network isolation, a local Docker daemon, KVM, jailer, and explicitly checksummed artifacts. Selecting this runtime never falls back to Docker.", - "additionalProperties": false, - "properties": { - "previewEnabled": { - "type": "boolean", - "default": false, - "description": "Explicitly enable Firecracker preview workload execution." - }, - "firecrackerBinary": { - "type": "string", - "description": "Absolute path to the Firecracker v1.16.1 binary. Defaults to /usr/local/bin/firecracker." - }, - "jailerBinary": { - "type": "string", - "description": "Absolute path to the matching v1.16.1 jailer binary. Defaults to /usr/local/bin/jailer." - }, - "kernelPath": { - "type": "string", - "description": "Absolute path to the trusted guest Linux kernel image." - }, - "rootfsPath": { - "type": "string", - "description": "Absolute path to the trusted guest root filesystem image." - }, - "supervisorPath": { - "type": "string", - "description": "Absolute path to the built AWF Firecracker guest supervisor." - }, - "vcpuCount": { - "type": "integer", - "minimum": 1, - "default": 2, - "description": "Number of guest virtual CPUs." - }, - "memoryMib": { - "type": "integer", - "minimum": 1, - "default": 512, - "description": "Guest memory in MiB." - }, - "apiTimeoutMs": { - "type": "integer", - "minimum": 1, - "default": 5000, - "description": "Bounded timeout in milliseconds for Firecracker API socket readiness and requests." - }, - "sha256": { - "type": "object", - "description": "Pinned SHA-256 digests for trusted Firecracker artifacts. All entries are required for preview workload execution.", - "additionalProperties": false, - "properties": { - "firecracker": { - "$ref": "#/$defs/sha256Digest" - }, - "jailer": { - "$ref": "#/$defs/sha256Digest" - }, - "kernel": { - "$ref": "#/$defs/sha256Digest" - }, - "rootfs": { - "$ref": "#/$defs/sha256Digest" - }, - "supervisor": { - "$ref": "#/$defs/sha256Digest" - } - } + "description": "Runtime for the primary agent. \"gvisor\" uses runsc in Docker Compose. \"sbx\" uses a Docker sbx microVM. \"cloud-hypervisor\" selects the explicit Linux/KVM Cloud Hypervisor v53.0 workload preview (GitHub-hosted Ubuntu x86_64 KVM runners only). Infrastructure containers always use the default runc runtime." } } }, @@ -753,7 +680,7 @@ }, "supervisorPath": { "type": "string", - "description": "Absolute path to the built AWF guest supervisor (shared with Firecracker)." + "description": "Absolute path to the built AWF guest supervisor." }, "vcpuCount": { "type": "integer", diff --git a/docs/cloud-hypervisor-foundation.md b/docs/cloud-hypervisor-foundation.md index 8178352ed..c54ec2a7e 100644 --- a/docs/cloud-hypervisor-foundation.md +++ b/docs/cloud-hypervisor-foundation.md @@ -1,978 +1,262 @@ --- -title: Cloud Hypervisor integration (preview) -description: Cloud Hypervisor v53.0 microVM backend — REST API client, secure launcher, manager/backend, GitHub-hosted Ubuntu x86_64 KVM runners only, Landlock/seccomp confinement in place of a jailer. +title: Cloud Hypervisor architecture +description: Architecture, security boundaries, artifacts, networking, lifecycle, CI, and troubleshooting for the Cloud Hypervisor microVM preview. --- -:::caution[Preview — GitHub-hosted Ubuntu x86_64 KVM runners only] -Cloud Hypervisor is an explicit-opt-in preview, exactly like -[Firecracker](./firecracker-integration.md). It requires -`--cloud-hypervisor-preview` **and** `--container-runtime cloud-hypervisor`, -runs only on GitHub-hosted Ubuntu x86_64 KVM runners (self-hosted runners -are rejected, unlike Firecracker's preview), and is otherwise fail-closed. -Firecracker continues to work unchanged and is unaffected by this backend. -::: +Cloud Hypervisor runs the primary agent in a hardware-isolated microVM while +AWF keeps Squid and the API proxy in Docker Compose on the host. -This document originated as **stack layer 3**, which built a complete, -runnable Cloud Hypervisor microVM backend on top of the layer 1 VMM-neutral -`src/microvm/` primitives and the layer 2 configuration/artifact/preflight -foundation. **Stack layer 4** (this layer) adds the live-KVM GitHub Actions -CI workflow (`test-cloud-hypervisor.yml`, Part 14), the parity/security -smoke suite (`scripts/ci/cloud-hypervisor-live-smoke.sh`), and this -documentation update; Firecracker remains present and unaffected. Firecracker -removal, if it ever happens, is an explicit later layer and is out of scope -here. - -## Part 1 — What Cloud Hypervisor adds and why it is a preview - -### What Cloud Hypervisor is - -[Cloud Hypervisor](https://www.cloudhypervisor.org/) is a Rust VMM built on -`rust-vmm` crates, offering a REST API (`/api/v1`) over a Unix domain socket -for VM lifecycle management. AWF uses it as a second, alternative microVM -backend to Firecracker — same threat model (hypervisor-isolated agent -execution, mandatory network egress control, mandatory API proxy credential -isolation), different VMM implementation and host launch strategy. - -### Why it is a preview - -- It is supported **only** on GitHub-hosted Ubuntu x86_64 KVM runners (see - Part 4). Self-hosted runners, other architectures, and other operating - systems are all explicitly rejected. -- Cloud Hypervisor has **no jailer-equivalent process**. AWF replaces - jailer's chroot+pivot_root+capability-drop with a different (not weaker) - boundary: network-namespace join, non-root privilege drop, and - kernel-enforced Landlock filesystem confinement (see Part 3). -- The live-KVM GitHub Actions workflow that exercises this backend end to - end on real hardware is `test-cloud-hypervisor.yml` (Part 14), added in - stack layer 4 alongside the parity/security smoke suite. - -### Comparison with Firecracker - -| Aspect | Firecracker | Cloud Hypervisor | -|---|---|---| -| Control plane | REST over UDS, Firecracker-specific endpoints | REST over UDS, `/api/v1`, upstream OpenAPI-documented | -| Privileged launcher | `jailer` binary (chroot, cgroup, netns join, uid/gid drop) | None — AWF's own launcher (netns join via `ip netns exec`, privilege drop via `setpriv`, Landlock via VM config) | -| Bus for block/net/vsock | MMIO (`pci=off`) | virtio-**pci** (PCI required; no MMIO transport) | -| Host support | Linux/KVM, GitHub-hosted or self-hosted | GitHub-hosted Ubuntu x86_64 KVM only | -| Guest kernel, rootfs, supervisor | Own pinned artifacts | Shared upstream kernel config plus `CONFIG_VIRTIO_FS=y`; shared supervisor | -| Workspace | Writable ext4 image with stop-time copy-back | Live read-write virtio-fs export at `/workspace` | -| Resource limits | jailer's own cgroup (no explicit quotas set by AWF) | AWF creates and assigns an explicit memory/CPU/PID cgroup | - -## Part 2 — Architecture - -### Host-side components - -1. **`src/cloud-hypervisor/api-client.ts`** — `CloudHypervisorApiClient`, a - typed REST client over the Unix domain socket, implementing exactly the - endpoints AWF needs: `vmm.ping`, `vm.create`, `vm.boot`, `vm.info`, - `vm.counters`, `vm.shutdown`, `vmm.shutdown`. Every request has a bounded - wall-clock timeout and a 1 MiB response cap; error bodies (Cloud - Hypervisor's chained-error-message JSON arrays) are parsed into a single - readable message. -2. **`src/cloud-hypervisor/launcher.ts`** — pure functions and small classes - for the secure host launch: - - `buildCloudHypervisorLaunchCommand()` builds the exact argv AWF spawns: - `ip netns exec setpriv --reuid= --regid= - --groups= --no-new-privs --inh-caps=-all,+net_admin - --bounding-set=-all,+net_admin --ambient-caps=+net_admin - -- cloud-hypervisor --api-socket path= --log-file -v - --seccomp true`. `--groups=` replaces the operator's full - supplementary group list with only the group that owns `/dev/kvm` - (resolved by preflight) — a blanket `--clear-groups` would also drop - kvm-group access and make every real launch fail with EACCES. The - capability set is empty except for `CAP_NET_ADMIN`, retained via the - bounding, inheritable, and ambient sets together — Cloud Hypervisor's - virtio-net backend needs it to finish configuring the already-created, - already-owned TAP device (`vm.boot` otherwise fails with "Failed to - read the TAP flags from sysfs: Permission denied"). No shell is ever - invoked — this argv is passed directly to `execa`, never interpolated - into a shell string. - - `computeCloudHypervisorLandlockRules()` computes the minimal - `landlock_rules` list sent in the `vm.create` payload. - - `CloudHypervisorCgroup` manages a cgroup v2 hierarchy: it enables - `cpu`/`memory`/`pids` delegation (`cgroup.subtree_control`) at the - cgroup root and the shared parent directory before creating the - per-run leaf cgroup (cgroup v2 only materializes a controller's - interface files in a child once the parent delegates it), writes - explicit `memory.max`/`cpu.max`/`pids.max`, and assigns the launched - process's PID to it. Cleanup uses a plain `rmdir` on the leaf — - cgroupfs's controller files are virtual and a recursive `rm` fails. - `runCloudHypervisorPreflight` rejects cgroup v1-only hosts explicitly - (see Part 4) rather than falling back to a v1 hierarchy this class - does not manage. -3. **`src/cloud-hypervisor/manager.ts`** — `CloudHypervisorManager` owns one - run end to end: preflight → network namespace setup (reusing - `src/microvm/network.ts` unchanged) → rootfs-only supervisor injection → - private run-directory staging → cgroup setup → VMM launch → API-socket - readiness → one sandboxed `virtiofsd` per validated export → `vm.create` → - (later) `vm.boot` → VSOCK guest-supervisor connect, retried with a fresh - client on the guest-boot-timing race documented in Part 3 (reusing - `src/microvm/vsock-client.ts` and `guest-protocol.ts` unchanged) → - execution → graceful `vm.shutdown`/`vmm.shutdown` → VMM termination - → virtiofsd termination → network/cgroup/run-directory cleanup, with - aggregated cleanup-error reporting matching Firecracker's manager. - The class itself is an orchestration facade; the supporting pieces live - beside it in `src/cloud-hypervisor/`: shared run paths/types in - `manager-types.ts`, the `vm.create` payload in `vm-config-builder.ts`, - run-directory staging plus failure diagnostics in `diagnostics.ts`, and - the guest VSOCK execution/IO surface in `guest-execution.ts`. -4. **`src/cloud-hypervisor-runtime-backend.ts`** — `CloudHypervisorRuntimeBackend` - implements `ExternalAgentRuntimeBackend`: infrastructure discovery - (`src/microvm/infrastructure.ts`, unchanged), credential-safe guest - environment construction, guest connectivity probing (Squid + API - proxy), cancellation/timeout/exit-code handling, and diagnostics - collection — structurally identical to `FirecrackerRuntimeBackend`. - -### Guest contents - -Shared with Firecracker except for a deterministic `CONFIG_VIRTIO_FS=y` -overlay on the PCI-capable guest kernel (the upstream -`microvm-kernel-ci-x86_64-6.1.config` SHA remains pinned), -a deterministic BusyBox + CA-bundle ext4 rootfs, and the VMM-neutral -shared `awf-supervisor` guest binary (`guest/firecracker-supervisor/`). - -### Control flow +:::caution[Preview support] +This runtime requires both `--container-runtime cloud-hypervisor` and +`--cloud-hypervisor-preview`. It supports only GitHub-hosted Ubuntu x86_64 +runners with KVM and fails closed on other hosts. +::: -``` -awf --container-runtime cloud-hypervisor --cloud-hypervisor-preview ... - ↓ -assertCloudHypervisorRuntimeCompatibility() — security mode, topology, GitHub-hosted host eligibility, artifact/digest completeness - ↓ -runCloudHypervisorPreflight() — Linux/KVM/x86_64, /dev/kvm + owning gid, cgroup v2 (v1-only hosts rejected), trusted host tools incl. setpriv, trusted+pinned+digest-verified artifacts - ↓ -CloudHypervisorManager.start() - ↓ -MicrovmNetworkManager.setup() (netns, veth, TAP, nftables — shared with Firecracker) - ↓ -MicrovmRootfsPreparer.prepare() (writable rootfs copy + supervisor injection; no workspace image) - ↓ -private run directory under /run/awf-cloud-hypervisor (0711 ancestors, 0700 leaf owned by the non-root identity) + per-run cgroup v2 (subtree_control delegated root→parent→leaf) - ↓ -buildCloudHypervisorLaunchCommand() → ip netns exec → setpriv --groups= --ambient-caps=+net_admin → cloud-hypervisor --api-socket ... --seccomp true (minimal PATH-only environment) - ↓ -wait for API socket → vmm.ping → start sandboxed virtiofsd daemons → vm.create (memory.shared=true, fs devices, landlock_enable=true) - ↓ -CloudHypervisorRuntimeBackend.start(): vm.boot → VSOCK connect (CID 3, CONNECT \n, retried on the guest-boot-timing race — see Part 3) → Squid/API-proxy connectivity probe - ↓ -Agent command executes inside the guest via the VSOCK guest-protocol transport (unchanged) - ↓ -guest sync + reverse unmount → vm.shutdown → vmm.shutdown → SIGTERM/SIGKILL fallback → stop/reap virtiofsd → network/cgroup/run-directory cleanup +## Architecture overview + +The runtime separates infrastructure from workload execution: + +```text +GitHub-hosted Ubuntu x86_64 runner +├── Docker Compose +│ ├── Squid proxy +│ └── API proxy +├── AWF control process +│ ├── artifact and host preflight +│ ├── Cloud Hypervisor REST client +│ ├── network namespace and nftables policy +│ ├── cgroup v2 resource limits +│ └── sandboxed virtiofsd processes +└── Cloud Hypervisor microVM + ├── pinned Linux kernel and rootfs + ├── shared AWF guest supervisor + ├── /workspace through virtio-fs + └── agent command ``` -## Part 3 — Security boundary: the launcher in place of a jailer - -Cloud Hypervisor ships as a single static binary with **no jailer -equivalent** — nothing that atomically joins a network namespace, chroots, -drops capabilities, and execs the VMM. Reimplementing jailer's chroot + -`pivot_root` for a foreign binary was judged impractical and risky within -this layer, so AWF instead documents and tests an explicit replacement -boundary: - -1. **Network namespace join.** `ip netns exec ...` execs - directly into the namespace `src/microvm/network.ts` already prepared - (the same TAP/veth/nftables topology Firecracker uses), without an - intermediate fork — the resulting process keeps the PID the host - observes for cgroup assignment. -2. **Privilege drop.** `setpriv --reuid= --regid= - --groups= --no-new-privs --inh-caps=-all,+net_admin - --bounding-set=-all,+net_admin --ambient-caps=+net_admin` - execs Cloud Hypervisor as the same non-root operator identity - Firecracker's jailer targets (`SUDO_UID`/`SUDO_GID`), with `no_new_privs` - set and an otherwise-empty capability set before any guest code runs. - `--groups=` replaces the operator's supplementary group - list with only the group that owns `/dev/kvm` (resolved by preflight); - a blanket `--clear-groups` would also drop that membership and make - every real launch fail opening `/dev/kvm` even though root-run - preflight passed. The capability set retains exactly one exception, - `CAP_NET_ADMIN` — via the bounding, inheritable, and ambient sets - together, so it survives the uid change and `execve()` of a plain, - non-file-capability binary even under `--no-new-privs` — because Cloud - Hypervisor's virtio-net backend needs it to finish configuring the - already-created, already-owned TAP device; without it, `vm.boot` fails - with "Failed to read the TAP flags from sysfs: Permission denied". -3. **Filesystem confinement.** In place of jailer's userspace chroot, AWF - combines: - - a **private run directory** under `/run/awf-cloud-hypervisor///` - — deliberately **outside** `workDir` (which is root-owned `0700` - because it holds `docker-compose.yml`'s plaintext secrets). Since - there is no `chroot()` to make host-side ancestor permissions - irrelevant, the non-root launched process must be able to really - traverse down to the run directory: the two ancestor levels are - `0711` (traversable/executable by any uid, but not listable), and - only the per-run leaf directory is chowned to the target identity - with `0700` (so only that identity, or root, can read its contents); - - **Landlock**, a Linux LSM, enabled via `landlock_enable: true` in the - `vm.create` payload with a minimal `landlock_rules` list (kernel image - read-only; rootfs and the run directory read-write; - `/dev/kvm` and `/dev/net/tun` read-write for KVM ioctls and TAP - attachment; the TAP's own `/sys/class/net/` directory - read-only, for the world-readable `tun_flags` sysfs attribute Cloud - Hypervisor's virtio-net setup reads to detect multi-queue support — - without this rule Landlock itself blocks that read, surfacing as - `vm.boot` failing with "Failed to read the TAP flags from sysfs: - Permission denied" even though ordinary Unix file permissions would - have allowed it). Any path not listed becomes inaccessible to the Cloud - Hypervisor process the instant Landlock is enabled. Export source trees - are deliberately absent: only the separate virtiofsd processes can open - them. This is enforced by the - kernel, not a userspace boundary a compromised process could bypass. - - Cloud Hypervisor's own **default seccomp filter** (`--seccomp true`, - its default kill-on-violation mode). - - This is a **different** boundary than jailer's chroot — kernel-LSM-based - rather than mount-namespace-based — not a weaker one. It is exercised by - `src/cloud-hypervisor/launcher.test.ts` (argv construction, Landlock rule - computation, cgroup lifecycle) rather than silently degraded to "no - filesystem confinement". -4. **Resource limits.** A dedicated cgroup **v2** hierarchy is created - before launch: `cpu`/`memory`/`pids` delegation is enabled at the - cgroup root and the shared parent directory (`cgroup.subtree_control`) - before the per-run leaf cgroup is created, then explicit - `memory.max`/`cpu.max`/`pids.max` are written and the launched - VMM and every virtiofsd PID are assigned to it immediately after spawn. Cgroup v1-only - hosts are rejected explicitly at preflight (see Part 4) rather than - silently constructing a broken multi-controller v1 hierarchy. -5. **The management API socket is never guest-accessible.** It lives only - in the host-side private run directory; it is never passed to the guest - as a drive, vsock, or virtio-fs device, and Landlock additionally blocks - any *new* open() of it by the Cloud Hypervisor process after `vm.create` - (the already-open listening socket is unaffected, matching how a - jailer-chrooted Firecracker keeps its already-open resources). -6. **Minimal launcher environment.** The launched process receives an - explicit minimal environment (just `PATH`), never `process.env` — - Cloud Hypervisor directly parses untrusted guest/device input, so a VMM - compromise reading its own inherited environment could otherwise read - provider/GitHub credentials and bypass the API-proxy credential - isolation boundary entirely. - -## Part 4 — Prerequisites and supported hosts - -### Supported host configurations - -| Requirement | Value | -|-------------|-------| -| Operating system | **Linux only**, and additionally **GitHub-hosted only** (`GITHUB_ACTIONS=true`, `RUNNER_ENVIRONMENT=github-hosted`) | -| Distribution | Ubuntu (`ImageOS` must start with `ubuntu`) | -| Architecture | x86_64 only | -| KVM device | `/dev/kvm` must exist and be readable + writable | -| Cgroup hierarchy | **cgroup v2 unified only** (`/sys/fs/cgroup/cgroup.controllers` must exist) — cgroup v1-only hosts are rejected explicitly; see `CloudHypervisorCgroup` in Part 3 | -| Self-hosted runners | **Explicitly rejected** — see `src/cloud-hypervisor/host-eligibility.ts` | - -Host eligibility is checked in two layers: `evaluateGithubHostedRunnerEligibility()` -(host identity only — cheap, environment-variable based) and the full -`runCloudHypervisorPreflight()` (live capability checks: `/dev/kvm`, cgroup -version, trusted host tools, artifact trust/digests). - -### Required host tools - -Same as Firecracker (`ip`, `nft`, `sysctl`, `mke2fs`, `debugfs`, `e2fsck`, -`rsync`), plus: - -| Tool | Purpose | -|------|---------| -| `setpriv` | Drops to the non-root operator uid/gid before Cloud Hypervisor execs, retaining only `CAP_NET_ADMIN` (util-linux; standard on Ubuntu) | - -### Operator account - -Same as Firecracker: AWF must be invoked through `sudo` from a **non-root** -account; `SUDO_UID`/`SUDO_GID` determine the target identity for both the -launcher's `setpriv` step and the guest execution identity. The target -account must have `/dev/kvm` access (typically via `kvm` group membership). - -## Part 5 — Artifact policy - -Identical trust model to Firecracker (see -[Firecracker's Part 5](./firecracker-integration.md#part-5--artifact-policy)): -root/operator-owned non-writable regular files, trusted ancestor -directories, pinned version, mandatory SHA-256 digests. Cloud Hypervisor has no jailer-equivalent binary. It additionally requires a -trusted executable sibling `virtiofsd`, pinned to Ubuntu Noble's v1.10.0. - -| Artifact | Version | SHA-256 | -|---|---|---| -| `cloud-hypervisor` (x86_64 static) | v53.0 | `448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc` | -| `virtiofsd` (Ubuntu Noble `/usr/libexec/virtiofsd`) | v1.10.0 | recorded in bundle `SHA256SUMS` | -| Linux kernel source | 6.1.141 | `bc3c45faf6f5f0450666c75fa9dad9bc7c0cf7c7cba0dbd94e5cfdc58229c116` | -| Upstream kernel config (from Firecracker v1.16.1) | `microvm-kernel-ci-x86_64-6.1.config` | `adbc70ab5e89213ba00594b12d25e09bdf8bb1ed3c252d7449326bb14c22963b` | -| Final kernel config | upstream plus `CONFIG_FUSE_FS=y` and `CONFIG_VIRTIO_FS=y`, then `olddefconfig` | emitted as `kernel.config` and recorded in `SHA256SUMS` | -| BusyBox source | 1.36.1 | `b8cc24c9574d809e7279c3be349795c5d5ceb6fdf19ca709f80cde50e47de314` | -| CA bundle | 2025-02-25 | `50a6277ec69113f00c5fd45f09e8b97a4b3e32daa35d3a95ab30137a55386cef` | - -## Part 6 — Devices, boot, and networking - -- **Boot**: direct kernel boot (no UEFI/firmware layer), root device - `/dev/vda`, `rootfstype=ext4`, `rw`, - `net.ifnames=0 biosdevname=0` for deterministic `eth0` naming. Unlike - Firecracker, `pci=off` is **not** set — Cloud Hypervisor requires PCI. -- **Devices**: one virtio-**pci** block device (rootfs), virtio-fs shares, - net (single TAP, - pre-created and owned exactly like Firecracker's), vsock (CID 3, same - `CONNECT \n` transport and guest-protocol framing as Firecracker), - serial console redirected to a bounded host log file, virtio-console - disabled (`mode: "Off"`). No snapshots, migration, hotplug, - VFIO, vhost-user, vDPA, TDX/SEV, or TPM. -- **Networking**: the same TAP/netns/nftables design as Firecracker - (`src/microvm/network.ts`) — mandatory network isolation and mandatory API - proxy credential isolation. Trusted `topologyAttach` peers are resolved from - the proven internal Docker network, revalidated before boot, injected into - the guest hosts file, and allowed only on TCP 8080 for the MCP gateway. - -## Part 7 — Bounded diagnostics - -`CloudHypervisorManager.collectDiagnostics()` writes, all under a -0700-mode directory with 0600-mode files bounded to 1 MiB each: launcher -stdout/stderr capture, the Cloud Hypervisor log file, the guest serial -console log, bounded per-export virtiofsd logs, `vm.counters()` output (best-effort — failures don't block -diagnostics), the resolved network plan, and a `runtime.json` summary. This -mirrors Firecracker's `collectDiagnostics()` shape exactly. - -## Part 8 — CLI reference +Cloud Hypervisor exposes its lifecycle API over a Unix domain socket. AWF uses +the `/api/v1` endpoints needed to create, boot, inspect, shut down, and delete a +VM. Every API request has a bounded timeout and response-size limit. + +## Host components + +The implementation is divided into focused modules: + +- `src/cloud-hypervisor-runtime-backend.ts` implements the external agent + runtime contract, builds the credential-safe guest environment, probes Squid + and the API proxy, executes the command, and preserves diagnostics. +- `src/cloud-hypervisor/manager.ts` orchestrates preflight, networking, rootfs + preparation, VMM startup, virtio-fs, guest execution, and cleanup. +- `src/cloud-hypervisor/api-client.ts` implements the REST client over the Unix + socket. +- `src/cloud-hypervisor/launcher.ts` builds the non-shell VMM command and + computes Landlock rules. +- `src/cloud-hypervisor/vm-config-builder.ts` constructs the `vm.create` + payload. +- `src/microvm/` contains shared network, workspace, VSOCK, guest-protocol, and + artifact primitives. +- `guest/microvm-supervisor/` contains the shared guest supervisor. +- `guest/cloud-hypervisor/` contains Cloud Hypervisor artifact build and + verification tooling. + +## Runtime lifecycle + +AWF performs these steps for each run: + +1. Validate the runtime flags, security mode, topology, host eligibility, and + required artifact paths and digests. +2. Verify Linux, x86_64, KVM, cgroup v2, Landlock, Docker, and required host + tools. +3. Start the Squid and API proxy infrastructure through Docker Compose. +4. Create a dedicated network namespace, veth pair, TAP device, and nftables + policy. +5. Copy the rootfs, inject the guest supervisor, and stage files in a private + run directory. +6. Create a bounded cgroup v2 leaf and launch Cloud Hypervisor as the invoking + non-root identity. +7. Start one sandboxed `virtiofsd` process for each validated export. +8. Create and boot the VM, connect to the guest supervisor over VSOCK, and + probe infrastructure connectivity. +9. Execute the agent command and propagate its exit code. Timeouts return + `124`. +10. Sync and unmount guest filesystems, stop the VM and VMM, reap `virtiofsd`, + and remove network, cgroup, and run-directory resources. + +Cleanup is idempotent and aggregates errors so one cleanup failure does not +skip later cleanup steps. + +## Security boundaries + +### Host eligibility and artifact trust + +The runtime accepts only GitHub-hosted Ubuntu x86_64 runners. Preflight verifies +the GitHub Actions environment markers, `/dev/kvm`, the KVM group, cgroup v2, +Landlock, and required tools before creating the VM. + +AWF never downloads runtime artifacts automatically. You must provide the +Cloud Hypervisor binary, guest kernel, rootfs, supervisor, and `virtiofsd` +paths together with their expected SHA-256 digests. Preflight rejects missing, +mutable, incorrectly owned, or digest-mismatched artifacts. + +:::danger[Fail-closed verification] +Do not bypass artifact verification. A substituted VMM, kernel, rootfs, +supervisor, or filesystem daemon runs inside a trusted part of the boundary. +::: -```bash -sudo awf \ - --container-runtime cloud-hypervisor \ - --cloud-hypervisor-preview \ - --cloud-hypervisor-binary /usr/local/bin/cloud-hypervisor \ - --cloud-hypervisor-kernel /opt/awf/vmlinux \ - --cloud-hypervisor-rootfs /opt/awf/rootfs.ext4 \ - --cloud-hypervisor-supervisor /opt/awf/awf-supervisor \ - --cloud-hypervisor-binary-sha256 \ - --cloud-hypervisor-virtiofsd-sha256 \ - --cloud-hypervisor-kernel-sha256 \ - --cloud-hypervisor-rootfs-sha256 \ - --cloud-hypervisor-supervisor-sha256 \ - --enable-api-proxy \ - --allow-domains github.com \ - -- npx @github/copilot --prompt "list files" +### VMM confinement + +AWF launches Cloud Hypervisor through `ip netns exec` and `setpriv` without a +shell. The process: + +- runs as the non-root identity recorded by `SUDO_UID` and `SUDO_GID`; +- keeps only the KVM supplementary group; +- sets `no_new_privs`; +- retains only `CAP_NET_ADMIN`, which the virtio-net TAP setup requires; +- uses Cloud Hypervisor's seccomp filter; +- receives a minimal Landlock filesystem allowlist; and +- belongs to a cgroup v2 leaf with explicit memory, CPU, and PID limits. + +The private run directory is under +`/run/awf-cloud-hypervisor///`. Its per-run leaf is accessible +only to the selected non-root identity and root. + +### Credential isolation + +The API proxy is mandatory. Provider credentials remain in the host-side proxy +and are not copied into the guest environment. The guest receives only the +proxy endpoint and non-secret execution settings. + +### Network egress + +Each run uses deterministic, length-bounded resource names: + +- namespace: `awfvm-` +- host veth: `vmh` +- namespace veth: `vmn` +- TAP device: `vmt` + +The namespace connects the guest TAP to AWF's host-side infrastructure. +nftables permits only the required paths to Squid and the API proxy and denies +direct internet, arbitrary TCP, direct DNS, and instance metadata access. +Guest proxy environment variables improve client compatibility, but the +namespace policy is the enforcement boundary. + +## Guest and workspace + +The guest boots a pinned PCI-capable Linux kernel and deterministic BusyBox +rootfs. AWF injects the binary built from `guest/microvm-supervisor/` into the +per-run rootfs. + +The workspace is a live read-write virtio-fs export mounted at `/workspace`. +Validated additional exports use separate sandboxed `virtiofsd` processes. +The workspace path is never exposed directly to the VMM process through its +Landlock rules. + +Temporary microVM workspace data lives under: + +```text +/microvm-images// ``` -See [`docs/awf-config-spec.md`](./awf-config-spec.md) §4.1 for the full -config-file/CLI mapping. - -## Part 9 — Explicit scope limits (this layer) - -- **Direct kernel boot only.** No UEFI/firmware layer. -- **One raw ext4 rootfs disk**, `backing_files: false`, plus the fixed - narrow virtio-fs export policy. No arbitrary shares, - snapshot/restore, hotplug, VFIO, vhost-user, vDPA, or confidential - computing. -- **virtio-pci transport only** for block, net, and vsock devices. -- **GitHub-hosted Ubuntu x86_64 KVM runners only.** Self-hosted runners and - non-Ubuntu/non-x86_64 hosts are explicitly rejected. -- **No TTY, DinD, host access, extra volume mounts, DIFC proxies, or - enclaves.** Trusted MCP gateway topology peers are supported only through the - exact discovered internal-network IP and TCP port 8080. -- **No vhost-net/vhost-user and no throughput claims.** The performance - baselines in Part 14 measure boot/readiness latency and cgroup-bounded - memory overhead only; this preview makes no network throughput guarantees. -- **Firecracker is unaffected and remains supported.** Removing Firecracker - is an explicit later layer, not this one. - -### Virtio-fs export policy and tradeoffs - -The workspace is exported live, read-write, as tag `workspace` at -`/workspace`; there is no ext4 workspace image, staging copy, or copy-back. -Host and guest therefore observe writes immediately. This avoids stale merge -semantics but also means a guest write changes the host workspace directly. - -Optional exports are limited to `RUNNER_TOOL_CACHE` (falling back to -`AGENT_TOOLSDIRECTORY`) read-only at the same absolute path, -`${RUNNER_TEMP}/gh-aw` read-only at the same absolute path, and `/tmp/gh-aw` -read-write. Missing optional directories are skipped. AWF does not export all -of `RUNNER_TEMP`, the host home, or arbitrary paths. Guest `HOME` is -workspace-local `/workspace/.awf-home`, so it is writable by the invoking -runner identity even when that identity is not UID 1000. - -Each export has its own pinned virtiofsd with namespace sandboxing, explicit -kill-on-violation seccomp, controlled caching, and disabled inode file handles. -Read-only exports are backed by host read-only bind mounts under a separate -private directory before virtiofsd starts; guest mount flags enforce the same -policy again. Sockets and bounded logs live in the VMM run directory, while -the bind mounts remain outside its Landlock rules. The VMM receives socket -paths only and cannot access host source trees directly. +With `--keep-containers`, AWF preserves this directory, the network namespace, +and runtime diagnostics for investigation. + +## Limitations + +The preview rejects configurations that weaken or conflict with its boundary, +including: + +- self-hosted, non-Ubuntu, non-x86_64, or non-KVM hosts; +- remote Docker daemons; +- TTY mode; +- Docker-in-Docker agent execution; +- topology peers; +- unsupported host mounts; and +- enclave combinations not supported by the external runtime contract. + +Selecting Cloud Hypervisor never falls back to Docker, gVisor, or sbx. ## Part 14 — CI workflow -The Cloud Hypervisor preview has its own dedicated CI workflow, -[`test-cloud-hypervisor.yml`](../.github/workflows/test-cloud-hypervisor.yml), -based on the retained -[Firecracker test conventions](./firecracker-integration.md#part-14--ci-workflow) -but adapted for this backend's GitHub-hosted-only support statement and -jailer-free launcher. The corresponding Firecracker Actions workflow is -disabled. - -### Trigger conditions - -The workflow triggers **only** on: - -- `workflow_dispatch` — manual trigger; `run_live_kvm: false` validates the - deterministic artifact build without queueing the KVM job. -- A pull request open, synchronize, or reopen event, **scoped to paths** - under `guest/cloud-hypervisor/`, `guest/firecracker-supervisor/` (shared - guest supervisor), `src/cloud-hypervisor/`, - `src/cloud-hypervisor-runtime-backend.ts`, `src/microvm/`, - `scripts/ci/cloud-hypervisor-*.sh`, this document, and the workflow file - itself — builds and verifies the deterministic guest artifacts on - `ubuntu-24.04`. -- A pull request being **labeled** with `cloud-hypervisor-kvm` additionally - enables the live KVM job. - -It does **not** run on push or schedule. The path scoping keeps CI cost -proportional to actual changes to this preview instead of running on every -unrelated pull request. - -### Jobs - -#### `build-test-artifacts` (runs on `ubuntu-24.04`, no KVM required) - -1. Checks out the repository and sets up Go 1.25.0 (cache keyed on the - shared `guest/firecracker-supervisor/go.mod`). -2. Installs the same deterministic kernel-build prerequisites Firecracker's - job uses (`bc`, `binutils`, `bison`, `build-essential`, `cpio`, - `e2fsprogs`, `file`, `flex`, `libelf-dev`, `libssl-dev`, `rsync`, `virtiofsd`, - `xz-utils`) — both backends use the same pinned upstream kernel config; - Cloud Hypervisor applies the documented virtio-fs overlay. -3. Builds the canonical ARC/DinD `build-tools` sysroot image, then runs - `guest/cloud-hypervisor/build-test-artifacts.sh` — downloads Cloud - Hypervisor v53.0 (SHA-256 verified) and Linux 6.1.141 (SHA-256 verified); - applies and verifies `CONFIG_VIRTIO_FS=y`; copies Ubuntu Noble - virtiofsd v1.10.0; builds the kernel and shared supervisor; exports the Ubuntu 22.04 - `build-tools` userspace into the guest rootfs; produces - `SHA256SUMS`, `manifest.json`, and `sbom.spdx.json`; archives everything - as `release/cloud-hypervisor-test-x86_64/awf-cloud-hypervisor-test-x86_64.tar.gz`. -4. Runs `guest/cloud-hypervisor/verify-test-artifacts.sh` against the output. -5. Attests artifact provenance via `actions/attest-build-provenance`. -6. Uploads `release/cloud-hypervisor-test-x86_64/` as artifact - `cloud-hypervisor-test-x86_64` with 7-day retention. - -#### `live-kvm` (runs on `ubuntu-24.04`) - -Gated on manual dispatch (`run_live_kvm: true`) or the `cloud-hypervisor-kvm` -pull request label — never on unlabeled pull requests, push, or schedule. -Its preflight fails closed (does not silently skip) if the runner lacks -usable KVM or any other required host capability. - -1. Downloads the `cloud-hypervisor-test-x86_64` artifact from the build job. -2. Runs `scripts/ci/cloud-hypervisor-host-preflight.sh` — verifies Linux, - x86_64, GitHub-hosted-only host eligibility (`GITHUB_ACTIONS`, - `RUNNER_ENVIRONMENT`, `ImageOS`), `/dev/kvm`, required host tools - including `setpriv`, Landlock LSM availability, the Cloud Hypervisor - and virtiofsd version strings, and all artifact SHA-256 digests via - `sha256sum --check --strict SHA256SUMS`. -3. Installs NPM dependencies, builds the AWF distribution, and builds the - Squid and API proxy container images locally. -4. Runs `scripts/ci/cloud-hypervisor-live-smoke.sh` — the live test suite. -5. Collects redacted diagnostics (audit, proxy-logs, stdout/stderr) and - scans for the secret sentinel before uploading, unconditionally - (`if: always()`). -6. Enforces final residue cleanup — network namespaces, the - `awf-cloud-hypervisor` cgroup tree, and any lingering `cloud-hypervisor` - process — unconditionally (`if: always()`). - -### Live smoke test assertions - -`cloud-hypervisor-live-smoke.sh` reproduces -[Firecracker's full 13-case contract](./firecracker-integration.md#live-smoke-test-assertions) -verbatim (same case names, same expected exit codes, same assertions), then -adds three Cloud Hypervisor-specific cases: - -| Case | Expected exit | Assertion | -|------|--------------|-----------| -| `allowed-https` | 0 | `wget https://example.com` succeeds and returns "Example Domain" | -| `blocked-domain` | 0 | `wget https://github.com` fails (not in `--allow-domains`) | -| `direct-egress` | 0 | Unsets proxy vars; direct `wget` fails | -| `arbitrary-tcp` | 0 | `nc -z 1.1.1.1 443` fails | -| `dns-denial` | 0 | `nslookup example.com 8.8.8.8` fails | -| `metadata-denial` | 0 | Unsets proxy vars; `wget http://169.254.169.254/latest/meta-data/` fails | -| `api-proxy-reflect` | 0 | API proxy `/reflect` reachable; secret sentinel absent from guest `env` | -| `workspace-live-share` | 0 | Guest writes, `chmod 755`, and a symlink are immediately visible on the host | -| `runtime-cache-readonly` | 0 | `${RUNNER_TEMP}/gh-aw` is readable but guest writes fail | -| `exit-code` | 37 | `exit 37` propagates as AWF exit code 37 | -| `timeout-124` | 124 | `sleep 90` with `--agent-timeout 1` exits 124 | -| `device-assumptions` **(new)** | 0 | `/dev/vda` is the sole block disk, `/workspace` is virtio-fs, and `eth0` exists | -| `partial-start-cleanup` | 1 | Corrupt rootfs (valid digest, invalid content) fails cleanly; no residue | -| `cancellation` | 143 | `SIGTERM` after namespace appears cleans up; cleanup time is measured against a non-flaky ceiling | -| `keep` (keep mode) | 0 | `--keep-containers` preserves namespace/run-directory/diagnostics; all bounded ≤1 MiB | -| `security-assertions` **(new)** | 143 | See below | - -The `security-assertions` case starts a long-lived guest command, then — -while the VM is live — inspects the host-visible Cloud Hypervisor process -and its own `vm.info` response to prove the launcher's jailer-replacement -boundary (Part 3) live, not just at the argv-construction unit-test level: - -- **Non-root identity**: `/proc/` is not owned by uid 0. -- **Minimal effective capability set**: `/proc//status`'s `CapEff` is - exactly `0000000000001000` — `CAP_NET_ADMIN` alone (needed for Cloud - Hypervisor's virtio-net backend to finish configuring the already-owned - TAP device; see Part 3) and nothing else. -- **`no_new_privs` is set**: `NoNewPrivs: 1`. -- **Active seccomp filter**: `Seccomp: 2` (filter mode), confirming - `--seccomp true` is in effect. -- **Cgroup membership and bounded limits**: the process's PID appears in - `/sys/fs/cgroup/awf-cloud-hypervisor//cgroup.procs`; `memory.max` - is a bounded positive number (cgroup v1 hosts are rejected outright by - preflight, so there is no v1 fallback to check); live `memory.current` - usage is greater than zero and does not exceed it (the "bounded memory - overhead" baseline from Part 4/9). -- **`landlock_enable` reflected in `vm.create`** and an **exactly-minimal - device set**: querying the Cloud Hypervisor process's own - `GET /api/v1/vm.info` over its private Unix domain socket confirms - exactly one rootfs disk, the expected narrow virtio-fs devices, exactly one net device, and a - vsock device — proving the host-only API socket path is never wired to - the guest as any device (structurally, not just by absence of a mount). - -Boot/readiness and cleanup-time are measured as non-flaky regression -baselines (generous ceilings tuned for shared GitHub-hosted runners, not -tight performance targets): the `allowed-https` case's total wall time is -checked against a 90-second boot+readiness+run+cleanup ceiling, and the -`cancellation` case's SIGTERM-to-clean-residue time is checked against a -20-second ceiling. - -### Secret sentinel check - -All smoke test cases set -`OPENAI_API_KEY=awf-cloud-hypervisor-real-secret-do-not-expose` (a value -distinct from Firecracker's sentinel so the two suites' diagnostics never -cross-contaminate if run in the same job). After each run, the test scans -`stdout.log`, `audit/`, and `proxy-logs/` for this sentinel string. - -### Shared vs. backend-specific residue naming - -`src/microvm/network.ts` is VMM-neutral and used unmodified by both -backends (Part 2), so the network namespace (`awffc-*`) and veth/TAP -(`fch*`/`fcn*`/`fct*`) residue checks are intentionally identical to -Firecracker's — this is shared infrastructure, not a Cloud Hypervisor gap. -The cgroup path (`/sys/fs/cgroup/awf-cloud-hypervisor/`) and process -name (`cloud-hypervisor`) residue checks are Cloud Hypervisor-specific. - -## Part 15 — Troubleshooting - -Most preflight, boot, and cleanup failure modes are identical to -Firecracker's — see -[Firecracker's Part 15](./firecracker-integration.md#part-15--troubleshooting) -for the general shape of each failure (host tool missing, digest mismatch, -API timeout, guest connectivity probe failure, Docker infrastructure not -cleaned up; Firecracker-only workspace copy-back recovery via `debugfs`). This section covers -only what differs for Cloud Hypervisor. - -**`Cloud Hypervisor is supported only on GitHub-hosted runners, not self-hosted`** - -Unlike Firecracker's preview, this backend rejects self-hosted runners -outright (`src/cloud-hypervisor/host-eligibility.ts`). There is no flag to -override this; run on a GitHub-hosted Ubuntu x86_64 runner instead. - -**`Cloud Hypervisor requires a GitHub-hosted Ubuntu runner image`** - -`ImageOS` did not start with `ubuntu` — you are likely on a non-Ubuntu -GitHub-hosted image (e.g. Windows or macOS runners, which also lack -`/dev/kvm`). Use an `ubuntu-24.04` (or later) runner. - -**`Cloud Hypervisor is pinned to v53.0; found v`** - -The Cloud Hypervisor binary is not v53.0. Do not bypass the version check; -obtain the pinned release. - -**`host tool "setpriv" was not found on PATH`** - -Install `util-linux` (`setpriv` ships in it; standard on Ubuntu, so this -should not occur on a genuine GitHub-hosted Ubuntu runner). - -**`The running kernel does not report Landlock in /sys/kernel/security/lsm`** - -Landlock requires Linux 5.13+ with `CONFIG_SECURITY_LANDLOCK=y` and the LSM -enabled at boot (`lsm=landlock,...` on the kernel cmdline, or Landlock -included in the distribution default). GitHub-hosted Ubuntu 24.04 runners -ship this by default; a custom or older kernel may not. - -**`guest disconnected before readiness` on `startInstance()`** - -This is expected transient behavior, not a bug: Cloud Hypervisor's own -vsock-over-UDS multiplexer closes the host-facing connection immediately if -the guest hasn't yet started listening on the target vsock port (kernel -boot + guest supervisor startup take a variable, host-load-dependent amount -of time). `CloudHypervisorManager.startInstance()` retries the connect with -a fresh client every 250 ms for up to 90 seconds before surfacing the error; -seeing it in this error message means that entire retry budget was -exhausted. If it persists, check the guest serial console log -(`/cloud-hypervisor/serial.log`) for a kernel panic or a -supervisor startup failure rather than assuming it is purely a timing -issue. - -**Known GitHub-hosted-runner limitation: multi-vCPU guest boot can stall -for 90+ seconds under nested virtualization.** Live validation captured a -guest boot that made no further progress for 90+ real seconds immediately -after its serial console logged `kvm-guest: setup PV IPIs` at kernel -virtual time ~0.13s — the point where a guest with more than one vCPU -begins bringing up its secondary (AP) CPUs via inter-processor interrupts -(INIT-SIPI-SIPI). GitHub-hosted runners execute Cloud Hypervisor under -*nested* virtualization (its own log reports `Running under nested -virtualisation. Hypervisor string: Microsoft Hv` there); local -APIC/IPI virtualization for a nested (L2) guest is not hardware-accelerated -the way it is for an L1 guest on this class of infrastructure, so every -AP-bring-up step traps all the way up to the L0 host and back — a -well-documented, order-of-magnitude nested-KVM SMP penalty, not a Cloud -Hypervisor or AWF defect. A single-vCPU guest never reaches that code path -at all and boots in the expected sub-second-to-few-seconds range even -nested. `scripts/ci/cloud-hypervisor-live-smoke.sh` therefore passes -`--cloud-hypervisor-vcpus 1`; the default (`--cloud-hypervisor-vcpus`, -default 2, see `docs/awf-config-spec.md`) is unchanged for other -environments, but operators running this preview on similarly nested -infrastructure (self-managed nested-KVM CI, for example) should expect the -same AP-bring-up penalty at more than one vCPU and may need to raise -`CLOUD_HYPERVISOR_GUEST_READY_MAX_WAIT_MS` further or pin to a single vCPU. -The 90-second retry budget above remains a deliberately generous safety -margin for ordinary boot-timing variance (kernel decompression + supervisor -startup), independent of this specific SMP pathology. - -When `--diagnostic-logs` is set, `CloudHypervisorRuntimeBackend.start()`'s -failure path collects diagnostics via a `beforeCleanup` hook passed to -`CloudHypervisorManager.stop()`, invoked after the Cloud Hypervisor process -is confirmed terminated but before `stop()` removes the private run -directory those diagnostic files live in. This ordering matters: Cloud -Hypervisor does not guarantee flushing buffered guest serial console -output to disk until its own process actually exits (`vmm.shutdown()` -alone is not sufficient), so collecting diagnostics any earlier — e.g. -before the process is terminated at all — can observe a still-empty -`serial.log` even when the guest did write to its console before crashing -or hanging. - -`CloudHypervisorRuntimeBackend.collectDiagnostics()` is idempotent -(collects at most once per instance): the CLI's own generic cleanup path -(`buildCleanupFn` in `commands/main-action.ts`) unconditionally calls -`externalRuntimeBackend.collectDiagnostics()` again during shutdown, -regardless of whether `start()`'s failure path already collected -diagnostics via the `beforeCleanup` hook above. Without the idempotency -guard, that second, redundant call ran *after* `stop()` had already torn -down the network/cgroup, silently clobbering the earlier, more useful -snapshot with an empty/unavailable one (observed live: -`network-diagnostics.txt` regressing from real content to "network -namespace not set up" between two collectDiagnostics() calls in the same -failed run). Likewise, `start()`'s failure path now marks the backend -`stopped` once its own internal `manager.stop()` call succeeds, so that -same generic cleanup path's `externalRuntimeBackend.stop()` call becomes a -no-op instead of invoking `manager.stop()` a second, redundant time (a -failed internal stop deliberately leaves `stopped` false, so the outer -cleanup path still gets a genuine retry attempt). - -The same "API socket only responsive before shutdown, files only flushed -after shutdown" tension applies to `vm.info`/`vm.counters`, not just the -serial console: `CloudHypervisorManager.stop()` snapshots both *before* -calling `vmm.shutdown()` (stored as `lastVmInfo`/`lastVmCounters`), since a -live call from inside `collectDiagnostics()` (invoked later, via -`beforeCleanup`, after the process has already been asked to exit) would -always fail against an already-unresponsive socket. `collectDiagnostics()` -prefers this snapshot and only falls back to a live call when invoked -directly outside of `stop()` (e.g. via `--diagnostic-logs` without a -failure). Written to `vm-info.json` alongside the existing -`counters.json`. - -`CloudHypervisorManager.collectDiagnostics()`'s host-side network capture -(`network-diagnostics.txt`) now includes, in addition to `nft list -ruleset` and `ip -s link show`: `nft -a list ruleset` (rule handles, plus -per-rule hit counters — `generateMicrovmNftRuleset()` attaches a `counter` -object to every forward-chain rule purely for this diagnostic visibility; -it does not change any accept/drop decision), `ip -d link show`, `ip route -show`, `ip neigh show` (inside the microVM's own namespace), and a -separate host-side (outside any namespace) `bridge fdb show` for the -infrastructure bridge Squid/API-proxy containers are attached to — MAC -learning for that traffic happens on the bridge, not inside the -namespace. The guest-side capture (`probeGuestConnectivity()`'s failure -path) also now includes `ip neigh show` to confirm ARP resolution. - -**Guest kernel panics with `Attempted to kill init!` on every boot (fixed; -historical)** - -Once the two boot-timing issues above were resolved, live-KVM validation -uncovered the real underlying blocker: the guest kernel's serial console -showed `Run /sbin/awf-supervisor as init process` immediately followed by -`firecracker-supervisor: mount workspace: no such device` and a kernel -panic. `mountWorkspace()` in -`guest/firecracker-supervisor/runtime_linux.go` called -`syscall.Mount(device, mount, "", 0, "")` — an **empty filesystem type**. -An empty fstype is only valid for bind/remount mounts (`MS_BIND`/ -`MS_REMOUNT`); a fresh mount of a raw block device with an empty fstype -fails with `ENODEV` ("no such device"), even though the device itself -exists and is a valid block device. The workspace image is always -formatted `ext4` (see `src/microvm/workspace.ts`'s `mkfs -t ext4`), so this -was fixed by passing `"ext4"` explicitly. This guest supervisor binary is -shared between the Firecracker and Cloud Hypervisor backends. Cloud Hypervisor -now selects its virtio-fs path instead, while Firecracker retains this ext4 -mount unchanged. This was a genuine historical defect affecting both -backends when both used the block-device path. A -regression test (`TestWorkspaceMountArgsUseExt4Filesystem` in -`runtime_linux_test.go`) and a CI step running `go test ./...` for this -package (see Part 14) now guard against a regression of this specific -class of bug. - -**`Cloud Hypervisor guest connectivity probe failed with exit code 127` -(fixed; historical)** - -`CloudHypervisorRuntimeBackend.probeGuestConnectivity()` originally shelled -out to `curl` inside the guest to verify Squid and (if enabled) API proxy -reachability before declaring the microVM ready. The guest rootfs used for -this backend's own live-KVM validation is a minimal BusyBox userland (see -`guest/cloud-hypervisor/build-test-artifacts.sh`) that provides `wget` and -`nc` but not `curl` — exit code 127 is the shell's "command not found". -Fixed by switching the Squid reachability check to `nc -z` (a raw TCP -check; a non-proxy-style HTTP request to Squid's own port intentionally -returns a 4xx error page by Squid's design, which BusyBox `wget`, unlike -`curl` without `--fail`, treats as a script failure by default — so `nc -z` -avoids that mismatch entirely) and the API proxy `/reflect` check to -`wget` with the guest's proxy environment variables unset for that one -request (matching the smoke test's own `api-proxy-reflect` case, and -replacing `curl --noproxy '*'`, which BusyBox `wget` has no equivalent -flag for). **Note:** `FirecrackerRuntimeBackend.probeGuestConnectivity()` -has the identical `curl`-based implementation and shares this same guest -rootfs build; it was not modified here (out of scope for this layer), but -is very likely affected identically on any real Firecracker live-KVM run -against this rootfs — see the layer 4 completion handoff. - -**Guest boots and gets a valid IP, but all TCP connections to Squid/API -proxy time out (fixed; historical)** - -Once the defects above were fixed, the guest reliably booted with a -correctly-configured `eth0` IP and default route, but every connection to -Squid or the API proxy still timed out. Live diagnostics (`nft list -ruleset` + `ip -s link show` inside the microVM's network namespace, -captured before teardown — see `network-diagnostics.txt` above) showed the -*host-side TAP device* with an asymmetric packet count: ~10 RX packets -(guest-to-host — working) but only 1 TX packet (host-to-guest — stalled), -even though 20+ response packets had already arrived on the host-side veth -from Squid. Traffic was reaching the host and being correctly forwarded by -nftables, but Cloud Hypervisor was not relaying it back into the guest. - -Root cause: Cloud Hypervisor's own tap handling (`Tap::open_named()` in -`net_util/src/tap.rs`) always re-opens its tap file descriptor requesting -`IFF_VNET_HDR` (a `struct virtio_net_hdr` prefix on every frame). The -shared TAP-creation code in `src/microvm/network.ts` (`ip tuntap add ... -mode tap`, used unmodified by both Firecracker and Cloud Hypervisor) never -requested that feature at *creation* time. When Cloud Hypervisor's re-open -requests a frame layout the tap wasn't created to support, the host kernel -and Cloud Hypervisor disagree on frame layout specifically for the -host-to-guest direction — guest-to-host traffic (and the entire host-side -veth/nftables layer) keeps working normally, masking the problem as a -"the guest just isn't receiving responses" mystery rather than an obvious -hard failure. - -Fixed by adding a `tapVnetHdr` field to `MicrovmNetworkPlanOptions`/ -`MicrovmNetworkPlan` (defaulting to `false`, preserving Firecracker's -existing, unaffected behavior exactly), which conditionally appends -`vnet_hdr` to the `ip tuntap add` invocation. `CloudHypervisorManager` -opts in explicitly (`tapVnetHdr: true`) when building its network plan; -Firecracker's own manager does not (Firecracker's tap handling does not -request `IFF_VNET_HDR`, so creating the tap with that feature available -would have been a no-op for Firecracker, but changing shared, working -code without a concrete reason is unnecessary risk). - -**Guest connectivity probe still times out even with a fully-correct -network path (fixed; historical)** - -After the `vnet_hdr` fix above, live network diagnostics conclusively -showed the tap/nftables/MAC path working correctly — Squid's response -packets reached the host-side veth — yet -`probeGuestConnectivity()`'s `nc -z -w 5` inside the guest still timed out. -This is the same nested-virtualization vCPU-scheduling phenomenon -documented for guest boot above (see -`CLOUD_HYPERVISOR_GUEST_READY_MAX_WAIT_MS`): the guest's vCPU can be -scheduled so rarely that a short-lived command doesn't get enough real CPU -time to complete a `connect()` within a tight budget, even though nothing -about the network path itself is broken. Raised `nc`'s own timeout to 60s, -`wget`'s to 20s, and the overall guest-exec budget -(`CLOUD_HYPERVISOR_PROBE_TIMEOUT_MS`) to 90s to match the same -nested-KVM-tolerant convention used elsewhere, and increased the live-KVM -workflow job's `timeout-minutes` accordingly. - -**Guest→Squid packets forwarded but the return path never matches -`established,related` (under investigation)** - -With per-rule nftables counters added (see the diagnostics-lifecycle entry -above), a live run showed the guest→Squid forward-chain rule matching real -traffic (`counter packets 6 bytes 360 accept`, from `nc`'s own SYN -retransmissions over its 60s budget), while the return-path accept rule -(`ether daddr ip daddr ct state established,related -accept`) stayed at **zero** hits, and none of the anti-spoof drop rules -matched either. This rules out both a misconfigured anti-spoof rule and a -`vnet_hdr`/tap-negotiation failure (both would show up as counter hits -somewhere); the traffic leaves the guest and is accepted outbound, but -Squid's reply is never recognized as belonging to that connection. - -Two changes were made to narrow this further, not yet confirmed as the -fix: -- Added a `counter` to the chain's very first rule, `ct state invalid - drop` — previously uncounted, so a reply being marked "invalid" by - conntrack (and dropped before ever reaching the return-path accept rule) - would have been invisible. If this rule's counter is nonzero on the next - live run, that is the confirmed root cause. -- Cloud Hypervisor's virtio-net device defaults all three offloads - (`offload_tso`, `offload_ufo`, `offload_csum`) to enabled (confirmed via - `vm.info`'s `net[0]` config, now captured in `vm-info.json`). This - network path is a fully-software bridge/veth/tap chain with no real NIC - downstream to finish partially-offloaded (unchecksummed / - not-yet-segmented) frames; conntrack's TCP state tracking needs a valid, - fully-computed checksum to correctly parse segment flags/sequence - numbers, so an offloaded-but-never-finished checksum is a plausible - cause for exactly this "accepted outbound, reply never tracked as - established" symptom. All three are now explicitly disabled in the net - device config (`CloudHypervisorManager.buildVmConfig()`) rather than - relying on Cloud Hypervisor's own defaults. - -**Update**: neither of the two changes above resolved it. A follow-up live -run with both applied showed the *identical* pattern — `ct state invalid` -still at zero hits (ruling out conntrack-invalid marking) and the return -accept rule still at zero hits (ruling out the offload/checksum theory, -since disabling all three offloads made no observable difference). Since -the microVM's own nftables table shows no rule matching the return traffic -at all (neither accepting nor dropping it), the packets may never be -reaching this bridge/veth from Squid's side in the first place, or may be -handled entirely by a *different* ruleset before ever reaching this one. -`captureHostBridgeDiagnostics()` was extended to also capture the -host/default-namespace `nft -a list ruleset` and `iptables -S` — Docker -manages its own iptables/nftables rules for its bridge networks in that -same root namespace, entirely separate from (and evaluated in addition -to) the microVM's own table, and could independently drop or redirect -traffic on this shared bridge in a way the microVM's own counters would -never reveal. This is the next concrete lead to check against a live run. - -**Update**: a follow-up live run's host-level ruleset showed Docker's -`DOCKER-ISOLATION-STAGE-1`/`DOCKER-FORWARD` chains present for the -infrastructure bridge (confirmed as ours by its subnet-based isolation -rule, `ip saddr != 172.30.0.0/24 ... drop`, matching AWF's real subnet), -but neither of its two explicit anti-cross-network drop rules showed any -hits (0/0), and the one same-bridge accept rule -(`iifname X oifname X accept`, matching guest↔Squid intra-bridge traffic) -also showed exactly zero hits across the whole run — inconclusive on its -own, since 0 hits doesn't distinguish "never reached this rule" from -"reached but something upstream already handled it". - -Given Firecracker uses this exact same shared netns/bridge/nftables/veth -code and its own live-KVM CI is green, the bridge/Docker-isolation path -itself is very unlikely to be broken in a way specific to this scenario. -The next most likely explanation, consistent with everything observed so -far (guest boot needing a 90s budget for what should be sub-second AP -bring-up; the connectivity probe needing the same generous budget with -only marginal improvement from 5s→90s; a bare handful of tap packets -relayed regardless of how long the test waits): `CloudHypervisorCgroup` -sized the CPU quota as exactly "1 CPU per configured vCPU" -(`vcpuCount * CGROUP_V2_PERIOD_US`), but Cloud Hypervisor's own I/O, -virtio device emulation (including the tap fd read/write loop for the -guest's network device), and API threads all run in that *same* cgroup as -the vCPU thread(s) and compete for the *same* quota. Under nested KVM on -GitHub-hosted runners (where vCPU exits are unusually expensive), the -vCPU thread alone can consume most of an already-tight, vCPU-only-sized -quota, starving the VMM's own non-vCPU threads (including the one -relaying guest network I/O) of their share — independent of wall-clock -timeout length, matching why raising timeouts alone barely helped. - -Added a fixed `CGROUP_CPU_HEADROOM_QUOTA_US` (one additional full -CPU-equivalent, `100_000`us per period) on top of the per-vCPU quota, -mirroring the existing `CGROUP_MEMORY_HEADROOM_MIB` pattern for the same -"VMM overhead needs room beyond what's sized for the guest alone" reason. - -**Update — root cause confirmed and fixed.** Neither the CPU headroom -change nor any of the previous attempts changed the observable pattern: -`ct state invalid` and the return-path accept rule both stayed at exactly -zero hits, run after run, and the guest's outbound packet count (6 -packets from `nc`'s own SYN-retry schedule) was identical regardless of -CPU quota, offloads, or `vnet_hdr`. That consistency was itself the clue: -this was never a scheduling/timing artifact. - -Squid's own `access.log` (captured in the diagnostics artifact) settled -it conclusively: it showed **zero** connection attempts from the guest's -address, across the entire test run — while genuine container-to-container -traffic on the exact same bridge (the API proxy reaching Squid) worked -fine. The microVM's own nftables table showed the outbound packet being -accepted (leaving via the host-side veth), but it never arrived at Squid. - -Root cause: Docker's host-level `DOCKER-FORWARD` chain includes a -generic same-bridge accept rule (`iifname oifname -accept`) intended to permit intra-bridge traffic, but it never matched -traffic to/from our manually-injected (non-Docker-managed) veth on this -GitHub-hosted runner's Docker/kernel/nftables combination — silently -falling through to the `FORWARD` chain's default-drop policy. Real -Docker-managed containers on the same bridge are unaffected (Docker -grants them rules of their own that an externally-injected veth never -receives). - -Fixed by inserting a scoped `ACCEPT` rule directly into Docker's -`DOCKER-USER` chain (which Docker evaluates *before* its own isolation -logic) — `-i -o -j ACCEPT`, both interfaces required to -be this exact per-run bridge. This is the same `DOCKER-USER` -customization point AWF's own container-based sandbox mode already uses -(`src/host-iptables-chain.ts`), just scoped for the microVM -network-isolation path instead. Inserted right after the host veth joins -the bridge in `MicrovmNetworkManager.setup()`, and removed in `cleanup()` -(tolerant of it already being gone). Scoped to exactly this per-run -bridge (Docker Compose assigns a unique bridge name per invocation) with -both interfaces required to match, so it does not weaken isolation for -any other bridge/network on the host, and it does not bypass the -microVM's own in-namespace nftables allowlist — that allowlist still -governs what the guest can send in the first place; this rule only fixes -the Docker-level pass-through for traffic that has already cleared it. - -**`Cloud Hypervisor requires a non-root target uid/gid`** - -Same as Firecracker's jailer requirement: run through `sudo` from a -non-root account so `SUDO_UID`/`SUDO_GID` are set — see -[Firecracker's Part 15](./firecracker-integration.md#preflight-failures). - -**Cgroup residue remains after cleanup (fixed; historical)** - -Live-KVM validation observed `Cloud Hypervisor cgroup residue remains -after cleanup` following a guest-connectivity-probe failure and immediate -teardown. `CloudHypervisorCgroup.cleanup()` called `rmdir()` on the leaf -cgroup exactly once; cgroup v2 rejects `rmdir()` on a non-empty cgroup -with `EBUSY` not only while a process is still a live member, but also for -a short window *after* that process has fully exited — the memory -controller's charge-migration teardown can lag process-exit by a handful -of milliseconds under load, even though `stop()` only calls `cleanup()` -once process termination is already confirmed. Fixed by retrying `rmdir()` -on `EBUSY` for up to 5 seconds (100ms interval) before giving up; any -other error (e.g. `EACCES`) still fails immediately, unretried. - -**Namespace, cgroup, and process residue after a failed run:** +`.github/workflows/test-cloud-hypervisor.yml` provides deterministic build and +live-KVM jobs. -```bash -# List namespace residue (shared naming with Firecracker) -sudo ip netns list | grep awffc +The build job: + +1. builds the pinned Cloud Hypervisor binary, Linux kernel, BusyBox rootfs, + shared guest supervisor, and `virtiofsd`; +2. verifies source and output digests; +3. attests provenance; and +4. uploads the `cloud-hypervisor-test-x86_64` workflow artifact. -# Remove all AWF microVM namespaces -sudo ip netns list | awk '/^awffc-/{print $1}' | \ - xargs -r -I{} sudo ip netns delete {} +The live job runs only when explicitly enabled by workflow dispatch or the +`cloud-hypervisor-kvm` pull-request label. It executes +`scripts/ci/cloud-hypervisor-live-smoke.sh`, which validates: -# List and remove Cloud Hypervisor-specific cgroup residue -sudo find /sys/fs/cgroup/awf-cloud-hypervisor -mindepth 1 -maxdepth 1 -sudo rmdir /sys/fs/cgroup/awf-cloud-hypervisor/ +- allowed HTTPS and blocked domains; +- direct-egress, arbitrary-TCP, DNS, and metadata denial; +- API proxy reachability and secret non-disclosure; +- workspace persistence; +- exit-code, timeout, and cancellation behavior; +- device assumptions; +- partial-start and normal cleanup; +- preserved-state behavior; and +- uid, capabilities, `no_new_privs`, seccomp, cgroup, Landlock, and VM-device + security assertions. -# Find a lingering Cloud Hypervisor process (do not blindly pkill by name; -# confirm the PID belongs to an AWF run before terminating it) -pgrep -af 'cloud-hypervisor --api-socket' +After each case, the suite checks for leaked `awfvm-*` namespaces, +`vmh*`/`vmn*`/`vmt*` interfaces, cgroups, and Cloud Hypervisor processes. + +## Troubleshooting + +### Preflight rejects the host + +Confirm the job runs on a GitHub-hosted Ubuntu x86_64 runner and that KVM is +usable: + +```bash +uname -m +test -r /dev/kvm && test -w /dev/kvm +stat -c '%A %U %G %n' /dev/kvm ``` -**Run directory residue (only if stop failed):** +The runtime intentionally rejects self-hosted runners even if they expose KVM. + +### Inspect preserved resources + +Run with `--keep-containers`, then inspect the namespace and interfaces: ```bash -ls /tmp/awf-*/cloud-hypervisor-run/ 2>/dev/null -sudo rm -rf /tmp/awf-/cloud-hypervisor-run/cloud-hypervisor/ +sudo ip netns list | grep '^awfvm-' +sudo ip -o link show | grep -E ' (vmh|vmn|vmt)[0-9a-f]{12}[:@]' +sudo nft list ruleset ``` -## Part 16 — Validation performed in this layer - -- `tsc --noEmit -p tsconfig.check.json`: clean. -- Full Jest suite: all suites passing, including layer 3's API client, - launcher (argv construction, kvm-gid retention, Landlock rule - computation, cgroup v2 `subtree_control` delegation ordering and - rmdir-only cleanup), manager, backend, and runtime-registration coverage, - plus new layer 4 coverage for the CI workflow's YAML structure (triggers, - permissions, concurrency, job gating, path scoping) and the new - `scripts/ci/cloud-hypervisor-*.sh` scripts' behavior (13-case parity with - Firecracker, device-assumption, read-only-cache, and security-assertion coverage, distinct - secret sentinel, shared-vs-specific residue naming, digest flag wiring). -- `bash -n` and `shellcheck` (severity=error) on both new scripts, plus - `bash -n` on every `run:` block in the new workflow YAML. -- `guest/firecracker-supervisor` Go tests (`go vet`, `go test`): unaffected, - confirming the shared guest supervisor still works for both backends. -- **Live-KVM validation**: the `test-cloud-hypervisor.yml` workflow's - `live-kvm` job runs the full smoke/security suite on real GitHub-hosted - KVM hardware when triggered (manual dispatch or the - `cloud-hypervisor-kvm` pull request label). This development environment - has no `/dev/kvm`, so the suite's actual pass/fail status must be - confirmed from the workflow run itself rather than reproduced locally. +Inspect preserved workspace data under +`/microvm-images//` and VMM diagnostics under the run's +preserved log directory. + +:::caution +Preserved namespaces and processes continue consuming host resources. Remove +them only after collecting the diagnostics you need. +::: + +### Guest cannot reach Squid or the API proxy + +Check the namespace nftables rules, TAP state, and Squid/API proxy health. The +guest must not have a direct route to the internet; fixing connectivity by +loosening the default-deny policy would break the security boundary. + +### VMM boot fails with TAP permission errors + +Verify the launcher retained only `CAP_NET_ADMIN`, the TAP belongs to the +expected namespace, and the Landlock allowlist includes the TAP's +`/sys/class/net/` directory read-only. + +## Related documentation + +- [Architecture](./architecture.md) +- [Integration tests](./INTEGRATION-TESTS.md) +- [Configuration specification](./awf-config-spec.md) +- [Docker Sandboxes integration](./sbx-integration.md) +- [gVisor integration](./gvisor-integration.md) diff --git a/docs/compatibility.md b/docs/compatibility.md index 45735302a..cb9a2d5a3 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -123,26 +123,6 @@ sudo systemctl start docker sudo systemctl enable docker ``` -## Firecracker preview host requirements - -The Firecracker microVM backend is an **explicit opt-in preview** available only on a -subset of host configurations. It is **not** the default and does **not** fall back to -any other runtime. - -| Requirement | Value | -|-------------|-------| -| Operating system | **Linux only** — macOS and Windows fail preflight immediately | -| Architecture | x86_64 primary (aarch64 accepted by preflight code; no pre-built aarch64 test artifact) | -| KVM device | `/dev/kvm` must exist and be readable + writable by the workflow user | -| CI runner | GitHub-hosted x64 `ubuntu-24.04` with readable + writable `/dev/kvm` | -| Firecracker version | **v1.16.1 exactly** — enforced at preflight; any other version fails | -| Jailer | Mandatory — the jailer binary must match the Firecracker binary version | -| Docker | Local Unix-socket Docker Engine required; remote Docker hosts rejected | -| Required tools | `ip`, `sysctl`, `nft`, `mke2fs`, `debugfs`, `e2fsck`, `rsync`, `sha256sum`, `timeout`, `docker`, `docker compose` v2, passwordless `sudo` | - -See [Firecracker integration (preview)](./firecracker-integration.md) for the complete -prerequisites, artifact policy, and CLI reference. - ## Reporting Compatibility Issues If you encounter compatibility issues with a supported configuration, please: diff --git a/docs/firecracker-integration.md b/docs/firecracker-integration.md deleted file mode 100644 index 7e51b89bb..000000000 --- a/docs/firecracker-integration.md +++ /dev/null @@ -1,1089 +0,0 @@ ---- -title: Firecracker microVM integration (preview) -description: Architecture, threat model, prerequisites, workspace semantics, networking, API proxy, lifecycle, diagnostics, and CLI reference for the Firecracker v1.16.1 microVM preview backend. ---- - -import { Aside } from '@astrojs/starlight/components'; - -:::danger[Explicit preview opt-in required — not a production default] -The Firecracker backend is a **preview** that requires every flag described in -this document. It is **not** enabled by default, does **not** auto-activate, and -is **never** a fallback for any other runtime. You must supply -`--container-runtime firecracker --firecracker-preview` together with all -artifact paths and their SHA-256 digests on every invocation. Omitting any -required input is a hard failure. - -**Linux/KVM only.** macOS and Windows are unsupported and will never silently -pass; the preflight fails immediately on non-Linux hosts. CI specifically -supports GitHub-hosted x64 `ubuntu-24.04`, which exposes `/dev/kvm`. KVM remains -mandatory, and any host without a readable and writable `/dev/kvm` fails closed. -::: - -This document covers the AWF Firecracker v1.16.1 microVM preview. It is -structured for two audiences: - -1. **Operators** who want to set up and run the preview on a capable Linux/KVM - host. -2. **Engineers** who want to understand the implementation design and trust model. - -For the Docker-compose defaults see [Architecture](./architecture.md). For -gVisor (OCI runtime, no separate kernel) see [gVisor integration](./gvisor-integration.md). -For Docker Sandboxes (sbx) see [sbx integration](./sbx-integration.md). For -sandbox design rationale see [Sandbox design](./sandbox-design.md). - ---- - -## Part 1 — What Firecracker adds and why it is a preview - -### What Firecracker is - -[Firecracker](https://firecracker-microvm.github.io/) is a minimal virtual -machine monitor (VMM) built by AWS for serverless/container workloads. It uses -Linux's KVM subsystem to boot a separate Linux kernel inside a hardware-isolated -virtual machine. The guest is completely isolated from the host kernel; the only -communication surfaces are the Firecracker control API socket, a vsock channel, -and explicit network interfaces. - -AWF's Firecracker preview runs the **agent command** inside the microVM while -keeping the host-side egress filtering infrastructure (Squid proxy, API proxy, -iptables/nftables rules) on the host. The guest kernel never touches host memory; -the agent can only reach host-side services through explicit permitted -network endpoints. - -### Why it is a preview - -The Firecracker backend adds meaningful defense-in-depth but also imposes -requirements that make it unsuitable as a universal default: - -- **Hard host requirements** — Linux kernel with KVM, specific system tools, - cgroup v1 or v2, passwordless sudo for the operator account. -- **Operator-managed artifacts** — there is no auto-download. Every artifact - (Firecracker binary, jailer binary, guest kernel, rootfs, guest supervisor) - must be supplied by the operator with an exact SHA-256 digest. -- **Pinned to Firecracker v1.16.1** — the version pin is enforced at runtime; - any other version fails preflight. -- **x86_64 only for test artifacts** — the released test artifact set targets - x86_64. aarch64 is supported at the code level (preflight will accept it) but - no pre-built aarch64 test artifact is shipped. -- **Topology restrictions** — topology peers, enclaves, Docker-in-Docker, - extra volume mounts, TTY, host access, and DNS-over-HTTPS all fail closed. - -These constraints are intentional. Relaxing them, especially artifact management -and topology completeness, is a prerequisite for promotion to non-preview. - -### Comparison with the other isolation backends - -| Property | Docker (default) | gVisor | sbx | Firecracker | -|----------|-----------------|--------|-----|-------------| -| Isolation mechanism | Linux namespaces/cgroups | Userspace application kernel | Docker Sandboxes microVM (KVM) | Firecracker microVM (KVM) | -| Separate Linux kernel | No | No (own syscall surface) | Yes | Yes | -| Requires KVM | No | No (systrap platform) | macOS/Win: platform hypervisor; Linux: KVM | Yes — hard requirement | -| Works on GitHub-hosted runners | Yes | Yes | macOS only | Yes — x64 `ubuntu-24.04` with KVM | -| Workspace delivery | bind mount | bind mount | virtiofs passthrough | ext4 image copy-in / copy-back | -| Virtiofs / live bind mounts | N/A | N/A | Yes (default) | **No** | -| Docker-in-Docker | Supported | Supported | Yes (in-VM engine) | **Not supported** | -| Topology peers / enclaves | Supported | Supported | Supported | **Not supported (preview)** | -| TTY | Supported | Supported | Supported | **Not supported (preview)** | -| Credential isolation | API proxy (optional/required in strict) | API proxy | API proxy (host-side injection) | API proxy — **mandatory** | -| Auto-download artifacts | N/A | N/A | Yes (sbx binary) | **No — operator-managed** | -| Version pin | N/A | N/A | N/A | v1.16.1 — hard enforcement | - ---- - -## Part 2 — Architecture - -### Host-side components - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ Host (Linux, x86_64, KVM-capable) │ -│ │ -│ ┌──────────────────────────────────────────────────────────────┐ │ -│ │ AWF CLI (runs as root via sudo from the non-root operator) │ │ -│ │ - Validates all artifacts and digests (preflight) │ │ -│ │ - Starts Docker Compose: Squid + API proxy containers │ │ -│ │ - Creates dedicated network namespace (awffc-) │ │ -│ │ - Sets up TAP device + nftables rules inside namespace │ │ -│ │ - Creates ext4 workspace image, copies workspace in │ │ -│ │ - Launches Firecracker via jailer in the namespace │ │ -│ │ - Waits for vsock supervisor handshake │ │ -│ │ - Executes agent command via vsock protocol │ │ -│ │ - Copies changed workspace files back after completion │ │ -│ │ - Cleans up namespace, jail, images │ │ -│ └──────────────────────────────────────────────────────────────┘ │ -│ │ │ │ -│ ▼ ▼ │ -│ ┌──────────────────┐ ┌──────────────────────────────────────┐ │ -│ │ Squid Proxy │ │ Firecracker jailer chroot │ │ -│ │ (Docker) │ │ (/tmp/awf-.../firecracker-jailer/ │ │ -│ │ 172.30.0.10:3128│ │ /root/) │ │ -│ └──────────────────┘ │ ┌──────────────────────────────────┐│ │ -│ │ │ │ Firecracker VMM (pid inside ││ │ -│ ┌──────────────────┐ │ │ netns awffc-) ││ │ -│ │ API Proxy │ │ │ Kernel: vmlinux.bin ││ │ -│ │ (Docker) │ │ │ Rootfs: rootfs.ext4 (priv copy) ││ │ -│ │ 172.30.0.30 │ │ │ Workspace: workspace.ext4 (rw) ││ │ -│ └──────────────────┘ │ │ Supervisor: vsock port 52 ││ │ -│ │ └──────────────────────────────────┘│ │ -│ └──────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────────┘ - ↕ nftables-enforced per-namespace egress - (only Squid port 3128 and API proxy reachable from guest) -``` - -### Guest contents - -The guest rootfs (`rootfs.ext4`) is a minimal ext4 image containing: - -- **BusyBox** — static binary providing `/bin/sh`, `wget`, `nc`, `nslookup`, - `ip`, `timeout`, and other standard utilities -- **AWF guest supervisor** (`/sbin/awf-supervisor`) — a statically compiled Go - binary that listens on vsock port 52, receives execution requests from the - host, runs the agent command, and streams stdin/stdout/stderr back -- **CA bundle** — a pinned Mozilla CA bundle (date-stamped, SHA-256 verified) - at `/etc/ssl/certs/ca-certificates.crt` -- **Minimal /etc/passwd, /etc/group** — defines `root`, `awf` (uid/gid 1000), - and `nobody` -- **Empty /etc/resolv.conf** — direct DNS is intentionally absent; the comment - states this explicitly (see §Networking) - -The workspace is a **separate** writable ext4 image (`workspace.ext4`) mounted -at `/workspace` inside the VM. It is not part of the shared rootfs. - -### Control flow - -1. **Preflight** — validate host (Linux, x86_64 or arm64, `/dev/kvm` r/w, - required tools, cgroup hierarchy, Docker, passwordless sudo), validate all - five artifact files (type, permissions, ownership, digest) -2. **Infrastructure start** — Docker Compose brings up Squid and the API proxy - in the same network as usual; the host bridge is discovered -3. **Network namespace** — a dedicated namespace `awffc-` is created with - a veth pair (host side joins the infrastructure bridge; namespace side is - wired to a TAP device for the VM) and nftables rules that allow only the - Squid port and the API proxy from the guest -4. **Workspace image** — `mke2fs` creates `workspace.ext4`; the host workspace - and a `.awf-home` directory are copied in via `rsync` then `debugfs`; a - pre-run manifest is recorded -5. **Jailer launch** — Firecracker is launched by the jailer inside the network - namespace; the jailer establishes a chroot under - `/firecracker-jailer//root/` and drops to a non-root uid/gid -6. **API configuration** — AWF configures the VMM via its Unix socket: - kernel boot params, vcpu count, memory, TAP network interface, rootfs block - device (private writable copy), workspace block device (read-write), vsock - device (CID 3, port 52) -7. **Boot** — the VM boots; the supervisor starts and waits on vsock port 52 -8. **Connectivity probe** — host confirms Squid reachability and API proxy - `/reflect` endpoint from inside the guest before the agent is started -9. **Agent execution** — host sends the execution request over vsock; supervisor - forks the agent command, forwards stdin/stdout/stderr; exit code returned -10. **Copy-back** — workspace.ext4 is read back via `debugfs`; files changed - relative to the pre-run manifest are extracted and conflict-checked against - the host workspace before writing back -11. **Cleanup** — network namespace, veth pair, TAP device, and jail directory - are removed; images are deleted unless `--keep-containers` was given - ---- - -## Part 3 — Threat model and trust boundaries - -### What the microVM boundary enforces - -- **Host kernel isolation** — agent processes are inside a separate kernel with - hardware memory isolation. A memory corruption exploit in the agent cannot - directly access host memory. -- **No direct host filesystem access** — the only data path between guest and - host is the workspace ext4 image (explicit, bounded, copy-in/copy-back with - conflict checking) and the vsock protocol (explicit RPC). -- **No virtiofs / live bind mounts** — the preview does not expose any live - filesystem passthrough. Guest filesystem writes are bounded to `workspace.ext4` - and are checked on copy-back. -- **No host daemon access** — the guest cannot reach the host Docker socket, - the host Docker Engine, or any host Unix sockets. -- **Egress fully mediated** — guest direct egress is blocked at the nftables - level inside the network namespace. The guest can reach only Squid (port 3128) - and the API proxy. Direct IP connections, arbitrary TCP/UDP, and raw DNS are - all denied. -- **Credential isolation** — real API credentials (OpenAI, Anthropic, Copilot, - Gemini, GitHub token) are **never** passed as guest environment variables. An - explicit assertion at boot time verifies none of the configured secret values - appear in any guest environment variable. The API proxy on the host side - injects credentials into provider calls at the HTTP layer. -- **No metadata reachability** — the EC2/GCP/Azure instance metadata IP - (`169.254.169.254`) and link-local range are blocked in the guest network - namespace. - -### What the microVM boundary does NOT protect - -- **Agent trust** — the agent command runs as the configured uid/gid (default: - the operator's uid from `SUDO_UID`). A malicious agent can read and modify - all files in the workspace image and communicate with Squid. -- **Workload secrets passed explicitly** — any environment variable the operator - explicitly passes to the guest (not the credential set above) is in-scope for - the guest agent. -- **CPU side-channels** — Firecracker does not mitigate Spectre/Meltdown-class - attacks beyond what the host kernel provides. Multi-tenant use on the same - physical core requires additional host-level mitigations. -- **Jailer uid/gid** — the jailer drops to a non-root uid, but this uid must - exist on the host. The jailer process itself runs in the operator's sudo - session. - -### Trust boundaries summary - -| Boundary | Enforced by | Notes | -|----------|------------|-------| -| Guest ↔ host kernel | KVM hardware isolation | Hard boundary; VM escape requires VMM/KVM vulnerability | -| Guest filesystem ↔ host filesystem | Explicit ext4 image + copy-back | No live mounts; changes are conflict-checked | -| Guest ↔ host network | nftables rules in dedicated network namespace | Only Squid + API proxy reachable from guest | -| Agent credentials ↔ guest env | Explicit assertion at boot | Fails hard if any secret would enter guest env | -| Guest DNS | DNS-over-Squid only | Direct DNS denied; resolv.conf empty | -| Guest direct egress | nftables DENY | All outbound except proxy ports blocked | - ---- - -## Part 4 — Prerequisites and supported hosts - -### Supported host configurations - -| Requirement | Value | -|-------------|-------| -| Operating system | **Linux only** — macOS and Windows are unsupported and fail preflight immediately | -| Architecture | x86_64 (primary) — aarch64 accepted by preflight code but no pre-built test artifacts are released | -| KVM device | `/dev/kvm` must exist and be readable + writable by the workflow user | -| CI runner | GitHub-hosted x64 `ubuntu-24.04` with readable + writable `/dev/kvm` | - -:::caution[KVM requirement] -The CI-supported host is GitHub-hosted x64 `ubuntu-24.04`. Do not infer support -for other GitHub-hosted images or architectures. The preflight checks the actual -host capabilities, including a readable and writable `/dev/kvm`, and fails -closed when any requirement is absent. -::: - -### Required host tools - -The following tools must be present on `PATH` and executable: - -| Tool | Purpose | -|------|---------| -| `ip` | Network namespace and veth management | -| `nft` | nftables rules inside the network namespace | -| `mke2fs` | Create the workspace ext4 image | -| `debugfs` | Populate and extract workspace image content | -| `e2fsck` | Verify the workspace image after creation | -| `rsync` | Stage workspace content for `debugfs` import | -| `sha256sum` | Artifact digest verification in CI scripts | -| `timeout` | Bounded tool invocations in CI scripts | -| `docker` | Docker Engine (host-visible, local Unix socket) | -| `docker compose` v2 | Compose v2 for Squid/API proxy infrastructure | -| `sudo` | Passwordless sudo for jailer and netns setup | - -:::note -The Docker daemon must be reachable via a local Unix socket. A remote Docker -host (e.g., `DOCKER_HOST=tcp://...`) is rejected at preflight because the -infrastructure bridge created by Docker Compose must be directly visible on the -host's network stack. -::: - -### Required kernel controls - -The following host kernel paths must be readable: - -- `/proc/sys/net/ipv4/ip_forward` -- `/proc/sys/net/ipv6/conf/all/disable_ipv6` -- `/proc/sys/kernel/seccomp/actions_avail` - -One of the following cgroup hierarchies must be available: - -- `/sys/fs/cgroup/cgroup.controllers` (cgroup v2) -- `/sys/fs/cgroup` writable (cgroup v1) - -### Operator account - -AWF must be invoked through `sudo` from a **non-root** account. The jailer -specifically rejects a root `SUDO_UID`/`SUDO_GID` (the actual uid inside the -jail must be non-root). The `SUDO_UID` and `SUDO_GID` environment variables set -by sudo are used to determine the jailer uid/gid and the guest execution identity. - ---- - -## Part 5 — Artifact policy - -### Why artifacts are operator-managed - -The Firecracker preview does **not** auto-download any artifacts. There is no -"latest" mode, no demo asset, and no unverified download path. Every artifact -must be: - -1. Obtained by the operator from a trusted source -2. Supplied by absolute path on the CLI or in the config file -3. Accompanied by its exact SHA-256 digest - -At runtime, AWF computes the SHA-256 of every artifact file and compares it to -the supplied digest before the VM boots. A mismatch is a hard failure with no -retry. - -### Required artifacts - -| Artifact | CLI flag | Digest flag | Description | -|----------|---------|-------------|-------------| -| Firecracker binary | `--firecracker-binary` | `--firecracker-binary-sha256` | Firecracker VMM binary, **must be v1.16.1** | -| Jailer binary | `--firecracker-jailer-binary` | `--firecracker-jailer-sha256` | Jailer binary, same version as Firecracker binary | -| Guest kernel | `--firecracker-kernel` | `--firecracker-kernel-sha256` | A KVM-compatible Linux bzImage | -| Guest rootfs | `--firecracker-rootfs` | `--firecracker-rootfs-sha256` | Ext4 base image; staged as a private writable copy per run | -| Guest supervisor | `--firecracker-supervisor` | `--firecracker-supervisor-sha256` | AWF vsock supervisor binary | - -All five digests are required. Supplying any subset causes a hard pre-boot -validation failure. - -### Version pin - -The Firecracker binary and jailer binary are both pinned to **v1.16.1**. The -preflight: - -1. Runs `firecracker --version` and `jailer --version` -2. Parses the version strings -3. Requires both to report exactly `1.16.1` -4. Requires both to match each other - -Any version mismatch — including a newer Firecracker release — fails preflight. - -### Artifact file security requirements - -Each artifact file is validated for: - -- **Absolute path** — relative paths are rejected -- **Regular file** — symbolic links are rejected -- **Not group- or world-writable** — mode bits `o022` must not be set -- **Owned by root or the operator uid** — other owners are rejected -- **Correct access mode** — Firecracker/jailer binaries must be executable; kernel/rootfs/supervisor must be readable - -### Building test artifacts - -Automated Firecracker artifact builds and release publication are disabled. -The reproducible artifact set can still be built explicitly with -`guest/firecracker/build-test-artifacts.sh`; its output includes -`release/firecracker-test-x86_64/awf-firecracker-test-x86_64.tar.gz`. - -This tarball contains: - -| File | Description | -|------|-------------| -| `firecracker` | Firecracker v1.16.1 binary (extracted from upstream release, SHA-256 verified) | -| `jailer` | Jailer v1.16.1 binary | -| `vmlinux.bin` | Linux 6.1.141 bzImage built from upstream source with a pinned kernel config | -| `rootfs.ext4` | Minimal BusyBox + supervisor rootfs image | -| `awf-firecracker-supervisor` | AWF guest supervisor binary | -| `SHA256SUMS` | SHA-256 digests for all five files | -| `manifest.json` | Human-readable manifest of versions and build inputs | -| `sbom.spdx.json` | SPDX 2.3 software bill of materials | - -:::danger[Test artifacts — not production defaults] -The release artifact set is an **x86_64 test/preview artifact set**. It is -purpose-built for integration testing and preview evaluation. It is: - -- **Not** an auto-downloaded default -- **Not** a production-ready distribution -- **Not** intended for use as a stable base for production workloads without - independent review -- **Not distributed by the AWF release workflow** and never selected or - downloaded automatically by AWF - -Production use requires operators to obtain, verify, and manage their own -kernel and rootfs images appropriate to their workload security requirements. -The generated `firecracker-test-x86_64/SHA256SUMS` covers the five extracted -runtime files, not the tarball itself; extract the bundle before checking it. -::: - -### Guest kernel requirements - -Operators who want to supply their own guest kernel (recommended for production -evaluation) must use a Linux kernel built with a KVM-microVM-compatible -configuration. The Firecracker project publishes reference kernel configs at -`resources/guest_configs/` in the Firecracker repository. The kernel pin in the -test artifact set is **Linux 6.1.141** with the upstream Firecracker CI config -`microvm-kernel-ci-x86_64-6.1.config`. - -Operators are responsible for ensuring their guest kernel is: - -- A KVM guest kernel (not a full machine kernel) -- Compatible with Firecracker v1.16.1 -- Sourced from a trusted build and verified by digest - -AWF does not impose a specific kernel version beyond the version of the -Firecracker binary that boots it. - ---- - -## Part 6 — Workspace image semantics - -### What gets copied into the VM - -Before boot, AWF creates a writable ext4 image (`workspace.ext4`) sized to hold -the workspace with 128 MiB headroom (minimum 256 MiB, maximum 8 GiB). The image -file is created exclusively with host mode `0600`, so a pre-existing path cannot -be reused and only the operator can read or modify the staged workspace image. It -contains: - -- The entire `$GITHUB_WORKSPACE` (or `cwd()` if the variable is unset) directory, - copied via `rsync` then loaded into the image via `debugfs` -- A `.awf-home` subdirectory at the workspace root, used as `$HOME` inside the - guest (`/workspace/.awf-home`) - -:::caution[No virtiofs, no live bind mounts] -The Firecracker preview does **not** use virtiofs or any live filesystem -passthrough. The host workspace is snapshotted at boot time into a bounded ext4 -image. Changes the host makes to the workspace directory after the VM boots are -not visible to the guest, and changes the guest makes are not visible to the -host until copy-back completes after the agent exits. -::: - -### Copy-back and conflict detection - -After the agent command exits, AWF extracts the workspace image content via -`debugfs` and compares it to the pre-run manifest: - -- **New or modified files** — written back to the host workspace -- **Deleted files** — removed from the host workspace by the authoritative - `rsync --delete` copy-back -- **Conflict detection** — any concurrent host-only or divergent host/guest - change aborts copy-back rather than overwriting host work; AWF preserves the - changed raw workspace image for recovery - -### Recovery image - -If copy-back fails partway through, AWF writes the raw `workspace.ext4` to a -recovery path (`/firecracker-recovery/-workspace.ext4`) before -cleanup. This allows manual recovery using standard `debugfs` or `mount` tools. -The recovery path is logged at `warn` level so it is visible even if the overall -command exits non-zero. - -### Bounded image size - -The workspace image is bounded to a maximum of **8 GiB** by default -(`FIRECRACKER_DEFAULT_MAX_WORKSPACE_IMAGE_BYTES`). If the workspace content -exceeds this bound (after accounting for the 128 MiB headroom), preflight fails -with an explicit size error before the VM is launched. - ---- - -## Part 7 — Networking and egress guarantees - -### Network topology - -Each Firecracker run gets a **dedicated network namespace** named `awffc-`. -Inside the namespace: - -- A **veth pair** connects the namespace to the host infrastructure bridge - (the same bridge used by the Docker Compose Squid/API-proxy services) -- A **TAP device** connects the namespace to the Firecracker VMM -- **nftables rules** are installed that: - - Allow guest → Squid (`172.30.0.10:3128`) - - Allow guest → API proxy (if enabled) - - Drop all other guest-initiated outbound connections - -### Guest IP addressing - -Guest VMs are assigned IPs from the `100.64.0.0/10` range (IANA shared address -space), using `/30` subnets. Each run gets a unique `/30` subnet derived from -a 20-bit hash of the run ID, providing up to ~1 million distinct slots. - -### What the guest cannot reach - -| Target | Status | -|--------|--------| -| Arbitrary internet hosts (direct) | **Blocked** — all direct egress denied | -| Arbitrary TCP/UDP (not via Squid) | **Blocked** | -| Raw DNS (port 53, including 8.8.8.8) | **Blocked** — `/etc/resolv.conf` is empty; direct DNS denied | -| EC2/GCP/Azure metadata (`169.254.169.254`) | **Blocked** — link-local range blocked | -| Multicast (`224.0.0.0/4`) | **Blocked** | -| Host Docker socket | **Not accessible** — no socket mount | -| Host network | **Not accessible** — isolated namespace | -| Squid proxy | **Allowed** — all HTTP/HTTPS goes through Squid; Squid enforces `--allow-domains` ACL | -| API proxy | **Allowed** (mandatory in Firecracker) — credential injection only | - -### DNS in the guest - -The guest rootfs has an intentionally empty `/etc/resolv.conf`. The comment in -the file states: _"Direct DNS is intentionally unavailable in the Firecracker -preview."_ - -Hostname resolution for allowed domains goes through the Squid proxy's own DNS -resolution on the host. The guest does not perform independent DNS lookups. This -design means the guest cannot bypass the Squid domain ACL by resolving allowed -domain IPs and connecting to them directly — any direct IP connection is blocked -at the nftables layer. - -### Egress is proxy-mandatory - -All outbound HTTP/HTTPS traffic from the guest must go through the Squid proxy. -AWF sets `HTTP_PROXY`, `HTTPS_PROXY`, and related variables in the guest -environment pointing to Squid. Tools that ignore these variables (or that make -direct TCP connections) will fail because the nftables rules block direct -outbound connections. - ---- - -## Part 8 — API proxy and credential isolation - -### Why the API proxy is mandatory - -Unlike older Docker and gVisor configurations where the API proxy could be optional in -non-strict mode), the Firecracker preview **requires** the API proxy. The -runtime validation enforces this: - -``` -Firecracker preview requires API proxy credential isolation -``` - -The rationale is that the API proxy is the only path by which provider -credentials (OpenAI, Anthropic, Copilot, Gemini, GitHub) can reach their -destinations. Real credential values are explicitly **never** placed in guest -environment variables. - -### Credential isolation enforcement - -At the moment the guest environment is assembled, AWF runs an explicit assertion -(`assertNoProviderSecrets`) that scans every guest environment variable value -for any substring matching a configured provider credential. If a match is found, -AWF throws: - -``` -Refusing to pass a real provider credential through Firecracker guest variable -``` - -This is a hard failure before the VM boots. The check covers: -- `openaiApiKey` -- `anthropicApiKey` -- `copilotGithubToken` -- `copilotProviderApiKey` -- `geminiApiKey` -- `googleApiKey` -- `githubToken` - -### How credentials flow - -Provider API calls made by the agent: -1. Guest provider base URLs point directly at the API proxy's IP:port (e.g., - `http://172.30.0.30:10001`); that IP is also listed in `NO_PROXY`, so the - request goes straight to the API proxy without traversing Squid -2. API proxy injects the real `Authorization` / `x-api-key` header -3. API proxy's own upstream request to the real provider goes through Squid - (domain-ACL enforced) -4. Response is relayed back to the agent - -The guest never sees the real credential value; it sees only the API proxy -endpoint, which is accessible only from inside the AWF network. Squid enforces -egress policy on the API proxy's outbound request, not on the guest's request -to the API proxy. - -### API proxy connectivity probe - -After the VM boots but before the agent command is sent, AWF probes the API -proxy's `/reflect` endpoint from inside the guest: - -```sh -curl --fail --silent --show-error --max-time 5 --noproxy '*' \ - --output /dev/null http://:10000/reflect -``` - -If this probe fails, the agent command is never started and the VM is shut down. -The probe timeout is bounded at 15 seconds total (shared with the Squid probe). - ---- - -## Part 9 — Lifecycle, signals, and partial-start cleanup - -### Normal lifecycle - -``` -preflight → infrastructure start → network namespace create → -workspace image create → jailer launch → VM boot → supervisor handshake → -connectivity probe → agent execution → copy-back → cleanup -``` - -### Signal handling and cancellation - -When the AWF process receives `SIGTERM` or is cancelled: - -1. AWF calls `manager.cancel()` with reason `"AWF cleanup"`, which sends a - cancellation message over vsock to the supervisor -2. AWF waits up to **3 seconds** (`FIRECRACKER_CANCEL_GRACE_MS`) for the active - execution to finish -3. AWF calls `manager.stop()` which terminates the Firecracker process and - removes the jail -4. The network namespace and all associated interfaces are cleaned up -5. The workspace image and staging directories are removed (unless - `--keep-containers` was given) - -The CI live-smoke test verifies this: it sends `SIGTERM` after the namespace -appears, waits for the process to exit, and asserts no namespace residue remains. -Expected exit code after `SIGTERM` is **143** (128 + 15). - -### Partial-start cleanup - -If the VM fails to start after the jailer has been launched (e.g., a bad rootfs, -an API timeout, or a vsock handshake failure), AWF performs the same cleanup -sequence as normal stop. Workspace images, the jail directory, and the network -namespace are all removed. The cleanup result is logged; if cleanup itself fails, -the combined error (startup failure + cleanup failure) is reported with both -causes. - -### Keep mode (`--keep-containers`) - -When `--keep-containers` is set: - -- The network namespace `awffc-` is **not** deleted -- The jail directory `/firecracker-jailer//` is **not** deleted -- The workspace image directory `/firecracker-images//` is **not** deleted -- AWF logs the jail root path, the image directory, and the network namespace name - -On successful `--keep-containers` completion, the live-smoke test verifies: - -``` -sudo ip netns list | grep -q '^awffc-' # namespace preserved -test -d "$keep_work/firecracker-jailer" # jail preserved -``` - -Preserved resources must be cleaned up manually. See §Troubleshooting for -manual cleanup commands. - -### Resource residue policy - -Any network namespace matching `awffc-*` that persists after a run is considered -residue. The CI workflow's final step enforces this: - -```bash -while read -r namespace _; do - case "$namespace" in - awffc-*) sudo ip netns delete "$namespace" ;; - esac -done < <(sudo ip netns list) -if sudo ip netns list | grep -q '^awffc-'; then - echo "::error::Firecracker namespace residue remains after cleanup" - exit 1 -fi -``` - -Operators should include a similar cleanup step in any runner maintenance workflow. - ---- - -## Part 10 — CPU and memory controls - -### Virtual CPUs - -Default: **2 vCPUs**. Configure with `--firecracker-vcpus `. - -The guest vCPU count maps directly to Firecracker's vCPU configuration. The host -kernel schedules these as regular Linux threads inside the jailer process. There -is no CPU pinning by default; operators who need it should apply host-side cgroup -CPU sets to the jailer cgroup. - -### Memory - -Default: **512 MiB**. Configure with `--firecracker-memory-mib `. - -The memory limit is enforced by Firecracker's VMM; the guest cannot exceed the -configured allocation. The host-side cgroup enforced by the jailer also applies -the allocation. Memory is not overcommitted within a single run. - -### Jailer cgroup enforcement - -The Firecracker jailer places the VMM process in a cgroup. AWF's preflight -detects whether cgroup v1 or v2 is available and passes the version to the -jailer. The jailer creates its own cgroup hierarchy under its assigned uid/gid -group. Operators can apply additional cgroup controls (memory limits, CPU shares) -at the runner level by configuring the jailer's parent cgroup. - ---- - -## Part 11 — Bounded diagnostics, logging, and metrics - -### What is collected - -When `--keep-containers` is used or when `collectDiagnostics()` is called (e.g., -on workflow failure), AWF writes the following to `/firecracker/`: - -| File | Content | Size bound | -|------|---------|-----------| -| `network-plan.json` | Full network plan including namespace name, veth names, TAP name, IP assignments, allowed endpoints | Small (JSON) | -| `firecracker.log` | Firecracker VMM log (passed via the Firecracker API) | **1 MiB total** (truncated by AWF capture) | -| `firecracker.metrics.jsonl` | Firecracker metrics JSONL stream | **1 MiB total** (truncated by AWF capture) | -| `jailer-stdout.log` | Jailer process stdout capture | **1 MiB total** | -| `jailer-stderr.log` | Jailer process stderr capture | **1 MiB total** | - -All captured diagnostic files are individually bounded at **1 MiB** -(`FIRECRACKER_CAPTURE_LIMIT_BYTES = 1024 * 1024`). Files that exceed the capture -limit are truncated; the truncation is silent (no error thrown). - -The CI live-smoke test asserts this bound: - -```bash -find "$keep_audit/firecracker" -type f -size +1048576c -print -quit \ - | grep -q . && { - echo "Firecracker diagnostic artifact exceeded the 1 MiB bound" >&2 - exit 1 - } -``` - -### What the CI diagnostic collection step gathers - -The CI workflow's "Collect redacted diagnostics" step gathers only: - -- Files under `*/audit/*` -- Files under `*/proxy-logs/*` -- `stdout.log` and `stderr.log` for each test case - -Before uploading, the step scans all collected files for the secret sentinel -string (`awf-firecracker-real-secret-do-not-expose`). If found, the step fails -the workflow with an error annotation. - -### Proxy logs - -Squid access logs (`proxy-logs/`) are collected as in all other AWF runs. These -contain the Squid access log with per-request domain decisions (`TCP_TUNNEL`, -`TCP_DENIED`, etc.). Since the Firecracker guest always proxies through Squid, -the proxy logs are the definitive record of what the agent accessed. - -### Structured logging - -AWF emits structured JSON logs at the host level (not inside the guest). Key -Firecracker-specific log entries include: - -- `[firecracker] Agent command exited with code ` — normal agent exit -- `[firecracker] Guest supervisor, Squid, and API proxy connectivity verified` — successful probe -- `[firecracker] Preserved jail: ` — keep-mode notification -- `[firecracker] Preserved images: ` — keep-mode notification -- `[firecracker] Preserved network namespace: ` — keep-mode notification - ---- - -## Part 12 — Topology and feature restrictions - -The following features are **fail-closed** in the Firecracker preview: requesting -any of them produces a hard validation failure before the VM is launched. - -| Feature | Status | Error | -|---------|--------|-------| -| Topology peers (`--topology-attach`) | **Rejected** | "topology peers and enclaves are disabled" | -| Enclaves (`enclaves.enabled`) | **Rejected** | Same as above | -| Docker-in-Docker (`--enable-dind`) | **Rejected** | "does not support Docker-in-Docker or split filesystems" | -| Split Docker host path prefix | **Rejected** | Same as above | -| ARC DinD runner topology | **Rejected** | Same as above | -| Host access (`--enable-host-access`) | **Rejected** | "does not support host access" | -| Host ports (`--allow-host-ports`) | **Rejected** | Same as above | -| Host service ports | **Rejected** | Same as above | -| Extra volume mounts (`--volume-mount`) | **Rejected** | "does not support additional host volume mounts" | -| DNS-over-HTTPS (`--dns-over-https`) | **Rejected** | "does not support DNS-over-HTTPS" | -| TTY (`--tty`) | **Rejected** | "guest supervisor does not support TTY execution" | -| Remote Docker host (non-Unix socket) | **Rejected** | "requires a local Unix-socket Docker daemon" | -| Disabled API proxy (`--no-enable-api-proxy`) | **Rejected** | "requires API proxy credential isolation" | -| Legacy security mode (`--legacy-security`) | **Rejected** | "requires strict --network-isolation security" | - -:::caution[Enclaves never fall back] -If enclaves are enabled in the config and `--container-runtime firecracker` is -selected, the validation failure is immediate and hard. There is no fallback to -a non-Firecracker runtime. If you need enclaves, use a different runtime. -::: - -### Primary agent only - -The Firecracker backend supports **only the primary agent** execution path. It -does not participate in the enclave executor stack (script executor or agent -executor). Enclave executors that select `firecracker` as their runtime will -fail validation. - ---- - -## Part 13 — CLI reference - -### Minimal invocation - -```bash -sudo -E awf \ - --container-runtime firecracker \ - --firecracker-preview \ - --firecracker-binary /path/to/firecracker \ - --firecracker-jailer-binary /path/to/jailer \ - --firecracker-kernel /path/to/vmlinux.bin \ - --firecracker-rootfs /path/to/rootfs.ext4 \ - --firecracker-supervisor /path/to/awf-firecracker-supervisor \ - --firecracker-binary-sha256 <64-hex-chars> \ - --firecracker-jailer-sha256 <64-hex-chars> \ - --firecracker-kernel-sha256 <64-hex-chars> \ - --firecracker-rootfs-sha256 <64-hex-chars> \ - --firecracker-supervisor-sha256 <64-hex-chars> \ - --allow-domains example.com \ - -- my-agent-command -``` - -:::note -`sudo -E` is required. The `-E` flag preserves environment variables (including -`GITHUB_WORKSPACE` and provider token variables). The jailer requires root. -Firecracker options derive the non-root jail identity from `SUDO_UID`/`SUDO_GID`. -::: - -### Full CLI option reference - -| Flag | Type | Default | Description | -|------|------|---------|-------------| -| `--container-runtime firecracker` | string | — | **Required.** Selects the Firecracker backend. | -| `--firecracker-preview` | boolean | false | **Required.** Explicit opt-in gate. Artifact flags do not bypass this gate; the runtime refuses to start without it. | -| `--firecracker-binary ` | string | `/usr/local/bin/firecracker` | Absolute path to the Firecracker v1.16.1 binary. | -| `--firecracker-jailer-binary ` | string | `/usr/local/bin/jailer` | Absolute path to the matching jailer binary. | -| `--firecracker-kernel ` | string | — | **Required.** Absolute path to the guest Linux kernel image. | -| `--firecracker-rootfs ` | string | — | **Required.** Absolute path to the guest rootfs ext4 image. | -| `--firecracker-supervisor ` | string | — | **Required.** Absolute path to the AWF guest supervisor binary. | -| `--firecracker-vcpus ` | integer | 2 | Guest virtual CPU count. | -| `--firecracker-memory-mib ` | integer | 512 | Guest memory in MiB. | -| `--firecracker-api-timeout-ms ` | integer | 5000 | Bounded Firecracker API socket readiness timeout in milliseconds. | -| `--firecracker-binary-sha256 ` | string | — | **Required.** 64-character hex SHA-256 of the Firecracker binary. | -| `--firecracker-jailer-sha256 ` | string | — | **Required.** 64-character hex SHA-256 of the jailer binary. | -| `--firecracker-kernel-sha256 ` | string | — | **Required.** 64-character hex SHA-256 of the guest kernel. | -| `--firecracker-rootfs-sha256 ` | string | — | **Required.** 64-character hex SHA-256 of the guest rootfs. | -| `--firecracker-supervisor-sha256 ` | string | — | **Required.** 64-character hex SHA-256 of the AWF guest supervisor. | - -### Using test artifacts - -If you have unpacked `awf-firecracker-test-x86_64.tar.gz` to `$ARTIFACTS`: - -```bash -# Read digests from SHA256SUMS -ARTIFACTS=/path/to/firecracker-test-x86_64 -digest() { awk -v f="$1" '$2==f{print $1;exit}' "$ARTIFACTS/SHA256SUMS"; } - -sudo -E awf \ - --container-runtime firecracker \ - --firecracker-preview \ - --firecracker-binary "$ARTIFACTS/firecracker" \ - --firecracker-jailer-binary "$ARTIFACTS/jailer" \ - --firecracker-kernel "$ARTIFACTS/vmlinux.bin" \ - --firecracker-rootfs "$ARTIFACTS/rootfs.ext4" \ - --firecracker-supervisor "$ARTIFACTS/awf-firecracker-supervisor" \ - --firecracker-binary-sha256 "$(digest firecracker)" \ - --firecracker-jailer-sha256 "$(digest jailer)" \ - --firecracker-kernel-sha256 "$(digest vmlinux.bin)" \ - --firecracker-rootfs-sha256 "$(digest rootfs.ext4)" \ - --firecracker-supervisor-sha256 "$(digest awf-firecracker-supervisor)" \ - --allow-domains example.com \ - -- curl -s https://example.com -``` - -### Config file equivalent - -```json -{ - "containerRuntime": "firecracker", - "firecracker": { - "previewEnabled": true, - "firecrackerBinary": "/path/to/firecracker", - "jailerBinary": "/path/to/jailer", - "kernelPath": "/path/to/vmlinux.bin", - "rootfsPath": "/path/to/rootfs.ext4", - "supervisorPath": "/path/to/awf-firecracker-supervisor", - "vcpuCount": 2, - "memoryMib": 512, - "apiTimeoutMs": 5000, - "sha256": { - "firecracker": "<64-hex-chars>", - "jailer": "<64-hex-chars>", - "kernel": "<64-hex-chars>", - "rootfs": "<64-hex-chars>", - "supervisor": "<64-hex-chars>" - } - } -} -``` - ---- - -## Part 14 — CI workflow - -The dedicated Firecracker Actions workflow is disabled. Artifact build, -verification, host preflight, and live smoke scripts remain in the repository -for explicit local use, but GitHub Actions does not invoke them. - -### Live smoke test assertions - -The live smoke test (`firecracker-live-smoke.sh`) runs these named cases: - -| Case | Expected exit | Assertion | -|------|--------------|-----------| -| `allowed-https` | 0 | `wget https://example.com` succeeds and returns "Example Domain" | -| `blocked-domain` | 0 | `wget https://github.com` fails (not in `--allow-domains`) | -| `direct-egress` | 0 | Unsets `HTTP_PROXY`/`HTTPS_PROXY`; direct `wget https://example.com` fails | -| `arbitrary-tcp` | 0 | `nc -z 1.1.1.1 443` fails (direct TCP blocked) | -| `dns-denial` | 0 | `nslookup example.com 8.8.8.8` fails (direct DNS blocked) | -| `metadata-denial` | 0 | Unsets proxy vars; `wget http://169.254.169.254/latest/meta-data/` fails | -| `api-proxy-reflect` | 0 | API proxy `/reflect` endpoint reachable from guest; response contains "providers"; `env` does not contain the secret sentinel | -| `workspace-copyback` | 0 | Guest writes `.hidden`, `bin/run` (chmod 755), `run-link` (symlink); all appear on host after run | -| `exit-code` | 37 | `exit 37` inside guest propagates as AWF exit code 37 | -| `timeout-124` | 124 | `sleep 90` with `--agent-timeout 1` exits with code 124 | -| `partial-start-cleanup` | 1 | Corrupt rootfs (valid digest, invalid content) causes startup failure; no namespace residue | -| `cancellation` | 143 | `SIGTERM` after namespace appears; no namespace residue | -| `keep` (keep mode) | 0 | `--keep-containers`; namespace preserved; jail preserved; diagnostic files present; all bounded ≤1 MiB | - -After every case (except `keep`), the test asserts: - -- No `awffc-*` network namespaces remain -- No `fch*`, `fcn*`, or `fct*` veth/TAP interfaces remain - -### Secret sentinel check - -All smoke test cases set `OPENAI_API_KEY=awf-firecracker-real-secret-do-not-expose`. -After each run, the test scans `stdout.log`, `audit/`, and `proxy-logs/` for -this sentinel string. Finding it means a real credential would have leaked into -guest-visible or diagnostic output, which is a test failure. - ---- - -## Part 15 — Troubleshooting - -### Preflight failures - -**`Firecracker requires Linux with KVM; found darwin`** - -You are running on macOS. Firecracker is Linux/KVM only. macOS and Windows are -permanently unsupported. This development session cannot perform a live KVM boot. - -**`Firecracker requires readable and writable /dev/kvm`** - -The runner either lacks `/dev/kvm` (not a KVM-capable host, or a GitHub-hosted -runner) or the operator account lacks read/write access. - -Check: `ls -la /dev/kvm` — if the device does not exist, the host lacks KVM -support. If it exists but is not accessible, add the user to the `kvm` group: -`sudo usermod -aG kvm $USER` (requires re-login). - -**`Firecracker is pinned to v1.16.1; found v`** - -The Firecracker binary on the supplied path is not v1.16.1. Obtain the correct -version. Do not attempt to bypass the version check. - -**` SHA-256 mismatch: expected , got `** - -The artifact file does not match the supplied digest. Either the wrong digest -was provided, the file was modified after download, or the file is corrupt. -Re-obtain the artifact from a trusted source and recompute the digest. - -**`Firecracker jailer requires a non-root target uid/gid`** - -The operator account is root (uid 0), or `SUDO_UID`/`SUDO_GID` were not set -(i.e., `sudo -E` was not used correctly). The jailer requires a non-root jail -identity. Run as a non-root user through `sudo`. - -**`required host tool "" was not found on PATH`** - -Install the missing tool. Common package names: -- `ip`, `sysctl`, `nft` — `iproute2`, `procps`, `nftables` -- `mke2fs`, `debugfs`, `e2fsck` — `e2fsprogs` -- `rsync` — `rsync` - -**`Firecracker and Docker-in-Docker are mutually exclusive`** - -The config or CLI has both Firecracker and DinD/ARC-DinD selected. Remove the -DinD-related flags. The Firecracker backend does not support Docker-in-Docker -in this preview. - -### Boot failures - -**`Firecracker guest connectivity probe failed with exit code `** - -The VM booted but the guest could not reach Squid or the API proxy within the -15-second probe timeout. Possible causes: - -1. The infrastructure bridge is not ready — check `docker inspect ` -2. The nftables rules were not installed correctly — check with - `sudo ip netns exec awffc- nft list ruleset` -3. The guest IP was not assigned — check the network plan in - `/firecracker/network-plan.json` - -Enable `--log-level debug` to see detailed network setup steps. - -**`Firecracker manager did not expose the configured guest IP`** - -The VM started but the guest supervisor did not report a guest IP via vsock -handshake. Check `/firecracker/jailer-stderr.log` for VMM errors -and `/firecracker/firecracker.log` for boot errors. - -**API timeout (startup)** - -If the Firecracker API socket does not become ready within `--firecracker-api-timeout-ms` -(default 5000 ms), startup fails. On slow hosts or under memory pressure, try -increasing to 10000 ms. - -### Cleanup / residue - -**Namespace residue after a failed run:** - -```bash -# List residue -sudo ip netns list | grep awffc - -# Remove specific namespace -sudo ip netns delete awffc- - -# Remove all AWF Firecracker namespaces -sudo ip netns list | awk '/^awffc-/{print $1}' | \ - xargs -r -I{} sudo ip netns delete {} -``` - -**Jailer directory residue (only if stop failed):** - -```bash -# Find residue -ls /tmp/awf-*/firecracker-jailer/ 2>/dev/null - -# Remove (be certain this is AWF residue before removing) -sudo rm -rf /tmp/awf-/firecracker-jailer/ -``` - -**Docker infrastructure (Squid/API proxy) not cleaned up:** - -```bash -sudo docker compose -f /tmp/awf-/docker-compose.yml down --volumes --remove-orphans -``` - -### Workspace copy-back recovery - -If copy-back fails and a recovery image was preserved: - -```bash -# Mount the recovery image (requires root or loop mount capability) -sudo mkdir -p /mnt/awf-recovery -sudo mount -o loop /firecracker-recovery/-workspace.ext4 /mnt/awf-recovery - -# Browse or extract files -ls /mnt/awf-recovery/workspace/ - -# Unmount when done -sudo umount /mnt/awf-recovery -``` - -Alternatively, use `debugfs` directly: - -```bash -debugfs -R 'ls /workspace' -debugfs -R 'dump /workspace/output.txt /tmp/recovered-output.txt' -``` - ---- - -## Part 16 — Known limitations - -This section documents known gaps in the current preview. Items here are expected -to be addressed before promotion out of preview. - -| Limitation | Details | -|-----------|---------| -| **x86_64 test artifacts only** | No pre-built aarch64 test artifact is released. aarch64 is accepted by preflight code but must be built by the operator. | -| **No topology peers or enclaves** | The MCP gateway path is not proved in the Firecracker network model. Topology attachment and enclave execution are disabled. | -| **No TTY** | The guest supervisor does not implement a PTY multiplexer. Interactive agents requiring TTY cannot run. | -| **No virtiofs / live bind mounts** | Workspace is snapshotted at boot. Files created on the host after VM start are not visible to the guest. | -| **No Docker-in-Docker** | The guest has no Docker daemon. Agents that build or run containers cannot use Docker inside the VM. | -| **Single agent only** | The Firecracker path supports one agent execution per VM. Multi-agent or parallel executor models are not supported. | -| **Narrow CI host support** | CI specifically supports GitHub-hosted x64 `ubuntu-24.04`; other hosts must satisfy every preflight requirement and fail closed otherwise. | -| **macOS/Windows unsupported** | Firecracker requires Linux KVM; the preview fails preflight on these hosts. | -| **No unverified latest/demo assets** | There is no "just try it" path. Operators must manage and verify all artifacts. | -| **8 GiB workspace image ceiling** | Workspaces larger than 8 GiB cannot be used with Firecracker. | -| **Workspace copy-back is authoritative** | Guest deletions are applied with `rsync --delete`. Any concurrent host-only or divergent change fails copy-back and preserves the changed image for recovery instead of overwriting host work. | -| **No DNS-over-HTTPS in guest** | The DoH proxy is a host-side service; the guest does not participate. | diff --git a/docs/gvisor-integration.md b/docs/gvisor-integration.md index 590de536f..5d2532af1 100644 --- a/docs/gvisor-integration.md +++ b/docs/gvisor-integration.md @@ -66,8 +66,8 @@ gVisor can intercept syscalls in more than one way — the "platform": This is the natural bridge to any "KVM microVM" evaluation: gVisor's KVM *platform* uses KVM for address-space isolation without booting a full guest -kernel/VMM per sandbox, which is a different trade-off from a true microVM (sbx, -Firecracker) that boots a separate Linux kernel. +kernel/VMM per sandbox, which is a different trade-off from a true microVM such +as sbx or Cloud Hypervisor, which boots a separate Linux kernel. ### What gVisor does *not* protect against @@ -83,7 +83,7 @@ Firecracker) that boots a separate Linux kernel. | --- | --- | --- | --- | | Plain container (runc) | namespaces + cgroups | shared host kernel | lowest | | gVisor (runsc) | userspace application kernel | separate Go kernel (Sentry) | low–moderate | -| microVM (sbx, Firecracker) | hypervisor | separate real Linux kernel | highest | +| microVM (sbx, Cloud Hypervisor) | hypervisor | separate real Linux kernel | highest | ## Part 2 — How AWF uses gVisor diff --git a/docs/releasing.md b/docs/releasing.md index 747dacbac..e88980b8d 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -112,13 +112,6 @@ These images are automatically pulled by the CLI when running commands. The `agent-act` image is used when running with `--agent-image act` for workflows that need closer parity with GitHub Actions runner environments. -### Firecracker preview test artifacts - -The release workflow does not build or publish Firecracker preview test -artifacts. The repository retains explicit local build and verification scripts; -see [Firecracker integration (preview) — Artifact policy](./firecracker-integration.md#part-5--artifact-policy) -for the artifact specification and digest requirements. - ## Testing a Release Locally Before releasing, you can test the build process locally: diff --git a/docs/sandbox-design.md b/docs/sandbox-design.md index 2b5a699ee..29520e695 100644 --- a/docs/sandbox-design.md +++ b/docs/sandbox-design.md @@ -3,7 +3,9 @@ title: Sandbox design description: Why the firewall uses Docker containers instead of microVMs for network sandboxing in CI/CD environments. --- -The firewall sandboxes AI agent network traffic using Docker containers and a Squid proxy. This document explains why Docker was chosen over alternative isolation technologies like microVMs (Firecracker, Kata Containers). +The firewall sandboxes AI agent network traffic using Docker containers and a +Squid proxy. This document explains why Docker is the default instead of a +microVM runtime such as Cloud Hypervisor or Kata Containers. ## Threat model @@ -31,7 +33,7 @@ Adding a microVM inside this VM would create **nested virtualization** — a VM ### MicroVMs require KVM access -MicroVM runtimes (Firecracker, Cloud Hypervisor) require: +MicroVM runtimes such as Cloud Hypervisor require: - KVM access (`/dev/kvm`), which standard GitHub-hosted runners don't expose - Custom kernel images and rootfs preparation @@ -46,7 +48,7 @@ Container startup time directly impacts every workflow run: | Approach | Typical startup | Notes | |----------|----------------|-------| | Docker container | ~1-2s | Base image often pre-cached on runner; pulls may be cached within a job | -| Firecracker microVM | ~3-5s | Kernel boot + rootfs mount | +| Cloud Hypervisor microVM | ~3-5s | Kernel boot + rootfs mount | | Kata Containers | ~5-10s | Full VM boot with guest kernel | For a firewall that wraps every AI agent invocation, these seconds compound across workflow steps. @@ -103,7 +105,10 @@ On bare metal or shared infrastructure without an outer VM, Docker alone would n ## When microVMs would be the right choice -MicroVMs (Firecracker, Kata Containers) provide stronger isolation at the cost of complexity and performance. Other sandboxing runtimes like gVisor (a userspace kernel) can also harden isolation without full VM overhead. These approaches would be appropriate when: +MicroVMs such as Cloud Hypervisor and Kata Containers provide stronger +isolation at the cost of complexity and performance. gVisor, a userspace +kernel, can also harden isolation without full VM overhead. These approaches +are appropriate when: - **Running untrusted binaries** that might attempt kernel exploits - **Multi-tenant isolation** on shared bare-metal infrastructure diff --git a/docs/sbx-integration.md b/docs/sbx-integration.md index e587f790c..22392b193 100644 --- a/docs/sbx-integration.md +++ b/docs/sbx-integration.md @@ -8,8 +8,8 @@ to run an agent inside a hypervisor-isolated microVM while keeping AWF's own egress-filtering infrastructure on the host. It is written for two audiences: 1. Engineers who want to understand *how the existing sbx integration works*. -2. Ourselves, if we later want to add **another microVM backend that runs on top - of KVM** (e.g. Firecracker, Cloud Hypervisor, or a bespoke krun-based runner). +2. Engineers who want to add another KVM-based microVM backend, such as a + bespoke krun-based runner. :::note This is distinct from [Sandbox design](./sandbox-design.md), which explains why @@ -122,9 +122,9 @@ an `executionModel` of either `compose` or `microvm`: ```ts const RUNTIME_REGISTRY = { - gvisor: { executionModel: 'compose', dockerRuntime: 'runsc', needsStaticDns: true, usesIptables: false }, - sbx: { executionModel: 'microvm', dockerRuntime: undefined, needsStaticDns: false, usesIptables: false }, - firecracker: { executionModel: 'microvm', dockerRuntime: undefined, needsStaticDns: false, usesIptables: false }, + gvisor: { executionModel: 'compose', dockerRuntime: 'runsc', needsStaticDns: true, usesIptables: false }, + sbx: { executionModel: 'microvm', dockerRuntime: undefined, needsStaticDns: false, usesIptables: false }, + 'cloud-hypervisor': { executionModel: 'microvm', dockerRuntime: undefined, needsStaticDns: false, usesIptables: false }, }; ``` @@ -334,11 +334,7 @@ ACL. - Strict-security note (`src/commands/validators/security-mode.ts`): sbx enforces isolation at the hypervisor layer via `DOCKER_SANDBOXES_PROXY`, so AWF's Docker **network-isolation topology is not forced on** for it - (`isMicroVmRuntime` skips that override). The Firecracker microVM backend is - the exception: it explicitly attaches its host-side veth to AWF's proven - internal bridge, so network-isolation topology **is** still forced on for - `--container-runtime firecracker`. The api-proxy is still always enabled for - both. + (`isMicroVmRuntime` skips that override). The api-proxy remains enabled. ### End-to-end traffic flow @@ -368,16 +364,10 @@ flowchart TB Because the microVM path is abstracted behind a small set of seams, adding a new KVM backend is mostly a matter of implementing a manager and registering it. -Firecracker (`src/firecracker-runtime-backend.ts`, `src/firecracker/`) is a real, -fail-closed **workload preview** built on this same seam (gated behind -`--firecracker-preview`) — unlike sbx, it attaches its host-side veth directly to -AWF's proven internal `awf-net` bridge and reaches Squid/api-proxy at their -normal internal IPs (`SQUID_IP`/`API_PROXY_IP` from -`src/config/network-policy.ts`) rather than a docker0 gateway IP, and it keeps -strict network-isolation topology forced on (see -`src/commands/validators/security-mode.ts`). Cloud Hypervisor, krun/libkrun, -QEMU/KVM, etc. remain hypothetical. Here is the -checklist. +Cloud Hypervisor is the repository's fail-closed KVM workload preview built on +this seam. It uses a dedicated network namespace and reaches Squid and the API +proxy through AWF-managed networking. The following checklist applies to other +backends. ### 1. Register the runtime @@ -402,19 +392,18 @@ override in strict mode. It does **not** select the new manager: register an Implement `ExternalAgentRuntimeBackend` from `src/external-runtime-backend.ts`, following `SbxRuntimeBackend` in `src/sbx-runtime-backend.ts` (docker0-gateway addressing) or -`FirecrackerRuntimeBackend` in `src/firecracker-runtime-backend.ts` -(jailed microVM attached directly to `awf-net`, using internal -`SQUID_IP`/`API_PROXY_IP` addressing instead). The backend owns preflight, -startup, execution, diagnostics, and idempotent stop state. Concretely, a KVM -backend must: +`CloudHypervisorRuntimeBackend` in `src/cloud-hypervisor-runtime-backend.ts` +(a microVM isolated in an AWF-managed network namespace). The backend owns +preflight, startup, execution, diagnostics, and idempotent stop state. +Concretely, a KVM backend must: - **Boot a microVM on `/dev/kvm`** with a kernel + rootfs. Confirm KVM is available (`/dev/kvm` present, user in the `kvm` group). On stock GitHub-hosted runners `/dev/kvm` is *not* exposed — this backend is only for self-hosted / nested-virt-capable environments. -- **Mount the workspace** at its host absolute path (virtiofs is the sbx choice; - Firecracker typically uses a virtio-blk device or a virtiofsd sidecar). Also - surface `/tmp`, `/usr/local/bin`, and `$HOME` like `createSandbox` does. +- **Mount the workspace** at its host absolute path. sbx and Cloud Hypervisor + use virtio-fs; another backend may use a block device or a filesystem + sidecar. Also surface `/tmp`, `/usr/local/bin`, and `$HOME` when required. - **Inject the agent environment**, and **sanitize secrets** first — reuse the `sanitizeEnvForSbx()` pattern (strip `TOKEN|SECRET|KEY|...`). - **Return the agent's exit code** faithfully (AWF propagates it), mapping @@ -434,11 +423,9 @@ sandbox egress through AWF's host-side Squid: - Reproduce the boundary-crossing addressing pattern: sbx reaches Squid at the **bridge gateway IP + published port** (not the internal `172.30.0.x`) via `SBX_GATEWAY_IP`/`SBX_HOST_DOCKER_INTERNAL` in `src/sbx-runtime-backend.ts`, - because the sbx microVM sits outside `awf-net`. Firecracker instead attaches - its veth directly to `awf-net`, so it addresses Squid/api-proxy at their - normal internal IPs (`SQUID_IP`/`API_PROXY_IP` from - `src/config/network-policy.ts`) — pick whichever addressing model matches how - your VMM's network attaches to the host. + because the sbx microVM sits outside `awf-net`. Cloud Hypervisor instead uses + an AWF-managed namespace, veth pair, TAP device, and nftables policy. Pick the + addressing model that matches how your VMM attaches to the host. ### 4. Register the backend @@ -486,5 +473,5 @@ a way to force egress through AWF's Squid. - `sbx` releases: - AWF source: `src/container-runtime.ts`, `src/sbx-manager.ts`, `src/commands/main-action.ts`, `src/commands/validators/security-mode.ts`, - `src/firecracker-runtime-backend.ts` (second microVM backend built on this seam) + `src/cloud-hypervisor-runtime-backend.ts` (KVM backend built on this seam) - Related: [Sandbox design](./sandbox-design.md), [Architecture](./architecture.md) diff --git a/guest/cloud-hypervisor/build-test-artifacts.sh b/guest/cloud-hypervisor/build-test-artifacts.sh index 16b5db591..ff34e6d4c 100755 --- a/guest/cloud-hypervisor/build-test-artifacts.sh +++ b/guest/cloud-hypervisor/build-test-artifacts.sh @@ -5,10 +5,8 @@ umask 077 # Cloud Hypervisor v53.0 foundation guest artifacts. # -# This mirrors guest/firecracker/build-test-artifacts.sh's conventions and -# intentionally reuses the *exact same* pinned Linux kernel source and -# Firecracker microvm-kernel-ci config as the Firecracker pipeline: that -# config already builds a PCI-capable kernel (CONFIG_PCI, CONFIG_VIRTIO_PCI, +# The pinned kernel config is stored alongside this script. It builds a +# PCI-capable kernel (CONFIG_PCI, CONFIG_VIRTIO_PCI, # CONFIG_PCI_MMCONFIG for ACPI MCFG/PCIe ECAM, CONFIG_VIRTIO_BLK, # CONFIG_VIRTIO_NET, CONFIG_VIRTIO_CONSOLE, CONFIG_VSOCKETS, # CONFIG_VIRTIO_VSOCKETS, CONFIG_EXT4_FS, CONFIG_PVH for firmware-less direct @@ -16,9 +14,8 @@ umask 077 # CONFIG_VIRTIO_FS=y, with the kernel's scripts/config before olddefconfig. The original upstream config # SHA remains recorded separately from the final emitted kernel.config. # -# guest/firecracker-supervisor/build.sh is reused unmodified: it documents -# itself as VMM-neutral (length-prefixed JSON framing over vsock/UDS), so no -# Cloud Hypervisor-specific supervisor is needed. +# The VMM-neutral guest supervisor uses length-prefixed JSON framing over +# vsock/UDS. # # NOTE: these artifacts back the real Cloud Hypervisor lifecycle backend in # src/cloud-hypervisor/ (preview, gated behind --cloud-hypervisor-preview @@ -85,19 +82,12 @@ virtiofsd_package_version=$(dpkg-query --show --showformat='${Version}' "$virtio install -m 0755 "$virtiofsd_source" "$OUTPUT/virtiofsd" linux_tar="$BUILD/downloads/linux-${LINUX_VERSION}.tar.xz" -kernel_config="$BUILD/downloads/cloud-hypervisor-kernel.config" +kernel_config="$ROOT/guest/cloud-hypervisor/kernel.config" download_verified \ "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${LINUX_VERSION}.tar.xz" \ "$LINUX_SHA256" \ "$linux_tar" -# Reuses Firecracker's pinned, PCI-capable microvm-kernel-ci config (see -# header comment): same kernel source + same config as -# guest/firecracker/build-test-artifacts.sh, pinned to the Firecracker -# v1.16.1 release tag for stable provenance. -download_verified \ - "https://raw.githubusercontent.com/firecracker-microvm/firecracker/v1.16.1/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config" \ - "$KERNEL_CONFIG_SHA256" \ - "$kernel_config" +printf '%s %s\n' "$KERNEL_CONFIG_SHA256" "$kernel_config" | sha256sum --check --status tar --extract --xz --file "$linux_tar" --directory "$BUILD" cp "$kernel_config" "$BUILD/linux-${LINUX_VERSION}/.config" "$BUILD/linux-${LINUX_VERSION}/scripts/config" \ @@ -125,13 +115,11 @@ install -m 0644 \ "$OUTPUT/vmlinux.bin" install -m 0644 "$BUILD/linux-${LINUX_VERSION}/.config" "$OUTPUT/kernel.config" -# The AWF guest supervisor is intentionally VMM-neutral (see -# guest/firecracker-supervisor/protocol.go) and is shared as-is between the -# Firecracker and Cloud Hypervisor guest pipelines. +# Build the VMM-neutral AWF guest supervisor. supervisor="$OUTPUT/awf-supervisor" VERSION="${VERSION:-v${CLOUD_HYPERVISOR_VERSION}}" \ OUTPUT="$supervisor" \ - "$ROOT/guest/firecracker-supervisor/build.sh" + "$ROOT/guest/microvm-supervisor/build.sh" rootfs_tree="$BUILD/rootfs" if ! docker image inspect "$BUILD_TOOLS_IMAGE" >/dev/null 2>&1; then @@ -234,7 +222,7 @@ cat >"$OUTPUT/manifest.json" <&2 - exit 1 -fi - -for tool in curl sha256sum tar make gcc ld mke2fs e2fsck go; do - command -v "$tool" >/dev/null || { - echo "required build tool not found: $tool" >&2 - exit 1 - } -done - -rm -rf "$BUILD" "$OUTPUT" -mkdir -p "$BUILD/downloads" "$OUTPUT" - -download_verified() { - local url=$1 - local expected=$2 - local destination=$3 - curl --fail --location --proto '=https' --tlsv1.2 "$url" --output "$destination" - printf '%s %s\n' "$expected" "$destination" | sha256sum --check --status -} - -archive="$BUILD/downloads/firecracker-v${FIRECRACKER_VERSION}-x86_64.tgz" -download_verified \ - "https://github.com/firecracker-microvm/firecracker/releases/download/v${FIRECRACKER_VERSION}/firecracker-v${FIRECRACKER_VERSION}-x86_64.tgz" \ - "$FIRECRACKER_ARCHIVE_SHA256" \ - "$archive" -tar --extract --gzip --file "$archive" --directory "$BUILD" -release_dir="$BUILD/release-v${FIRECRACKER_VERSION}-x86_64" -( - cd "$release_dir" - sha256sum --check --ignore-missing SHA256SUMS -) -install -m 0755 \ - "$release_dir/firecracker-v${FIRECRACKER_VERSION}-x86_64" \ - "$OUTPUT/firecracker" -install -m 0755 \ - "$release_dir/jailer-v${FIRECRACKER_VERSION}-x86_64" \ - "$OUTPUT/jailer" - -linux_tar="$BUILD/downloads/linux-${LINUX_VERSION}.tar.xz" -kernel_config="$BUILD/downloads/firecracker-kernel.config" -download_verified \ - "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${LINUX_VERSION}.tar.xz" \ - "$LINUX_SHA256" \ - "$linux_tar" -download_verified \ - "https://raw.githubusercontent.com/firecracker-microvm/firecracker/v${FIRECRACKER_VERSION}/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config" \ - "$KERNEL_CONFIG_SHA256" \ - "$kernel_config" -tar --extract --xz --file "$linux_tar" --directory "$BUILD" -cp "$kernel_config" "$BUILD/linux-${LINUX_VERSION}/.config" -make -C "$BUILD/linux-${LINUX_VERSION}" \ - ARCH=x86_64 \ - KBUILD_BUILD_TIMESTAMP="@${SOURCE_DATE_EPOCH}" \ - KBUILD_BUILD_USER=awf \ - KBUILD_BUILD_HOST=github \ - LOCALVERSION=-awf-firecracker \ - olddefconfig -make -C "$BUILD/linux-${LINUX_VERSION}" \ - -j"$JOBS" \ - ARCH=x86_64 \ - KBUILD_BUILD_TIMESTAMP="@${SOURCE_DATE_EPOCH}" \ - KBUILD_BUILD_USER=awf \ - KBUILD_BUILD_HOST=github \ - LOCALVERSION=-awf-firecracker \ - bzImage -install -m 0644 \ - "$BUILD/linux-${LINUX_VERSION}/arch/x86/boot/bzImage" \ - "$OUTPUT/vmlinux.bin" - -busybox_tar="$BUILD/downloads/busybox-${BUSYBOX_VERSION}.tar.bz2" -download_verified \ - "https://busybox.net/downloads/busybox-${BUSYBOX_VERSION}.tar.bz2" \ - "$BUSYBOX_SHA256" \ - "$busybox_tar" -tar --extract --bzip2 --file "$busybox_tar" --directory "$BUILD" -busybox_dir="$BUILD/busybox-${BUSYBOX_VERSION}" -make -C "$busybox_dir" defconfig -enable_busybox_option() { - local option=$1 - if grep -q "^CONFIG_${option}=" "$busybox_dir/.config"; then - sed -i "s/^CONFIG_${option}=.*/CONFIG_${option}=y/" "$busybox_dir/.config" - elif grep -q "^# CONFIG_${option} is not set$" "$busybox_dir/.config"; then - sed -i "s/^# CONFIG_${option} is not set$/CONFIG_${option}=y/" "$busybox_dir/.config" - else - printf 'CONFIG_%s=y\n' "$option" >>"$busybox_dir/.config" - fi -} -disable_busybox_option() { - local option=$1 - if grep -q "^CONFIG_${option}=" "$busybox_dir/.config"; then - sed -i "s/^CONFIG_${option}=.*/# CONFIG_${option} is not set/" "$busybox_dir/.config" - elif ! grep -q "^# CONFIG_${option} is not set$" "$busybox_dir/.config"; then - printf '# CONFIG_%s is not set\n' "$option" >>"$busybox_dir/.config" - fi -} -for option in \ - STATIC \ - WGET \ - FEATURE_WGET_HTTPS \ - TLS \ - IP \ - IPADDR \ - IPLINK \ - IPROUTE \ - NC \ - NSLOOKUP \ - TIMEOUT; do - enable_busybox_option "$option" -done -# BusyBox 1.36.1 tc depends on CBQ UAPI definitions removed from newer build hosts. -# The minimal guest never uses traffic control; AWF enforces policy in the host netns. -disable_busybox_option TC -# FEATURE_WGET_OPENSSL defaults to enabled and, when active, makes wget -# handle every https:// URL by shelling out directly to -# `openssl s_client -connect :443` -- entirely bypassing wget's -# own HTTP(S)_PROXY-aware connection logic. That requires the guest to -# resolve DNS and reach arbitrary hosts on port 443 directly, both of -# which this network policy deliberately blocks (the guest is only ever -# supposed to reach Squid/API-proxy on their fixed IPs; Squid alone -# resolves/enforces the allowed-domain list). Disabling this makes wget -# fall back to FEATURE_WGET_HTTPS's internal TLS code, which correctly -# tunnels through HTTPS_PROXY/https_proxy via a CONNECT request using the -# hostname string, never needing guest-side DNS resolution at all. -disable_busybox_option FEATURE_WGET_OPENSSL -make -C "$busybox_dir" -j"$JOBS" - -supervisor="$OUTPUT/awf-firecracker-supervisor" -VERSION="${VERSION:-v${FIRECRACKER_VERSION}}" \ - OUTPUT="$supervisor" \ - "$ROOT/guest/firecracker-supervisor/build.sh" - -rootfs_tree="$BUILD/rootfs" -mkdir -p \ - "$rootfs_tree/bin" \ - "$rootfs_tree/dev" \ - "$rootfs_tree/etc/ssl/certs" \ - "$rootfs_tree/proc" \ - "$rootfs_tree/root" \ - "$rootfs_tree/sbin" \ - "$rootfs_tree/sys" \ - "$rootfs_tree/tmp" \ - "$rootfs_tree/usr/bin" \ - "$rootfs_tree/usr/sbin" \ - "$rootfs_tree/workspace" -make -C "$busybox_dir" CONFIG_PREFIX="$rootfs_tree" install -install -m 0755 "$supervisor" "$rootfs_tree/sbin/awf-supervisor" -cat >"$rootfs_tree/etc/passwd" <<'EOF' -root:x:0:0:root:/root:/bin/sh -awf:x:1000:1000:AWF guest:/workspace:/bin/sh -nobody:x:65534:65534:nobody:/:/bin/false -EOF -cat >"$rootfs_tree/etc/group" <<'EOF' -root:x:0: -awf:x:1000: -nogroup:x:65534: -EOF -cat >"$rootfs_tree/etc/resolv.conf" <<'EOF' -# Direct DNS is intentionally unavailable in the Firecracker preview. -EOF -ca_bundle="$BUILD/downloads/cacert-${CA_BUNDLE_DATE}.pem" -download_verified \ - "https://curl.se/ca/cacert-${CA_BUNDLE_DATE}.pem" \ - "$CA_BUNDLE_SHA256" \ - "$ca_bundle" -install -m 0644 "$ca_bundle" "$rootfs_tree/etc/ssl/certs/ca-certificates.crt" -chmod 01777 "$rootfs_tree/tmp" -find "$rootfs_tree" -print0 | xargs -0 touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - -rootfs="$OUTPUT/rootfs.ext4" -E2FSPROGS_FAKE_TIME="$SOURCE_DATE_EPOCH" mke2fs \ - -t ext4 \ - -F \ - -q \ - -b 4096 \ - -d "$rootfs_tree" \ - -U 7b6680c1-1e8c-4aac-a04e-95b8f36ff8ee \ - -E lazy_itable_init=0,lazy_journal_init=0 \ - "$rootfs" \ - 32768 -E2FSPROGS_FAKE_TIME="$SOURCE_DATE_EPOCH" e2fsck -f -y "$rootfs" >/dev/null - -( - cd "$OUTPUT" - sha256sum \ - firecracker \ - jailer \ - vmlinux.bin \ - rootfs.ext4 \ - awf-firecracker-supervisor \ - > SHA256SUMS -) - -cat >"$OUTPUT/manifest.json" <"$OUTPUT/sbom.spdx.json" <&2 - exit 1 - } -done - -( - cd "$ARTIFACT_DIR" - sha256sum --check SHA256SUMS -) - -"$ARTIFACT_DIR/firecracker" --version | grep -F '1.16.1' -"$ARTIFACT_DIR/jailer" --version | grep -F '1.16.1' -file "$ARTIFACT_DIR/vmlinux.bin" | grep -E 'Linux kernel|boot executable' -e2fsck -f -n "$ARTIFACT_DIR/rootfs.ext4" -debugfs -R 'stat /sbin/awf-supervisor' "$ARTIFACT_DIR/rootfs.ext4" 2>&1 \ - | grep -F 'Type: regular' -grep -F '"purpose": "AWF Firecracker preview test artifacts; not production defaults"' \ - "$ARTIFACT_DIR/manifest.json" -grep -F '"spdxVersion": "SPDX-2.3"' "$ARTIFACT_DIR/sbom.spdx.json" diff --git a/guest/firecracker-supervisor/build.sh b/guest/microvm-supervisor/build.sh similarity index 91% rename from guest/firecracker-supervisor/build.sh rename to guest/microvm-supervisor/build.sh index 8d8acc678..cbb5197ee 100755 --- a/guest/firecracker-supervisor/build.sh +++ b/guest/microvm-supervisor/build.sh @@ -4,7 +4,7 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) GO_VERSION=go1.25.0 VERSION=${VERSION:-dev} -OUTPUT=${OUTPUT:-"$ROOT/firecracker-supervisor"} +OUTPUT=${OUTPUT:-"$ROOT/microvm-supervisor"} actual=$(go env GOVERSION) if [ "$actual" != "$GO_VERSION" ]; then diff --git a/guest/firecracker-supervisor/config.go b/guest/microvm-supervisor/config.go similarity index 100% rename from guest/firecracker-supervisor/config.go rename to guest/microvm-supervisor/config.go diff --git a/guest/firecracker-supervisor/config_test.go b/guest/microvm-supervisor/config_test.go similarity index 100% rename from guest/firecracker-supervisor/config_test.go rename to guest/microvm-supervisor/config_test.go diff --git a/guest/microvm-supervisor/go.mod b/guest/microvm-supervisor/go.mod new file mode 100644 index 000000000..0f20e7336 --- /dev/null +++ b/guest/microvm-supervisor/go.mod @@ -0,0 +1,5 @@ +module github.com/github/gh-aw-firewall/microvm-supervisor + +go 1.24.0 + +toolchain go1.25.0 diff --git a/guest/firecracker-supervisor/main.go b/guest/microvm-supervisor/main.go similarity index 67% rename from guest/firecracker-supervisor/main.go rename to guest/microvm-supervisor/main.go index 602ab4e09..ff3ff790a 100644 --- a/guest/firecracker-supervisor/main.go +++ b/guest/microvm-supervisor/main.go @@ -1,4 +1,4 @@ -// firecracker-supervisor is the minimal guest-side command supervisor. +// microvm-supervisor is the minimal guest-side command supervisor. package main import ( @@ -17,7 +17,7 @@ func main() { return } if err := runSupervisor(); err != nil { - fmt.Fprintln(os.Stderr, "firecracker-supervisor:", err) + fmt.Fprintln(os.Stderr, "microvm-supervisor:", err) os.Exit(1) } } diff --git a/guest/firecracker-supervisor/protocol.go b/guest/microvm-supervisor/protocol.go similarity index 97% rename from guest/firecracker-supervisor/protocol.go rename to guest/microvm-supervisor/protocol.go index 5b85b7e5c..fb7ce90b1 100644 --- a/guest/firecracker-supervisor/protocol.go +++ b/guest/microvm-supervisor/protocol.go @@ -3,10 +3,8 @@ package main // This file implements the guest side of the AWF framed guest-supervisor // protocol. It is intentionally VMM-neutral: the length-prefixed JSON // framing and frame types here mirror src/microvm/guest-protocol.ts on the -// host side, and this binary (despite its package's historical -// "firecracker-supervisor" name/path) does not depend on any -// Firecracker-specific transport. A future VMM backend can reuse this -// supervisor as-is, addressed through the same vsock/UDS compatibility +// host side. This binary does not depend on a specific VMM transport and can +// be addressed through the same vsock/UDS compatibility // boundary, without protocol changes. import ( @@ -64,7 +62,7 @@ func (e *protocolError) Is(target error) bool { return ok && e.code == other.code } -// Frame is the exact JSON shape accepted by src/firecracker/vsock-protocol.ts. +// Frame is the exact JSON shape accepted by the host guest protocol. type Frame struct { Version int `json:"version"` Type string `json:"type"` diff --git a/guest/firecracker-supervisor/protocol_test.go b/guest/microvm-supervisor/protocol_test.go similarity index 100% rename from guest/firecracker-supervisor/protocol_test.go rename to guest/microvm-supervisor/protocol_test.go diff --git a/guest/firecracker-supervisor/runtime_linux.go b/guest/microvm-supervisor/runtime_linux.go similarity index 99% rename from guest/firecracker-supervisor/runtime_linux.go rename to guest/microvm-supervisor/runtime_linux.go index a223d4f3b..03292e84f 100644 --- a/guest/firecracker-supervisor/runtime_linux.go +++ b/guest/microvm-supervisor/runtime_linux.go @@ -238,8 +238,7 @@ func mountWorkspace(config bootConfig) error { // which made this supervisor's init process return an error and the // kernel panic with "Attempted to kill init!" on every single guest // boot. Discovered via live-KVM validation (a genuine, pre-existing - // defect shared by both the Firecracker and Cloud Hypervisor backends, - // since they share this guest supervisor binary). + // defect in the shared guest supervisor binary). source, target, fstype, flags := workspaceMountArgs(config) if err := mountFilesystem(source, target, fstype, flags, ""); err != nil { return fmt.Errorf("mount workspace: %w", err) diff --git a/guest/firecracker-supervisor/runtime_linux_test.go b/guest/microvm-supervisor/runtime_linux_test.go similarity index 98% rename from guest/firecracker-supervisor/runtime_linux_test.go rename to guest/microvm-supervisor/runtime_linux_test.go index 1c476a0ff..da0054acc 100644 --- a/guest/firecracker-supervisor/runtime_linux_test.go +++ b/guest/microvm-supervisor/runtime_linux_test.go @@ -39,8 +39,7 @@ func TestWorkspaceMountArgsUseExt4Filesystem(t *testing.T) { // device"), which made the guest supervisor's init process return an // error and the kernel panic with "Attempted to kill init!" on every // single guest boot — discovered via live-KVM validation, a genuine, - // pre-existing defect shared by both the Firecracker and Cloud - // Hypervisor backends (they share this guest supervisor binary). + // pre-existing defect in the shared guest supervisor binary. config := bootConfig{WorkspaceDevice: "/dev/vdb", WorkspaceMount: "/workspace"} source, target, fstype, flags := workspaceMountArgs(config) if source != config.WorkspaceDevice { diff --git a/guest/firecracker-supervisor/runtime_other.go b/guest/microvm-supervisor/runtime_other.go similarity index 53% rename from guest/firecracker-supervisor/runtime_other.go rename to guest/microvm-supervisor/runtime_other.go index db1370d97..7fb209642 100644 --- a/guest/firecracker-supervisor/runtime_other.go +++ b/guest/microvm-supervisor/runtime_other.go @@ -5,5 +5,5 @@ package main import "errors" func runSupervisor() error { - return errors.New("the Firecracker guest supervisor requires Linux") + return errors.New("the microVM guest supervisor requires Linux") } diff --git a/scripts/ci/cloud-hypervisor-ci-scripts.test.ts b/scripts/ci/cloud-hypervisor-ci-scripts.test.ts index 12b2689aa..ec9efc5f7 100644 --- a/scripts/ci/cloud-hypervisor-ci-scripts.test.ts +++ b/scripts/ci/cloud-hypervisor-ci-scripts.test.ts @@ -4,8 +4,6 @@ import { execFileSync } from 'child_process'; const preflightPath = path.resolve(__dirname, 'cloud-hypervisor-host-preflight.sh'); const smokePath = path.resolve(__dirname, 'cloud-hypervisor-live-smoke.sh'); -const firecrackerPreflightPath = path.resolve(__dirname, 'firecracker-host-preflight.sh'); -const firecrackerSmokePath = path.resolve(__dirname, 'firecracker-live-smoke.sh'); function shellcheckAvailable(): boolean { try { @@ -116,10 +114,10 @@ describe('cloud-hypervisor-live-smoke.sh', () => { expect(source).toContain('awf-cloud-hypervisor-real-secret-do-not-expose'); }); - it('checks netns/veth/TAP residue (shared with Firecracker) plus Cloud Hypervisor-specific cgroup/process residue', () => { + it('checks netns/veth/TAP plus Cloud Hypervisor-specific cgroup/process residue', () => { const source = fs.readFileSync(smokePath, 'utf-8'); - expect(source).toContain("grep -q '^awffc-'"); - expect(source).toContain('(fch|fcn|fct)'); + expect(source).toContain("grep -q '^awfvm-'"); + expect(source).toContain('(vmh|vmn|vmt)'); expect(source).toContain('CGROUP_ROOT'); expect(source).toContain("pgrep -f 'cloud-hypervisor --api-socket'"); }); @@ -143,30 +141,9 @@ describe('cloud-hypervisor-live-smoke.sh', () => { expect(source).toMatch(/COMMON=\(\n(?:.*\n)*?\s*--network-isolation\n/); }); - (shellcheckAvailable() ? it : it.skip)('has no new shellcheck errors beyond the Firecracker baseline', () => { + (shellcheckAvailable() ? it : it.skip)('has no shellcheck errors', () => { expect(() => execFileSync('shellcheck', ['--severity=error', smokePath]), ).not.toThrow(); }); }); - -describe('parity with the Firecracker live-smoke conventions', () => { - it('both Firecracker and Cloud Hypervisor scripts define the same run_case/assert_no_residue helpers', () => { - const firecracker = fs.readFileSync(firecrackerSmokePath, 'utf-8'); - const cloudHypervisor = fs.readFileSync(smokePath, 'utf-8'); - for (const helper of ['run_case()', 'assert_no_residue()']) { - expect(firecracker).toContain(helper); - expect(cloudHypervisor).toContain(helper); - } - }); - - it('both preflight scripts fail closed on missing Linux/x86_64/KVM host requirements', () => { - const firecracker = fs.readFileSync(firecrackerPreflightPath, 'utf-8'); - const cloudHypervisor = fs.readFileSync(preflightPath, 'utf-8'); - for (const script of [firecracker, cloudHypervisor]) { - expect(script).toContain('/dev/kvm'); - expect(script).toContain('x86_64'); - expect(script).toContain('set -euo pipefail'); - } - }); -}); diff --git a/scripts/ci/cloud-hypervisor-host-preflight.sh b/scripts/ci/cloud-hypervisor-host-preflight.sh index 8065fbce6..c7d9c1f02 100755 --- a/scripts/ci/cloud-hypervisor-host-preflight.sh +++ b/scripts/ci/cloud-hypervisor-host-preflight.sh @@ -3,12 +3,11 @@ set -euo pipefail # Fail-closed CI host preflight for the Cloud Hypervisor live-KVM job. # -# Mirrors scripts/ci/firecracker-host-preflight.sh's checks (Linux/x86_64, -# /dev/kvm, kernel controls, cgroup hierarchy, required host tools, docker, -# artifact digests), with two Cloud Hypervisor-specific differences: +# Checks Linux/x86_64, /dev/kvm, kernel controls, cgroup hierarchy, required +# host tools, Docker, and artifact digests. Two backend-specific requirements: # 1. GitHub-hosted-only host eligibility is enforced here too (this -# backend rejects self-hosted runners, unlike Firecracker's preview — -# see src/cloud-hypervisor/host-eligibility.ts), so a misconfigured +# backend rejects self-hosted runners; see +# src/cloud-hypervisor/host-eligibility.ts), so a misconfigured # self-hosted runner fails fast in CI instead of failing later inside # the CLI. # 2. `setpriv` is required (the launcher's jailer replacement — see diff --git a/scripts/ci/cloud-hypervisor-live-smoke.sh b/scripts/ci/cloud-hypervisor-live-smoke.sh index fa1643d2c..97acca7b6 100755 --- a/scripts/ci/cloud-hypervisor-live-smoke.sh +++ b/scripts/ci/cloud-hypervisor-live-smoke.sh @@ -4,13 +4,11 @@ set -euo pipefail # Live GitHub-hosted Ubuntu x86_64 KVM smoke/security suite for the Cloud # Hypervisor preview backend. # -# This reproduces the same 13-case behavioral/security contract as -# scripts/ci/firecracker-live-smoke.sh (allowed/blocked domains, direct +# This covers allowed/blocked domains, direct # egress, arbitrary TCP, DNS, metadata IP, mandatory API-proxy reflect with # secret-sentinel absence, live workspace sharing incl. symlinks/permissions, # exit-code propagation, timeout, SIGTERM cancellation, partial-start -# rollback, keep/preserve diagnostics), then adds Cloud Hypervisor-specific -# live checks that have no Firecracker/jailer equivalent: +# rollback, keep/preserve diagnostics, plus backend-specific live checks: # # - device-assumptions: confirms eth0, the sole /dev/vda block disk, and # virtio-fs workspace layout documented in Part 6. @@ -25,11 +23,9 @@ set -euo pipefail # set with no path to the host-only API socket) — see # src/cloud-hypervisor/launcher.ts. # -# NOTE on shared namespace/interface naming: src/microvm/network.ts is -# VMM-neutral and used unmodified by both the Firecracker and Cloud -# Hypervisor backends (see docs/cloud-hypervisor-foundation.md Part 2), so -# the network namespace (`awffc-*`) and veth/TAP (`fch*`/`fcn*`/`fct*`) -# naming below is intentionally identical to Firecracker's, not a defect. +# NOTE on namespace/interface naming: src/microvm/network.ts is VMM-neutral. +# The network namespace uses `awfvm-*`; veth/TAP devices use +# `vmh*`/`vmn*`/`vmt*`. # The cgroup path (`awf-cloud-hypervisor/`) and process name # (`cloud-hypervisor`) residue checks below ARE Cloud Hypervisor-specific. @@ -89,12 +85,12 @@ COMMON=( ) assert_no_residue() { - if sudo ip netns list | grep -q '^awffc-'; then + if sudo ip netns list | grep -q '^awfvm-'; then sudo ip netns list >&2 echo "Cloud Hypervisor network namespace residue detected" >&2 return 1 fi - if sudo ip -o link show | grep -Eq ' (fch|fcn|fct)[0-9a-f]{12}[:@]'; then + if sudo ip -o link show | grep -Eq ' (vmh|vmn|vmt)[0-9a-f]{12}[:@]'; then sudo ip -o link show >&2 echo "Cloud Hypervisor veth/TAP residue detected" >&2 return 1 @@ -222,7 +218,7 @@ ns_watcher_pid= if command -v tcpdump >/dev/null 2>&1; then ( for _ in $(seq 1 200); do - ns=$(sudo ip netns list 2>/dev/null | awk '{print $1}' | grep -m1 '^awffc-' || true) + ns=$(sudo ip netns list 2>/dev/null | awk '{print $1}' | grep -m1 '^awfvm-' || true) if [ -n "$ns" ]; then exec sudo ip netns exec "$ns" tcpdump -i any -w "$ns_tcpdump_out" \ 'port 3128 or arp or icmp' >/dev/null 2>&1 @@ -314,7 +310,7 @@ mkdir -p "$cancel_work" "$cancel_workspace" "$cancel_audit" ) >"$RUN_ROOT/cancellation/stdout.log" 2>"$RUN_ROOT/cancellation/stderr.log" & cancel_pid=$! for _ in $(seq 1 60); do - sudo ip netns list | grep -q '^awffc-' && break + sudo ip netns list | grep -q '^awfvm-' && break sleep 1 done cleanup_start_ns=$(date +%s%N) @@ -363,7 +359,7 @@ if [ "$keep_status" -ne 0 ]; then tail -200 "$RUN_ROOT/keep/stderr.log" >&2 exit 1 fi -sudo ip netns list | grep -q '^awffc-' || { +sudo ip netns list | grep -q '^awfvm-' || { echo "keep mode did not preserve the run network namespace" >&2 exit 1 } @@ -391,7 +387,7 @@ sudo find "$keep_audit/cloud-hypervisor" -type f -size +1048576c -print -quit \ while read -r namespace _; do case "$namespace" in - awffc-*) sudo ip netns delete "$namespace" ;; + awfvm-*) sudo ip netns delete "$namespace" ;; esac done < <(sudo ip netns list) sudo docker compose -f "$keep_work/docker-compose.yml" down --volumes --remove-orphans @@ -522,15 +518,15 @@ esac # also proves these assertions inspect a live VM, not merely a launched VMM. # # The expected TAP name is derived exactly like -# createMicrovmNetworkPlan() (src/microvm/network.ts): `fct` + the first 12 +# createMicrovmNetworkPlan() (src/microvm/network.ts): `vmt` + the first 12 # hex characters of sha256(runId). The per-run network namespace shares -# the same token (`awffc-` + token) and is where the TAP device actually +# the same token (`awfvm-` + token) and is where the TAP device actually # lives -- checking for it in the root/default namespace, which is what # actually hosts this script, would never find a namespace-scoped # interface regardless of whether it truly exists. run_token=$(printf '%s' "$run_id" | sha256sum | cut -c1-12) -expected_tap="fct$run_token" -expected_namespace="awffc-$run_token" +expected_tap="vmt$run_token" +expected_namespace="awfvm-$run_token" expected_rootfs_path="$run_directory/rootfs.ext4" expected_vsock_socket="$run_directory/awf-vsock.socket" diff --git a/scripts/ci/firecracker-host-preflight.sh b/scripts/ci/firecracker-host-preflight.sh deleted file mode 100755 index 7a29155f4..000000000 --- a/scripts/ci/firecracker-host-preflight.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ARTIFACT_DIR=${1:?usage: firecracker-host-preflight.sh ARTIFACT_DIR} - -fail() { - echo "::error title=Firecracker host preflight::$*" >&2 - exit 1 -} - -[ "$(uname -s)" = Linux ] || fail "Linux is required; macOS and Windows are unsupported." -[ "$(uname -m)" = x86_64 ] || fail "This CI artifact set requires an x86_64 host." -[ -c /dev/kvm ] || fail "/dev/kvm is missing; use a KVM-capable host." -[ -r /dev/kvm ] && [ -w /dev/kvm ] \ - || fail "/dev/kvm must be readable and writable by the workflow user." - -for control in \ - /proc/sys/net/ipv4/ip_forward \ - /proc/sys/net/ipv6/conf/all/disable_ipv6 \ - /proc/sys/kernel/seccomp/actions_avail; do - [ -r "$control" ] || fail "Required host kernel control is unavailable: $control" -done -[ -r /sys/fs/cgroup/cgroup.controllers ] || [ -w /sys/fs/cgroup ] \ - || fail "A usable cgroup v1 or v2 hierarchy is required by jailer." - -for tool in nft ip sysctl mke2fs debugfs e2fsck rsync docker sha256sum timeout; do - command -v "$tool" >/dev/null || fail "Required host tool is missing: $tool" -done -command -v sudo >/dev/null || fail "Passwordless sudo is required for jailer and netns setup." -sudo -n true || fail "Passwordless sudo is required for jailer and netns setup." -docker info >/dev/null || fail "A host-visible Docker Engine is required." -docker compose version >/dev/null || fail "Docker Compose v2 is required." - -"$ARTIFACT_DIR/firecracker" --version | grep -Fq '1.16.1' \ - || fail "Firecracker v1.16.1 is required." -"$ARTIFACT_DIR/jailer" --version | grep -Fq '1.16.1' \ - || fail "jailer v1.16.1 is required." -( - cd "$ARTIFACT_DIR" - sha256sum --check --strict SHA256SUMS -) || fail "Artifact digest verification failed." - -echo "Firecracker host preflight passed on Linux/x86_64 with accessible KVM." diff --git a/scripts/ci/firecracker-live-smoke.sh b/scripts/ci/firecracker-live-smoke.sh deleted file mode 100755 index acf0d5cde..000000000 --- a/scripts/ci/firecracker-live-smoke.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ARTIFACT_DIR=${1:?usage: firecracker-live-smoke.sh ARTIFACT_DIR} -ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) -RUN_ROOT=${RUNNER_TEMP:-/tmp}/awf-firecracker-live -SECRET_SENTINEL=awf-firecracker-real-secret-do-not-expose - -ARTIFACT_DIR=$(cd "$ARTIFACT_DIR" && pwd) -rm -rf "$RUN_ROOT" -mkdir -p "$RUN_ROOT" - -digest() { - awk -v file="$1" '$2 == file { print $1; exit }' "$ARTIFACT_DIR/SHA256SUMS" -} - -COMMON=( - --container-runtime firecracker - --firecracker-preview - --firecracker-binary "$ARTIFACT_DIR/firecracker" - --firecracker-jailer-binary "$ARTIFACT_DIR/jailer" - --firecracker-kernel "$ARTIFACT_DIR/vmlinux.bin" - --firecracker-rootfs "$ARTIFACT_DIR/rootfs.ext4" - --firecracker-supervisor "$ARTIFACT_DIR/awf-firecracker-supervisor" - --firecracker-binary-sha256 "$(digest firecracker)" - --firecracker-jailer-sha256 "$(digest jailer)" - --firecracker-kernel-sha256 "$(digest vmlinux.bin)" - --firecracker-rootfs-sha256 "$(digest rootfs.ext4)" - --firecracker-supervisor-sha256 "$(digest awf-firecracker-supervisor)" - --allow-domains example.com - --skip-pull - --diagnostic-logs -) - -assert_no_residue() { - if sudo ip netns list | grep -q '^awffc-'; then - sudo ip netns list >&2 - echo "Firecracker network namespace residue detected" >&2 - return 1 - fi - if sudo ip -o link show | grep -Eq ' (fch|fcn|fct)[0-9a-f]{12}[:@]'; then - sudo ip -o link show >&2 - echo "Firecracker veth/TAP residue detected" >&2 - return 1 - fi -} - -run_case() { - local name=$1 - local expected=$2 - local command=$3 - shift 3 - local work="$RUN_ROOT/$name/work" - local workspace="$RUN_ROOT/$name/workspace" - local audit="$RUN_ROOT/$name/audit" - local proxy_logs="$RUN_ROOT/$name/proxy-logs" - mkdir -p "$work" "$workspace" "$audit" "$proxy_logs" - printf 'host-input\n' >"$workspace/input.txt" - - set +e - ( - export GITHUB_WORKSPACE="$workspace" - export OPENAI_API_KEY="$SECRET_SENTINEL" - sudo -E node "$ROOT/dist/cli.js" \ - "${COMMON[@]}" \ - --work-dir "$work" \ - --audit-dir "$audit" \ - --proxy-logs-dir "$proxy_logs" \ - "$@" \ - -- "$command" - ) >"$RUN_ROOT/$name/stdout.log" 2>"$RUN_ROOT/$name/stderr.log" - local status=$? - set -e - - if [ "$status" -ne "$expected" ]; then - echo "case $name: expected exit $expected, got $status" >&2 - tail -200 "$RUN_ROOT/$name/stderr.log" >&2 - return 1 - fi - # awf-resolved-config.json's own agentCommand field always contains - # this case's shell command verbatim; for api-proxy-reflect that - # command intentionally references the sentinel string itself (the - # pattern it greps for, to assert the sentinel's absence from `env`). - # That is expected, self-referential test source text, not a leak of - # the sentinel value into somewhere it shouldn't be -- guest stdout, - # proxy logs, and every other diagnostic file are still fully scanned. - if grep -R --binary-files=without-match -F "$SECRET_SENTINEL" \ - --exclude='awf-resolved-config.json' \ - "$RUN_ROOT/$name/stdout.log" \ - "$audit" \ - "$proxy_logs" >/dev/null 2>&1; then - echo "case $name: secret sentinel leaked into guest-visible or diagnostic output" >&2 - return 1 - fi - assert_no_residue -} - -assert_no_residue - -run_case allowed-https 0 \ - 'wget -qO- https://example.com | grep -q "Example Domain"' -run_case blocked-domain 0 \ - '! wget -qO- https://github.com' -run_case direct-egress 0 \ - 'unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy; ! wget -qO- https://example.com' -run_case arbitrary-tcp 0 \ - '! nc -z -w 3 1.1.1.1 443' -run_case dns-denial 0 \ - '! nslookup example.com 8.8.8.8' -run_case metadata-denial 0 \ - 'unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy; ! wget -T 3 -qO- http://169.254.169.254/latest/meta-data/' -run_case api-proxy-reflect 0 \ - 'wget -qO /tmp/reflect http://172.30.0.30:10000/reflect && grep -q "providers" /tmp/reflect && ! env | grep -F "awf-firecracker-real-secret-do-not-expose"' - -run_case workspace-copyback 0 \ - 'printf changed > .hidden && mkdir -p bin && printf "#!/bin/sh\necho ok\n" > bin/run && chmod 755 bin/run && ln -s bin/run run-link' -test "$(cat "$RUN_ROOT/workspace-copyback/workspace/.hidden")" = changed -test -x "$RUN_ROOT/workspace-copyback/workspace/bin/run" -test "$(readlink "$RUN_ROOT/workspace-copyback/workspace/run-link")" = bin/run - -run_case exit-code 37 'exit 37' -run_case timeout-124 124 'sleep 90' --agent-timeout 1 - -corrupt="$RUN_ROOT/corrupt-rootfs.ext4" -printf 'not-an-ext4-image\n' >"$corrupt" -corrupt_digest=$(sha256sum "$corrupt" | awk '{print $1}') -run_case partial-start-cleanup 1 'true' \ - --firecracker-rootfs "$corrupt" \ - --firecracker-rootfs-sha256 "$corrupt_digest" \ - --firecracker-api-timeout-ms 3000 - -cancel_work="$RUN_ROOT/cancellation/work" -cancel_workspace="$RUN_ROOT/cancellation/workspace" -cancel_audit="$RUN_ROOT/cancellation/audit" -mkdir -p "$cancel_work" "$cancel_workspace" "$cancel_audit" -( - export GITHUB_WORKSPACE="$cancel_workspace" - export OPENAI_API_KEY="$SECRET_SENTINEL" - exec sudo -E node "$ROOT/dist/cli.js" \ - "${COMMON[@]}" \ - --work-dir "$cancel_work" \ - --audit-dir "$cancel_audit" \ - -- 'sleep 300' -) >"$RUN_ROOT/cancellation/stdout.log" 2>"$RUN_ROOT/cancellation/stderr.log" & -cancel_pid=$! -for _ in $(seq 1 60); do - sudo ip netns list | grep -q '^awffc-' && break - sleep 1 -done -kill -TERM "$cancel_pid" -set +e -wait "$cancel_pid" -cancel_status=$? -set -e -[ "$cancel_status" -eq 143 ] || { - echo "cancellation: expected exit 143, got $cancel_status" >&2 - exit 1 -} -assert_no_residue - -keep_work="$RUN_ROOT/keep/work" -keep_workspace="$RUN_ROOT/keep/workspace" -keep_audit="$RUN_ROOT/keep/audit" -mkdir -p "$keep_work" "$keep_workspace" "$keep_audit" -set +e -( - export GITHUB_WORKSPACE="$keep_workspace" - export OPENAI_API_KEY="$SECRET_SENTINEL" - sudo -E node "$ROOT/dist/cli.js" \ - "${COMMON[@]}" \ - --keep-containers \ - --work-dir "$keep_work" \ - --audit-dir "$keep_audit" \ - -- 'true' -) >"$RUN_ROOT/keep/stdout.log" 2>"$RUN_ROOT/keep/stderr.log" -keep_status=$? -set -e -if [ "$keep_status" -ne 0 ]; then - # See the identical fix in cloud-hypervisor-live-smoke.sh: unlike - # run_case, this invocation is not wrapped by a helper that tails its - # own log on failure, so under `set -e` a non-zero exit here previously - # aborted the whole suite silently. - echo "keep-containers invocation: expected exit 0, got $keep_status" >&2 - tail -200 "$RUN_ROOT/keep/stderr.log" >&2 - exit 1 -fi -sudo ip netns list | grep -q '^awffc-' || { - echo "keep mode did not preserve the run network namespace" >&2 - exit 1 -} -sudo test -d "$keep_work/firecracker-jailer" || { - echo "keep mode did not preserve the firecracker-jailer work directory" >&2 - exit 1 -} -sudo test -f "$keep_audit/firecracker/network-plan.json" || { - echo "keep mode did not preserve network-plan.json" >&2 - exit 1 -} -sudo test -f "$keep_audit/firecracker/firecracker.log" || { - echo "keep mode did not preserve firecracker.log" >&2 - exit 1 -} -sudo test -f "$keep_audit/firecracker/firecracker.metrics.jsonl" || { - echo "keep mode did not preserve firecracker.metrics.jsonl" >&2 - exit 1 -} -sudo find "$keep_audit/firecracker" -type f -size +1048576c -print -quit \ - | grep -q . && { - echo "Firecracker diagnostic artifact exceeded the 1 MiB bound" >&2 - exit 1 - } - -while read -r namespace _; do - case "$namespace" in - awffc-*) sudo ip netns delete "$namespace" ;; - esac -done < <(sudo ip netns list) -sudo docker compose -f "$keep_work/docker-compose.yml" down --volumes --remove-orphans -assert_no_residue - -echo "Firecracker live smoke/security suite passed." diff --git a/scripts/ci/test-cloud-hypervisor-workflow.test.ts b/scripts/ci/test-cloud-hypervisor-workflow.test.ts index b6de554fd..5b6103f3f 100644 --- a/scripts/ci/test-cloud-hypervisor-workflow.test.ts +++ b/scripts/ci/test-cloud-hypervisor-workflow.test.ts @@ -136,7 +136,7 @@ describe('Cloud Hypervisor CI workflow', () => { const cleanupStep = steps.find((step) => step.name === 'Enforce final residue cleanup'); expect(cleanupStep?.if).toBe('always()'); - expect(cleanupStep?.run).toContain('awffc-'); + expect(cleanupStep?.run).toContain('awfvm-'); expect(cleanupStep?.run).toContain('awf-cloud-hypervisor'); const diagnosticsStep = steps.find((step) => step.name === 'Collect redacted diagnostics'); diff --git a/src/awf-config-schema.json b/src/awf-config-schema.json index 58b4dfea9..60d95db24 100644 --- a/src/awf-config-schema.json +++ b/src/awf-config-schema.json @@ -650,82 +650,9 @@ "enum": [ "gvisor", "sbx", - "firecracker", "cloud-hypervisor" ], - "description": "Runtime for the primary agent. \"gvisor\" uses runsc in Docker Compose. \"sbx\" uses a Docker sbx microVM. \"firecracker\" selects the explicit Linux/KVM Firecracker v1.16.1 workload preview. \"cloud-hypervisor\" selects the explicit Linux/KVM Cloud Hypervisor v53.0 workload preview (GitHub-hosted Ubuntu x86_64 KVM runners only). Infrastructure containers always use the default runc runtime." - } - } - }, - "firecracker": { - "type": "object", - "description": "Firecracker v1.16.1 workload preview configuration. Requires strict network isolation, a local Docker daemon, KVM, jailer, and explicitly checksummed artifacts. Selecting this runtime never falls back to Docker.", - "additionalProperties": false, - "properties": { - "previewEnabled": { - "type": "boolean", - "default": false, - "description": "Explicitly enable Firecracker preview workload execution." - }, - "firecrackerBinary": { - "type": "string", - "description": "Absolute path to the Firecracker v1.16.1 binary. Defaults to /usr/local/bin/firecracker." - }, - "jailerBinary": { - "type": "string", - "description": "Absolute path to the matching v1.16.1 jailer binary. Defaults to /usr/local/bin/jailer." - }, - "kernelPath": { - "type": "string", - "description": "Absolute path to the trusted guest Linux kernel image." - }, - "rootfsPath": { - "type": "string", - "description": "Absolute path to the trusted guest root filesystem image." - }, - "supervisorPath": { - "type": "string", - "description": "Absolute path to the built AWF Firecracker guest supervisor." - }, - "vcpuCount": { - "type": "integer", - "minimum": 1, - "default": 2, - "description": "Number of guest virtual CPUs." - }, - "memoryMib": { - "type": "integer", - "minimum": 1, - "default": 512, - "description": "Guest memory in MiB." - }, - "apiTimeoutMs": { - "type": "integer", - "minimum": 1, - "default": 5000, - "description": "Bounded timeout in milliseconds for Firecracker API socket readiness and requests." - }, - "sha256": { - "type": "object", - "description": "Pinned SHA-256 digests for trusted Firecracker artifacts. All entries are required for preview workload execution.", - "additionalProperties": false, - "properties": { - "firecracker": { - "$ref": "#/$defs/sha256Digest" - }, - "jailer": { - "$ref": "#/$defs/sha256Digest" - }, - "kernel": { - "$ref": "#/$defs/sha256Digest" - }, - "rootfs": { - "$ref": "#/$defs/sha256Digest" - }, - "supervisor": { - "$ref": "#/$defs/sha256Digest" - } - } + "description": "Runtime for the primary agent. \"gvisor\" uses runsc in Docker Compose. \"sbx\" uses a Docker sbx microVM. \"cloud-hypervisor\" selects the explicit Linux/KVM Cloud Hypervisor v53.0 workload preview (GitHub-hosted Ubuntu x86_64 KVM runners only). Infrastructure containers always use the default runc runtime." } } }, @@ -753,7 +680,7 @@ }, "supervisorPath": { "type": "string", - "description": "Absolute path to the built AWF guest supervisor (shared with Firecracker)." + "description": "Absolute path to the built AWF guest supervisor." }, "vcpuCount": { "type": "integer", diff --git a/src/cli-options.ts b/src/cli-options.ts index 732a6f7bd..03c173b46 100644 --- a/src/cli-options.ts +++ b/src/cli-options.ts @@ -11,7 +11,6 @@ const optionGroupHeaders: Record = { 'config': 'Configuration:', 'allow-domains': 'Domain Filtering:', 'build-local': 'Image Management:', - 'firecracker-preview': 'Firecracker Preview:', 'cloud-hypervisor-preview': 'Cloud Hypervisor Preview (GitHub-hosted Ubuntu x86_64 KVM only):', 'env': 'Container Configuration:', 'dns-servers': 'Network & Security:', @@ -177,36 +176,11 @@ program 'Container runtime for the agent container.\n' + ' "gvisor" — OCI runtime via Docker Compose (translates to runsc).\n' + ' "sbx" — Docker sbx microVM with hypervisor isolation.\n' + - ' "firecracker" — explicit Linux/KVM Firecracker v1.16.1 preview.\n' + ' "cloud-hypervisor" — explicit GitHub-hosted Ubuntu x86_64 KVM\n' + ' Cloud Hypervisor v53.0 preview.\n' + ' Unknown values are passed through as raw Docker runtime names.' ) - .option( - '--firecracker-preview', - 'Enable the Firecracker v1.16.1 workload-execution preview.\n' + - ' Requires Linux/KVM, local Docker, jailer, and pinned guest artifacts.', - false - ) - .option('--firecracker-binary ', 'Path to the Firecracker v1.16.1 binary.') - .option('--firecracker-jailer-binary ', 'Path to the matching Firecracker v1.16.1 jailer binary.') - .option('--firecracker-kernel ', 'Path to the guest Linux kernel image.') - .option('--firecracker-rootfs ', 'Path to the guest root filesystem image.') - .option('--firecracker-supervisor ', 'Path to the built AWF Firecracker guest supervisor.') - .option('--firecracker-vcpus ', 'Guest virtual CPU count (default: 2).') - .option('--firecracker-memory-mib ', 'Guest memory in MiB (default: 512).') - .option('--firecracker-api-timeout-ms ', 'Bounded API socket readiness timeout in milliseconds (default: 5000).') - .option('--firecracker-binary-sha256 ', 'Expected SHA-256 digest of the Firecracker binary.') - .option('--firecracker-jailer-sha256 ', 'Expected SHA-256 digest of the jailer binary.') - .option('--firecracker-kernel-sha256 ', 'Expected SHA-256 digest of the guest kernel.') - .option('--firecracker-rootfs-sha256 ', 'Expected SHA-256 digest of the guest rootfs.') - .option('--firecracker-supervisor-sha256 ', 'Expected SHA-256 digest of the AWF guest supervisor.') - // -- Cloud Hypervisor Preview -- - // - // NOTE: like Firecracker, this requires explicit --cloud-hypervisor-preview - // opt-in plus --container-runtime cloud-hypervisor and is supported only - // on GitHub-hosted Ubuntu x86_64 KVM runners (self-hosted is unsupported). .option( '--cloud-hypervisor-preview', `Enable the Cloud Hypervisor v${CLOUD_HYPERVISOR_RELEASE_VERSION} workload-execution preview.\n` + @@ -217,7 +191,7 @@ program .option('--cloud-hypervisor-binary ', `Path to the Cloud Hypervisor v${CLOUD_HYPERVISOR_RELEASE_VERSION} binary.`) .option('--cloud-hypervisor-kernel ', 'Path to the PCI-capable guest Linux kernel image.') .option('--cloud-hypervisor-rootfs ', 'Path to the guest root filesystem image.') - .option('--cloud-hypervisor-supervisor ', 'Path to the built AWF guest supervisor (shared with Firecracker).') + .option('--cloud-hypervisor-supervisor ', 'Path to the built AWF guest supervisor.') .option('--cloud-hypervisor-vcpus ', 'Guest virtual CPU count (default: 2).') .option('--cloud-hypervisor-memory-mib ', 'Guest memory in MiB (default: 512).') .option('--cloud-hypervisor-api-timeout-ms ', 'Bounded API socket readiness timeout in milliseconds (default: 5000).') diff --git a/src/cloud-hypervisor-runtime-backend.test.ts b/src/cloud-hypervisor-runtime-backend.test.ts index ce94f4107..b61cf4027 100644 --- a/src/cloud-hypervisor-runtime-backend.test.ts +++ b/src/cloud-hypervisor-runtime-backend.test.ts @@ -98,7 +98,7 @@ function harness(overrides: Partial = const manager = { paths: { runDirectory: '/tmp/awf/cloud-hypervisor-run/cloud-hypervisor/test' }, guestIp: '100.64.0.2', - networkNamespace: 'awffc-test', + networkNamespace: 'awfvm-test', start: jest.fn(async () => { order.push('vm-config'); }), startInstance: jest.fn(async () => { order.push('vm-start'); }), execute: jest.fn() @@ -607,7 +607,7 @@ describe('Cloud Hypervisor runtime backend', () => { expect(manager.stop).toHaveBeenCalledWith({ preserve: true }); expect(manager.stop).toHaveBeenCalledTimes(1); expect(deps.logger.info).toHaveBeenCalledWith( - '[cloud-hypervisor] Preserved network namespace: awffc-test', + '[cloud-hypervisor] Preserved network namespace: awfvm-test', ); }); diff --git a/src/cloud-hypervisor-runtime-backend.ts b/src/cloud-hypervisor-runtime-backend.ts index 8082522fa..607e25a6b 100644 --- a/src/cloud-hypervisor-runtime-backend.ts +++ b/src/cloud-hypervisor-runtime-backend.ts @@ -408,7 +408,7 @@ export class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBacken `[cloud-hypervisor] Preserved run directory: ${this.manager.paths.runDirectory}`, ); this.dependencies.logger.info( - `[cloud-hypervisor] Preserved images: ${this.config.workDir}/firecracker-images`, + `[cloud-hypervisor] Preserved images: ${this.config.workDir}/microvm-images`, ); if (this.manager.networkNamespace) { this.dependencies.logger.info( diff --git a/src/cloud-hypervisor/launcher.test.ts b/src/cloud-hypervisor/launcher.test.ts index a837b1fc5..18acf2c9b 100644 --- a/src/cloud-hypervisor/launcher.test.ts +++ b/src/cloud-hypervisor/launcher.test.ts @@ -68,7 +68,7 @@ describe('computeCloudHypervisorLandlockRules', () => { runDirectory: '/run/awf/run', apiSocketPath: '/run/awf/run/api.socket', vsockSocketPath: '/run/awf/run/vsock.socket', - tapName: 'fctabc123', + tapName: 'vmtabc123', }); expect(rules).toEqual([ @@ -77,7 +77,7 @@ describe('computeCloudHypervisorLandlockRules', () => { { path: '/run/awf/run', access: 'rw' }, { path: '/dev/kvm', access: 'rw' }, { path: '/dev/net/tun', access: 'rw' }, - { path: '/sys/class/net/fctabc123', access: 'r' }, + { path: '/sys/class/net/vmtabc123', access: 'r' }, ]); expect(rules).not.toEqual(expect.arrayContaining([ expect.objectContaining({ path: '/host/workspace' }), @@ -91,7 +91,7 @@ describe('computeCloudHypervisorLandlockRules', () => { runDirectory: '/run/awf/run', apiSocketPath: '/run/awf/run/api.socket', vsockSocketPath: '/run/awf/run/vsock.socket', - tapName: 'fctabc123', + tapName: 'vmtabc123', }); expect(rules.some((rule) => rule.path.includes('workspace'))).toBe(false); @@ -109,10 +109,10 @@ describe('computeCloudHypervisorLandlockRules', () => { runDirectory: '/run/awf/run', apiSocketPath: '/run/awf/run/api.socket', vsockSocketPath: '/run/awf/run/vsock.socket', - tapName: 'fctabc123', + tapName: 'vmtabc123', }); - expect(rules).toContainEqual({ path: '/sys/class/net/fctabc123', access: 'r' }); + expect(rules).toContainEqual({ path: '/sys/class/net/vmtabc123', access: 'r' }); }); }); diff --git a/src/cloud-hypervisor/launcher.ts b/src/cloud-hypervisor/launcher.ts index 7fcf52505..cf9588475 100644 --- a/src/cloud-hypervisor/launcher.ts +++ b/src/cloud-hypervisor/launcher.ts @@ -6,8 +6,7 @@ import type { CloudHypervisorLandlockRule } from './api-client'; * Secure host launch-argv construction and resource-confinement helpers for * Cloud Hypervisor. * - * Cloud Hypervisor has no jailer-equivalent process (unlike Firecracker), so - * there is nothing that natively joins a prepared network namespace, + * Cloud Hypervisor has no native process that joins a prepared network namespace, * chroots, drops capabilities, and execs the VMM as one atomic operation. * This module documents and implements the exact replacement boundary AWF * uses instead: @@ -21,9 +20,7 @@ import type { CloudHypervisorLandlockRule } from './api-client'; * --no-new-privs --inh-caps=-all --bounding-set=-all` execs the Cloud * Hypervisor binary as the non-root operator uid/gid with an empty * capability bounding set and `no_new_privs` set, before any guest code - * runs. This is the same non-root identity Firecracker's jailer targets - * (see `resolveJailerIdentity` in `src/firecracker/manager.ts`) and - * requires the same operator preconditions (kvm-group membership, + * runs. This requires operator preconditions including kvm-group membership, * `/dev/kvm` access). * 3. **Filesystem confinement** — Cloud Hypervisor has no chroot of its * own, and jailer's userspace chroot+pivot_root cannot be replicated @@ -59,7 +56,7 @@ export interface CloudHypervisorLaunchPaths { readonly runDirectory: string; readonly apiSocketPath: string; readonly vsockSocketPath: string; - /** Host TAP interface name (e.g. `fct`), for the + /** Host TAP interface name (e.g. `vmt`), for the * `/sys/class/net/` Landlock rule — see * {@link computeCloudHypervisorLandlockRules}. */ readonly tapName: string; @@ -85,8 +82,7 @@ export interface CloudHypervisorLaunchToolPaths { * network namespace, drop to the non-root operator identity retaining * exactly two things it needs to configure its own virtio-net TAP device, * then exec the pinned Cloud Hypervisor binary with only its API socket - * configured (the VM itself is created and booted afterwards over that - * socket, mirroring Firecracker's `--api-sock`-only jailer invocation). + * configured; the VM itself is created and booted afterwards over that socket. * * The launched process retains exactly one supplementary group: the group * that owns `/dev/kvm` (resolved by preflight). A blanket `--clear-groups` diff --git a/src/cloud-hypervisor/manager.test.ts b/src/cloud-hypervisor/manager.test.ts index d2b9910e3..e0b3ee396 100644 --- a/src/cloud-hypervisor/manager.test.ts +++ b/src/cloud-hypervisor/manager.test.ts @@ -248,7 +248,7 @@ describe('CloudHypervisorManager', () => { expect(deps.launch).toHaveBeenCalledWith( '/usr/bin/ip', expect.arrayContaining([ - 'netns', 'exec', expect.stringMatching(/^awffc-/), + 'netns', 'exec', expect.stringMatching(/^awfvm-/), '/usr/bin/setpriv', '--reuid=1000', '--regid=1000', diff --git a/src/cloud-hypervisor/manager.ts b/src/cloud-hypervisor/manager.ts index c70571293..1b78a88cf 100644 --- a/src/cloud-hypervisor/manager.ts +++ b/src/cloud-hypervisor/manager.ts @@ -156,8 +156,8 @@ function parsePositiveIdentity(value: string | undefined): number | undefined { /** * Owns one Cloud Hypervisor process launched via the secure host launcher in - * `./launcher.ts` (network-namespace join + privilege drop + Landlock, in - * place of Firecracker's jailer) and its partial-start cleanup. + * `./launcher.ts` (network-namespace join + privilege drop + Landlock) and + * its partial-start cleanup. * * This class is an orchestration facade: VM boot configuration lives in * `./vm-config-builder.ts`, run-directory staging plus failure diagnostics in @@ -229,9 +229,7 @@ export class CloudHypervisorManager { // tap RX=10 packets (guest-to-host, unaffected) vs. TX=1 packet // (host-to-guest, effectively stalled) despite response // packets already having arrived on the host-side veth. - // Firecracker's own tap handling does not request - // IFF_VNET_HDR, so this is opted in here only, not changed for - // the shared default. + // This backend requires IFF_VNET_HDR, so it opts in explicitly. tapVnetHdr: true, }); this.networkPlan = networkPlan; diff --git a/src/cloud-hypervisor/preflight.ts b/src/cloud-hypervisor/preflight.ts index 5a8ffcb4a..2612a6bca 100644 --- a/src/cloud-hypervisor/preflight.ts +++ b/src/cloud-hypervisor/preflight.ts @@ -9,9 +9,8 @@ import { /** * Fail-closed host and artifact validation for the Cloud Hypervisor v53.0 - * runtime. This module intentionally mirrors - * `src/firecracker/preflight.ts`'s trust-check patterns (absolute paths, - * root/operator-owned non-writable regular files, trusted ancestor + * runtime. Trust checks cover absolute paths, root/operator-owned + * non-writable regular files, trusted ancestor * directories, digest pinning, PATH-resolved but ownership-verified host * tools) so both VMM backends share the same fail-closed posture. * @@ -328,9 +327,8 @@ async function assertDigest( /** * Fail-closed host and artifact validation for Cloud Hypervisor v53.0. * - * This performs the same categories of checks as - * `runFirecrackerPreflight` — Linux/KVM host requirements, trusted - * artifact ownership/permissions, pinned version, and pinned digests — + * This checks Linux/KVM host requirements, trusted artifact + * ownership/permissions, pinned version, and pinned digests, * adapted for Cloud Hypervisor's single-binary VMM (no jailer). */ export async function runCloudHypervisorPreflight( diff --git a/src/cloud-hypervisor/runtime-validation.ts b/src/cloud-hypervisor/runtime-validation.ts index 233d6953b..2de5463d4 100644 --- a/src/cloud-hypervisor/runtime-validation.ts +++ b/src/cloud-hypervisor/runtime-validation.ts @@ -4,14 +4,13 @@ import { assertGithubHostedRunnerEligibility } from './host-eligibility'; /** * Explicit, fail-closed compatibility guards for the Cloud Hypervisor - * v53.0 microVM runtime. This module mirrors - * `src/firecracker/runtime-validation.ts` closely — same security-mode - * and Docker-host requirements — with two Cloud + * v53.0 microVM runtime. This module enforces security-mode + * and Docker-host requirements, with two Cloud * Hypervisor-specific additions: no jailer digest is required (Cloud * Hypervisor has no jailer-equivalent process) and host eligibility is * additionally restricted to GitHub-hosted Ubuntu x86_64 KVM runners via - * {@link assertGithubHostedRunnerEligibility} (self-hosted runners are not - * supported, unlike Firecracker's preview). + * {@link assertGithubHostedRunnerEligibility}; self-hosted runners are not + * supported. */ export function assertCloudHypervisorSelection(config: WrapperConfig): void { diff --git a/src/cloud-hypervisor/vm-config-builder.ts b/src/cloud-hypervisor/vm-config-builder.ts index fca119fda..72cc2f1f5 100644 --- a/src/cloud-hypervisor/vm-config-builder.ts +++ b/src/cloud-hypervisor/vm-config-builder.ts @@ -122,8 +122,7 @@ export function buildSupervisorBootArgs( 'rootfstype=ext4', 'rootflags=data=ordered', 'rw', - // Cloud Hypervisor requires PCI (no `pci=off` MMIO-only mode like - // Firecracker); pin legacy `ethN` interface naming so the guest's + // Cloud Hypervisor requires PCI; pin legacy `ethN` interface naming so the guest's // single virtio-pci NIC has a deterministic name across boots. 'net.ifnames=0', 'biosdevname=0', diff --git a/src/commands/build-config.ts b/src/commands/build-config.ts index e4d1046fe..42232bab0 100644 --- a/src/commands/build-config.ts +++ b/src/commands/build-config.ts @@ -4,11 +4,6 @@ import { resolveApiCredentials } from './resolve-credentials'; import { normalizeEnclavesConfig } from '../parsers/enclave-parser'; import { logger } from '../logger'; import { - FIRECRACKER_DEFAULT_API_TIMEOUT_MS, - FIRECRACKER_DEFAULT_BINARY, - FIRECRACKER_DEFAULT_JAILER_BINARY, - FIRECRACKER_DEFAULT_MEMORY_MIB, - FIRECRACKER_DEFAULT_VCPU_COUNT, CLOUD_HYPERVISOR_DEFAULT_API_TIMEOUT_MS, CLOUD_HYPERVISOR_DEFAULT_BINARY, CLOUD_HYPERVISOR_DEFAULT_MEMORY_MIB, @@ -128,7 +123,6 @@ export function buildConfig(inputs: BuildConfigInputs): WrapperConfig { const chrootIdentity = buildChrootIdentity(options); const dind = buildDindConfig(options); - const firecracker = buildFirecrackerConfig(options); const cloudHypervisor = buildCloudHypervisorConfig(options); const apiCredentials = resolveApiCredentials(options, { resolvedCopilotApiTarget, @@ -231,7 +225,6 @@ export function buildConfig(inputs: BuildConfigInputs): WrapperConfig { chrootBinariesSourcePath: options.chrootBinariesSourcePath as string | undefined, chrootIdentity, dind, - firecracker, cloudHypervisor, enclaves: normalizeEnclavesConfig( options.enclaves as AwfFileConfig['enclaves'] | undefined, @@ -239,67 +232,6 @@ export function buildConfig(inputs: BuildConfigInputs): WrapperConfig { }; } -function buildFirecrackerConfig( - options: Record, -): WrapperConfig['firecracker'] { - const selected = options.containerRuntime === 'firecracker'; - const configured = options.firecrackerPreview === true - || [ - 'firecrackerBinary', - 'firecrackerJailerBinary', - 'firecrackerKernel', - 'firecrackerRootfs', - 'firecrackerSupervisor', - 'firecrackerVcpus', - 'firecrackerMemoryMib', - 'firecrackerApiTimeoutMs', - 'firecrackerBinarySha256', - 'firecrackerJailerSha256', - 'firecrackerKernelSha256', - 'firecrackerRootfsSha256', - 'firecrackerSupervisorSha256', - ].some((key) => options[key] !== undefined); - if (!selected && !configured) return undefined; - - const sha256 = { - firecracker: options.firecrackerBinarySha256 as string | undefined, - jailer: options.firecrackerJailerSha256 as string | undefined, - kernel: options.firecrackerKernelSha256 as string | undefined, - rootfs: options.firecrackerRootfsSha256 as string | undefined, - supervisor: options.firecrackerSupervisorSha256 as string | undefined, - }; - - return { - previewEnabled: options.firecrackerPreview === true, - firecrackerBinary: - (options.firecrackerBinary as string | undefined) ?? FIRECRACKER_DEFAULT_BINARY, - jailerBinary: - (options.firecrackerJailerBinary as string | undefined) ?? - FIRECRACKER_DEFAULT_JAILER_BINARY, - kernelPath: options.firecrackerKernel as string | undefined, - rootfsPath: options.firecrackerRootfs as string | undefined, - supervisorPath: options.firecrackerSupervisor as string | undefined, - vcpuCount: parsePositiveIntegerOption( - options.firecrackerVcpus, - '--firecracker-vcpus', - FIRECRACKER_DEFAULT_VCPU_COUNT, - ), - memoryMib: parsePositiveIntegerOption( - options.firecrackerMemoryMib, - '--firecracker-memory-mib', - FIRECRACKER_DEFAULT_MEMORY_MIB, - ), - apiTimeoutMs: parsePositiveIntegerOption( - options.firecrackerApiTimeoutMs, - '--firecracker-api-timeout-ms', - FIRECRACKER_DEFAULT_API_TIMEOUT_MS, - ), - sha256: Object.values(sha256).some((value) => value !== undefined) - ? sha256 - : undefined, - }; -} - function parsePositiveIntegerOption( value: unknown, optionName: string, @@ -315,8 +247,8 @@ function parsePositiveIntegerOption( /** * Builds the Cloud Hypervisor microVM runtime config (artifacts/digests - * plus vcpu/memory/timeout settings). `selected` mirrors the Firecracker - * pattern: `--container-runtime cloud-hypervisor` requires explicit + * plus vcpu/memory/timeout settings). `--container-runtime cloud-hypervisor` + * requires explicit * `--cloud-hypervisor-preview` opt-in and full artifact/digest * configuration, enforced by * `assertCloudHypervisorRuntimeCompatibility` in diff --git a/src/commands/main-action.test.ts b/src/commands/main-action.test.ts index a4ee228d3..690de16e7 100644 --- a/src/commands/main-action.test.ts +++ b/src/commands/main-action.test.ts @@ -461,7 +461,7 @@ describe('createMainAction', () => { it('quiesces an external runtime through its preserve hook', async () => { const preserve = jest.fn().mockResolvedValue(undefined); const backend = { - runtime: 'firecracker', + runtime: 'cloud-hypervisor', preflight: jest.fn(), start: jest.fn(), exec: jest.fn(), @@ -490,9 +490,9 @@ describe('createMainAction', () => { describe('external runtime cleanup failures', () => { it('continues generic cleanup and then rethrows the runtime failure', async () => { - const runtimeError = new Error('Firecracker teardown failed'); + const runtimeError = new Error('External runtime teardown failed'); const backend = { - runtime: 'firecracker', + runtime: 'cloud-hypervisor', preflight: jest.fn(), start: jest.fn(), exec: jest.fn(), @@ -523,7 +523,7 @@ describe('createMainAction', () => { describe('external runtime diagnostics', () => { it('aggregates backend and Docker diagnostics with explicit failures', async () => { const backend = { - runtime: 'firecracker', + runtime: 'cloud-hypervisor', preflight: jest.fn().mockResolvedValue(undefined), start: jest.fn(), exec: jest.fn(), diff --git a/src/commands/validate-options.test.ts b/src/commands/validate-options.test.ts index 7940c3473..d3c642b83 100644 --- a/src/commands/validate-options.test.ts +++ b/src/commands/validate-options.test.ts @@ -472,75 +472,6 @@ describe('validateOptions', () => { // Post-config validations (docker host, rate limits, feature flags, ports) // --------------------------------------------------------------------------- - describe('Firecracker runtime validation', () => { - const digest = 'a'.repeat(64); - const firecracker = { - previewEnabled: true, - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/kernel', - rootfsPath: '/opt/rootfs', - supervisorPath: '/opt/supervisor', - vcpuCount: 2, - memoryMib: 512, - apiTimeoutMs: 5000, - sha256: { - firecracker: digest, - jailer: digest, - kernel: digest, - rootfs: digest, - supervisor: digest, - }, - }; - - function firecrackerConfig(overrides: Record = {}) { - return { - ...STUB_CONFIG, - containerRuntime: 'firecracker', - legacySecurity: false, - networkIsolation: undefined, - enableApiProxy: undefined, - firecracker, - ...overrides, - }; - } - - it('accepts a complete strict preview configuration', () => { - mockedBuildConfig.buildConfig.mockReturnValue(firecrackerConfig()); - expect(() => validateOptions(validOptions(), 'echo hi')).not.toThrow(); - }); - - it('rejects Firecracker options for another runtime', () => { - mockedBuildConfig.buildConfig.mockReturnValue(firecrackerConfig({ - containerRuntime: 'gvisor', - })); - expect(() => validateOptions(validOptions(), 'echo hi')).toThrow('process.exit called'); - expect(mockedLogger.error).toHaveBeenCalledWith( - expect.stringContaining('Firecracker options require'), - ); - }); - - it('rejects unsupported Firecracker policy before strict-mode coercion', () => { - mockedBuildConfig.buildConfig.mockReturnValue(firecrackerConfig({ - enableDind: true, - })); - expect(() => validateOptions(validOptions(), 'echo hi')).toThrow('process.exit called'); - expect(mockedLogger.error).toHaveBeenCalledWith( - expect.stringContaining('does not support Docker-in-Docker'), - ); - }); - - it('rejects an incomplete Firecracker runtime after security defaults', () => { - mockedBuildConfig.buildConfig.mockReturnValue(firecrackerConfig({ - firecracker: { ...firecracker, previewEnabled: false }, - })); - expect(() => validateOptions(validOptions(), 'echo hi')).toThrow('process.exit called'); - expect(mockedLogger.error).toHaveBeenCalledWith( - expect.stringContaining('requires explicit --firecracker-preview'), - ); - }); - }); - describe('--docker-host validation', () => { it('exits when --docker-host is not a unix:// URI', () => { mockedBuildConfig.buildConfig.mockReturnValue({ diff --git a/src/commands/validators/config-assembly.ts b/src/commands/validators/config-assembly.ts index 24a8962fe..0134062df 100644 --- a/src/commands/validators/config-assembly.ts +++ b/src/commands/validators/config-assembly.ts @@ -9,11 +9,6 @@ import { validateInfrastructureOptions, applyRateLimitConfig, validateFeatureFla import { applySecurityMode } from './security-mode'; import { validateHostAccessConfig } from './network-access-validator'; import { validateApiProxyOptions, validateCopilotModelOption } from './api-proxy-validator'; -import { - assertFirecrackerPreSecurityCompatibility, - assertFirecrackerRuntimeCompatibility, - assertFirecrackerSelection, -} from '../../firecracker/runtime-validation'; import { assertCloudHypervisorPreSecurityCompatibility, assertCloudHypervisorRuntimeCompatibility, @@ -81,20 +76,11 @@ export function assembleAndValidateConfig( validateInfrastructureOptions(config); try { - assertFirecrackerSelection(config); assertCloudHypervisorSelection(config); } catch (error) { logger.error(`❌ ${error instanceof Error ? error.message : String(error)}`); process.exit(1); } - if (config.containerRuntime === 'firecracker') { - try { - assertFirecrackerPreSecurityCompatibility(config); - } catch (error) { - logger.error(`❌ ${error instanceof Error ? error.message : String(error)}`); - process.exit(1); - } - } if (config.containerRuntime === 'cloud-hypervisor') { try { assertCloudHypervisorPreSecurityCompatibility(config); @@ -104,14 +90,6 @@ export function assembleAndValidateConfig( } } applySecurityMode(config); - if (config.containerRuntime === 'firecracker') { - try { - assertFirecrackerRuntimeCompatibility(config); - } catch (error) { - logger.error(`❌ ${error instanceof Error ? error.message : String(error)}`); - process.exit(1); - } - } if (config.containerRuntime === 'cloud-hypervisor') { try { assertCloudHypervisorRuntimeCompatibility(config); diff --git a/src/commands/validators/security-mode.ts b/src/commands/validators/security-mode.ts index 88d2e5fd6..fa02849ed 100644 --- a/src/commands/validators/security-mode.ts +++ b/src/commands/validators/security-mode.ts @@ -30,9 +30,7 @@ export function applySecurityMode(config: WrapperConfig): void { // --- strict security (default) --- - // Docker sbx enforces isolation through its hypervisor proxy and does not use - // Docker topology. Firecracker is also a microVM, but explicitly attaches its - // host-side veth to AWF's proven internal bridge, so topology remains required. + // MicroVM runtimes enforce isolation outside Docker topology. const isMicroVmRuntime = !runtimeUsesComposeAgent(config.containerRuntime); if (isMicroVmRuntime && config.pidsLimit !== undefined) { @@ -42,7 +40,7 @@ export function applySecurityMode(config: WrapperConfig): void { ); } - if (!isMicroVmRuntime || config.containerRuntime === 'firecracker') { + if (!isMicroVmRuntime) { // Force network-isolation on. // Only warn when explicitly disabled (=== false); undefined means "not set by user". if (!config.networkIsolation) { diff --git a/src/compose-generator.ts b/src/compose-generator.ts index 6a479c833..22e21165f 100644 --- a/src/compose-generator.ts +++ b/src/compose-generator.ts @@ -156,7 +156,7 @@ export function generateDockerCompose( // In network-isolation mode the internal network blocks host→container traffic, // so we also attach api-proxy to the external bridge (`awf-ext`) — same as // Squid — so published ports are reachable from outside Docker. - if (!includeAgent && config.containerRuntime !== 'firecracker' && services['api-proxy']) { + if (!includeAgent && services['api-proxy']) { const proxyService = services['api-proxy']; if (!proxyService.ports) { proxyService.ports = []; diff --git a/src/config-file.ts b/src/config-file.ts index aae61eeb8..484842a10 100644 --- a/src/config-file.ts +++ b/src/config-file.ts @@ -3,7 +3,7 @@ import * as path from 'path'; import * as yaml from 'js-yaml'; import { validateWithSchema } from './schema-validator'; import type { RawEnclavesConfig } from './types/enclave-options'; -import type { FirecrackerArtifactDigests, CloudHypervisorArtifactDigests } from './types/runtime-options'; +import type { CloudHypervisorArtifactDigests } from './types/runtime-options'; /** @internal Used only by config-file helpers — not part of public API */ // ts-prune-ignore-next @@ -123,18 +123,6 @@ export interface AwfFileConfig { runnerToolCachePath?: string; mounts?: string[]; }; - firecracker?: { - previewEnabled?: boolean; - firecrackerBinary?: string; - jailerBinary?: string; - kernelPath?: string; - rootfsPath?: string; - supervisorPath?: string; - vcpuCount?: number; - memoryMib?: number; - apiTimeoutMs?: number; - sha256?: FirecrackerArtifactDigests; - }; /** * Cloud Hypervisor v53.0 preview microVM runtime. * Selectable via `container.containerRuntime: "cloud-hypervisor"`, gated diff --git a/src/config-mapper.ts b/src/config-mapper.ts index 97378b498..c5f1a8258 100644 --- a/src/config-mapper.ts +++ b/src/config-mapper.ts @@ -114,20 +114,6 @@ export function mapAwfFileConfigToCliOptions(config: AwfFileConfig): Record { expect(resolveDockerRuntime('sbx')).toBeUndefined(); }); - it('returns undefined for Firecracker (no OCI runtime)', () => { - expect(resolveDockerRuntime('firecracker')).toBeUndefined(); - }); - it('returns undefined for Cloud Hypervisor (no OCI runtime)', () => { expect(resolveDockerRuntime('cloud-hypervisor')).toBeUndefined(); }); @@ -40,10 +36,6 @@ describe('container-runtime', () => { expect(runtimeNeedsStaticDns('sbx')).toBe(false); }); - it('returns false for Firecracker', () => { - expect(runtimeNeedsStaticDns('firecracker')).toBe(false); - }); - it('returns false for Cloud Hypervisor', () => { expect(runtimeNeedsStaticDns('cloud-hypervisor')).toBe(false); }); @@ -71,10 +63,6 @@ describe('container-runtime', () => { expect(runtimeUsesIptables('sbx')).toBe(false); }); - it('returns false for Firecracker (no host-agent iptables)', () => { - expect(runtimeUsesIptables('firecracker')).toBe(false); - }); - it('returns false for Cloud Hypervisor (no host-agent iptables)', () => { expect(runtimeUsesIptables('cloud-hypervisor')).toBe(false); }); @@ -105,10 +93,6 @@ describe('container-runtime', () => { expect(runtimeUsesComposeAgent('sbx')).toBe(false); }); - it('returns false for the Firecracker microVM model', () => { - expect(runtimeUsesComposeAgent('firecracker')).toBe(false); - }); - it('returns false for the Cloud Hypervisor microVM model', () => { expect(runtimeUsesComposeAgent('cloud-hypervisor')).toBe(false); }); diff --git a/src/container-runtime.ts b/src/container-runtime.ts index 3c19ca619..7352ff92a 100644 --- a/src/container-runtime.ts +++ b/src/container-runtime.ts @@ -110,12 +110,6 @@ const RUNTIME_REGISTRY: Readonly> = { needsStaticDns: false, // sbx manages its own DNS usesIptables: false, // microVM manages its own network egress }, - firecracker: { - executionModel: 'microvm', - dockerRuntime: undefined, - needsStaticDns: false, - usesIptables: false, - }, 'cloud-hypervisor': { executionModel: 'microvm', dockerRuntime: undefined, diff --git a/src/enclave/agent-runner-spec.test.ts b/src/enclave/agent-runner-spec.test.ts index b205ed34e..459aa45ed 100644 --- a/src/enclave/agent-runner-spec.test.ts +++ b/src/enclave/agent-runner-spec.test.ts @@ -133,7 +133,7 @@ describe('unified enclave agent runner specification', () => { }); it('fails closed for an unimplemented enclave backend', () => { - expect(() => createEnclaveRunner({ ...trustedConfig, backend: 'firecracker' })) + expect(() => createEnclaveRunner({ ...trustedConfig, backend: 'unsupported' })) .toThrow(/Unsupported enclave-agent backend/); }); diff --git a/src/enclave/runtime-preflight.test.ts b/src/enclave/runtime-preflight.test.ts index 06d56420b..824cbac5e 100644 --- a/src/enclave/runtime-preflight.test.ts +++ b/src/enclave/runtime-preflight.test.ts @@ -26,7 +26,6 @@ describe('enclave runtime preflight', () => { ['gvisor', 'gvisor'], ['runsc', 'gvisor'], ['sbx', 'sbx'], - ['firecracker', 'firecracker'], ] as const)('normalizes primary runtime %s to %s', (runtime, expected) => { expect(resolvePrimaryRuntimeBackend(runtime)).toBe(expected); }); @@ -52,18 +51,6 @@ describe('enclave runtime preflight', () => { )).rejects.toThrow(/sbx.*unavailable.*never fall back/); }); - it('recognizes Firecracker but rejects enclave integration without probing fallbacks', async () => { - await expect(assertPrimaryRuntimeAvailable( - 'firecracker', - runtimeAvailable, - dockerAvailable, - sbxAvailable, - )).rejects.toThrow(/control-plane preview.*not implemented.*never fall back/); - expect(runtimeAvailable).not.toHaveBeenCalled(); - expect(dockerAvailable).not.toHaveBeenCalled(); - expect(sbxAvailable).not.toHaveBeenCalled(); - }); - it('fails closed when Docker is unavailable for either executor', async () => { dockerAvailable.mockResolvedValue(false); await expect(assertScriptRuntimeAvailable( diff --git a/src/enclave/runtime-preflight.ts b/src/enclave/runtime-preflight.ts index 76d5f7700..e722cd472 100644 --- a/src/enclave/runtime-preflight.ts +++ b/src/enclave/runtime-preflight.ts @@ -14,14 +14,13 @@ import { const RUNSC_RUNTIME = 'runsc'; -export type PrimaryRuntimeBackend = 'docker' | 'gvisor' | 'sbx' | 'firecracker'; +export type PrimaryRuntimeBackend = 'docker' | 'gvisor' | 'sbx'; export function resolvePrimaryRuntimeBackend( containerRuntime: string | undefined, ): PrimaryRuntimeBackend { if (containerRuntime === 'gvisor' || containerRuntime === RUNSC_RUNTIME) return 'gvisor'; if (containerRuntime === 'sbx') return 'sbx'; - if (containerRuntime === 'firecracker') return 'firecracker'; return 'docker'; } @@ -31,12 +30,6 @@ export async function assertPrimaryRuntimeAvailable( queryDockerAvailable: DockerAvailabilityQuery = defaultDockerAvailabilityQuery, querySbxAvailable: SbxAvailabilityQuery = defaultSbxAvailabilityQuery, ): Promise { - if (containerRuntime === 'firecracker') { - throw new Error( - 'Primary-agent runtime "firecracker" is a control-plane preview; ' + - 'enclave integration is not implemented and enclaves never fall back', - ); - } if (containerRuntime === 'sbx') { if (!(await querySbxAvailable())) { throw new Error('Primary-agent runtime "sbx" is unavailable; enclaves never fall back'); diff --git a/src/external-runtime-backend-resolver.ts b/src/external-runtime-backend-resolver.ts index 96b9b6f79..d0a1ff075 100644 --- a/src/external-runtime-backend-resolver.ts +++ b/src/external-runtime-backend-resolver.ts @@ -2,7 +2,6 @@ import type { WorkflowDependencies } from './cli-workflow'; import { runtimeUsesComposeAgent } from './container-runtime'; import type { ExternalAgentRuntimeBackend } from './external-runtime-backend'; import { createSbxRuntimeBackend } from './sbx-runtime-backend'; -import { createFirecrackerRuntimeBackend } from './firecracker-runtime-backend'; import { createCloudHypervisorRuntimeBackend } from './cloud-hypervisor-runtime-backend'; import type { WrapperConfig } from './types'; @@ -22,8 +21,6 @@ type ExternalRuntimeBackendRegistry = Readonly< const EXTERNAL_RUNTIME_BACKENDS: ExternalRuntimeBackendRegistry = { sbx: ({ config, startInfrastructure }) => createSbxRuntimeBackend(config, startInfrastructure), - firecracker: ({ config, startInfrastructure }) => - createFirecrackerRuntimeBackend(config, startInfrastructure), 'cloud-hypervisor': ({ config, startInfrastructure }) => createCloudHypervisorRuntimeBackend(config, startInfrastructure), }; @@ -44,11 +41,6 @@ export function resolveExternalRuntimeBackend( } const runtime = config.containerRuntime; - if (runtime === 'firecracker' && !config.firecracker?.previewEnabled) { - throw new Error( - 'Firecracker workload execution requires explicit --firecracker-preview opt-in', - ); - } if (runtime === 'cloud-hypervisor' && !config.cloudHypervisor?.previewEnabled) { throw new Error( 'Cloud Hypervisor workload execution requires explicit --cloud-hypervisor-preview opt-in', diff --git a/src/external-runtime-backend.test.ts b/src/external-runtime-backend.test.ts index 750a44e69..ac6e5be3e 100644 --- a/src/external-runtime-backend.test.ts +++ b/src/external-runtime-backend.test.ts @@ -59,25 +59,6 @@ describe('external runtime backend', () => { )).toThrow('No external agent runtime backend is registered for "sbx"'); }); - it('requires explicit Firecracker preview opt-in during resolution', () => { - const config = { - containerRuntime: 'firecracker', - firecracker: { previewEnabled: false }, - } as WrapperConfig; - expect(() => resolveExternalRuntimeBackend(config, startInfrastructure)) - .toThrow(/explicit --firecracker-preview/); - expect(startInfrastructure).not.toHaveBeenCalled(); - }); - - it('uses the registered Firecracker factory after preview opt-in', () => { - const backend = resolveExternalRuntimeBackend({ - containerRuntime: 'firecracker', - firecracker: { previewEnabled: true }, - } as WrapperConfig, startInfrastructure); - - expect(backend?.runtime).toBe('firecracker'); - }); - it('requires explicit Cloud Hypervisor preview opt-in during resolution', () => { const config = { containerRuntime: 'cloud-hypervisor', diff --git a/src/firecracker-runtime-backend.test.ts b/src/firecracker-runtime-backend.test.ts deleted file mode 100644 index 8ba48a8a7..000000000 --- a/src/firecracker-runtime-backend.test.ts +++ /dev/null @@ -1,440 +0,0 @@ -import { PassThrough } from 'stream'; -import type { WrapperConfig } from './types'; -import { - FirecrackerRuntimeBackend, - assertFirecrackerPreSecurityCompatibility, - buildFirecrackerGuestEnvironment, - createFirecrackerRuntimeBackend, - firecrackerRuntimeTestHelpers, - type FirecrackerRuntimeBackendDependencies, -} from './firecracker-runtime-backend'; -import { assertFirecrackerSelection } from './firecracker/runtime-validation'; -import type { MicrovmInfrastructureSnapshot } from './microvm/infrastructure'; - -const digest = 'a'.repeat(64); - -function config(overrides: Partial = {}): WrapperConfig { - return { - containerRuntime: 'firecracker', - firecracker: { - previewEnabled: true, - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/kernel', - rootfsPath: '/opt/rootfs', - supervisorPath: '/opt/supervisor', - vcpuCount: 2, - memoryMib: 512, - apiTimeoutMs: 5000, - sha256: { - firecracker: digest, - jailer: digest, - kernel: digest, - rootfs: digest, - supervisor: digest, - }, - }, - agentCommand: 'printf hello', - allowedDomains: ['github.com'], - workDir: '/tmp/awf', - keepContainers: false, - networkIsolation: true, - legacySecurity: false, - enableApiProxy: true, - enableDind: false, - enableHostAccess: false, - tty: false, - logLevel: 'info', - buildLocal: false, - skipPull: true, - imageRegistry: 'registry', - imageTag: 'tag', - envAll: false, - sslBump: false, - enableDlp: false, - ...overrides, - } as WrapperConfig; -} - -function infrastructure(): MicrovmInfrastructureSnapshot { - return { - networkId: 'a'.repeat(64), - bridgeName: 'br-aaaaaaaaaaaa', - subnet: '172.30.0.0/24', - gateway: '172.30.0.1', - squidIp: '172.30.0.10', - apiProxyIp: '172.30.0.30', - topologyPeerIps: {}, - revalidate: jest.fn().mockResolvedValue(undefined), - }; -} - -const preflightResult = { - version: '1.16.1', - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/kernel', - rootfsPath: '/opt/rootfs', - supervisorPath: '/opt/supervisor', - cgroupVersion: 2 as const, - tools: { - ip: '/usr/bin/ip', - nft: '/usr/sbin/nft', - sysctl: '/usr/sbin/sysctl', - mke2fs: '/usr/sbin/mke2fs', - debugfs: '/usr/sbin/debugfs', - e2fsck: '/usr/sbin/e2fsck', - rsync: '/usr/bin/rsync', - }, -}; - -function harness(overrides: Partial = {}) { - const order: string[] = []; - const stdin = new PassThrough(); - const manager = { - paths: { jailRoot: '/tmp/awf/jail' }, - guestIp: '100.64.0.2', - networkNamespace: 'awffc-test', - start: jest.fn(async () => { order.push('vm-config'); }), - startInstance: jest.fn(async () => { order.push('vm-start'); }), - execute: jest.fn() - .mockImplementationOnce(async () => { - order.push('probe'); - return { requestId: 'probe', exitCode: 0, signal: null, timedOut: false }; - }) - .mockImplementationOnce(async () => ({ - requestId: 'agent', - exitCode: 23, - signal: null, - timedOut: false, - })), - cancel: jest.fn().mockResolvedValue(undefined), - writeStdin: jest.fn().mockResolvedValue(undefined), - endStdin: jest.fn().mockResolvedValue(undefined), - collectDiagnostics: jest.fn().mockResolvedValue(undefined), - stop: jest.fn(async () => { order.push('vm-stop'); }), - }; - const infra = infrastructure(); - (infra.revalidate as jest.Mock).mockImplementation(async () => { - order.push('revalidate'); - }); - const deps: FirecrackerRuntimeBackendDependencies = { - startInfrastructure: jest.fn(async () => { order.push('compose'); }), - preflight: jest.fn(async () => { order.push('preflight'); return preflightResult; }), - resolveInfrastructure: jest.fn(async () => infra), - createManager: jest.fn(() => manager), - workspacePath: () => '/workspace-host', - homePath: () => '/home/runner', - identity: () => ({ uid: 1000, gid: 1000 }), - stdin, - stdout: new PassThrough(), - stderr: new PassThrough(), - logger: { - debug: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - }, - ...overrides, - }; - return { order, manager, infra, deps, stdin }; -} - -describe('Firecracker runtime backend', () => { - it('constructs default backend dependencies and manager policy', () => { - const startInfrastructure = jest.fn(); - const defaults = firecrackerRuntimeTestHelpers.defaultDependencies(startInfrastructure); - const previousWorkspace = process.env.GITHUB_WORKSPACE; - process.env.GITHUB_WORKSPACE = '/github/workspace'; - try { - expect(defaults.workspacePath()).toBe('/github/workspace'); - delete process.env.GITHUB_WORKSPACE; - expect(defaults.workspacePath()).toBe(process.cwd()); - expect(defaults.homePath()).toBeTruthy(); - expect(defaults.identity()).toEqual({ - uid: expect.any(Number), - gid: expect.any(Number), - }); - expect(defaults.createManager( - config().firecracker!, - '/tmp/awf', - infrastructure(), - '/workspace', - '/home/runner', - { uid: 1000, gid: 1000 }, - )).toBeDefined(); - expect(createFirecrackerRuntimeBackend(config(), startInfrastructure)) - .toBeInstanceOf(FirecrackerRuntimeBackend); - } finally { - if (previousWorkspace === undefined) delete process.env.GITHUB_WORKSPACE; - else process.env.GITHUB_WORKSPACE = previousWorkspace; - } - }); - - it('starts infrastructure, revalidates it, boots and probes before execution', async () => { - const { order, manager, deps, stdin } = harness(); - const backend = new FirecrackerRuntimeBackend(config(), deps); - - await backend.start('/tmp/awf', ['github.com']); - const execution = backend.exec('/tmp/awf', ['github.com'], undefined, 1); - stdin.end('input'); - await expect(execution).resolves.toEqual({ exitCode: 23 }); - await backend.stop(); - - expect(order).toEqual([ - 'preflight', - 'compose', - 'revalidate', - 'vm-config', - 'vm-start', - 'probe', - 'vm-stop', - ]); - expect(manager.execute).toHaveBeenNthCalledWith(2, expect.objectContaining({ - argv: ['/bin/sh', '-lc', 'printf hello'], - cwd: '/workspace', - uid: 1000, - gid: 1000, - timeoutMs: 60_000, - })); - expect(manager.writeStdin).toHaveBeenCalledWith( - Buffer.from('input'), - expect.stringMatching(/^agent-/), - ); - }); - - it('rejects timeouts beyond the guest supervisor limit before infrastructure startup', async () => { - const { deps } = harness(); - const backend = new FirecrackerRuntimeBackend(config({ agentTimeout: 1441 }), deps); - - await expect(backend.start('/tmp/awf', ['github.com'])) - .rejects.toThrow(/up to 1440 minutes/); - expect(deps.startInfrastructure).not.toHaveBeenCalled(); - }); - - it('serializes stdin chunks before sending EOF', async () => { - const { manager, deps, stdin } = harness(); - let releaseFirstWrite!: () => void; - let resolveExecution!: (value: { - requestId: string; - exitCode: number; - signal: null; - timedOut: boolean; - }) => void; - manager.execute - .mockReset() - .mockResolvedValueOnce({ - requestId: 'probe', - exitCode: 0, - signal: null, - timedOut: false, - }) - .mockReturnValueOnce(new Promise((resolve) => { - resolveExecution = resolve; - })); - manager.writeStdin.mockImplementationOnce(() => new Promise((resolve) => { - releaseFirstWrite = resolve; - })); - const backend = new FirecrackerRuntimeBackend(config(), deps); - await backend.start('/tmp/awf', ['github.com']); - const execution = backend.exec('/tmp/awf', ['github.com']); - stdin.write(Buffer.alloc(70_000, 1)); - stdin.end('second'); - await new Promise((resolve) => setImmediate(resolve)); - - expect(manager.endStdin).not.toHaveBeenCalled(); - releaseFirstWrite(); - await new Promise((resolve) => setImmediate(resolve)); - resolveExecution({ - requestId: 'agent', - exitCode: 0, - signal: null, - timedOut: false, - }); - await execution; - expect(manager.writeStdin.mock.invocationCallOrder[0]) - .toBeLessThan(manager.writeStdin.mock.invocationCallOrder[1]); - expect(manager.writeStdin.mock.invocationCallOrder[1]) - .toBeLessThan(manager.endStdin.mock.invocationCallOrder[0]); - }); - - it('stops the partial VM when readiness probing fails', async () => { - const { manager, deps } = harness(); - manager.execute.mockReset().mockResolvedValue({ - requestId: 'probe', - exitCode: 41, - signal: null, - timedOut: false, - }); - const backend = new FirecrackerRuntimeBackend(config(), deps); - - await expect(backend.start('/tmp/awf', ['github.com'])) - .rejects.toThrow(/connectivity probe failed/); - expect(manager.stop).toHaveBeenCalledTimes(1); - }); - - it('fails closed when manager readiness or startup cleanup is unavailable', async () => { - const missingIp = harness(); - Reflect.set(missingIp.manager, 'guestIp', undefined); - const backend = new FirecrackerRuntimeBackend(config(), missingIp.deps); - await expect(backend.start('/tmp/awf', ['github.com'])) - .rejects.toThrow(/did not expose the configured guest IP/); - expect(missingIp.manager.stop).toHaveBeenCalledTimes(1); - - const dualFailure = harness(); - (dualFailure.infra.revalidate as jest.Mock).mockRejectedValue('topology moved'); - dualFailure.manager.stop.mockRejectedValue('cleanup failed'); - const failing = new FirecrackerRuntimeBackend(config(), dualFailure.deps); - await expect(failing.start('/tmp/awf', ['github.com'])).rejects.toMatchObject({ - message: expect.stringContaining('topology moved'), - cause: 'topology moved', - cleanupCause: 'cleanup failed', - }); - }); - - it('rejects execution before readiness and unsupported TTY execution', async () => { - const cold = harness(); - await expect(new FirecrackerRuntimeBackend(config(), cold.deps).exec( - '/tmp/awf', - ['github.com'], - )).rejects.toThrow(/microVM is not ready/); - - const ttyHarness = harness(); - const ttyConfig = config(); - const ttyBackend = new FirecrackerRuntimeBackend(ttyConfig, ttyHarness.deps); - await ttyBackend.start('/tmp/awf', ['github.com']); - ttyConfig.tty = true; - await expect(ttyBackend.exec('/tmp/awf', ['github.com'])) - .rejects.toThrow(/does not support TTY execution/); - await ttyBackend.stop(); - }); - - it('preserves a stopped VM once and logs retained artifacts', async () => { - const { manager, deps } = harness(); - const backend = new FirecrackerRuntimeBackend(config(), deps); - await backend.start('/tmp/awf', ['github.com']); - - await backend.preserve(); - await backend.preserve(); - await backend.collectDiagnostics(); - - expect(manager.stop).toHaveBeenCalledWith({ preserve: true }); - expect(manager.stop).toHaveBeenCalledTimes(1); - expect(deps.logger.info).toHaveBeenCalledWith( - '[firecracker] Preserved network namespace: awffc-test', - ); - }); - - it('cancels an active guest command before stopping', async () => { - const { manager, deps } = harness(); - let resolveExecution!: (value: { - requestId: string; - exitCode: number; - signal: null; - timedOut: boolean; - }) => void; - manager.execute - .mockReset() - .mockResolvedValueOnce({ - requestId: 'probe', - exitCode: 0, - signal: null, - timedOut: false, - }) - .mockReturnValueOnce(new Promise((resolve) => { - resolveExecution = resolve; - })); - manager.cancel.mockImplementationOnce(async () => { - resolveExecution({ - requestId: 'agent', - exitCode: 130, - signal: null, - timedOut: false, - }); - }); - const backend = new FirecrackerRuntimeBackend(config(), deps); - await backend.start('/tmp/awf', ['github.com']); - const execution = backend.exec('/tmp/awf', ['github.com']); - - await backend.stop(); - await expect(execution).resolves.toEqual({ exitCode: 130 }); - await backend.stop(); - expect(manager.cancel).toHaveBeenCalledWith( - 'AWF cleanup', - expect.stringMatching(/^agent-/), - ); - expect(manager.stop).toHaveBeenCalledWith({ preserve: false }); - }); - - it('cancels after stdin forwarding failure without changing command output', async () => { - const { manager, deps, stdin } = harness(); - manager.writeStdin.mockRejectedValueOnce(new Error('closed stdin')); - const backend = new FirecrackerRuntimeBackend(config(), deps); - await backend.start('/tmp/awf', ['github.com']); - const execution = backend.exec('/tmp/awf', ['github.com']); - stdin.write('input'); - await new Promise((resolve) => setImmediate(resolve)); - - await expect(execution).resolves.toEqual({ exitCode: 23 }); - expect(deps.logger.warn).toHaveBeenCalledWith( - expect.stringContaining('stdin forwarding failed'), - ); - expect(manager.cancel).toHaveBeenCalledWith( - 'stdin forwarding failure', - expect.stringMatching(/^agent-/), - ); - await backend.stop(); - }); - - it('preserves sanitized env values without leaking real provider secrets', () => { - const secret = 'sk-real-provider-secret'; - const environment = buildFirecrackerGuestEnvironment( - config({ - openaiApiKey: secret, - additionalEnv: { - SAFE_SETTING: 'enabled', - OPENAI_API_KEY: secret, - }, - }), - infrastructure(), - ); - - expect(environment.SAFE_SETTING).toBe('enabled'); - expect(environment.OPENAI_API_KEY).not.toBe(secret); - expect(Object.values(environment)).not.toContain(secret); - expect(environment.HTTP_PROXY).toBe('http://172.30.0.10:3128'); - expect(environment.HOME).toBe('/workspace/.awf-home'); - - expect(() => buildFirecrackerGuestEnvironment( - config({ - openaiApiKey: 'enabled', - additionalEnv: { SAFE_SETTING: 'enabled' }, - }), - infrastructure(), - )).toThrow(/Refusing to pass a real provider credential/); - }); - - it('sets lowercase http_proxy so BusyBox wget honors the proxy for https:// too', () => { - // See the identical regression test in - // cloud-hypervisor-runtime-backend.test.ts: BusyBox wget (shared by - // both microVM guests) reads only lowercase "http_proxy" for every - // protocol including https, with no https_proxy check at all. - const environment = buildFirecrackerGuestEnvironment(config(), infrastructure()); - - expect(environment.http_proxy).toBe('http://172.30.0.10:3128'); - }); - - it('rejects unsupported strict-security and topology combinations', () => { - expect(() => assertFirecrackerPreSecurityCompatibility( - config({ enableDind: true }), - )).toThrow(/Docker-in-Docker/); - expect(() => assertFirecrackerPreSecurityCompatibility( - config({ enableHostAccess: true }), - )).toThrow(/host access/); - expect(() => assertFirecrackerPreSecurityCompatibility( - config({ enclaves: { enabled: true } } as Partial), - )).toThrow(/MCP gateway path/); - expect(() => assertFirecrackerSelection( - config({ containerRuntime: 'gvisor' }), - )).toThrow(/require --container-runtime firecracker/); - }); -}); diff --git a/src/firecracker-runtime-backend.ts b/src/firecracker-runtime-backend.ts deleted file mode 100644 index 16522bb2a..000000000 --- a/src/firecracker-runtime-backend.ts +++ /dev/null @@ -1,430 +0,0 @@ -import type { Readable, Writable } from 'stream'; -import type { WorkflowDependencies } from './cli-workflow'; -import type { ExternalAgentRuntimeBackend } from './external-runtime-backend'; -import { - API_PROXY_IP, - NETWORK_SUBNET, - SQUID_IP, -} from './config/network-policy'; -import { - resolveMicrovmInfrastructure, - type MicrovmInfrastructureSnapshot, -} from './microvm/infrastructure'; -import type { - GuestExecutionRequest, - GuestExecutionResult, -} from './microvm/vsock-client'; -import type { FirecrackerPreflightResult } from './firecracker/preflight'; -import { FirecrackerManager } from './firecracker/manager'; -import { runFirecrackerPreflight } from './firecracker/preflight'; -import { getRealUserHome, getSafeHostGid, getSafeHostUid } from './host-identity'; -import { logger } from './logger'; -import { buildGuestEnvironment } from './microvm/guest-environment'; -import type { FirecrackerOptions, WrapperConfig } from './types'; -import { - assertFirecrackerRuntimeCompatibility, - requireFirecrackerConfig, -} from './firecracker/runtime-validation'; -export { - assertFirecrackerPreSecurityCompatibility, - assertFirecrackerRuntimeCompatibility, -} from './firecracker/runtime-validation'; - -const FIRECRACKER_GUEST_WORKSPACE = '/workspace'; -const FIRECRACKER_GUEST_HOME = `${FIRECRACKER_GUEST_WORKSPACE}/.awf-home`; -const FIRECRACKER_PROBE_TIMEOUT_MS = 15_000; -const FIRECRACKER_CANCEL_GRACE_MS = 3_000; -const FIRECRACKER_MAX_TIMEOUT_MS = 86_400_000; - -interface FirecrackerBackendLogger { - debug(message: string, ...args: unknown[]): void; - info(message: string, ...args: unknown[]): void; - warn(message: string, ...args: unknown[]): void; -} - -interface FirecrackerManagerAdapter { - readonly paths: Pick; - readonly guestIp?: string; - readonly networkNamespace?: string; - start(): Promise; - startInstance(): Promise; - execute(request: GuestExecutionRequest): Promise; - cancel(reason?: string, requestId?: string): Promise; - writeStdin(data: Buffer, requestId?: string): Promise; - endStdin(requestId?: string): Promise; - stop(options?: { preserve?: boolean }): Promise; - collectDiagnostics(directory: string): Promise; -} - -export interface FirecrackerRuntimeBackendDependencies { - startInfrastructure: WorkflowDependencies['startContainers']; - preflight(config: FirecrackerOptions): Promise; - resolveInfrastructure(enableApiProxy: boolean, ipPath?: string): Promise; - createManager( - config: FirecrackerOptions, - workDir: string, - infrastructure: MicrovmInfrastructureSnapshot, - workspacePath: string, - homePath: string, - identity: { uid: number; gid: number }, - ): FirecrackerManagerAdapter; - workspacePath(): string; - homePath(): string; - identity(): { uid: number; gid: number }; - stdin: Readable & { isTTY?: boolean }; - stdout: Writable; - stderr: Writable; - logger: FirecrackerBackendLogger; -} - -function defaultDependencies( - startInfrastructure: WorkflowDependencies['startContainers'], -): FirecrackerRuntimeBackendDependencies { - return { - startInfrastructure, - preflight: runFirecrackerPreflight, - resolveInfrastructure: (enableApiProxy, ipPath) => - resolveMicrovmInfrastructure(enableApiProxy, undefined, ipPath), - createManager: (config, workDir, infrastructure, workspacePath, homePath, identity) => - new FirecrackerManager( - config, - workDir, - undefined, - undefined, - { - infrastructureBridge: infrastructure.bridgeName, - enableApiProxy: Boolean(infrastructure.apiProxyIp), - }, - { - workspacePath, - homePath, - supervisorBinaryPath: config.supervisorPath!, - supervisorSha256: config.sha256!.supervisor!, - identity, - }, - ), - workspacePath: () => process.env.GITHUB_WORKSPACE || process.cwd(), - homePath: getRealUserHome, - identity: () => ({ - uid: Number(getSafeHostUid()), - gid: Number(getSafeHostGid()), - }), - stdin: process.stdin, - stdout: process.stdout, - stderr: process.stderr, - logger, - }; -} - -/** @internal Exposed only for focused default-policy tests. */ -export const firecrackerRuntimeTestHelpers = { defaultDependencies }; - -/** Stateful adapter for an explicitly enabled, fail-closed Firecracker microVM. */ -export class FirecrackerRuntimeBackend implements ExternalAgentRuntimeBackend { - readonly runtime = 'firecracker'; - - private manager: FirecrackerManagerAdapter | undefined; - private environment: Record | undefined; - private activeExecution: - | { requestId: string; promise: Promise } - | undefined; - private stopped = false; - private stopping: Promise | undefined; - private identity: { uid: number; gid: number } | undefined; - private preflightResult: FirecrackerPreflightResult | undefined; - - constructor( - private readonly config: WrapperConfig, - private readonly dependencies: FirecrackerRuntimeBackendDependencies, - ) {} - - async preflight(): Promise { - const firecracker = requireFirecrackerConfig(this.config); - if ( - this.config.agentTimeout !== undefined && - this.config.agentTimeout * 60_000 > FIRECRACKER_MAX_TIMEOUT_MS - ) { - throw new Error( - `Firecracker preview supports --agent-timeout values up to ${ - FIRECRACKER_MAX_TIMEOUT_MS / 60_000 - } minutes`, - ); - } - assertFirecrackerRuntimeCompatibility(this.config, firecracker); - this.preflightResult = await this.dependencies.preflight(firecracker); - } - - readonly start: WorkflowDependencies['startContainers'] = async ( - workDir, - allowedDomains, - proxyLogsDir, - skipPull, - onNetworkReady, - onInfrastructureReady, - ) => { - let stage = 'preflight'; - this.dependencies.logger.info( - '[firecracker] runtime=firecracker maturity=preview fallback=disabled', - ); - try { - await this.preflight(); - stage = 'compose-infrastructure'; - await this.dependencies.startInfrastructure( - workDir, - allowedDomains, - proxyLogsDir, - skipPull, - onNetworkReady, - onInfrastructureReady, - ); - - stage = 'infrastructure-discovery'; - const firecracker = requireFirecrackerConfig(this.config); - const infrastructure = await this.dependencies.resolveInfrastructure( - Boolean(this.config.enableApiProxy), - this.preflightResult?.tools.ip, - ); - this.identity = this.dependencies.identity(); - this.manager = this.dependencies.createManager( - firecracker, - workDir, - infrastructure, - this.dependencies.workspacePath(), - this.dependencies.homePath(), - this.identity, - ); - - stage = 'topology-revalidation'; - await infrastructure.revalidate(); - stage = 'jailer-configuration'; - await this.manager.start(); - if (!this.manager.guestIp) { - throw new Error('Firecracker manager did not expose the configured guest IP'); - } - this.environment = buildFirecrackerGuestEnvironment( - this.config, - infrastructure, - this.manager.guestIp, - ); - stage = 'guest-boot'; - await this.manager.startInstance(); - stage = 'guest-connectivity'; - await this.probeGuestConnectivity(); - this.dependencies.logger.info('[firecracker] stage=ready'); - } catch (error) { - this.dependencies.logger.warn( - `[firecracker] stage=${stage} status=failed: ${formatError(error)}`, - ); - try { - await this.manager?.stop(); - } catch (cleanupError) { - const combined = new Error( - `Firecracker startup failed: ${formatError(error)}; ` + - `microVM cleanup also failed: ${formatError(cleanupError)}`, - ); - Object.defineProperty(combined, 'cause', { value: error }); - Object.assign(combined, { cleanupCause: cleanupError }); - throw combined; - } - throw error; - } - }; - - readonly exec: WorkflowDependencies['runAgentCommand'] = async ( - _workDir, - _allowedDomains, - _proxyLogsDir, - agentTimeoutMinutes, - ) => { - const manager = this.manager; - const environment = this.environment; - const identity = this.identity; - if (!manager || !environment || !identity) { - throw new Error('Firecracker microVM is not ready'); - } - if (this.config.tty) { - throw new Error( - 'Firecracker preview guest supervisor does not support TTY execution', - ); - } - - const requestId = `agent-${process.pid}-${Date.now()}`; - const timeoutMs = agentTimeoutMinutes === undefined - ? undefined - : agentTimeoutMinutes * 60_000; - const execution = manager.execute({ - requestId, - argv: ['/bin/sh', '-lc', this.config.agentCommand], - env: environment, - cwd: FIRECRACKER_GUEST_WORKSPACE, - ...identity, - tty: false, - ...(timeoutMs === undefined ? {} : { timeoutMs }), - stdout: this.dependencies.stdout, - stderr: this.dependencies.stderr, - }); - this.activeExecution = { requestId, promise: execution }; - - let forwarding = Promise.resolve(); - let stdinEnded = false; - const forward = (operation: () => Promise): void => { - forwarding = forwarding.then(operation).catch((error) => { - this.dependencies.logger.warn( - `Firecracker guest stdin forwarding failed: ${formatError(error)}`, - ); - return manager.cancel('stdin forwarding failure', requestId).catch(() => undefined); - }); - }; - const onData = (chunk: Buffer | string): void => { - const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - forward(() => manager.writeStdin(data, requestId)); - }; - const onEnd = (): void => { - if (stdinEnded) return; - stdinEnded = true; - forward(() => manager.endStdin(requestId)); - }; - this.dependencies.stdin.on('data', onData); - this.dependencies.stdin.once('end', onEnd); - if (this.dependencies.stdin.readableEnded) onEnd(); - - try { - const result = await execution; - this.dependencies.logger.info( - `[firecracker] Agent command exited with code ${result.exitCode}` + - (result.signal ? ` (${result.signal})` : ''), - ); - return { exitCode: result.exitCode }; - } finally { - this.dependencies.stdin.off('data', onData); - this.dependencies.stdin.off('end', onEnd); - await forwarding; - this.activeExecution = undefined; - } - }; - - async collectDiagnostics(): Promise { - if (!this.manager) return; - const directory = this.config.auditDir - ? `${this.config.auditDir}/firecracker` - : `${this.config.workDir}/diagnostics/firecracker`; - await this.manager.collectDiagnostics(directory); - } - - async stop(): Promise { - if (this.stopped) return; - if (this.stopping) return this.stopping; - this.stopping = this.stopManager(false); - try { - await this.stopping; - this.stopped = true; - } finally { - this.stopping = undefined; - } - } - - async preserve(): Promise { - if (this.stopped) return; - if (this.stopping) return this.stopping; - this.stopping = this.stopManager(true); - try { - await this.stopping; - this.stopped = true; - if (this.manager) { - this.dependencies.logger.info( - `[firecracker] Preserved jail: ${this.manager.paths.jailRoot}`, - ); - this.dependencies.logger.info( - `[firecracker] Preserved images: ${this.config.workDir}/firecracker-images`, - ); - if (this.manager.networkNamespace) { - this.dependencies.logger.info( - `[firecracker] Preserved network namespace: ${this.manager.networkNamespace}`, - ); - } - } - } finally { - this.stopping = undefined; - } - } - - private async stopManager(preserve: boolean): Promise { - const active = this.activeExecution; - if (active && this.manager) { - try { - await this.manager.cancel('AWF cleanup', active.requestId); - } catch { - // Process termination below remains authoritative. - } - await Promise.race([ - active.promise.catch(() => undefined), - new Promise((resolve) => setTimeout(resolve, FIRECRACKER_CANCEL_GRACE_MS)), - ]); - } - await this.manager?.stop({ preserve }); - } - - private async probeGuestConnectivity(): Promise { - const manager = this.manager!; - const environment = this.environment!; - const identity = this.identity; - if (!identity) { - throw new Error('Firecracker guest identity is not ready'); - } - const squidProbe = - `curl --silent --show-error --max-time 5 --output /dev/null ` + - `http://${SQUID_IP}:3128/`; - const apiProxyProbe = this.config.enableApiProxy - ? ` && curl --fail --silent --show-error --max-time 5 --noproxy '*' ` + - `--output /dev/null http://${API_PROXY_IP}:10000/reflect` - : ''; - const result = await manager.execute({ - requestId: `probe-${process.pid}-${Date.now()}`, - argv: ['/bin/sh', '-c', `set -eu; ${squidProbe}${apiProxyProbe}`], - env: environment, - cwd: FIRECRACKER_GUEST_WORKSPACE, - ...identity, - timeoutMs: FIRECRACKER_PROBE_TIMEOUT_MS, - }); - if (result.exitCode !== 0) { - throw new Error( - `Firecracker guest connectivity probe failed with exit code ${result.exitCode}`, - ); - } - this.dependencies.logger.info( - '[firecracker] Guest supervisor, Squid, and API proxy connectivity verified', - ); - } -} - -export function buildFirecrackerGuestEnvironment( - config: WrapperConfig, - infrastructure: Pick, - guestIp = '100.64.0.2', -): Record { - const networkConfig = { - subnet: NETWORK_SUBNET, - squidIp: infrastructure.squidIp, - agentIp: guestIp, - proxyIp: infrastructure.apiProxyIp, - }; - return buildGuestEnvironment({ - config, - networkConfig, - home: FIRECRACKER_GUEST_HOME, - workspace: FIRECRACKER_GUEST_WORKSPACE, - runtimeName: 'firecracker', - runtimeDisplayName: 'Firecracker', - }); -} - -function formatError(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -export function createFirecrackerRuntimeBackend( - config: WrapperConfig, - startInfrastructure: WorkflowDependencies['startContainers'], -): FirecrackerRuntimeBackend { - return new FirecrackerRuntimeBackend(config, defaultDependencies(startInfrastructure)); -} diff --git a/src/firecracker/api-client.test.ts b/src/firecracker/api-client.test.ts deleted file mode 100644 index 3515c6268..000000000 --- a/src/firecracker/api-client.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -import * as http from 'http'; -import { promises as fs } from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { - FirecrackerApiClient, - FirecrackerApiError, -} from './api-client'; - -describe('FirecrackerApiClient', () => { - let directory: string; - let socketPath: string; - let server: http.Server; - - beforeEach(async () => { - directory = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-fc-api-')); - socketPath = path.join(directory, 'api.socket'); - }); - - afterEach(async () => { - if (server?.listening) { - await new Promise((resolve, reject) => { - server.close((error) => error ? reject(error) : resolve()); - }); - } - await fs.rm(directory, { recursive: true, force: true }); - }); - - async function listen( - handler: http.RequestListener, - ): Promise { - server = http.createServer(handler); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(socketPath, resolve); - }); - } - - it('sends typed JSON requests over the Unix socket', async () => { - const received: Array<{ method?: string; url?: string; body: string }> = []; - await listen((request, response) => { - const chunks: Buffer[] = []; - request.on('data', (chunk: Buffer) => chunks.push(chunk)); - request.on('end', () => { - received.push({ - method: request.method, - url: request.url, - body: Buffer.concat(chunks).toString('utf8'), - }); - response.writeHead(204).end(); - }); - }); - - const client = new FirecrackerApiClient({ socketPath }); - await client.putMachineConfig({ vcpu_count: 2, mem_size_mib: 512 }); - await client.putDrive({ - drive_id: 'root drive', - path_on_host: '/rootfs', - is_root_device: true, - is_read_only: false, - }); - await client.putNetworkInterface({ - iface_id: 'primary interface', - host_dev_name: 'fct123456789012', - guest_mac: '02:00:00:00:00:01', - }); - await client.putLogger({ - log_path: '/run/firecracker.log', - level: 'Info', - show_level: true, - }); - await client.putMetrics({ - metrics_path: '/run/firecracker.metrics.jsonl', - }); - await client.putAction('FlushMetrics'); - await client.instanceStart(); - - expect(received).toEqual([ - { - method: 'PUT', - url: '/machine-config', - body: JSON.stringify({ vcpu_count: 2, mem_size_mib: 512 }), - }, - { - method: 'PUT', - url: '/drives/root%20drive', - body: JSON.stringify({ - drive_id: 'root drive', - path_on_host: '/rootfs', - is_root_device: true, - is_read_only: false, - }), - }, - { - method: 'PUT', - url: '/network-interfaces/primary%20interface', - body: JSON.stringify({ - iface_id: 'primary interface', - host_dev_name: 'fct123456789012', - guest_mac: '02:00:00:00:00:01', - }), - }, - { - method: 'PUT', - url: '/logger', - body: JSON.stringify({ - log_path: '/run/firecracker.log', - level: 'Info', - show_level: true, - }), - }, - { - method: 'PUT', - url: '/metrics', - body: JSON.stringify({ - metrics_path: '/run/firecracker.metrics.jsonl', - }), - }, - { - method: 'PUT', - url: '/actions', - body: JSON.stringify({ action_type: 'FlushMetrics' }), - }, - { - method: 'PUT', - url: '/actions', - body: JSON.stringify({ action_type: 'InstanceStart' }), - }, - ]); - }); - - it('returns structured Firecracker API errors', async () => { - await listen((_request, response) => { - response.writeHead(400, { 'Content-Type': 'application/json' }); - response.end(JSON.stringify({ fault_message: 'invalid machine config' })); - }); - - const client = new FirecrackerApiClient({ socketPath }); - const error = await client.putMachineConfig({ - vcpu_count: 0, - mem_size_mib: 512, - }).catch((caught) => caught); - - expect(error).toBeInstanceOf(FirecrackerApiError); - expect(error).toMatchObject({ - method: 'PUT', - requestPath: '/machine-config', - statusCode: 400, - }); - expect(error.message).toContain('invalid machine config'); - }); - - it('enforces a wall-clock timeout even when the peer keeps sending data', async () => { - await listen((_request, response) => { - response.writeHead(200, { 'Content-Type': 'application/json' }); - const interval = setInterval(() => { - response.write(' '); - }, 5); - response.on('close', () => clearInterval(interval)); - }); - - const client = new FirecrackerApiClient({ socketPath, timeoutMs: 30 }); - await expect(client.getInstanceInfo()).rejects.toThrow(/timed out after 30ms/); - }); - - it('rejects when the response stream errors before completion', async () => { - await listen((_request, response) => { - response.writeHead(200, { 'Content-Type': 'application/json' }); - response.write('{"id":"vm"'); - response.destroy(new Error('socket closed')); - }); - - const client = new FirecrackerApiClient({ socketPath }); - await expect(client.getInstanceInfo()).rejects.toThrow(); - }); -}); diff --git a/src/firecracker/api-client.ts b/src/firecracker/api-client.ts deleted file mode 100644 index 5478f54d5..000000000 --- a/src/firecracker/api-client.ts +++ /dev/null @@ -1,247 +0,0 @@ -import * as http from 'http'; - -export interface FirecrackerMachineConfig { - vcpu_count: number; - mem_size_mib: number; - smt?: boolean; - track_dirty_pages?: boolean; -} - -export interface FirecrackerBootSource { - kernel_image_path: string; - boot_args?: string; - initrd_path?: string; -} - -export interface FirecrackerRateLimiter { - bandwidth?: { size: number; refill_time: number; one_time_burst?: number }; - ops?: { size: number; refill_time: number; one_time_burst?: number }; -} - -export interface FirecrackerDrive { - drive_id: string; - path_on_host: string; - is_root_device: boolean; - is_read_only: boolean; - cache_type?: 'Unsafe' | 'Writeback'; - io_engine?: 'Sync' | 'Async'; - rate_limiter?: FirecrackerRateLimiter; -} - -export interface FirecrackerVsock { - guest_cid: number; - uds_path: string; -} - -export interface FirecrackerNetworkInterface { - iface_id: string; - host_dev_name: string; - guest_mac?: string; - rx_rate_limiter?: FirecrackerRateLimiter; - tx_rate_limiter?: FirecrackerRateLimiter; -} - -export interface FirecrackerLoggerConfig { - log_path: string; - level?: 'Error' | 'Warning' | 'Info' | 'Debug' | 'Trace'; - show_level?: boolean; - show_log_origin?: boolean; -} - -export interface FirecrackerMetricsConfig { - metrics_path: string; -} - -export type FirecrackerActionType = - | 'InstanceStart' - | 'SendCtrlAltDel' - | 'FlushMetrics'; - -export interface FirecrackerInstanceInfo { - id: string; - state: 'Not started' | 'Running' | 'Paused'; - vmm_version: string; - app_name: string; -} - -export type FirecrackerVmState = 'Paused' | 'Resumed'; - -interface FirecrackerErrorBody { - fault_message?: string; -} - -export class FirecrackerApiError extends Error { - constructor( - readonly method: string, - readonly requestPath: string, - readonly statusCode: number, - readonly responseBody: string, - message: string, - ) { - super(message); - this.name = 'FirecrackerApiError'; - } -} - -export interface FirecrackerApiClientOptions { - socketPath: string; - timeoutMs?: number; -} - -/** - * Typed client for Firecracker's REST API over its Unix domain socket. - */ -export class FirecrackerApiClient { - private readonly timeoutMs: number; - - constructor(private readonly options: FirecrackerApiClientOptions) { - this.timeoutMs = options.timeoutMs ?? 5_000; - } - - putMachineConfig(config: FirecrackerMachineConfig): Promise { - return this.request('PUT', '/machine-config', config); - } - - putBootSource(source: FirecrackerBootSource): Promise { - return this.request('PUT', '/boot-source', source); - } - - putDrive(drive: FirecrackerDrive): Promise { - return this.request('PUT', `/drives/${encodeURIComponent(drive.drive_id)}`, drive); - } - - putVsock(vsock: FirecrackerVsock): Promise { - return this.request('PUT', '/vsock', vsock); - } - - putNetworkInterface(networkInterface: FirecrackerNetworkInterface): Promise { - return this.request( - 'PUT', - `/network-interfaces/${encodeURIComponent(networkInterface.iface_id)}`, - networkInterface, - ); - } - - putLogger(config: FirecrackerLoggerConfig): Promise { - return this.request('PUT', '/logger', config); - } - - putMetrics(config: FirecrackerMetricsConfig): Promise { - return this.request('PUT', '/metrics', config); - } - - instanceStart(): Promise { - return this.putAction('InstanceStart'); - } - - putAction(actionType: FirecrackerActionType): Promise { - return this.request('PUT', '/actions', { action_type: actionType }); - } - - getInstanceInfo(): Promise { - return this.request('GET', '/'); - } - - patchVmState(state: FirecrackerVmState): Promise { - return this.request('PATCH', '/vm', { state }); - } - - private request( - method: string, - requestPath: string, - payload?: object, - ): Promise { - const body = payload === undefined ? undefined : JSON.stringify(payload); - - return new Promise((resolve, reject) => { - let settled = false; - const timer = setTimeout(() => { - const error = new Error( - `Firecracker API ${method} ${requestPath} timed out after ${this.timeoutMs}ms`, - ); - rejectOnce(error); - request.destroy(error); - }, this.timeoutMs); - const clearTimer = () => clearTimeout(timer); - const resolveOnce = (value: TResponse) => { - if (settled) return; - settled = true; - clearTimer(); - resolve(value); - }; - const rejectOnce = (error: unknown) => { - if (settled) return; - settled = true; - clearTimer(); - reject(error); - }; - - const request = http.request({ - socketPath: this.options.socketPath, - path: requestPath, - method, - headers: body === undefined - ? undefined - : { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - }, - }, (response) => { - const chunks: Buffer[] = []; - let totalBytes = 0; - response.on('error', rejectOnce); - response.on('aborted', () => { - rejectOnce(new Error(`Firecracker API ${method} ${requestPath} response was aborted`)); - }); - response.on('data', (chunk: Buffer) => { - totalBytes += chunk.length; - if (totalBytes > 1024 * 1024) { - const error = new Error('Firecracker API response exceeded 1 MiB'); - rejectOnce(error); - request.destroy(error); - return; - } - chunks.push(chunk); - }); - response.on('end', () => { - const responseBody = Buffer.concat(chunks).toString('utf8'); - const statusCode = response.statusCode ?? 0; - if (statusCode < 200 || statusCode >= 300) { - let parsed: FirecrackerErrorBody | undefined; - try { - parsed = responseBody ? JSON.parse(responseBody) as FirecrackerErrorBody : undefined; - } catch { - parsed = undefined; - } - const detail = parsed?.fault_message || responseBody || 'empty response'; - rejectOnce(new FirecrackerApiError( - method, - requestPath, - statusCode, - responseBody, - `Firecracker API ${method} ${requestPath} failed with HTTP ${statusCode}: ${detail}`, - )); - return; - } - - if (!responseBody) { - resolveOnce(undefined as TResponse); - return; - } - try { - resolveOnce(JSON.parse(responseBody) as TResponse); - } catch (error) { - rejectOnce(new Error( - `Firecracker API ${method} ${requestPath} returned invalid JSON: ` + - `${error instanceof Error ? error.message : String(error)}`, - )); - } - }); - }); - - request.on('error', rejectOnce); - if (body !== undefined) request.write(body); - request.end(); - }); - } -} diff --git a/src/firecracker/config.test.ts b/src/firecracker/config.test.ts deleted file mode 100644 index 883037d09..000000000 --- a/src/firecracker/config.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { buildConfig } from '../commands/build-config'; -import { mapAwfFileConfigToCliOptions } from '../config-mapper'; -import { validateAwfFileConfig } from '../config-file'; -import { - FIRECRACKER_DEFAULT_API_TIMEOUT_MS, - FIRECRACKER_DEFAULT_BINARY, - FIRECRACKER_DEFAULT_JAILER_BINARY, - FIRECRACKER_DEFAULT_MEMORY_MIB, - FIRECRACKER_DEFAULT_VCPU_COUNT, -} from '../types/runtime-options'; - -function buildFirecrackerConfig(options: Record) { - return buildConfig({ - options: { - keepContainers: false, - buildLocal: false, - skipPull: false, - imageRegistry: 'registry', - imageTag: 'latest', - envAll: false, - sslBump: false, - enableDind: false, - enableDlp: false, - ...options, - }, - agentCommand: 'echo test', - logLevel: 'info', - allowedDomains: [], - blockedDomains: [], - localhostDetected: false, - additionalEnv: {}, - volumeMounts: undefined, - upstreamProxy: undefined, - dnsServers: [], - dnsOverHttps: undefined, - allowedUrls: undefined, - memoryLimit: undefined, - pidsLimit: undefined, - agentImage: undefined, - modelAliases: undefined, - allowedModels: undefined, - disallowedModels: undefined, - maxEffectiveTokens: undefined, - maxAiCredits: undefined, - effectiveTokenModelMultipliers: undefined, - effectiveTokenDefaultModelMultiplier: undefined, - maxRuns: undefined, - maxPermissionDenied: undefined, - maxCacheMisses: undefined, - resolvedCopilotApiTarget: undefined, - resolvedCopilotApiBasePath: undefined, - dockerHostPathPrefix: undefined, - }).firecracker; -} - -describe('Firecracker configuration', () => { - it('maps the cohesive config-file surface to CLI option semantics', () => { - const digest = 'a'.repeat(64); - const mapped = mapAwfFileConfigToCliOptions({ - firecracker: { - previewEnabled: true, - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/vmlinux', - rootfsPath: '/opt/rootfs.ext4', - supervisorPath: '/opt/awf-supervisor', - vcpuCount: 4, - memoryMib: 1024, - apiTimeoutMs: 8000, - sha256: { kernel: digest, supervisor: digest }, - }, - }); - - expect(mapped).toEqual(expect.objectContaining({ - firecrackerPreview: true, - firecrackerBinary: '/opt/firecracker', - firecrackerJailerBinary: '/opt/jailer', - firecrackerKernel: '/opt/vmlinux', - firecrackerRootfs: '/opt/rootfs.ext4', - firecrackerSupervisor: '/opt/awf-supervisor', - firecrackerVcpus: 4, - firecrackerMemoryMib: 1024, - firecrackerApiTimeoutMs: 8000, - firecrackerKernelSha256: digest, - firecrackerSupervisorSha256: digest, - })); - }); - - it('applies explicit safe defaults when Firecracker is selected', () => { - expect(buildFirecrackerConfig({ containerRuntime: 'firecracker' })).toEqual({ - previewEnabled: false, - firecrackerBinary: FIRECRACKER_DEFAULT_BINARY, - jailerBinary: FIRECRACKER_DEFAULT_JAILER_BINARY, - kernelPath: undefined, - rootfsPath: undefined, - supervisorPath: undefined, - vcpuCount: FIRECRACKER_DEFAULT_VCPU_COUNT, - memoryMib: FIRECRACKER_DEFAULT_MEMORY_MIB, - apiTimeoutMs: FIRECRACKER_DEFAULT_API_TIMEOUT_MS, - sha256: undefined, - }); - }); - - it('does not populate Firecracker defaults for unrelated runtimes', () => { - expect(buildFirecrackerConfig({ - containerRuntime: 'gvisor', - firecrackerPreview: false, - })).toBeUndefined(); - }); - - it('validates runtime names, positive resources, digests, and unknown keys', () => { - expect(validateAwfFileConfig({ - container: { containerRuntime: 'firecracker' }, - firecracker: { - vcpuCount: 2, - memoryMib: 512, - sha256: { rootfs: '0'.repeat(64) }, - }, - })).toEqual([]); - expect(validateAwfFileConfig({ firecracker: { vcpuCount: 0 } })) - .toContain('config.firecracker.vcpuCount must be a positive integer'); - expect(validateAwfFileConfig({ firecracker: { sha256: { kernel: 'bad' } } })) - .toContain('config.firecracker.sha256.kernel must match pattern "^[A-Fa-f0-9]{64}$"'); - expect(validateAwfFileConfig({ firecracker: { unsupported: true } })) - .toContain('config.firecracker.unsupported is not supported'); - }); -}); diff --git a/src/firecracker/manager.test.ts b/src/firecracker/manager.test.ts deleted file mode 100644 index 990aee41c..000000000 --- a/src/firecracker/manager.test.ts +++ /dev/null @@ -1,782 +0,0 @@ -import type { ExecaChildProcess } from 'execa'; -import { PassThrough } from 'stream'; -import type { - MicrovmNetworkLifecycle, - MicrovmNetworkPlan, -} from '../microvm/network'; -import type { MicrovmVsockClient } from '../microvm/vsock-client'; -import type { MicrovmWorkspaceImage } from '../microvm/workspace'; -import type { FirecrackerOptions } from '../types/runtime-options'; -import type { FirecrackerApiClient } from './api-client'; -import { - FirecrackerManager, - buildSupervisorBootArgs, - createFirecrackerRunPaths, - firecrackerManagerTestHelpers, - type FirecrackerManagerDependencies, - type FirecrackerManagerNetworkConfig, -} from './manager'; -import type { FirecrackerHostToolPaths } from './preflight'; - -const hostTools: FirecrackerHostToolPaths = { - ip: '/usr/bin/ip', - nft: '/usr/sbin/nft', - sysctl: '/usr/sbin/sysctl', - mke2fs: '/usr/sbin/mke2fs', - debugfs: '/usr/sbin/debugfs', - e2fsck: '/usr/sbin/e2fsck', - rsync: '/usr/bin/rsync', -}; - -function config(overrides: Partial = {}): FirecrackerOptions { - return { - previewEnabled: true, - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/vmlinux', - rootfsPath: '/opt/rootfs.ext4', - supervisorPath: '/opt/awf-supervisor', - vcpuCount: 2, - memoryMib: 512, - apiTimeoutMs: 1, - ...overrides, - }; -} - -function processMock(): ExecaChildProcess { - const child = Promise.resolve({ exitCode: 0 }) as unknown as ExecaChildProcess; - Object.assign(child, { - exitCode: null, - signalCode: null, - killed: false, - kill: jest.fn(() => { - Object.assign(child, { exitCode: 0, killed: true }); - return true; - }), - }); - return child; -} - -function networkConfig( - overrides: Partial = {}, -): FirecrackerManagerNetworkConfig { - return { - infrastructureBridge: 'awfbr0', - enableApiProxy: true, - ...overrides, - }; -} - -function networkLifecycle(plan: MicrovmNetworkPlan): MicrovmNetworkLifecycle { - return { - plan, - setup: jest.fn().mockResolvedValue(plan), - cleanup: jest.fn().mockResolvedValue(undefined), - }; -} - -function dependencies( - overrides: Partial = {}, -): FirecrackerManagerDependencies { - const client = { - putMachineConfig: jest.fn().mockResolvedValue(undefined), - putBootSource: jest.fn().mockResolvedValue(undefined), - putDrive: jest.fn().mockResolvedValue(undefined), - putVsock: jest.fn().mockResolvedValue(undefined), - putNetworkInterface: jest.fn().mockResolvedValue(undefined), - putLogger: jest.fn().mockResolvedValue(undefined), - putMetrics: jest.fn().mockResolvedValue(undefined), - putAction: jest.fn().mockResolvedValue(undefined), - instanceStart: jest.fn().mockResolvedValue(undefined), - } as unknown as FirecrackerApiClient; - return { - preflight: jest.fn().mockResolvedValue({ - version: '1.16.1', - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/vmlinux', - rootfsPath: '/opt/rootfs.ext4', - tools: hostTools, - supervisorPath: '/opt/awf-supervisor', - cgroupVersion: 2, - }), - launch: jest.fn().mockReturnValue(processMock()), - mkdir: jest.fn().mockResolvedValue(undefined), - copyFile: jest.fn().mockResolvedValue(undefined), - chmod: jest.fn().mockResolvedValue(undefined), - chown: jest.fn().mockResolvedValue(undefined), - writeFile: jest.fn().mockResolvedValue(undefined), - readFileTail: jest.fn().mockResolvedValue(Buffer.alloc(0)), - access: jest.fn().mockResolvedValue(undefined), - rm: jest.fn().mockResolvedValue(undefined), - sleep: jest.fn().mockResolvedValue(undefined), - createClient: jest.fn().mockReturnValue(client), - createNetwork: jest.fn((plan) => networkLifecycle(plan)), - createWorkspaceImage: jest.fn(), - createVsockClient: jest.fn(), - resolveIdentity: jest.fn().mockReturnValue({ uid: 1000, gid: 1000 }), - ...overrides, - }; -} - -describe('FirecrackerManager', () => { - it('constructs the default host adapters and jailer identity', async () => { - const defaults = firecrackerManagerTestHelpers.defaultDependencies; - const child = defaults.launch(process.execPath, ['-e', ''], { - reject: false, - stdio: ['ignore', 'pipe', 'pipe'], - env: process.env, - }); - await expect(child).resolves.toMatchObject({ exitCode: 0 }); - await expect(defaults.sleep(0)).resolves.toBeUndefined(); - expect(defaults.createClient('/tmp/firecracker.socket', 100)).toBeDefined(); - expect(defaults.createNetwork({} as MicrovmNetworkPlan, hostTools)).toBeDefined(); - expect(defaults.createWorkspaceImage({ - runId: 'adapter-test', - workDir: '/tmp/awf', - workspacePath: '/workspace', - homePath: '/home/runner', - baseRootfsPath: '/opt/rootfs', - supervisorBinaryPath: '/opt/supervisor', - supervisorSha256: 'a'.repeat(64), - uid: 1000, - gid: 1000, - }, hostTools)).toBeDefined(); - expect(defaults.createVsockClient('/tmp/vsock.socket', 52, 100)).toBeDefined(); - - const originalSudoUid = process.env.SUDO_UID; - const originalSudoGid = process.env.SUDO_GID; - const uidSpy = jest.spyOn(process, 'getuid').mockReturnValue(0); - const gidSpy = jest.spyOn(process, 'getgid').mockReturnValue(0); - try { - process.env.SUDO_UID = '2001'; - process.env.SUDO_GID = '2002'; - expect(firecrackerManagerTestHelpers.resolveJailerIdentity()).toEqual({ - uid: 2001, - gid: 2002, - }); - - delete process.env.SUDO_UID; - delete process.env.SUDO_GID; - expect(firecrackerManagerTestHelpers.resolveJailerIdentity) - .toThrow(/non-root target uid\/gid/); - } finally { - uidSpy.mockRestore(); - gidSpy.mockRestore(); - if (originalSudoUid === undefined) delete process.env.SUDO_UID; - else process.env.SUDO_UID = originalSudoUid; - if (originalSudoGid === undefined) delete process.env.SUDO_GID; - else process.env.SUDO_GID = originalSudoGid; - } - }); - - it('constructs unique, contained jail paths', () => { - const first = createFirecrackerRunPaths('/tmp/awf', '/opt/firecracker'); - const second = createFirecrackerRunPaths('/tmp/awf', '/opt/firecracker'); - expect(first.runId).not.toBe(second.runId); - expect(first.jailRoot).toContain('/tmp/awf/firecracker-jailer/firecracker/'); - expect(() => createFirecrackerRunPaths( - '/tmp/awf', - '/opt/firecracker', - '../escape', - )).toThrow(/Unsafe microVM run id/); - expect(() => createFirecrackerRunPaths( - '/tmp/awf', - '/opt/firecracker', - 'run_1', - )).toThrow(/Unsafe microVM run id/); - expect(() => createFirecrackerRunPaths( - '/tmp/awf', - '/opt/firecracker', - `run-${'a'.repeat(61)}`, - )).toThrow(/Unsafe microVM run id/); - }); - - it('launches jailer and configures machine, kernel, and root drive', async () => { - const deps = dependencies(); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'run-1', - networkConfig(), - ); - const client = await manager.start(); - - expect(deps.launch).toHaveBeenCalledWith( - '/opt/jailer', - expect.arrayContaining([ - '--id', 'run-1', - '--exec-file', '/opt/firecracker', - '--netns', expect.stringMatching(/^\/var\/run\/netns\/awffc-/), - '--api-sock', '/run/firecracker.socket', - ]), - expect.objectContaining({ reject: false }), - ); - expect(client.putMachineConfig).toHaveBeenCalledWith({ - vcpu_count: 2, - mem_size_mib: 512, - }); - expect(client.putBootSource).toHaveBeenCalledWith({ - kernel_image_path: '/kernel', - }); - expect(client.putDrive).toHaveBeenCalledWith(expect.objectContaining({ - drive_id: 'rootfs', - path_on_host: '/rootfs', - is_root_device: true, - })); - expect(client.putNetworkInterface).toHaveBeenCalledWith({ - iface_id: 'eth0', - host_dev_name: expect.stringMatching(/^fct[0-9a-f]{12}$/), - guest_mac: expect.any(String), - }); - const configuredNetwork = (client.putNetworkInterface as jest.Mock) - .mock.calls[0][0] as { guest_mac: string }; - expect(configuredNetwork.guest_mac.split(':')).toHaveLength(6); - expect(configuredNetwork.guest_mac.startsWith('02:')).toBe(true); - expect(deps.createNetwork).toHaveBeenCalledWith( - expect.objectContaining({ - infrastructureBridge: 'awfbr0', - tapOwnerUid: 1000, - tapOwnerGid: 1000, - tapVnetHdr: false, - }), - hostTools, - ); - const lifecycle = (deps.createNetwork as jest.Mock).mock.results[0] - .value as MicrovmNetworkLifecycle; - expect(lifecycle.setup).toHaveBeenCalledTimes(1); - }); - - it('terminates the partial process and removes its jail on readiness failure', async () => { - const child = processMock(); - const missing = Object.assign(new Error('missing'), { code: 'ENOENT' }); - const deps = dependencies({ - launch: jest.fn().mockReturnValue(child), - access: jest.fn().mockRejectedValue(missing), - sleep: jest.fn(async () => new Promise((resolve) => setTimeout(resolve, 2))), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'partial', - networkConfig(), - ); - - await expect(manager.start()).rejects.toThrow(/API socket was not ready/); - expect(child.kill).toHaveBeenCalledWith( - 'SIGTERM', - { forceKillAfterTimeout: 2_000 }, - ); - expect(deps.rm).toHaveBeenCalledWith( - '/tmp/awf/firecracker-jailer/firecracker/partial', - { recursive: true, force: true }, - ); - const lifecycle = (deps.createNetwork as jest.Mock).mock.results[0] - .value as MicrovmNetworkLifecycle; - expect(lifecycle.cleanup).toHaveBeenCalledTimes(1); - }); - - it('refuses to launch without host-side network enforcement', async () => { - const deps = dependencies(); - const manager = new FirecrackerManager(config(), '/tmp/awf', deps, 'unsafe'); - - await expect(manager.start()).rejects.toThrow(/unfiltered microVM/); - expect(deps.preflight).not.toHaveBeenCalled(); - expect(deps.launch).not.toHaveBeenCalled(); - }); - - it('cleans up the network before removing the jail', async () => { - const order: string[] = []; - const deps = dependencies({ - createNetwork: jest.fn((plan) => ({ - plan, - setup: jest.fn().mockResolvedValue(plan), - cleanup: jest.fn(async () => { - order.push('network'); - }), - })), - rm: jest.fn(async () => { - order.push('jail'); - }), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'cleanup', - networkConfig(), - ); - - await manager.start(); - await manager.stop(); - - expect(order).toEqual(['network', 'jail']); - }); - - it('retains failed network cleanup for a later stop retry', async () => { - const cleanup = jest.fn() - .mockRejectedValueOnce(new Error('network cleanup failed')) - .mockResolvedValue(undefined); - const deps = dependencies({ - createNetwork: jest.fn((plan) => ({ - plan, - setup: jest.fn().mockResolvedValue(plan), - cleanup, - })), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'cleanup-retry', - networkConfig(), - ); - - await manager.start(); - await expect(manager.stop()).rejects.toThrow('network cleanup failed'); - await expect(manager.stop()).resolves.toBeUndefined(); - - expect(cleanup).toHaveBeenCalledTimes(2); - }); - - it('configures the workspace drive and vsock, then extracts only after VM termination', async () => { - const order: string[] = []; - const child = processMock(); - const workspace = { - prepare: jest.fn().mockResolvedValue({ - workspaceImagePath: '/tmp/prepared-workspace.ext4', - rootfsImagePath: '/tmp/prepared-rootfs.ext4', - imageBytes: 1024, - originalManifest: new Map(), - }), - extractAfterStop: jest.fn(async () => { - order.push('extract'); - expect(child.exitCode).toBe(0); - }), - cleanup: jest.fn().mockResolvedValue(undefined), - } as unknown as MicrovmWorkspaceImage; - const guestClient = { - connect: jest.fn().mockResolvedValue({ - version: 1, - type: 'ready', - requestId: 'control', - capabilities: { stdin: true, tty: false, resize: false }, - }), - execute: jest.fn().mockResolvedValue({ - requestId: 'command', - exitCode: 0, - signal: null, - timedOut: false, - }), - shutdown: jest.fn().mockResolvedValue(undefined), - destroy: jest.fn(), - } as unknown as MicrovmVsockClient; - const deps = dependencies({ - launch: jest.fn().mockReturnValue(child), - createWorkspaceImage: jest.fn().mockReturnValue(workspace), - createVsockClient: jest.fn().mockReturnValue(guestClient), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'guest', - networkConfig(), - { - workspacePath: '/workspace', - homePath: '/home/runner', - supervisorBinaryPath: '/opt/awf-supervisor', - supervisorSha256: 'a'.repeat(64), - }, - ); - - const client = await manager.start(); - expect(client.putBootSource).toHaveBeenCalledWith(expect.objectContaining({ - kernel_image_path: '/kernel', - boot_args: expect.stringContaining('init=/sbin/awf-supervisor'), - })); - expect(client.putDrive).toHaveBeenCalledWith({ - drive_id: 'workspace', - path_on_host: '/workspace.ext4', - is_root_device: false, - is_read_only: false, - }); - expect(client.putVsock).toHaveBeenCalledWith({ - guest_cid: 3, - uds_path: '/run/awf-vsock.socket', - }); - await manager.startInstance(); - expect(deps.createVsockClient).toHaveBeenCalledWith( - '/tmp/awf/firecracker-jailer/firecracker/guest/root/run/awf-vsock.socket', - 52, - 1, - ); - await expect(manager.execute({ - requestId: 'command', - argv: ['true'], - env: {}, - cwd: '/workspace', - uid: 1000, - gid: 1000, - })).resolves.toEqual(expect.objectContaining({ exitCode: 0 })); - await manager.stop(); - - expect(guestClient.shutdown).toHaveBeenCalledTimes(1); - expect(workspace.extractAfterStop).toHaveBeenCalledWith( - '/tmp/awf/firecracker-jailer/firecracker/guest/root/workspace.ext4', - ); - expect(order).toEqual(['extract']); - }); - - it('delegates guest cancellation, stdin, and resize only after readiness', async () => { - const cold = new FirecrackerManager( - config(), - '/tmp/awf', - dependencies(), - 'cold-guest', - networkConfig(), - ); - await expect(cold.cancel()).rejects.toThrow(/supervisor is not ready/); - await expect(cold.writeStdin(Buffer.from('input'))).rejects.toThrow(/supervisor is not ready/); - await expect(cold.endStdin()).rejects.toThrow(/supervisor is not ready/); - await expect(cold.resize(80, 24)).rejects.toThrow(/supervisor is not ready/); - - const guestClient = { - connect: jest.fn().mockResolvedValue(undefined), - execute: jest.fn(), - cancel: jest.fn().mockResolvedValue(undefined), - writeStdin: jest.fn().mockResolvedValue(undefined), - endStdin: jest.fn().mockResolvedValue(undefined), - resize: jest.fn().mockResolvedValue(undefined), - shutdown: jest.fn().mockResolvedValue(undefined), - destroy: jest.fn(), - } as unknown as MicrovmVsockClient; - const workspace = { - prepare: jest.fn().mockResolvedValue({ - workspaceImagePath: '/tmp/workspace.ext4', - rootfsImagePath: '/tmp/rootfs.ext4', - imageBytes: 1024, - originalManifest: new Map(), - }), - extractAfterStop: jest.fn().mockResolvedValue(undefined), - cleanup: jest.fn().mockResolvedValue(undefined), - } as unknown as MicrovmWorkspaceImage; - const deps = dependencies({ - createVsockClient: jest.fn().mockReturnValue(guestClient), - createWorkspaceImage: jest.fn().mockReturnValue(workspace), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'ready-guest', - networkConfig(), - { - workspacePath: '/workspace', - homePath: '/home/runner', - supervisorBinaryPath: '/opt/supervisor', - supervisorSha256: 'a'.repeat(64), - }, - ); - await manager.start(); - await manager.startInstance(); - await manager.cancel('test', 'request'); - await manager.writeStdin(Buffer.from('input'), 'request'); - await manager.endStdin('request'); - await manager.resize(80, 24, 'request'); - - expect(guestClient.cancel).toHaveBeenCalledWith('test', 'request'); - expect(guestClient.writeStdin).toHaveBeenCalledWith(Buffer.from('input'), 'request'); - expect(guestClient.endStdin).toHaveBeenCalledWith('request'); - expect(guestClient.resize).toHaveBeenCalledWith(80, 24, 'request'); - await manager.stop(); - }); - - it('quiesces and copies back while preserving jail, images, and network in keep mode', async () => { - const child = processMock(); - const workspace = { - prepare: jest.fn().mockResolvedValue({ - workspaceImagePath: '/tmp/prepared-workspace.ext4', - rootfsImagePath: '/tmp/prepared-rootfs.ext4', - imageBytes: 1024, - originalManifest: new Map(), - }), - extractAfterStop: jest.fn().mockResolvedValue(undefined), - cleanup: jest.fn().mockResolvedValue(undefined), - } as unknown as MicrovmWorkspaceImage; - const guestClient = { - connect: jest.fn().mockResolvedValue(undefined), - shutdown: jest.fn().mockResolvedValue(undefined), - destroy: jest.fn(), - } as unknown as MicrovmVsockClient; - const deps = dependencies({ - launch: jest.fn().mockReturnValue(child), - createWorkspaceImage: jest.fn().mockReturnValue(workspace), - createVsockClient: jest.fn().mockReturnValue(guestClient), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'keep', - networkConfig(), - { - workspacePath: '/workspace', - homePath: '/home/runner', - supervisorBinaryPath: '/opt/awf-supervisor', - supervisorSha256: 'a'.repeat(64), - }, - ); - await manager.start(); - await manager.startInstance(); - - await manager.stop({ preserve: true }); - - const lifecycle = (deps.createNetwork as jest.Mock).mock.results[0] - .value as MicrovmNetworkLifecycle; - expect(workspace.extractAfterStop).toHaveBeenCalledTimes(1); - expect(lifecycle.cleanup).not.toHaveBeenCalled(); - expect(workspace.cleanup).not.toHaveBeenCalled(); - expect(deps.rm).not.toHaveBeenCalled(); - }); - - it('builds explicit supervisor boot networking without widening policy', () => { - const args = buildSupervisorBootArgs({ - runId: 'run', - namespaceName: 'ns', - netnsPath: '/var/run/netns/ns', - nftTableName: 'table', - infrastructureBridge: 'awfbr0', - hostVethName: 'host', - namespaceVethName: 'namespace', - tapName: 'tap', - infrastructureIp: '172.30.0.20', - infrastructureCidr: '172.30.0.0/24', - hostGatewayIp: '172.30.0.1', - guestSubnet: '100.64.0.0/30', - guestIp: '100.64.0.2', - guestGatewayIp: '100.64.0.1', - guestPrefixLength: 30, - guestMac: '02:00:00:00:00:01', - tapOwnerUid: 1000, - tapOwnerGid: 1000, - tapVnetHdr: false, - allowedEndpoints: [], - networkInterface: { iface_id: 'eth0', host_dev_name: 'tap' }, - }, { - workspacePath: '/workspace', - homePath: '/home/runner', - supervisorBinaryPath: '/opt/supervisor', - supervisorSha256: 'a'.repeat(64), - }); - expect(args).toContain('awf.guest-ip=100.64.0.2'); - expect(args).toContain('awf.guest-gateway=100.64.0.1'); - expect(args).toContain('awf.workspace-device=/dev/vdb'); - expect(args).not.toContain('8.8.8.8'); - }); - - it('retains the workspace and network until process termination is confirmed', async () => { - const child = Promise.resolve({ exitCode: null }) as unknown as ExecaChildProcess; - Object.assign(child, { - exitCode: null, - signalCode: null, - killed: false, - kill: jest.fn(() => { - Object.assign(child, { killed: true }); - return true; - }), - }); - const workspace = { - prepare: jest.fn().mockResolvedValue({ - workspaceImagePath: '/tmp/prepared-workspace.ext4', - rootfsImagePath: '/tmp/prepared-rootfs.ext4', - imageBytes: 1024, - originalManifest: new Map(), - }), - extractAfterStop: jest.fn().mockResolvedValue(undefined), - cleanup: jest.fn().mockResolvedValue(undefined), - } as unknown as MicrovmWorkspaceImage; - const guestClient = { - connect: jest.fn().mockResolvedValue(undefined), - shutdown: jest.fn().mockResolvedValue(undefined), - destroy: jest.fn(), - } as unknown as MicrovmVsockClient; - const deps = dependencies({ - launch: jest.fn().mockReturnValue(child), - createWorkspaceImage: jest.fn().mockReturnValue(workspace), - createVsockClient: jest.fn().mockReturnValue(guestClient), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'termination', - networkConfig(), - { - workspacePath: '/workspace', - homePath: '/home/runner', - supervisorBinaryPath: '/opt/awf-supervisor', - supervisorSha256: 'a'.repeat(64), - }, - ); - await manager.start(); - await manager.startInstance(); - - await expect(manager.stop()).rejects.toThrow(/stopped before workspace\/network removal/); - const lifecycle = (deps.createNetwork as jest.Mock).mock.results[0] - .value as MicrovmNetworkLifecycle; - expect(lifecycle.cleanup).not.toHaveBeenCalled(); - expect(workspace.extractAfterStop).not.toHaveBeenCalled(); - expect(deps.rm).not.toHaveBeenCalled(); - - Object.assign(child, { exitCode: 0 }); - await expect(manager.stop()).resolves.toBeUndefined(); - expect(workspace.extractAfterStop).toHaveBeenCalledTimes(1); - expect(lifecycle.cleanup).toHaveBeenCalledTimes(1); - }); - - it('waits briefly for natural VM exit after guest shutdown before sending SIGTERM', async () => { - const child = processMock(); - const workspace = { - prepare: jest.fn().mockResolvedValue({ - workspaceImagePath: '/tmp/prepared-workspace.ext4', - rootfsImagePath: '/tmp/prepared-rootfs.ext4', - imageBytes: 1024, - originalManifest: new Map(), - }), - extractAfterStop: jest.fn().mockResolvedValue(undefined), - cleanup: jest.fn().mockResolvedValue(undefined), - } as unknown as MicrovmWorkspaceImage; - const guestClient = { - connect: jest.fn().mockResolvedValue(undefined), - shutdown: jest.fn().mockResolvedValue(undefined), - destroy: jest.fn(), - } as unknown as MicrovmVsockClient; - let sleepCalls = 0; - const deps = dependencies({ - launch: jest.fn().mockReturnValue(child), - createWorkspaceImage: jest.fn().mockReturnValue(workspace), - createVsockClient: jest.fn().mockReturnValue(guestClient), - sleep: jest.fn(async () => { - sleepCalls += 1; - if (sleepCalls === 3) Object.assign(child, { exitCode: 0 }); - }), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'natural-exit', - networkConfig(), - { - workspacePath: '/workspace', - homePath: '/home/runner', - supervisorBinaryPath: '/opt/awf-supervisor', - supervisorSha256: 'a'.repeat(64), - }, - ); - await manager.start(); - await manager.startInstance(); - await manager.stop(); - expect(child.kill).not.toHaveBeenCalled(); - expect(sleepCalls).toBeGreaterThan(0); - }); - - it('rolls back the network when typed NIC configuration fails', async () => { - const client = { - putMachineConfig: jest.fn().mockResolvedValue(undefined), - putBootSource: jest.fn().mockResolvedValue(undefined), - putDrive: jest.fn().mockResolvedValue(undefined), - putLogger: jest.fn().mockResolvedValue(undefined), - putMetrics: jest.fn().mockResolvedValue(undefined), - putAction: jest.fn().mockResolvedValue(undefined), - putNetworkInterface: jest.fn().mockRejectedValue(new Error('invalid NIC')), - } as unknown as FirecrackerApiClient; - const deps = dependencies({ - createClient: jest.fn().mockReturnValue(client), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'nic-failure', - networkConfig(), - ); - - await expect(manager.start()).rejects.toThrow('invalid NIC'); - - const lifecycle = (deps.createNetwork as jest.Mock).mock.results[0] - .value as MicrovmNetworkLifecycle; - expect(lifecycle.cleanup).toHaveBeenCalledTimes(1); - expect(deps.rm).toHaveBeenCalled(); - }); - - it('fails fast when jailer exits by signal before API readiness', async () => { - const child = processMock(); - Object.assign(child, { signalCode: 'SIGKILL', kill: jest.fn() }); - const missing = Object.assign(new Error('missing'), { code: 'ENOENT' }); - const deps = dependencies({ - launch: jest.fn().mockReturnValue(child), - access: jest.fn().mockRejectedValue(missing), - sleep: jest.fn().mockResolvedValue(undefined), - }); - const manager = new FirecrackerManager( - config({ apiTimeoutMs: 2000 }), - '/tmp/awf', - deps, - 'signal', - networkConfig(), - ); - - await expect(manager.start()).rejects.toThrow( - /exited before API readiness with code null and signal SIGKILL/, - ); - expect(deps.sleep).not.toHaveBeenCalled(); - }); - - it('flushes metrics and bounds diagnostic files before persistence', async () => { - const oversized = Buffer.alloc(1024 * 1024 + 128, 0x61); - const child = processMock(); - const stdout = new PassThrough(); - const stderr = new PassThrough(); - Object.assign(child, { stdout, stderr }); - const deps = dependencies({ - launch: jest.fn().mockReturnValue(child), - readFileTail: jest.fn().mockImplementation((_source: string, maxBytes: number) => - Promise.resolve(oversized.subarray(oversized.length - maxBytes)), - ), - }); - const manager = new FirecrackerManager( - config(), - '/tmp/awf', - deps, - 'diagnostics', - networkConfig(), - ); - - const client = await manager.start(); - stdout.write(oversized); - stderr.write('jailer error'); - await manager.startInstance(); - await manager.collectDiagnostics('/tmp/diagnostics'); - - expect(client.putAction).toHaveBeenCalledWith('FlushMetrics'); - expect(deps.writeFile).toHaveBeenCalledWith( - '/tmp/diagnostics/firecracker.metrics.jsonl', - expect.objectContaining({ length: 1024 * 1024 }), - { mode: 0o600 }, - ); - expect(deps.writeFile).toHaveBeenCalledWith( - '/tmp/diagnostics/jailer-stdout.log', - expect.objectContaining({ length: 1024 * 1024 }), - { mode: 0o600 }, - ); - expect(deps.writeFile).toHaveBeenCalledWith( - '/tmp/diagnostics/jailer-stderr.log', - Buffer.from('jailer error'), - { mode: 0o600 }, - ); - }); -}); diff --git a/src/firecracker/manager.ts b/src/firecracker/manager.ts deleted file mode 100644 index cc45230ce..000000000 --- a/src/firecracker/manager.ts +++ /dev/null @@ -1,699 +0,0 @@ -import { randomBytes } from 'crypto'; -import { constants, promises as fs } from 'fs'; -import * as path from 'path'; -import execa, { type ExecaChildProcess } from 'execa'; -import { - FIRECRACKER_RELEASE_VERSION, - type FirecrackerOptions, -} from '../types/runtime-options'; -import { getSafeHostGid, getSafeHostUid } from '../host-identity'; -import { - LinuxNetworkCommands, - MicrovmNetworkManager, - assertSafeMicrovmRunId, - createMicrovmNetworkPlan, - type MicrovmControlPeer, - type MicrovmNetworkLifecycle, - type MicrovmNetworkPlan, -} from '../microvm/network'; -import { - MicrovmVsockClient, - type GuestExecutionRequest, - type GuestExecutionResult, -} from '../microvm/vsock-client'; -import { - MicrovmWorkspaceImage, - type MicrovmWorkspaceImageConfig, -} from '../microvm/workspace'; -import { FirecrackerApiClient } from './api-client'; -import { runFirecrackerPreflight } from './preflight'; -import type { FirecrackerHostToolPaths } from './preflight'; - -const API_SOCKET_NAME = 'firecracker.socket'; -const VSOCK_SOCKET_NAME = 'awf-vsock.socket'; -const WORKSPACE_IMAGE_NAME = 'workspace.ext4'; -const FIRECRACKER_LOG_NAME = 'firecracker.log'; -const FIRECRACKER_METRICS_NAME = 'firecracker.metrics.jsonl'; -const FIRECRACKER_CAPTURE_LIMIT_BYTES = 1024 * 1024; -const KERNEL_JAIL_PATH = '/kernel'; -const ROOTFS_JAIL_PATH = '/rootfs'; -const WORKSPACE_JAIL_PATH = '/workspace.ext4'; -const VSOCK_JAIL_PATH = `/run/${VSOCK_SOCKET_NAME}`; -export const FIRECRACKER_GUEST_VSOCK_PORT = 52; -const FIRECRACKER_GUEST_SHUTDOWN_GRACE_MS = 5_000; - -export interface FirecrackerRunPaths { - runId: string; - chrootBaseDir: string; - jailRoot: string; - apiSocketPath: string; - kernelPath: string; - rootfsPath: string; - workspacePath: string; - vsockSocketPath: string; - logPath: string; - metricsPath: string; -} - -export interface FirecrackerManagerDependencies { - preflight: typeof runFirecrackerPreflight; - launch( - command: string, - args: string[], - options: { - reject: false; - stdio: ['ignore', 'pipe', 'pipe']; - env: NodeJS.ProcessEnv; - }, - ): ExecaChildProcess; - mkdir(directory: string, options: { recursive: true; mode: number }): Promise; - copyFile(source: string, destination: string, flags: number): Promise; - chmod(filePath: string, mode: number): Promise; - chown(filePath: string, uid: number, gid: number): Promise; - writeFile: typeof fs.writeFile; - readFileTail(filePath: string, maxBytes: number): Promise; - access(filePath: string): Promise; - rm(directory: string, options: { recursive: true; force: true }): Promise; - sleep(milliseconds: number): Promise; - createClient(socketPath: string, timeoutMs: number): FirecrackerApiClient; - createNetwork(plan: MicrovmNetworkPlan, tools: FirecrackerHostToolPaths): MicrovmNetworkLifecycle; - createWorkspaceImage(config: MicrovmWorkspaceImageConfig, tools: FirecrackerHostToolPaths): MicrovmWorkspaceImage; - createVsockClient(socketPath: string, guestPort: number, timeoutMs: number): MicrovmVsockClient; - resolveIdentity(): { uid: number; gid: number }; -} - -export interface FirecrackerManagerNetworkConfig { - infrastructureBridge: string; - enableApiProxy: boolean; - controlPeer?: MicrovmControlPeer; -} - -export interface FirecrackerManagerGuestConfig { - readonly workspacePath: string; - readonly homePath: string; - readonly supervisorBinaryPath: string; - readonly supervisorSha256: string; - readonly maxWorkspaceImageBytes?: number; - readonly vsockPort?: number; - readonly identity?: { uid: number; gid: number }; -} - -async function readBoundedTail(filePath: string, maxBytes: number): Promise { - const handle = await fs.open(filePath, 'r'); - try { - const { size } = await handle.stat(); - const length = Math.min(size, maxBytes); - const buffer = Buffer.alloc(length); - if (length > 0) { - await handle.read(buffer, 0, length, size - length); - } - return buffer; - } finally { - await handle.close(); - } -} - -const defaultDependencies: FirecrackerManagerDependencies = { - preflight: runFirecrackerPreflight, - launch: (command, args, options) => execa(command, args, options), - mkdir: fs.mkdir, - copyFile: fs.copyFile, - chmod: fs.chmod, - chown: fs.chown, - writeFile: fs.writeFile, - readFileTail: (filePath, maxBytes) => readBoundedTail(filePath, maxBytes), - access: fs.access, - rm: fs.rm, - sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), - createClient: (socketPath, timeoutMs) => new FirecrackerApiClient({ socketPath, timeoutMs }), - createNetwork: (plan, tools) => new MicrovmNetworkManager( - plan, - new LinuxNetworkCommands(undefined, tools), - ), - createWorkspaceImage: (config, tools) => new MicrovmWorkspaceImage(config, undefined, tools), - createVsockClient: (socketPath, guestPort, timeoutMs) => new MicrovmVsockClient({ - socketPath, - guestPort, - connectTimeoutMs: timeoutMs, - readTimeoutMs: Math.max(timeoutMs, 30_000), - writeTimeoutMs: timeoutMs, - }), - resolveIdentity: resolveJailerIdentity, -}; - -/** @internal Exposed only for focused host-adapter tests. */ -export const firecrackerManagerTestHelpers = { - defaultDependencies, - resolveJailerIdentity, -}; - -function resolveJailerIdentity(): { uid: number; gid: number } { - const operatorUid = parsePositiveIdentity(process.env.SUDO_UID) ?? process.getuid?.(); - const operatorGid = parsePositiveIdentity(process.env.SUDO_GID) ?? process.getgid?.(); - if ( - operatorUid === undefined || - operatorGid === undefined || - operatorUid === 0 || - operatorGid === 0 - ) { - throw new Error( - 'Firecracker jailer requires a non-root target uid/gid; run through sudo from a non-root account', - ); - } - const uid = Number(getSafeHostUid()); - const gid = Number(getSafeHostGid()); - if (!Number.isSafeInteger(uid) || !Number.isSafeInteger(gid) || uid < 1 || gid < 1) { - throw new Error( - 'Firecracker jailer requires a non-root target uid/gid; run through sudo from a non-root account', - ); - } - return { uid, gid }; -} - -function parsePositiveIdentity(value: string | undefined): number | undefined { - if (!value || !/^[1-9]\d*$/.test(value)) return undefined; - return Number(value); -} - -export function createFirecrackerRunPaths( - workDir: string, - firecrackerBinary: string, - runId = `awf-${process.pid}-${randomBytes(6).toString('hex')}`, -): FirecrackerRunPaths { - assertSafeMicrovmRunId(runId); - const chrootBaseDir = path.join(workDir, 'firecracker-jailer'); - const jailRoot = path.join( - chrootBaseDir, - path.basename(firecrackerBinary), - runId, - 'root', - ); - return { - runId, - chrootBaseDir, - jailRoot, - apiSocketPath: path.join(jailRoot, 'run', API_SOCKET_NAME), - kernelPath: path.join(jailRoot, KERNEL_JAIL_PATH), - rootfsPath: path.join(jailRoot, ROOTFS_JAIL_PATH), - workspacePath: path.join(jailRoot, WORKSPACE_IMAGE_NAME), - vsockSocketPath: path.join(jailRoot, 'run', VSOCK_SOCKET_NAME), - logPath: path.join(jailRoot, 'run', FIRECRACKER_LOG_NAME), - metricsPath: path.join(jailRoot, 'run', FIRECRACKER_METRICS_NAME), - }; -} - -/** - * Owns one jailer-launched Firecracker process and its partial-start cleanup. - */ -export class FirecrackerManager { - readonly paths: FirecrackerRunPaths; - private process: ExecaChildProcess | undefined; - private client: FirecrackerApiClient | undefined; - private network: MicrovmNetworkLifecycle | undefined; - private workspace: MicrovmWorkspaceImage | undefined; - private guestClient: MicrovmVsockClient | undefined; - private networkPlan: MicrovmNetworkPlan | undefined; - private instanceStarted = false; - private readonly stdoutCapture = new BoundedOutputCapture(FIRECRACKER_CAPTURE_LIMIT_BYTES); - private readonly stderrCapture = new BoundedOutputCapture(FIRECRACKER_CAPTURE_LIMIT_BYTES); - - get guestIp(): string | undefined { - return this.networkPlan?.guestIp; - } - - get networkNamespace(): string | undefined { - return this.networkPlan?.namespaceName; - } - - constructor( - private readonly config: FirecrackerOptions, - private readonly workDir: string, - private readonly dependencies: FirecrackerManagerDependencies = defaultDependencies, - runId?: string, - private readonly networkConfig?: FirecrackerManagerNetworkConfig, - private readonly guestConfig?: FirecrackerManagerGuestConfig, - ) { - this.paths = createFirecrackerRunPaths(this.workDir, config.firecrackerBinary, runId); - } - - async start(): Promise { - if (!this.networkConfig) { - throw new Error( - 'Firecracker network configuration is required; refusing to launch an unfiltered microVM', - ); - } - - let startupError: unknown; - try { - const artifacts = await this.dependencies.preflight(this.config); - const identity = this.guestConfig?.identity ?? this.dependencies.resolveIdentity(); - const networkPlan = createMicrovmNetworkPlan(this.paths.runId, { - ...this.networkConfig, - tapOwnerUid: identity.uid, - tapOwnerGid: identity.gid, - }); - this.networkPlan = networkPlan; - this.network = this.dependencies.createNetwork(networkPlan, artifacts.tools); - await this.network.setup(); - let rootfsSource = artifacts.rootfsPath; - let workspaceSource: string | undefined; - if (this.guestConfig) { - this.workspace = this.dependencies.createWorkspaceImage({ - runId: this.paths.runId, - workDir: this.workDir, - workspacePath: this.guestConfig.workspacePath, - homePath: this.guestConfig.homePath, - baseRootfsPath: artifacts.rootfsPath, - supervisorBinaryPath: this.guestConfig.supervisorBinaryPath, - supervisorSha256: this.guestConfig.supervisorSha256, - ...(this.guestConfig.maxWorkspaceImageBytes === undefined - ? {} - : { maxImageBytes: this.guestConfig.maxWorkspaceImageBytes }), - uid: identity.uid, - gid: identity.gid, - }, artifacts.tools); - const preparation = await this.workspace.prepare(); - rootfsSource = preparation.rootfsImagePath; - workspaceSource = preparation.workspaceImagePath; - } - await this.dependencies.mkdir(this.paths.chrootBaseDir, { - recursive: true, - mode: 0o700, - }); - - this.process = this.dependencies.launch( - this.config.jailerBinary, - [ - '--id', this.paths.runId, - '--exec-file', this.config.firecrackerBinary, - '--uid', String(identity.uid), - '--gid', String(identity.gid), - '--chroot-base-dir', this.paths.chrootBaseDir, - '--netns', networkPlan.netnsPath, - '--cgroup-version', String(artifacts.cgroupVersion), - '--', - '--api-sock', `/run/${API_SOCKET_NAME}`, - ], - { - reject: false, - stdio: ['ignore', 'pipe', 'pipe'], - env: { ...process.env }, - }, - ); - this.process.stdout?.on('data', (chunk: Buffer | string) => { - this.stdoutCapture.append(chunk); - }); - this.process.stderr?.on('data', (chunk: Buffer | string) => { - this.stderrCapture.append(chunk); - }); - - await this.waitForApiSocket(); - await this.stageArtifact(artifacts.kernelPath, this.paths.kernelPath, 0o400, identity); - await this.stageArtifact(rootfsSource, this.paths.rootfsPath, 0o600, identity); - if (workspaceSource) { - await this.stageArtifact(workspaceSource, this.paths.workspacePath, 0o600, identity); - } - - this.client = this.dependencies.createClient( - this.paths.apiSocketPath, - this.config.apiTimeoutMs, - ); - await this.stageDiagnosticFile(this.paths.logPath, identity); - await this.stageDiagnosticFile(this.paths.metricsPath, identity); - await this.client.putLogger({ - log_path: `/run/${FIRECRACKER_LOG_NAME}`, - level: 'Info', - show_level: true, - show_log_origin: true, - }); - await this.client.putMetrics({ - metrics_path: `/run/${FIRECRACKER_METRICS_NAME}`, - }); - await this.client.putMachineConfig({ - vcpu_count: this.config.vcpuCount, - mem_size_mib: this.config.memoryMib, - }); - await this.client.putBootSource({ - kernel_image_path: KERNEL_JAIL_PATH, - ...(this.guestConfig - ? { boot_args: buildSupervisorBootArgs(networkPlan, this.guestConfig) } - : {}), - }); - await this.client.putDrive({ - drive_id: 'rootfs', - path_on_host: ROOTFS_JAIL_PATH, - is_root_device: true, - is_read_only: false, - }); - await this.client.putNetworkInterface(networkPlan.networkInterface); - if (this.guestConfig) { - await this.client.putDrive({ - drive_id: 'workspace', - path_on_host: WORKSPACE_JAIL_PATH, - is_root_device: false, - is_read_only: false, - }); - await this.client.putVsock({ - guest_cid: 3, - uds_path: VSOCK_JAIL_PATH, - }); - } - return this.client; - } catch (error) { - startupError = error; - } - - try { - await this.stop(); - } catch (cleanupError) { - throw new Error( - `Firecracker startup failed: ${formatError(startupError)}; ` + - `partial-start cleanup also failed: ${formatError(cleanupError)}`, - ); - } - throw startupError; - } - - async startInstance(): Promise { - if (!this.client) throw new Error('Firecracker API is not configured'); - await this.client.instanceStart(); - this.instanceStarted = true; - if (this.guestConfig) { - this.guestClient = this.dependencies.createVsockClient( - this.paths.vsockSocketPath, - this.guestConfig.vsockPort ?? FIRECRACKER_GUEST_VSOCK_PORT, - this.config.apiTimeoutMs, - ); - await this.guestClient.connect(); - } - } - - async execute( - request: GuestExecutionRequest, - ): Promise { - if (!this.guestClient) { - throw new Error('Firecracker guest supervisor is not ready'); - } - return this.guestClient.execute(request); - } - - cancel(reason = 'host cancellation', requestId?: string): Promise { - if (!this.guestClient) { - return Promise.reject(new Error('Firecracker guest supervisor is not ready')); - } - return this.guestClient.cancel(reason, requestId); - } - - writeStdin(data: Buffer, requestId?: string): Promise { - if (!this.guestClient) { - return Promise.reject(new Error('Firecracker guest supervisor is not ready')); - } - return this.guestClient.writeStdin(data, requestId); - } - - endStdin(requestId?: string): Promise { - if (!this.guestClient) { - return Promise.reject(new Error('Firecracker guest supervisor is not ready')); - } - return this.guestClient.endStdin(requestId); - } - - resize(columns: number, rows: number, requestId?: string): Promise { - if (!this.guestClient) { - return Promise.reject(new Error('Firecracker guest supervisor is not ready')); - } - return this.guestClient.resize(columns, rows, requestId); - } - - async stop(options: { preserve?: boolean } = {}): Promise { - const errors: unknown[] = []; - const instanceWasStarted = this.instanceStarted; - let guestShutdownAcknowledged = false; - if (this.guestClient) { - try { - await this.guestClient.shutdown(); - guestShutdownAcknowledged = true; - } catch (error) { - if ( - !(error instanceof Error) || - error.message !== 'Cannot shut down Firecracker guest while a request is running' - ) { - errors.push(error); - } - this.guestClient.destroy(); - } - } - this.guestClient = undefined; - - let terminationConfirmed = !this.process || - this.process.exitCode !== null || - this.process.signalCode !== null; - if ( - this.process && - this.process.exitCode === null && - this.process.signalCode === null - ) { - const child = this.process; - try { - if (guestShutdownAcknowledged) { - terminationConfirmed = await this.waitForProcessExit( - child, - FIRECRACKER_GUEST_SHUTDOWN_GRACE_MS, - ); - } - if (!child.killed) { - if (child.exitCode === null && child.signalCode === null) { - child.kill('SIGTERM', { forceKillAfterTimeout: 2_000 }); - } - } - if (!terminationConfirmed) { - await child; - if (child.exitCode === null && child.signalCode === null) { - throw new Error('Firecracker process termination was not confirmed'); - } - } - terminationConfirmed = true; - } catch (error) { - terminationConfirmed = child.exitCode !== null || child.signalCode !== null; - errors.push(error); - } - } - if (!terminationConfirmed && this.process) { - if (errors.length === 0) { - errors.push(new Error('Firecracker process termination was not confirmed')); - } - throw new Error( - `Firecracker cleanup stopped before workspace/network removal: ` + - `${errors.map(formatError).join('; ')}`, - ); - } - this.process = undefined; - this.client = undefined; - - if (this.workspace && instanceWasStarted) { - try { - await this.workspace.extractAfterStop(this.paths.workspacePath); - } catch (error) { - errors.push(error); - } - } - this.instanceStarted = false; - - if (options.preserve) { - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) { - throw new Error( - `Firecracker preservation failed: ${errors.map(formatError).join('; ')}`, - ); - } - return; - } - - try { - await this.network?.cleanup(); - this.network = undefined; - this.networkPlan = undefined; - } catch (error) { - errors.push(error); - } - - if (!instanceWasStarted || terminationConfirmed) { - try { - await this.dependencies.rm( - path.join( - this.paths.chrootBaseDir, - path.basename(this.config.firecrackerBinary), - this.paths.runId, - ), - { recursive: true, force: true }, - ); - } catch (error) { - errors.push(error); - } - } - - try { - await this.workspace?.cleanup(!instanceWasStarted); - } catch (error) { - errors.push(error); - } - this.workspace = undefined; - - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) { - throw new Error( - `Firecracker cleanup failed: ${errors.map(formatError).join('; ')}`, - ); - } - } - - private async waitForProcessExit( - child: ExecaChildProcess, - timeoutMs: number, - ): Promise { - const pollIntervalMs = 25; - const attempts = Math.max(1, Math.ceil(timeoutMs / pollIntervalMs)); - for (let attempt = 0; attempt < attempts; attempt += 1) { - if (child.exitCode !== null || child.signalCode !== null) return true; - await this.dependencies.sleep(pollIntervalMs); - } - return child.exitCode !== null || child.signalCode !== null; - } - - async collectDiagnostics(directory: string): Promise { - await this.dependencies.mkdir(directory, { recursive: true, mode: 0o700 }); - if (this.client && this.instanceStarted) { - await this.client.putAction('FlushMetrics'); - await this.dependencies.sleep(25); - } - const writeBounded = async (fileName: string, contents: Buffer): Promise => { - const destination = path.join(directory, fileName); - await this.dependencies.writeFile(destination, contents, { mode: 0o600 }); - }; - await writeBounded('jailer-stdout.log', this.stdoutCapture.contents()); - await writeBounded('jailer-stderr.log', this.stderrCapture.contents()); - await this.copyBoundedDiagnostic( - this.paths.logPath, - path.join(directory, FIRECRACKER_LOG_NAME), - ); - await this.copyBoundedDiagnostic( - this.paths.metricsPath, - path.join(directory, FIRECRACKER_METRICS_NAME), - ); - await this.dependencies.writeFile( - path.join(directory, 'network-plan.json'), - `${JSON.stringify(this.networkPlan ?? null, null, 2)}\n`, - { mode: 0o600 }, - ); - await this.dependencies.writeFile( - path.join(directory, 'runtime.json'), - `${JSON.stringify({ - runtime: 'firecracker', - version: FIRECRACKER_RELEASE_VERSION, - runId: this.paths.runId, - vcpuCount: this.config.vcpuCount, - memoryMib: this.config.memoryMib, - instanceStarted: this.instanceStarted, - }, null, 2)}\n`, - { mode: 0o600 }, - ); - } - - private async waitForApiSocket(): Promise { - const deadline = Date.now() + this.config.apiTimeoutMs; - while (Date.now() < deadline) { - if (this.process && (this.process.exitCode != null || this.process.signalCode != null)) { - throw new Error( - `Firecracker jailer exited before API readiness with code ${this.process.exitCode ?? 'null'} ` + - `and signal ${this.process.signalCode ?? 'null'}`, - ); - } - try { - await this.dependencies.access(this.paths.apiSocketPath); - return; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== 'ENOENT') throw error; - } - await this.dependencies.sleep(25); - } - throw new Error( - `Firecracker API socket was not ready after ${this.config.apiTimeoutMs}ms: ` + - this.paths.apiSocketPath, - ); - } - - private async stageArtifact( - source: string, - destination: string, - mode: number, - identity: { uid: number; gid: number }, - ): Promise { - await this.dependencies.copyFile(source, destination, constants.COPYFILE_EXCL); - await this.dependencies.chown(destination, identity.uid, identity.gid); - await this.dependencies.chmod(destination, mode); - } - - private async stageDiagnosticFile( - destination: string, - identity: { uid: number; gid: number }, - ): Promise { - await this.dependencies.writeFile(destination, '', { flag: 'wx', mode: 0o600 }); - await this.dependencies.chown(destination, identity.uid, identity.gid); - } - - private async copyBoundedDiagnostic(source: string, destination: string): Promise { - try { - const bounded = await this.dependencies.readFileTail(source, FIRECRACKER_CAPTURE_LIMIT_BYTES); - await this.dependencies.writeFile(destination, bounded, { mode: 0o600 }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - } -} - -export function buildSupervisorBootArgs( - networkPlan: MicrovmNetworkPlan, - guestConfig: FirecrackerManagerGuestConfig, -): string { - const port = guestConfig.vsockPort ?? FIRECRACKER_GUEST_VSOCK_PORT; - if (!Number.isInteger(port) || port < 1 || port > 65_535) { - throw new Error(`Firecracker guest vsock port must be in 1-65535: ${port}`); - } - return [ - 'console=ttyS0', - 'reboot=k', - 'panic=1', - 'pci=off', - 'init=/sbin/awf-supervisor', - 'awf.workspace-device=/dev/vdb', - 'awf.workspace-mount=/workspace', - `awf.vsock-port=${port}`, - `awf.guest-ip=${networkPlan.guestIp}`, - `awf.guest-prefix=${networkPlan.guestPrefixLength}`, - `awf.guest-gateway=${networkPlan.guestGatewayIp}`, - 'awf.guest-interface=eth0', - ].join(' '); -} - -function formatError(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -class BoundedOutputCapture { - private buffer = Buffer.alloc(0); - - constructor(private readonly maximumBytes: number) {} - - append(chunk: Buffer | string): void { - const next = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - this.buffer = Buffer.concat([this.buffer, next]); - if (this.buffer.length > this.maximumBytes) { - this.buffer = this.buffer.subarray(this.buffer.length - this.maximumBytes); - } - } - - contents(): Buffer { - return this.buffer; - } -} diff --git a/src/firecracker/preflight.test.ts b/src/firecracker/preflight.test.ts deleted file mode 100644 index a82c2b6f6..000000000 --- a/src/firecracker/preflight.test.ts +++ /dev/null @@ -1,401 +0,0 @@ -import { constants } from 'fs'; -import { createHash } from 'crypto'; -import { promises as fs } from 'fs'; -import execa from 'execa'; -import * as os from 'os'; -import * as path from 'path'; -import type { FirecrackerOptions } from '../types/runtime-options'; -import { - calculateSha256, - firecrackerPreflightTestHelpers, - parseFirecrackerVersion, - runFirecrackerPreflight, - type FirecrackerPreflightDependencies, -} from './preflight'; - -jest.mock('execa'); - -const digest = 'a'.repeat(64); -const mockedExeca = execa as jest.MockedFunction; - -function config(overrides: Partial = {}): FirecrackerOptions { - return { - previewEnabled: true, - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/vmlinux', - rootfsPath: '/opt/rootfs.ext4', - supervisorPath: '/opt/awf-supervisor', - vcpuCount: 2, - memoryMib: 512, - apiTimeoutMs: 5000, - ...overrides, - }; -} - -function dependencies( - overrides: Partial = {}, -): Partial { - return { - platform: 'linux', - arch: 'x64', - uid: 1000, - access: jest.fn().mockResolvedValue(undefined), - lstat: jest.fn().mockResolvedValue({ - isFile: () => true, - isSymbolicLink: () => false, - mode: 0o100755, - uid: 0, - }), - runVersion: jest.fn().mockResolvedValue('Firecracker v1.16.1'), - sha256: jest.fn().mockResolvedValue(digest), - assertToolAvailable: jest.fn(async (tool: string) => `/usr/bin/${tool}`), - assertHostPolicy: jest.fn().mockResolvedValue(2), - assertDockerInfrastructure: jest.fn().mockResolvedValue(undefined), - ...overrides, - }; -} - -describe('Firecracker preflight', () => { - let originalPath: string | undefined; - - beforeEach(() => { - originalPath = process.env.PATH; - mockedExeca.mockReset(); - }); - - afterEach(() => { - delete process.env.SUDO_UID; - jest.restoreAllMocks(); - if (originalPath === undefined) delete process.env.PATH; - else process.env.PATH = originalPath; - }); - - it('runs default version, tool, and digest host probes', async () => { - const defaults = firecrackerPreflightTestHelpers.defaultDependencies; - mockedExeca - .mockResolvedValueOnce({ - exitCode: 0, - stdout: `node ${process.version.slice(1)}`, - stderr: '', - } as never) - .mockResolvedValueOnce({ - exitCode: 1, - stdout: '', - stderr: 'unsupported flag', - } as never); - await expect(defaults.runVersion(process.execPath)).resolves.toContain( - process.version.slice(1), - ); - await expect(defaults.runVersion('/bin/false')).rejects.toThrow( - /--version" exited with code/, - ); - - process.env.PATH = `${path.delimiter}/usr/bin`; - await expect(defaults.assertToolAvailable('false')) - .resolves.toBe('/usr/bin/false'); - await expect(defaults.assertToolAvailable('definitely-not-an-awf-tool')) - .rejects.toThrow(/was not found on PATH/); - - const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-preflight-digest-')); - const target = path.join(directory, 'artifact'); - try { - await fs.writeFile(target, 'verified artifact'); - await expect(calculateSha256(target)).resolves.toBe( - createHash('sha256').update('verified artifact').digest('hex'), - ); - } finally { - await fs.rm(directory, { recursive: true, force: true }); - } - }); - - it('runs layer-6 host policy and Docker probes through the default helper', async () => { - const defaults = firecrackerPreflightTestHelpers.defaultDependencies; - jest.spyOn(process, 'getuid').mockReturnValue(0); - const access = jest.spyOn(fs, 'access').mockResolvedValue(undefined); - await expect(defaults.assertHostPolicy()).resolves.toBe(2); - expect(access).toHaveBeenCalledWith( - '/proc/sys/kernel/seccomp/actions_avail', - constants.R_OK, - ); - - mockedExeca.mockResolvedValue({ - exitCode: 0, - stdout: 'available', - stderr: '', - } as never); - await expect(defaults.assertDockerInfrastructure()).resolves.toBeUndefined(); - expect(mockedExeca).toHaveBeenCalledWith( - 'docker', - ['compose', 'version'], - expect.objectContaining({ timeout: 10_000, reject: false }), - ); - }); - - it('reports layer-6 host policy and Docker probe failures', async () => { - const defaults = firecrackerPreflightTestHelpers.defaultDependencies; - jest.spyOn(process, 'getuid').mockReturnValue(1000); - await expect(defaults.assertHostPolicy()).rejects.toThrow(/require root/); - - mockedExeca.mockResolvedValue({ - exitCode: 1, - stdout: '', - stderr: 'daemon unavailable', - } as never); - await expect(defaults.assertDockerInfrastructure()) - .rejects.toThrow(/docker info failed.*daemon unavailable/); - }); - - it('supports cgroup v1 and reports missing kernel or cgroup controls', async () => { - const defaults = firecrackerPreflightTestHelpers.defaultDependencies; - jest.spyOn(process, 'getuid').mockReturnValue(0); - const access = jest.spyOn(fs, 'access'); - access.mockImplementation(async (filePath) => { - if (filePath === '/sys/fs/cgroup/cgroup.controllers') { - throw new Error('no cgroup v2'); - } - }); - await expect(defaults.assertHostPolicy()).resolves.toBe(1); - - access.mockReset().mockRejectedValue(new Error('kernel control denied')); - await expect(defaults.assertHostPolicy()) - .rejects.toThrow(/host kernel policy.*kernel control denied/); - - access.mockReset().mockImplementation(async (filePath) => { - if (String(filePath).startsWith('/proc/')) return; - throw new Error('no usable cgroup'); - }); - await expect(defaults.assertHostPolicy()) - .rejects.toThrow(/requires a writable cgroup v1 hierarchy.*no usable cgroup/); - - access.mockReset().mockRejectedValue('kernel control string failure'); - await expect(defaults.assertHostPolicy()) - .rejects.toThrow(/host kernel policy.*kernel control string failure/); - - access.mockReset().mockImplementation(async (filePath) => { - if (String(filePath).startsWith('/proc/')) return; - return Promise.reject('cgroup string failure'); - }); - await expect(defaults.assertHostPolicy()) - .rejects.toThrow(/requires a writable cgroup v1 hierarchy.*cgroup string failure/); - }); - - it('parses Firecracker and jailer release output', () => { - expect(parseFirecrackerVersion('Firecracker v1.16.1')).toBe('1.16.1'); - expect(parseFirecrackerVersion('Jailer v1.16.1')).toBe('1.16.1'); - expect(() => parseFirecrackerVersion('unknown')).toThrow(/Could not parse/); - }); - - it('pins matching Firecracker and jailer v1.16.1 and verifies configured digests', async () => { - const deps = dependencies(); - const result = await runFirecrackerPreflight(config({ - sha256: { - firecracker: digest, - jailer: digest, - kernel: digest, - rootfs: digest, - supervisor: digest, - }, - }), deps); - - expect(result.version).toBe('1.16.1'); - expect(result.cgroupVersion).toBe(2); - expect(deps.access).toHaveBeenCalledWith( - '/dev/kvm', - constants.R_OK | constants.W_OK, - ); - expect(deps.sha256).toHaveBeenCalledTimes(5); - expect(deps.assertToolAvailable).toHaveBeenCalledTimes(7); - expect(result.tools).toEqual({ - ip: '/usr/bin/ip', - nft: '/usr/bin/nft', - sysctl: '/usr/bin/sysctl', - mke2fs: '/usr/bin/mke2fs', - debugfs: '/usr/bin/debugfs', - e2fsck: '/usr/bin/e2fsck', - rsync: '/usr/bin/rsync', - }); - expect(result.cgroupVersion).toBe(2); - }); - - it('rejects inaccessible KVM without checking artifacts', async () => { - const access = jest.fn().mockRejectedValue(new Error('EACCES')); - const lstat = jest.fn(); - await expect(runFirecrackerPreflight( - config(), - dependencies({ access, lstat }), - )).rejects.toThrow(/readable and writable \/dev\/kvm.*EACCES/); - expect(lstat).not.toHaveBeenCalled(); - }); - - it('rejects mismatched versions, unsafe permissions, and digest mismatches', async () => { - const runVersion = jest.fn() - .mockResolvedValueOnce('Firecracker v1.16.1') - .mockResolvedValueOnce('Jailer v1.15.0'); - await expect(runFirecrackerPreflight( - config(), - dependencies({ runVersion }), - )).rejects.toThrow(/versions must match/); - - await expect(runFirecrackerPreflight( - config(), - dependencies({ - lstat: jest.fn().mockResolvedValue({ - isFile: () => true, - isSymbolicLink: () => false, - mode: 0o100777, - uid: 1000, - }), - }), - )).rejects.toThrow(/must not be group- or world-writable/); - - await expect(runFirecrackerPreflight( - config({ sha256: { kernel: digest } }), - dependencies({ sha256: jest.fn().mockResolvedValue('b'.repeat(64)) }), - )).rejects.toThrow(/SHA-256 mismatch/); - - await expect(runFirecrackerPreflight( - config({ sha256: { kernel: 'bad' } }), - dependencies(), - )).rejects.toThrow(/must contain exactly 64 hexadecimal/); - }); - - it('rejects missing artifacts, unsupported hosts, and unavailable tools', async () => { - await expect(runFirecrackerPreflight( - config({ supervisorPath: undefined }), - dependencies(), - )).rejects.toThrow(/requires guest kernel, rootfs, and supervisor/); - await expect(runFirecrackerPreflight( - config(), - dependencies({ platform: 'darwin' }), - )).rejects.toThrow(/requires Linux with KVM/); - await expect(runFirecrackerPreflight( - config(), - dependencies({ arch: 'ia32' }), - )).rejects.toThrow(/supports only x86_64 and aarch64/); - await expect(runFirecrackerPreflight( - config(), - dependencies({ - assertToolAvailable: jest.fn().mockRejectedValue('missing'), - }), - )).rejects.toThrow(/requires host tool "ip": missing/); - }); - - it('rejects untrusted artifact files and inaccessible paths', async () => { - await expect(runFirecrackerPreflight( - config({ firecrackerBinary: 'relative/firecracker' }), - dependencies(), - )).rejects.toThrow(/path must be absolute/); - await expect(runFirecrackerPreflight( - config(), - dependencies({ - lstat: jest.fn(async (filePath: string) => ( - filePath === '/opt/firecracker' - ? { - isFile: () => false, - isSymbolicLink: () => true, - mode: 0o120777, - uid: 0, - } - : { - isFile: () => false, - isSymbolicLink: () => false, - mode: 0o040755, - uid: 0, - } - )), - }), - )).rejects.toThrow(/regular file and not a symbolic link/); - await expect(runFirecrackerPreflight( - config(), - dependencies({ - lstat: jest.fn().mockResolvedValue({ - isFile: () => true, - isSymbolicLink: () => false, - mode: 0o100755, - uid: 4000, - }), - }), - )).rejects.toThrow(/must be owned by root or uid/); - await expect(runFirecrackerPreflight( - config(), - dependencies({ - access: jest.fn(async (filePath: string) => { - if (filePath !== '/dev/kvm') throw new Error('EACCES'); - }), - }), - )).rejects.toThrow(/does not have the required host access/); - }); - - it('uses SUDO_UID as trusted owner when running under sudo', async () => { - process.env.SUDO_UID = '2001'; - const lstat = jest.fn().mockResolvedValue({ - isFile: () => true, - isSymbolicLink: () => false, - mode: 0o100755, - uid: 2001, - }); - await expect(runFirecrackerPreflight( - config(), - dependencies({ uid: undefined, lstat }), - )).resolves.toMatchObject({ version: '1.16.1' }); - }); - - it('rejects writable or symlinked parent directories', async () => { - const lstat = jest.fn(async (filePath: string) => { - if (filePath === '/opt') { - return { - isFile: () => false, - isSymbolicLink: () => false, - mode: 0o040777, - uid: 0, - }; - } - return { - isFile: () => true, - isSymbolicLink: () => false, - mode: 0o100755, - uid: 0, - }; - }); - await expect(runFirecrackerPreflight( - config(), - dependencies({ lstat }), - )).rejects.toThrow(/parent directory must not be group- or world-writable/); - - const symlinkParent = jest.fn(async (filePath: string) => { - if (filePath === '/opt') { - return { - isFile: () => false, - isSymbolicLink: () => true, - mode: 0o040755, - uid: 0, - }; - } - return { - isFile: () => true, - isSymbolicLink: () => false, - mode: 0o100755, - uid: 0, - }; - }); - await expect(runFirecrackerPreflight( - config(), - dependencies({ lstat: symlinkParent }), - )).rejects.toThrow(/parent directory must not be a symbolic link/); - }); - - it('rejects user-controlled PATH tools', async () => { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-preflight-tool-')); - const tool = path.join(directory, 'ip'); - await fs.writeFile(tool, '#!/bin/sh\n'); - await fs.chmod(tool, 0o755); - process.env.PATH = directory; - try { - await expect(firecrackerPreflightTestHelpers.defaultDependencies.assertToolAvailable('ip')) - .rejects.toThrow(/trusted host tool "ip"/); - } finally { - await fs.rm(directory, { recursive: true, force: true }); - } - }); -}); diff --git a/src/firecracker/preflight.ts b/src/firecracker/preflight.ts deleted file mode 100644 index 6b89d2555..000000000 --- a/src/firecracker/preflight.ts +++ /dev/null @@ -1,405 +0,0 @@ -import { createHash } from 'crypto'; -import { createReadStream, constants, promises as fs } from 'fs'; -import * as path from 'path'; -import execa from 'execa'; -import { - FIRECRACKER_RELEASE_VERSION, - type FirecrackerOptions, -} from '../types/runtime-options'; - -export interface FirecrackerPreflightDependencies { - platform: NodeJS.Platform; - arch: string; - uid: number; - access(filePath: string, mode: number): Promise; - lstat(filePath: string): Promise<{ - isFile(): boolean; - isSymbolicLink(): boolean; - mode: number; - uid: number; - }>; - runVersion(binaryPath: string): Promise; - sha256(filePath: string): Promise; - assertToolAvailable(tool: string): Promise; - assertHostPolicy(): Promise<1 | 2>; - assertDockerInfrastructure(): Promise; -} - -export type FirecrackerHostToolPaths = Readonly<{ - ip: string; - nft: string; - sysctl: string; - mke2fs: string; - debugfs: string; - e2fsck: string; - rsync: string; -}>; -const FIRECRACKER_HOST_TOOLS: (keyof FirecrackerHostToolPaths)[] = [ - 'ip', 'nft', 'sysctl', 'mke2fs', 'debugfs', 'e2fsck', 'rsync', -]; - -const defaultDependencies: FirecrackerPreflightDependencies = { - platform: process.platform, - arch: process.arch, - uid: -1, - access: fs.access, - lstat: fs.lstat, - runVersion: async (binaryPath) => { - const result = await execa(binaryPath, ['--version'], { - reject: false, - timeout: 5_000, - stdio: ['ignore', 'pipe', 'pipe'], - }); - if (result.exitCode !== 0) { - throw new Error( - `"${binaryPath} --version" exited with code ${result.exitCode}: ${result.stderr.trim()}`, - ); - } - return `${result.stdout}\n${result.stderr}`.trim(); - }, - sha256: calculateSha256, - assertToolAvailable: async (tool) => { - const searchPath = process.env.PATH ?? ''; - for (const directory of searchPath.split(path.delimiter)) { - if (!directory) continue; - try { - const candidate = path.join(directory, tool); - await assertTrustedHostTool(tool, candidate); - return candidate; - } catch { - // Continue searching the bounded host PATH. - } - } - throw new Error(`required trusted host tool "${tool}" was not found on PATH`); - }, - assertHostPolicy: async () => { - if (process.getuid?.() !== 0) { - throw new Error( - 'Firecracker jailer and network setup require root; invoke awf through sudo from a non-root account', - ); - } - try { - await fs.access('/proc/sys/net/ipv4/ip_forward', constants.R_OK); - await fs.access('/proc/sys/net/ipv6/conf/all/disable_ipv6', constants.R_OK); - await fs.access('/proc/sys/kernel/seccomp/actions_avail', constants.R_OK); - } catch (error) { - throw new Error( - 'host kernel policy does not expose required network namespace and seccomp controls: ' + - `${error instanceof Error ? error.message : String(error)}`, - ); - } - try { - await fs.access('/sys/fs/cgroup/cgroup.controllers', constants.R_OK); - return 2; - } catch { - try { - await fs.access('/sys/fs/cgroup', constants.R_OK | constants.W_OK); - return 1; - } catch (error) { - throw new Error( - 'Firecracker jailer requires a writable cgroup v1 hierarchy or cgroup v2 controllers: ' + - `${error instanceof Error ? error.message : String(error)}`, - ); - } - } - }, - assertDockerInfrastructure: async () => { - for (const args of [['info'], ['compose', 'version']] as const) { - const result = await execa('docker', [...args], { - reject: false, - timeout: 10_000, - stdio: ['ignore', 'pipe', 'pipe'], - }); - if (result.exitCode !== 0) { - throw new Error( - `docker ${args.join(' ')} failed with code ${result.exitCode}: ${result.stderr.trim()}`, - ); - } - } - }, -}; - -/** @internal Exposed only for focused host-probe tests. */ -export const firecrackerPreflightTestHelpers = { defaultDependencies }; - -export interface FirecrackerPreflightResult { - version: string; - firecrackerBinary: string; - jailerBinary: string; - kernelPath: string; - rootfsPath: string; - supervisorPath: string; - tools: FirecrackerHostToolPaths; - cgroupVersion: 1 | 2; -} - -async function assertTrustedHostTool(label: string, filePath: string): Promise { - if (!path.isAbsolute(filePath)) { - throw new Error(`host tool "${label}" path must be absolute: ${filePath}`); - } - const { root } = path.parse(filePath); - const segments = filePath.slice(root.length).split('/').filter(Boolean); - let ancestor = root; - for (const segment of segments.slice(0, -1)) { - ancestor = path.join(ancestor, segment); - const stat = await fs.lstat(ancestor); - if (stat.isSymbolicLink() || (stat.mode & 0o022) !== 0 || stat.uid !== 0) { - throw new Error(`host tool "${label}" has an untrusted parent directory: ${ancestor}`); - } - } - const stat = await fs.lstat(filePath); - if ( - stat.isSymbolicLink() || - !stat.isFile() || - (stat.mode & 0o022) !== 0 || - stat.uid !== 0 - ) { - throw new Error(`host tool "${label}" must be a root-owned non-writable regular file: ${filePath}`); - } - await fs.access(filePath, constants.X_OK); -} - -export function parseFirecrackerVersion(output: string): string { - const match = output.match(/\bv?(\d+\.\d+\.\d+)\b/); - if (!match) { - throw new Error(`Could not parse Firecracker version from: ${JSON.stringify(output)}`); - } - return match[1]; -} - -export async function calculateSha256(filePath: string): Promise { - const hash = createHash('sha256'); - const stream = createReadStream(filePath); - for await (const chunk of stream) { - hash.update(chunk as Buffer); - } - return hash.digest('hex'); -} - -async function assertTrustedRegularFile( - label: string, - filePath: string, - accessMode: number, - dependencies: FirecrackerPreflightDependencies, -): Promise { - if (!path.isAbsolute(filePath)) { - throw new Error(`${label} path must be absolute: ${filePath}`); - } - await assertTrustedAncestorChain(label, filePath, dependencies); - const stat = await dependencies.lstat(filePath); - if (stat.isSymbolicLink() || !stat.isFile()) { - throw new Error(`${label} must be a regular file and not a symbolic link: ${filePath}`); - } - if ((stat.mode & 0o022) !== 0) { - throw new Error(`${label} must not be group- or world-writable: ${filePath}`); - } - if (stat.uid !== 0 && stat.uid !== dependencies.uid) { - throw new Error( - `${label} must be owned by root or uid ${dependencies.uid}; found uid ${stat.uid}: ${filePath}`, - ); - } - try { - await dependencies.access(filePath, accessMode); - } catch (error) { - throw new Error( - `${label} does not have the required host access: ${filePath}: ` + - `${error instanceof Error ? error.message : String(error)}`, - ); - } -} - -function parsePositiveUid(value: string | undefined): number | undefined { - if (!value || !/^[1-9]\d*$/.test(value)) return undefined; - return Number(value); -} - -function resolveTrustedOperatorUid(): number { - return parsePositiveUid(process.env.SUDO_UID) ?? (process.getuid?.() ?? -1); -} - -async function assertTrustedAncestorChain( - label: string, - filePath: string, - dependencies: FirecrackerPreflightDependencies, -): Promise { - const { root } = path.parse(filePath); - const segments = filePath.slice(root.length).split('/').filter((segment) => segment.length > 0); - let ancestor = root; - for (const segment of segments.slice(0, -1)) { - ancestor = path.join(ancestor, segment); - const stat = await dependencies.lstat(ancestor); - if (stat.isSymbolicLink()) { - throw new Error( - `${label} parent directory must not be a symbolic link: ${ancestor}`, - ); - } - if ((stat.mode & 0o022) !== 0) { - throw new Error( - `${label} parent directory must not be group- or world-writable: ${ancestor}`, - ); - } - if (stat.uid !== 0 && stat.uid !== dependencies.uid) { - throw new Error( - `${label} parent directory must be owned by root or uid ${dependencies.uid}; ` + - `found uid ${stat.uid}: ${ancestor}`, - ); - } - } -} - -async function assertDigest( - label: string, - filePath: string, - expected: string | undefined, - dependencies: FirecrackerPreflightDependencies, -): Promise { - if (!expected) return; - if (!/^[a-fA-F0-9]{64}$/.test(expected)) { - throw new Error(`${label} SHA-256 must contain exactly 64 hexadecimal characters`); - } - const actual = await dependencies.sha256(filePath); - if (actual.toLowerCase() !== expected.toLowerCase()) { - throw new Error( - `${label} SHA-256 mismatch: expected ${expected.toLowerCase()}, got ${actual.toLowerCase()}`, - ); - } -} - -/** - * Fail-closed host and artifact validation for Firecracker v1.16.1. - */ -export async function runFirecrackerPreflight( - config: FirecrackerOptions, - overrides: Partial = {}, -): Promise { - const dependencies = { - ...defaultDependencies, - ...overrides, - uid: overrides.uid ?? resolveTrustedOperatorUid(), - }; - if (dependencies.platform !== 'linux') { - throw new Error(`Firecracker requires Linux with KVM; found ${dependencies.platform}`); - } - if (dependencies.arch !== 'x64' && dependencies.arch !== 'arm64') { - throw new Error( - `Firecracker supports only x86_64 and aarch64; found Node architecture ${dependencies.arch}`, - ); - } - if (!config.kernelPath || !config.rootfsPath || !config.supervisorPath) { - throw new Error( - 'Firecracker requires guest kernel, rootfs, and supervisor artifact paths', - ); - } - - try { - await dependencies.access('/dev/kvm', constants.R_OK | constants.W_OK); - } catch (error) { - throw new Error( - 'Firecracker requires readable and writable /dev/kvm: ' + - `${error instanceof Error ? error.message : String(error)}`, - ); - } - const cgroupVersion = await dependencies.assertHostPolicy(); - await dependencies.assertDockerInfrastructure(); - await assertTrustedRegularFile( - 'Firecracker binary', - config.firecrackerBinary, - constants.R_OK | constants.X_OK, - dependencies, - ); - - const tools = {} as Record; - for (const tool of FIRECRACKER_HOST_TOOLS) { - try { - tools[tool] = await dependencies.assertToolAvailable(tool); - } catch (error) { - throw new Error( - `Firecracker requires host tool "${tool}": ` + - `${error instanceof Error ? error.message : String(error)}`, - ); - } - } - await assertTrustedRegularFile( - 'Firecracker jailer binary', - config.jailerBinary, - constants.R_OK | constants.X_OK, - dependencies, - ); - await assertTrustedRegularFile( - 'Firecracker guest kernel', - config.kernelPath, - constants.R_OK, - dependencies, - ); - await assertTrustedRegularFile( - 'Firecracker rootfs', - config.rootfsPath, - constants.R_OK, - dependencies, - ); - await assertTrustedRegularFile( - 'Firecracker guest supervisor', - config.supervisorPath, - constants.R_OK, - dependencies, - ); - - const firecrackerVersion = parseFirecrackerVersion( - await dependencies.runVersion(config.firecrackerBinary), - ); - const jailerVersion = parseFirecrackerVersion( - await dependencies.runVersion(config.jailerBinary), - ); - if (firecrackerVersion !== jailerVersion) { - throw new Error( - `Firecracker and jailer versions must match; found ${firecrackerVersion} and ${jailerVersion}`, - ); - } - if (firecrackerVersion !== FIRECRACKER_RELEASE_VERSION) { - throw new Error( - `Firecracker is pinned to v${FIRECRACKER_RELEASE_VERSION}; found v${firecrackerVersion}`, - ); - } - - await assertDigest( - 'Firecracker binary', - config.firecrackerBinary, - config.sha256?.firecracker, - dependencies, - ); - await assertDigest( - 'Firecracker jailer binary', - config.jailerBinary, - config.sha256?.jailer, - dependencies, - ); - await assertDigest( - 'Firecracker guest kernel', - config.kernelPath, - config.sha256?.kernel, - dependencies, - ); - await assertDigest( - 'Firecracker rootfs', - config.rootfsPath, - config.sha256?.rootfs, - dependencies, - ); - await assertDigest( - 'Firecracker guest supervisor', - config.supervisorPath, - config.sha256?.supervisor, - dependencies, - ); - - return { - version: firecrackerVersion, - firecrackerBinary: config.firecrackerBinary, - jailerBinary: config.jailerBinary, - kernelPath: config.kernelPath, - rootfsPath: config.rootfsPath, - supervisorPath: config.supervisorPath, - tools, - cgroupVersion, - }; -} diff --git a/src/firecracker/runtime-validation.test.ts b/src/firecracker/runtime-validation.test.ts deleted file mode 100644 index 9204e4967..000000000 --- a/src/firecracker/runtime-validation.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { WrapperConfig } from '../types'; -import { - assertFirecrackerPreSecurityCompatibility, - assertFirecrackerRuntimeCompatibility, - assertFirecrackerSelection, - requireFirecrackerConfig, -} from './runtime-validation'; - -const digest = 'a'.repeat(64); - -function config(overrides: Partial = {}): WrapperConfig { - return { - containerRuntime: 'firecracker', - networkIsolation: true, - legacySecurity: false, - enableApiProxy: true, - enableDind: false, - enableHostAccess: false, - tty: false, - firecracker: { - previewEnabled: true, - firecrackerBinary: '/opt/firecracker', - jailerBinary: '/opt/jailer', - kernelPath: '/opt/kernel', - rootfsPath: '/opt/rootfs', - supervisorPath: '/opt/supervisor', - vcpuCount: 2, - memoryMib: 512, - apiTimeoutMs: 5000, - sha256: { - firecracker: digest, - jailer: digest, - kernel: digest, - rootfs: digest, - supervisor: digest, - }, - }, - ...overrides, - } as WrapperConfig; -} - -describe('Firecracker runtime validation', () => { - it('accepts only a complete explicitly selected preview', () => { - const valid = config(); - expect(() => assertFirecrackerSelection(valid)).not.toThrow(); - expect(() => assertFirecrackerRuntimeCompatibility(valid)).not.toThrow(); - expect(requireFirecrackerConfig(valid)).toBe(valid.firecracker); - - expect(() => assertFirecrackerSelection(config({ - containerRuntime: 'gvisor', - }))).toThrow(/require --container-runtime firecracker/); - expect(() => requireFirecrackerConfig(config({ - containerRuntime: 'gvisor', - }))).toThrow(/resolved without Firecracker runtime configuration/); - }); - - it.each([ - [{ firecracker: { ...config().firecracker!, previewEnabled: false } }, /explicit --firecracker-preview/], - [{ networkIsolation: false }, /strict --network-isolation/], - [{ legacySecurity: true }, /strict --network-isolation/], - [{ enableApiProxy: false }, /API proxy credential isolation/], - [{ - firecracker: { - ...config().firecracker!, - supervisorPath: undefined, - }, - }, /explicit kernel, rootfs, and guest supervisor/], - [{ - firecracker: { - ...config().firecracker!, - sha256: { ...config().firecracker!.sha256, supervisor: undefined }, - }, - }, /requires SHA-256 digests/], - ] as const)('rejects incomplete runtime configuration %#', (overrides, error) => { - expect(() => assertFirecrackerRuntimeCompatibility( - config(overrides as Partial), - )).toThrow(error); - }); - - it.each([ - [{ networkIsolation: false }, /cannot disable --network-isolation/], - [{ enableDind: true }, /Docker-in-Docker/], - [{ dockerHostPathPrefix: '/host' }, /split filesystems/], - [{ runnerTopology: 'arc-dind' }, /split filesystems/], - [{ enableHostAccess: true }, /host access/], - [{ allowHostPorts: ['8080'] }, /host access/], - [{ allowHostServicePorts: ['5432'] }, /host access/], - [{ volumeMounts: ['/tmp:/tmp'] }, /additional host volume mounts/], - [{ topologyAttach: ['gateway'] }, /MCP gateway path/], - [{ difcProxyHost: 'proxy:443' }, /MCP gateway path/], - [{ enclaves: { enabled: true } }, /MCP gateway path/], - [{ dnsOverHttps: 'https://dns.example/dns-query' }, /DNS-over-HTTPS/], - [{ tty: true }, /does not support --tty/], - [{ awfDockerHost: 'tcp://localhost:2375' }, /local Unix-socket Docker daemon/], - ] as const)('rejects unsupported preview policy %#', (overrides, error) => { - expect(() => assertFirecrackerPreSecurityCompatibility( - config(overrides as Partial), - )).toThrow(error); - }); - - it('accepts a local Unix Docker socket', () => { - expect(() => assertFirecrackerPreSecurityCompatibility(config({ - awfDockerHost: 'unix:///var/run/docker.sock', - }))).not.toThrow(); - }); -}); diff --git a/src/firecracker/runtime-validation.ts b/src/firecracker/runtime-validation.ts deleted file mode 100644 index 3ca237b36..000000000 --- a/src/firecracker/runtime-validation.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { getLocalDockerEnv } from '../docker-host'; -import type { FirecrackerOptions, WrapperConfig } from '../types'; - -export function assertFirecrackerSelection(config: WrapperConfig): void { - if (config.firecracker && config.containerRuntime !== 'firecracker') { - throw new Error( - 'Firecracker options require --container-runtime firecracker', - ); - } -} - -export function assertFirecrackerRuntimeCompatibility( - config: WrapperConfig, - firecracker = requireFirecrackerConfig(config), -): void { - if (!firecracker.previewEnabled) { - throw new Error( - 'Firecracker workload execution requires explicit --firecracker-preview opt-in', - ); - } - if (!config.networkIsolation || config.legacySecurity) { - throw new Error('Firecracker preview requires strict --network-isolation security'); - } - if (!config.enableApiProxy) { - throw new Error('Firecracker preview requires API proxy credential isolation'); - } - assertFirecrackerPreSecurityCompatibility(config); - if (!firecracker.kernelPath || !firecracker.rootfsPath || !firecracker.supervisorPath) { - throw new Error( - 'Firecracker preview requires explicit kernel, rootfs, and guest supervisor artifacts', - ); - } - const digests = firecracker.sha256; - if ( - !digests?.firecracker || - !digests.jailer || - !digests.kernel || - !digests.rootfs || - !digests.supervisor - ) { - throw new Error( - 'Firecracker preview requires SHA-256 digests for firecracker, jailer, kernel, rootfs, and supervisor', - ); - } -} - -export function assertFirecrackerPreSecurityCompatibility(config: WrapperConfig): void { - if (config.networkIsolation === false) { - throw new Error('Firecracker preview cannot disable --network-isolation'); - } - if ( - config.enableDind || - config.dockerHostPathPrefix || - config.runnerTopology === 'arc-dind' - ) { - throw new Error('Firecracker preview does not support Docker-in-Docker or split filesystems'); - } - if (config.enableHostAccess || config.allowHostPorts || config.allowHostServicePorts) { - throw new Error('Firecracker preview does not support host access'); - } - if (config.volumeMounts?.length) { - throw new Error('Firecracker preview does not support additional host volume mounts'); - } - if ( - config.topologyAttach?.length || - config.difcProxyHost || - config.enclaves?.enabled - ) { - throw new Error( - 'Firecracker preview does not yet prove the MCP gateway path; topology peers and enclaves are disabled', - ); - } - if (config.dnsOverHttps) { - throw new Error('Firecracker preview does not support DNS-over-HTTPS'); - } - if (config.tty) { - throw new Error('Firecracker preview guest supervisor does not support --tty'); - } - const dockerHost = config.awfDockerHost ?? getLocalDockerEnv().DOCKER_HOST; - if (dockerHost && !dockerHost.startsWith('unix://')) { - throw new Error( - 'Firecracker preview requires a local Unix-socket Docker daemon so its bridge is host-visible', - ); - } -} - -export function requireFirecrackerConfig(config: WrapperConfig): FirecrackerOptions { - if (config.containerRuntime !== 'firecracker' || !config.firecracker) { - throw new Error('Firecracker backend resolved without Firecracker runtime configuration'); - } - return config.firecracker; -} diff --git a/src/microvm/guest-protocol.ts b/src/microvm/guest-protocol.ts index 479e117ce..a689596f9 100644 --- a/src/microvm/guest-protocol.ts +++ b/src/microvm/guest-protocol.ts @@ -1,7 +1,7 @@ /** * AWF framed guest-supervisor protocol. Transport-independent: the same * length-prefixed JSON framing is used regardless of which VMM backend - * (Firecracker today, others later) carries the bytes over vsock/UDS. + * carries the bytes over vsock/UDS. */ export const GUEST_PROTOCOL_VERSION = 1 as const; export const GUEST_MAX_FRAME_BYTES = 1024 * 1024; diff --git a/src/microvm/network-plan.ts b/src/microvm/network-plan.ts index ff45fdd28..61038211f 100644 --- a/src/microvm/network-plan.ts +++ b/src/microvm/network-plan.ts @@ -46,10 +46,10 @@ export function createMicrovmNetworkPlan( digest[8], ].map((byte) => byte.toString(16).padStart(2, '0')).join(':'); - const namespaceName = `awffc-${token}`; - const tapName = `fct${token}`; - const hostVethName = `fch${token}`; - const namespaceVethName = `fcn${token}`; + const namespaceName = `awfvm-${token}`; + const tapName = `vmt${token}`; + const hostVethName = `vmh${token}`; + const namespaceVethName = `vmn${token}`; const nftTableName = `awf_fc_${token}`; for (const [label, name] of [ ['TAP', tapName], diff --git a/src/microvm/network-types.ts b/src/microvm/network-types.ts index 9fae9e0a5..be7ea46eb 100644 --- a/src/microvm/network-types.ts +++ b/src/microvm/network-types.ts @@ -6,9 +6,7 @@ export interface MicrovmNetworkHostTools { /** * Generic tap-device descriptor a VMM's network-interface configuration API - * needs. Field names intentionally match the wire shape already used by - * Firecracker's `PUT /network-interfaces`; a future backend with a - * differently-shaped API translates from this structural descriptor. + * needs. Backends translate this structural descriptor to their API shape. */ export interface MicrovmTapInterface { readonly iface_id: string; @@ -46,9 +44,7 @@ export interface MicrovmNetworkPlanOptions { * layer) keeps working normally -- exactly the asymmetric RX-works/ * TX-stalls pattern observed live (tap RX=10 packets, TX=1 packet, * despite 23 response packets already having arrived on the veth). - * Firecracker's own tap handling does not request `IFF_VNET_HDR`, so - * this defaults to `false` (this shared code's prior, Firecracker-only - * behavior) and Cloud Hypervisor opts in explicitly. + * This defaults to `false`; Cloud Hypervisor opts in explicitly. */ readonly tapVnetHdr?: boolean; } diff --git a/src/microvm/network.test.ts b/src/microvm/network.test.ts index 0a79ab97c..93de83153 100644 --- a/src/microvm/network.test.ts +++ b/src/microvm/network.test.ts @@ -323,7 +323,7 @@ describe('microVM network lifecycle', () => { expect(calls).toEqual([]); }); - it('creates the TAP with vnet_hdr only when the plan opts in (Cloud Hypervisor requires it; Firecracker does not)', async () => { + it('creates the TAP with vnet_hdr only when the plan opts in', async () => { // Regression test: Cloud Hypervisor's own tap handling // (Tap::open_named() in net_util/src/tap.rs) always re-opens the tap // with IFF_VNET_HDR requested. If the tap wasn't *created* with that @@ -332,9 +332,8 @@ describe('microVM network lifecycle', () => { // the host-side veth/nft layer) keeps working, but host-to-guest // traffic silently never reaches the guest -- observed live as a tap // RX=10 packets / TX=1 packet asymmetry despite response packets - // already having arrived on the host-side veth. Firecracker's own tap - // handling does not request IFF_VNET_HDR, so this flag defaults to - // false (unchanged prior behavior) and is opted into explicitly. + // already having arrived on the host-side veth. The flag defaults to false + // and is opted into explicitly by runtimes that require it. const withVnetHdr = createPlan('run-vnet-hdr', { tapVnetHdr: true }); const { calls: vnetHdrCalls, commands: vnetHdrCommands } = commandHarness(); await new MicrovmNetworkManager(withVnetHdr, vnetHdrCommands).setup(); @@ -645,19 +644,19 @@ describe('LinuxNetworkCommands.nftInNamespace ruleset file handling', () => { it('writes the ruleset to a file and passes its path instead of "-f -"', async () => { const { rulesetFile, calls, commands } = rulesetFileHarness(); - await commands.nftInNamespace('awffc-test', ['-f', '-'], 'table inet awf {}'); + await commands.nftInNamespace('awfvm-test', ['-f', '-'], 'table inet awf {}'); expect(rulesetFile.write).toHaveBeenCalledWith('table inet awf {}'); expect(calls).toEqual([{ command: 'ip', - args: ['netns', 'exec', 'awffc-test', 'nft', '-f', '/tmp/awf-nft-fake.nft:17'], + args: ['netns', 'exec', 'awfvm-test', 'nft', '-f', '/tmp/awf-nft-fake.nft:17'], options: { reject: true }, }]); }); it('removes the temp file after a successful nft invocation', async () => { const { rulesetFile, commands } = rulesetFileHarness(); - await commands.nftInNamespace('awffc-test', ['-f', '-'], 'table inet awf {}'); + await commands.nftInNamespace('awfvm-test', ['-f', '-'], 'table inet awf {}'); expect(rulesetFile.remove).toHaveBeenCalledWith('/tmp/awf-nft-fake.nft:17'); }); @@ -676,20 +675,20 @@ describe('LinuxNetworkCommands.nftInNamespace ruleset file handling', () => { ); await expect( - commands.nftInNamespace('awffc-test', ['-f', '-'], 'table inet awf {}'), + commands.nftInNamespace('awfvm-test', ['-f', '-'], 'table inet awf {}'), ).rejects.toThrow('nft rejected the ruleset'); expect(rulesetFile.remove).toHaveBeenCalledWith('/tmp/awf-nft-fake.nft'); }); it('skips ruleset file handling entirely when no input is given', async () => { const { rulesetFile, calls, commands } = rulesetFileHarness(); - await commands.nftInNamespace('awffc-test', ['list', 'ruleset']); + await commands.nftInNamespace('awfvm-test', ['list', 'ruleset']); expect(rulesetFile.write).not.toHaveBeenCalled(); expect(rulesetFile.remove).not.toHaveBeenCalled(); expect(calls).toEqual([{ command: 'ip', - args: ['netns', 'exec', 'awffc-test', 'nft', 'list', 'ruleset'], + args: ['netns', 'exec', 'awfvm-test', 'nft', 'list', 'ruleset'], options: { reject: true }, }]); }); @@ -702,7 +701,7 @@ describe('LinuxNetworkCommands.nftInNamespace ruleset file handling', () => { jest.fn(async () => undefined), ); await expect( - commands.nftInNamespace('awffc-real-fs-test', ['-f', '-'], 'table inet awf { }'), + commands.nftInNamespace('awfvm-real-fs-test', ['-f', '-'], 'table inet awf { }'), ).resolves.toBeUndefined(); }); }); @@ -744,7 +743,7 @@ describe('LinuxNetworkCommands.captureDiagnosticsInNamespace', () => { }), ); - const result = await commands.captureDiagnosticsInNamespace('awffc-test'); + const result = await commands.captureDiagnosticsInNamespace('awfvm-test'); expect(result).toContain('--- nft -a list ruleset (handles + hit counters) ---'); expect(result).toContain('table inet awf_fc_abc123'); @@ -771,7 +770,7 @@ describe('LinuxNetworkCommands.captureDiagnosticsInNamespace', () => { }), ); - const result = await commands.captureDiagnosticsInNamespace('awffc-test'); + const result = await commands.captureDiagnosticsInNamespace('awfvm-test'); expect(result).toContain('(empty or unavailable)'); }); diff --git a/src/microvm/vsock-client.ts b/src/microvm/vsock-client.ts index f75d97aae..990431ef3 100644 --- a/src/microvm/vsock-client.ts +++ b/src/microvm/vsock-client.ts @@ -66,8 +66,8 @@ export class GuestExecutionError extends Error { /** * Host endpoint for a VMM's CONNECT-over-UDS vsock mapping (the convention - * used by Firecracker and other VMMs that expose vsock via a host UDS - * socket). Speaks the AWF framed guest protocol once the handshake + * used by VMMs that expose vsock via a host UDS socket). Speaks the AWF + * framed guest protocol once the handshake * completes; independent of which VMM backend owns the socket. */ export class MicrovmVsockClient { diff --git a/src/microvm/workspace.ts b/src/microvm/workspace.ts index 66392b08a..fe6782188 100644 --- a/src/microvm/workspace.ts +++ b/src/microvm/workspace.ts @@ -16,8 +16,7 @@ const WORKSPACE_BLOCK_BYTES = 4096; const E2FSCK_REPAIR_EXIT_CODE = 1; /** Minimal host tool paths this module needs; a structural subset so callers - * (e.g. Firecracker's preflight-derived tool paths) can pass their own - * richer tool-path record without this module depending on it. */ + * can pass richer tool-path records without this module depending on them. */ export interface MicrovmWorkspaceHostTools { readonly mke2fs: string; readonly debugfs: string; @@ -104,13 +103,13 @@ export class MicrovmWorkspaceImage { private readonly tools?: MicrovmWorkspaceHostTools, ) { assertSafeRunId(config.runId); - this.runDirectory = path.join(config.workDir, 'firecracker-images', config.runId); + this.runDirectory = path.join(config.workDir, 'microvm-images', config.runId); this.stagingDirectory = path.join(this.runDirectory, 'staging'); this.workspaceImagePath = path.join(this.runDirectory, 'workspace.ext4'); this.rootfsImagePath = path.join(this.runDirectory, 'rootfs.ext4'); this.recoveryImagePath = path.join( config.workspacePath, - '.awf-firecracker-recovery', + '.awf-microvm-recovery', `${config.runId}-workspace.ext4`, ); } diff --git a/src/services/agent-service-build.test.ts b/src/services/agent-service-build.test.ts index 209f1ec9e..8f0b29afa 100644 --- a/src/services/agent-service-build.test.ts +++ b/src/services/agent-service-build.test.ts @@ -636,15 +636,5 @@ describe('agent service', () => { expect(agent.extra_hosts?.['api-proxy']).toBeUndefined(); }); - it('keeps Firecracker API proxy ports and networks internal', () => { - const result = generateDockerCompose( - { ...mockConfig, containerRuntime: 'firecracker', enableApiProxy: true }, - { ...mockNetworkConfig, proxyIp: '172.30.0.30' }, - ); - const proxy = result.services['api-proxy'] as any; - - expect(proxy.ports).toBeUndefined(); - expect(proxy.networks?.['awf-ext']).toBeUndefined(); - }); }); }); diff --git a/src/types/index.ts b/src/types/index.ts index 932cb9cbe..c17abf9a9 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -13,14 +13,6 @@ export type * from './wrapper-config'; export { type UpstreamProxyConfig } from './upstream-proxy'; export { type LogLevel } from './log-level'; export { - type FirecrackerArtifactDigests, - type FirecrackerOptions, - FIRECRACKER_RELEASE_VERSION, - FIRECRACKER_DEFAULT_BINARY, - FIRECRACKER_DEFAULT_JAILER_BINARY, - FIRECRACKER_DEFAULT_VCPU_COUNT, - FIRECRACKER_DEFAULT_MEMORY_MIB, - FIRECRACKER_DEFAULT_API_TIMEOUT_MS, type CloudHypervisorArtifactDigests, type CloudHypervisorOptions, CLOUD_HYPERVISOR_RELEASE_VERSION, diff --git a/src/types/runtime-options.ts b/src/types/runtime-options.ts index a0e9719e8..a48a5e155 100644 --- a/src/types/runtime-options.ts +++ b/src/types/runtime-options.ts @@ -4,40 +4,6 @@ import type { LogLevel } from './log-level'; -export const FIRECRACKER_RELEASE_VERSION = '1.16.1'; -export const FIRECRACKER_DEFAULT_BINARY = '/usr/local/bin/firecracker'; -export const FIRECRACKER_DEFAULT_JAILER_BINARY = '/usr/local/bin/jailer'; -export const FIRECRACKER_DEFAULT_VCPU_COUNT = 2; -export const FIRECRACKER_DEFAULT_MEMORY_MIB = 512; -export const FIRECRACKER_DEFAULT_API_TIMEOUT_MS = 5_000; - -export interface FirecrackerArtifactDigests { - firecracker?: string; - jailer?: string; - kernel?: string; - rootfs?: string; - supervisor?: string; -} - -/** - * Preview workload configuration for the Firecracker microVM runtime. - * - * Host-side network enforcement and guest execution inputs are supplied - * directly to FirecrackerManager after live infrastructure discovery. - */ -export interface FirecrackerOptions { - previewEnabled: boolean; - firecrackerBinary: string; - jailerBinary: string; - kernelPath?: string; - rootfsPath?: string; - supervisorPath?: string; - vcpuCount: number; - memoryMib: number; - apiTimeoutMs: number; - sha256?: FirecrackerArtifactDigests; -} - // ─── Cloud Hypervisor (v53.0 preview lifecycle backend) ──────────────────── // // This configuration surface pins trusted artifacts and configures the @@ -239,9 +205,6 @@ export interface RuntimeOptions { }; }; - /** Firecracker microVM control-plane settings. */ - firecracker?: FirecrackerOptions; - /** * Cloud Hypervisor v53.0 preview microVM runtime settings. * From 4ee1e429de8181b7b53780c3de3e3e81df3740f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:31:17 +0000 Subject: [PATCH 2/2] Rename microVM nftables table prefix Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com> --- src/microvm/network-plan.ts | 2 +- src/microvm/network.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/microvm/network-plan.ts b/src/microvm/network-plan.ts index 61038211f..eb93a57d8 100644 --- a/src/microvm/network-plan.ts +++ b/src/microvm/network-plan.ts @@ -50,7 +50,7 @@ export function createMicrovmNetworkPlan( const tapName = `vmt${token}`; const hostVethName = `vmh${token}`; const namespaceVethName = `vmn${token}`; - const nftTableName = `awf_fc_${token}`; + const nftTableName = `awf_vm_${token}`; for (const [label, name] of [ ['TAP', tapName], ['host veth', hostVethName], diff --git a/src/microvm/network.test.ts b/src/microvm/network.test.ts index 93de83153..095206fe9 100644 --- a/src/microvm/network.test.ts +++ b/src/microvm/network.test.ts @@ -716,7 +716,7 @@ describe('LinuxNetworkCommands.captureDiagnosticsInNamespace', () => { const commands = new LinuxNetworkCommands( jest.fn(async (_command, args) => { if (args.includes('nft')) { - return { stdout: 'table inet awf_fc_abc123 { chain forward { ... } }' }; + return { stdout: 'table inet awf_vm_abc123 { chain forward { ... } }' }; } if (args.includes('-s') && args.includes('link')) { return { stdout: '2: eth0: ... RX: 0 bytes 0 packets' }; @@ -746,7 +746,7 @@ describe('LinuxNetworkCommands.captureDiagnosticsInNamespace', () => { const result = await commands.captureDiagnosticsInNamespace('awfvm-test'); expect(result).toContain('--- nft -a list ruleset (handles + hit counters) ---'); - expect(result).toContain('table inet awf_fc_abc123'); + expect(result).toContain('table inet awf_vm_abc123'); expect(result).toContain('--- ip -s link show (packet/byte/error counters) ---'); expect(result).toContain('RX: 0 bytes 0 packets'); expect(result).toContain('--- ip -d link show (detailed link info, incl. vnet_hdr/multiqueue flags) ---');