diff --git a/.github/workflows/test-firecracker.yml b/.github/workflows/test-firecracker.yml index 82c0860ab..c746c5c53 100644 --- a/.github/workflows/test-firecracker.yml +++ b/.github/workflows/test-firecracker.yml @@ -98,6 +98,13 @@ jobs: name: firecracker-test-x86_64 path: ${{ runner.temp }}/firecracker-test-x86_64 + - name: Restore executable artifact modes + run: | + chmod 755 \ + "$RUNNER_TEMP/firecracker-test-x86_64/firecracker" \ + "$RUNNER_TEMP/firecracker-test-x86_64/jailer" \ + "$RUNNER_TEMP/firecracker-test-x86_64/awf-firecracker-supervisor" + - name: Grant workflow user access to KVM run: | if [ -e /dev/kvm ]; then @@ -133,15 +140,16 @@ jobs: while IFS= read -r -d '' file; do relative=${file#"$source_root/"} mkdir -p "$destination/$(dirname "$relative")" - cp "$file" "$destination/$relative" + sudo cp "$file" "$destination/$relative" done < <( - find "$source_root" -type f \ + sudo find "$source_root" -type f \ \( -path '*/audit/*' \ -o -path '*/proxy-logs/*' \ -o -name 'stdout.log' \ -o -name 'stderr.log' \) \ -print0 ) + sudo chown -R "$(id -u):$(id -g)" "$destination" fi if grep -R --binary-files=without-match \ -F 'awf-firecracker-real-secret-do-not-expose' \ diff --git a/docs/INTEGRATION-TESTS.md b/docs/INTEGRATION-TESTS.md index 39b401d86..63357e7d4 100644 --- a/docs/INTEGRATION-TESTS.md +++ b/docs/INTEGRATION-TESTS.md @@ -210,9 +210,11 @@ supervisor — from pinned, SHA-256 verified sources. Attests provenance. Upload as a 7-day workflow artifact. **Live job** (`ubuntu-24.04`): Runs on a GitHub-hosted x64 runner, downloads the -build artifact, verifies all five SHA-256 digests, and runs the live -smoke/security suite. The preflight requires usable KVM and fails closed if -`/dev/kvm` or another required host capability is unavailable. +build artifact, restores executable modes stripped by artifact transport, +grants the workflow user access to `/dev/kvm`, verifies all five SHA-256 +digests, and runs the live smoke/security suite. The preflight requires usable +KVM and fails closed if `/dev/kvm` or another required host capability is +unavailable. Live assertions (see `scripts/ci/firecracker-live-smoke.sh`): diff --git a/docs/firecracker-integration.md b/docs/firecracker-integration.md index e23dee588..c6fe098e4 100644 --- a/docs/firecracker-integration.md +++ b/docs/firecracker-integration.md @@ -327,7 +327,7 @@ retry. |----------|---------|-------------|-------------| | Firecracker binary | `--firecracker-binary` | `--firecracker-binary-sha256` | Firecracker VMM binary, **must be v1.16.1** | | Jailer binary | `--firecracker-jailer-binary` | `--firecracker-jailer-sha256` | Jailer binary, same version as Firecracker binary | -| Guest kernel | `--firecracker-kernel` | `--firecracker-kernel-sha256` | A KVM-compatible Linux bzImage | +| Guest kernel | `--firecracker-kernel` | `--firecracker-kernel-sha256` | A KVM-compatible uncompressed Linux ELF kernel | | Guest rootfs | `--firecracker-rootfs` | `--firecracker-rootfs-sha256` | Ext4 base image; staged as a private writable copy per run | | Guest supervisor | `--firecracker-supervisor` | `--firecracker-supervisor-sha256` | AWF vsock supervisor binary | @@ -371,7 +371,7 @@ This tarball contains: |------|-------------| | `firecracker` | Firecracker v1.16.1 binary (extracted from upstream release, SHA-256 verified) | | `jailer` | Jailer v1.16.1 binary | -| `vmlinux.bin` | Linux 6.1.141 bzImage built from upstream source with a pinned kernel config | +| `vmlinux.bin` | Uncompressed Linux 6.1.141 ELF kernel built from upstream source with a pinned kernel config | | `rootfs.ext4` | Minimal BusyBox + supervisor rootfs image | | `awf-firecracker-supervisor` | AWF guest supervisor binary | | `SHA256SUMS` | SHA-256 digests for all five files | @@ -948,16 +948,18 @@ host capability. It: -1. Downloads the `firecracker-test-x86_64` artifact from the build job -2. Runs `scripts/ci/firecracker-host-preflight.sh` — verifies Linux, x86_64, +1. Downloads the `firecracker-test-x86_64` artifact from the build job and + restores executable modes stripped by GitHub artifact transport +2. Grants the workflow user access to the hosted runner's `/dev/kvm` +3. Runs `scripts/ci/firecracker-host-preflight.sh` — verifies Linux, x86_64, `/dev/kvm`, required tools, Firecracker/jailer version strings, and all five SHA-256 digests via `sha256sum --check --strict SHA256SUMS` -3. Installs NPM dependencies, builds the AWF distribution, and builds the +4. Installs NPM dependencies, builds the AWF distribution, and builds the Squid and API proxy container images locally -4. Runs `scripts/ci/firecracker-live-smoke.sh` — the live test suite -5. Collects redacted diagnostics (audit, proxy-logs, stdout/stderr) and scans +5. Runs `scripts/ci/firecracker-live-smoke.sh` — the live test suite +6. Collects redacted diagnostics (audit, proxy-logs, stdout/stderr) and scans for the secret sentinel before uploading -6. Enforces final residue cleanup of all `awffc-*` network namespaces +7. Enforces final residue cleanup of all `awffc-*` network namespaces ### Live smoke test assertions diff --git a/guest/firecracker-supervisor/runtime_linux.go b/guest/firecracker-supervisor/runtime_linux.go index 2153b016c..ba1e91f85 100644 --- a/guest/firecracker-supervisor/runtime_linux.go +++ b/guest/firecracker-supervisor/runtime_linux.go @@ -137,7 +137,7 @@ func mountWorkspace(config bootConfig) error { if err := os.MkdirAll(config.WorkspaceMount, 0755); err != nil { return fmt.Errorf("create workspace mount: %w", err) } - if err := syscall.Mount(config.WorkspaceDevice, config.WorkspaceMount, "", 0, ""); err != nil { + if err := syscall.Mount(config.WorkspaceDevice, config.WorkspaceMount, "ext4", 0, ""); err != nil { return fmt.Errorf("mount workspace: %w", err) } return nil diff --git a/guest/firecracker/build-test-artifacts.sh b/guest/firecracker/build-test-artifacts.sh index e9d138c35..3a1209ab6 100755 --- a/guest/firecracker/build-test-artifacts.sh +++ b/guest/firecracker/build-test-artifacts.sh @@ -86,9 +86,9 @@ make -C "$BUILD/linux-${LINUX_VERSION}" \ KBUILD_BUILD_USER=awf \ KBUILD_BUILD_HOST=github \ LOCALVERSION=-awf-firecracker \ - bzImage + vmlinux install -m 0644 \ - "$BUILD/linux-${LINUX_VERSION}/arch/x86/boot/bzImage" \ + "$BUILD/linux-${LINUX_VERSION}/vmlinux" \ "$OUTPUT/vmlinux.bin" busybox_tar="$BUILD/downloads/busybox-${BUSYBOX_VERSION}.tar.bz2" diff --git a/guest/firecracker/verify-test-artifacts.sh b/guest/firecracker/verify-test-artifacts.sh index 328119769..9d9514eca 100755 --- a/guest/firecracker/verify-test-artifacts.sh +++ b/guest/firecracker/verify-test-artifacts.sh @@ -25,7 +25,7 @@ done "$ARTIFACT_DIR/firecracker" --version | grep -F '1.16.1' "$ARTIFACT_DIR/jailer" --version | grep -F '1.16.1' -file "$ARTIFACT_DIR/vmlinux.bin" | grep -E 'Linux kernel|boot executable' +file "$ARTIFACT_DIR/vmlinux.bin" | grep -F 'ELF 64-bit LSB executable' e2fsck -f -n "$ARTIFACT_DIR/rootfs.ext4" debugfs -R 'stat /sbin/awf-supervisor' "$ARTIFACT_DIR/rootfs.ext4" 2>&1 \ | grep -F 'Type: regular' diff --git a/scripts/ci/firecracker-host-preflight.sh b/scripts/ci/firecracker-host-preflight.sh index 7a29155f4..8f85164b4 100755 --- a/scripts/ci/firecracker-host-preflight.sh +++ b/scripts/ci/firecracker-host-preflight.sh @@ -31,6 +31,12 @@ sudo -n true || fail "Passwordless sudo is required for jailer and netns setup." docker info >/dev/null || fail "A host-visible Docker Engine is required." docker compose version >/dev/null || fail "Docker Compose v2 is required." +[ -x "$ARTIFACT_DIR/firecracker" ] \ + || fail "The Firecracker artifact must be executable." +[ -x "$ARTIFACT_DIR/jailer" ] \ + || fail "The jailer artifact must be executable." +[ -x "$ARTIFACT_DIR/awf-firecracker-supervisor" ] \ + || fail "The guest supervisor artifact must be executable." "$ARTIFACT_DIR/firecracker" --version | grep -Fq '1.16.1' \ || fail "Firecracker v1.16.1 is required." "$ARTIFACT_DIR/jailer" --version | grep -Fq '1.16.1' \ diff --git a/src/compose-generator.test.ts b/src/compose-generator.test.ts index 1c4ad642d..a7906eefb 100644 --- a/src/compose-generator.test.ts +++ b/src/compose-generator.test.ts @@ -274,6 +274,7 @@ describe('generateDockerCompose', () => { expect(result.networks['awf-net'].external).toBeUndefined(); expect(result.networks['awf-net'].name).toBe('awf-net'); expect(result.networks['awf-net'].ipam?.config?.[0]?.subnet).toBe(mockNetworkConfig.subnet); + expect(result.networks['awf-net'].ipam?.config?.[0]?.gateway).toBe('172.30.0.1'); expect(result.networks['awf-ext'].driver).toBe('bridge'); }); diff --git a/src/compose-network.ts b/src/compose-network.ts index 52072c233..23face39d 100644 --- a/src/compose-network.ts +++ b/src/compose-network.ts @@ -1,6 +1,10 @@ import { DockerComposeConfig } from './types'; import { TOPOLOGY_NETWORK_NAME } from './topology'; -import { EXTERNAL_BRIDGE_NAME, EMBEDDED_DNS_RESOLVER } from './config/network-policy'; +import { + EMBEDDED_DNS_RESOLVER, + EXTERNAL_BRIDGE_NAME, + NETWORK_GATEWAY, +} from './config/network-policy'; import { NetworkConfig } from './services/squid-service'; interface BuildComposeNetworksParams { @@ -52,7 +56,7 @@ export function buildComposeNetworks(params: BuildComposeNetworksParams): Docker name: TOPOLOGY_NETWORK_NAME, internal: true, ipam: { - config: [{ subnet: networkConfig.subnet }], + config: [{ subnet: networkConfig.subnet, gateway: NETWORK_GATEWAY }], }, }, [EXTERNAL_BRIDGE_NAME]: { diff --git a/src/config/network-policy-validation.test.ts b/src/config/network-policy-validation.test.ts index 3d339138e..d0e8da1eb 100644 --- a/src/config/network-policy-validation.test.ts +++ b/src/config/network-policy-validation.test.ts @@ -19,6 +19,7 @@ const BASE_POLICY = { networkName: 'awf-net', externalBridgeName: 'awf-ext', subnet: '172.30.0.0/24', + gateway: '172.30.0.1', hosts: { squid: { ip: '172.30.0.10', role: 'egress-proxy', required: true, dualHomed: true }, agent: { ip: '172.30.0.20', role: 'workload', required: true }, @@ -113,6 +114,12 @@ describe('network-policy validation errors', () => { }); describe('assertIpv4', () => { + it('throws when the topology gateway is invalid', () => { + const policy = deepClone(BASE_POLICY); + policy.topology.gateway = 'not-an-ip'; + expect(loadWithPolicy(policy)).toThrow('topology.gateway must be a valid IPv4 address'); + }); + it('throws when IP has an octet > 255', () => { const policy = deepClone(BASE_POLICY); policy.topology.hosts.squid.ip = '172.30.0.300'; diff --git a/src/config/network-policy.test.ts b/src/config/network-policy.test.ts index 0783931a8..907a37b9e 100644 --- a/src/config/network-policy.test.ts +++ b/src/config/network-policy.test.ts @@ -3,6 +3,7 @@ import { NETWORK_NAME, EXTERNAL_BRIDGE_NAME, NETWORK_SUBNET, + NETWORK_GATEWAY, SQUID_IP, AGENT_IP, API_PROXY_IP, @@ -161,6 +162,7 @@ describe('network-policy', () => { expect(NETWORK_NAME).toBe('awf-net'); expect(EXTERNAL_BRIDGE_NAME).toBe('awf-ext'); expect(NETWORK_SUBNET).toBe('172.30.0.0/24'); + expect(NETWORK_GATEWAY).toBe('172.30.0.1'); expect(HOST_GATEWAY).toBe('172.30.0.1'); expect(SQUID_IP).toBe('172.30.0.10'); expect(AGENT_IP).toBe('172.30.0.20'); diff --git a/src/config/network-policy.ts b/src/config/network-policy.ts index b4d524a19..69f7ce285 100644 --- a/src/config/network-policy.ts +++ b/src/config/network-policy.ts @@ -65,6 +65,7 @@ export interface NetworkPolicy { readonly networkName: string; readonly externalBridgeName: string; readonly subnet: string; + readonly gateway: string; readonly hosts: { readonly squid: NetworkHost; readonly agent: NetworkHost; @@ -201,6 +202,7 @@ function parseTopology(value: unknown): NetworkPolicy['topology'] { networkName: assertString(t.networkName, 'topology.networkName'), externalBridgeName: assertString(t.externalBridgeName, 'topology.externalBridgeName'), subnet: assertCidr(t.subnet, 'topology.subnet'), + gateway: assertIpv4(t.gateway, 'topology.gateway'), hosts: parseHosts(t.hosts), }; } @@ -325,6 +327,9 @@ export const EXTERNAL_BRIDGE_NAME: string = networkPolicy.topology.externalBridg /** IPv4 subnet (CIDR) of the internal topology network, shared by all runtimes. */ export const NETWORK_SUBNET: string = networkPolicy.topology.subnet; +/** Fixed gateway of the internal Docker topology network. */ +export const NETWORK_GATEWAY: string = networkPolicy.topology.gateway; + /** Fixed IP of the Squid egress proxy on the internal network. */ export const SQUID_IP: string = networkPolicy.topology.hosts.squid.ip; diff --git a/src/config/sandbox-network-policy.json b/src/config/sandbox-network-policy.json index d395dc300..787dda146 100644 --- a/src/config/sandbox-network-policy.json +++ b/src/config/sandbox-network-policy.json @@ -2,10 +2,11 @@ "$comment": "Central declarative NETWORK policy shared by ALL runtimes. The PRIMARY, default path is network-isolation (topology) mode: the agent runs on an `internal` Docker network whose only egress is the dual-homed Squid proxy — NO iptables (host or container), no NET_ADMIN. In that mode egress restriction is STRUCTURAL (default-deny by routing): blocked ports/CIDRs have no route out, so they are denied by construction rather than by rules. The `legacyIptables` section below is consumed ONLY by the uncommon --legacy-security / --enable-host-access path (host iptables + the container setup-iptables.sh NAT script). Every value here is a logical Docker network address (subnet, static IP, port, network name) that the daemon resolves — it carries NO host-filesystem or host-network assumptions, so the policy is ARC/DinD safe and needs no --docker-host-path-prefix translation. Loaded and validated by src/config/network-policy.ts.", "topology": { - "$comment": "CORE — consumed by the default iptables-less path. Compose IPAM assigns these static IPs on the internal `networkName`; Squid is dual-homed onto `externalBridgeName` as the sole egress. `subnet` is the fixed subnet all runtimes use. `hostGateway` lives in `legacyIptables`, not here — the default path never touches the host network.", + "$comment": "CORE — consumed by the default iptables-less path. Compose IPAM assigns the fixed subnet, bridge gateway, and static service IPs on the internal `networkName`; Squid is dual-homed onto `externalBridgeName` as the sole egress. `gateway` is the internal Docker bridge address. The separate `hostGateway` in `legacyIptables` is consumed by host-network rules only.", "networkName": "awf-net", "externalBridgeName": "awf-ext", "subnet": "172.30.0.0/24", + "gateway": "172.30.0.1", "hosts": { "squid": { "ip": "172.30.0.10", "role": "egress-proxy", "required": true, "dualHomed": true }, "agent": { "ip": "172.30.0.20", "role": "workload", "required": true }, diff --git a/src/firecracker-runtime-backend.test.ts b/src/firecracker-runtime-backend.test.ts index e17c5ddb0..dd301cff4 100644 --- a/src/firecracker-runtime-backend.test.ts +++ b/src/firecracker-runtime-backend.test.ts @@ -188,6 +188,20 @@ describe('Firecracker runtime backend', () => { 'probe', 'vm-stop', ]); + expect(manager.execute).toHaveBeenNthCalledWith(1, expect.objectContaining({ + argv: [ + '/bin/sh', + '-c', + expect.stringContaining('timeout 5 nc 172.30.0.10 3128'), + ], + })); + expect(manager.execute).toHaveBeenNthCalledWith(1, expect.objectContaining({ + argv: [ + '/bin/sh', + '-c', + expect.stringContaining('wget -q -T 5 -O /dev/null http://172.30.0.30:10000/reflect'), + ], + })); expect(manager.execute).toHaveBeenNthCalledWith(2, expect.objectContaining({ argv: ['/bin/sh', '-lc', 'printf hello'], cwd: '/workspace', @@ -271,6 +285,23 @@ describe('Firecracker runtime backend', () => { expect(manager.stop).toHaveBeenCalledTimes(1); }); + it('collects startup diagnostics before tearing down a failed VM', async () => { + const { manager, deps } = harness(); + manager.startInstance.mockRejectedValue(new Error('guest boot failed')); + manager.collectDiagnostics.mockImplementation(async () => { + expect(manager.stop).not.toHaveBeenCalled(); + }); + const backend = new FirecrackerRuntimeBackend( + config({ diagnosticLogs: true, auditDir: '/tmp/audit' }), + deps, + ); + + await expect(backend.start('/tmp/awf', ['github.com'])) + .rejects.toThrow(/guest boot failed/); + expect(manager.collectDiagnostics).toHaveBeenCalledWith('/tmp/audit/firecracker'); + expect(manager.stop).toHaveBeenCalledTimes(1); + }); + it('fails closed when manager readiness or startup cleanup is unavailable', async () => { const missingIp = harness(); Reflect.set(missingIp.manager, 'guestIp', undefined); @@ -401,6 +432,8 @@ describe('Firecracker runtime backend', () => { expect(environment.OPENAI_API_KEY).not.toBe(secret); expect(Object.values(environment)).not.toContain(secret); expect(environment.HTTP_PROXY).toBe('http://172.30.0.10:3128'); + expect(environment.http_proxy).toBe(environment.HTTP_PROXY); + expect(environment.https_proxy).toBe(environment.HTTPS_PROXY); expect(environment.HOME).toBe('/workspace/.awf-home'); expect(() => buildFirecrackerGuestEnvironment( diff --git a/src/firecracker-runtime-backend.ts b/src/firecracker-runtime-backend.ts index 9dc3a6738..12a68f279 100644 --- a/src/firecracker-runtime-backend.ts +++ b/src/firecracker-runtime-backend.ts @@ -1,4 +1,4 @@ -import type { Readable, Writable } from 'stream'; +import { PassThrough, type Readable, type Writable } from 'stream'; import type { WorkflowDependencies } from './cli-workflow'; import type { ExternalAgentRuntimeBackend } from './external-runtime-backend'; import { @@ -216,6 +216,15 @@ export class FirecrackerRuntimeBackend implements ExternalAgentRuntimeBackend { this.dependencies.logger.warn( `[firecracker] stage=${stage} status=failed: ${formatError(error)}`, ); + if (this.config.diagnosticLogs && this.manager) { + try { + await this.collectDiagnostics(); + } catch (diagnosticError) { + this.dependencies.logger.warn( + `[firecracker] startup diagnostic collection failed: ${formatError(diagnosticError)}`, + ); + } + } try { await this.manager?.stop(); } catch (cleanupError) { @@ -373,23 +382,40 @@ export class FirecrackerRuntimeBackend implements ExternalAgentRuntimeBackend { throw new Error('Firecracker guest identity is not ready'); } const squidProbe = - `curl --silent --show-error --max-time 5 --output /dev/null ` + - `http://${SQUID_IP}:3128/`; + `printf 'GET http://localhost/ HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n' ` + + `| timeout 5 nc ${SQUID_IP} 3128 | grep -q '^HTTP/'`; const apiProxyProbe = this.config.enableApiProxy - ? ` && curl --fail --silent --show-error --max-time 5 --noproxy '*' ` + - `--output /dev/null http://${API_PROXY_IP}:10000/reflect` + ? ` && wget -q -T 5 -O /dev/null http://${API_PROXY_IP}:10000/reflect` : ''; + const probeOutput = new PassThrough(); + const probeError = new PassThrough(); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + probeOutput.on('data', (chunk: Buffer) => stdout.push(chunk)); + probeError.on('data', (chunk: Buffer) => stderr.push(chunk)); const result = await manager.execute({ requestId: `probe-${process.pid}-${Date.now()}`, - argv: ['/bin/sh', '-c', `set -eu; ${squidProbe}${apiProxyProbe}`], + argv: [ + '/bin/sh', + '-c', + `set -eu; { ${squidProbe}${apiProxyProbe}; } || ` + + `{ status=$?; ip address show; ip route show; exit "$status"; }`, + ], env: environment, cwd: FIRECRACKER_GUEST_WORKSPACE, ...identity, timeoutMs: FIRECRACKER_PROBE_TIMEOUT_MS, + stdout: probeOutput, + stderr: probeError, }); if (result.exitCode !== 0) { + const diagnostics = Buffer.concat([...stdout, ...stderr]) + .subarray(0, 4096) + .toString('utf8') + .trim(); throw new Error( - `Firecracker guest connectivity probe failed with exit code ${result.exitCode}`, + `Firecracker guest connectivity probe failed with exit code ${result.exitCode}` + + (diagnostics ? `:\n${diagnostics}` : ''), ); } this.dependencies.logger.info( @@ -421,6 +447,8 @@ export function buildFirecrackerGuestEnvironment( HOME: FIRECRACKER_GUEST_HOME, PWD: FIRECRACKER_GUEST_WORKSPACE, AWF_WORKDIR: FIRECRACKER_GUEST_WORKSPACE, + http_proxy: `http://${infrastructure.squidIp}:3128`, + https_proxy: `http://${infrastructure.squidIp}:3128`, SQUID_PROXY_HOST: infrastructure.squidIp, HOSTNAME: 'awf-firecracker', AWF_RUNTIME: 'firecracker', diff --git a/src/firecracker/infrastructure.ts b/src/firecracker/infrastructure.ts index 1a97f4cc8..9470e53a0 100644 --- a/src/firecracker/infrastructure.ts +++ b/src/firecracker/infrastructure.ts @@ -1,7 +1,7 @@ import execa from 'execa'; import { API_PROXY_IP, - HOST_GATEWAY, + NETWORK_GATEWAY, NETWORK_NAME, NETWORK_SUBNET, SQUID_IP, @@ -150,11 +150,11 @@ async function inspectInfrastructure( if ( ipv4Configs.length !== 1 || ipv4Configs[0].Subnet !== NETWORK_SUBNET || - ipv4Configs[0].Gateway !== HOST_GATEWAY + ipv4Configs[0].Gateway !== NETWORK_GATEWAY ) { throw new Error( `Docker network "${NETWORK_NAME}" must have exactly ${NETWORK_SUBNET} ` + - `with gateway ${HOST_GATEWAY}`, + `with gateway ${NETWORK_GATEWAY}`, ); } @@ -182,7 +182,7 @@ async function inspectInfrastructure( networkId: network.Id, bridgeName, subnet: NETWORK_SUBNET, - gateway: HOST_GATEWAY, + gateway: NETWORK_GATEWAY, squidIp, ...(apiProxyIp ? { apiProxyIp } : {}), }; diff --git a/src/firecracker/manager.test.ts b/src/firecracker/manager.test.ts index 639b427d2..26c77e503 100644 --- a/src/firecracker/manager.test.ts +++ b/src/firecracker/manager.test.ts @@ -102,6 +102,8 @@ function dependencies( }), launch: jest.fn().mockReturnValue(processMock()), mkdir: jest.fn().mockResolvedValue(undefined), + mkdtemp: jest.fn().mockResolvedValue('/tmp/awf-fc-sockets-test'), + symlink: jest.fn().mockResolvedValue(undefined), copyFile: jest.fn().mockResolvedValue(undefined), chmod: jest.fn().mockResolvedValue(undefined), chown: jest.fn().mockResolvedValue(undefined), @@ -408,7 +410,7 @@ describe('FirecrackerManager', () => { }); await manager.startInstance(); expect(deps.createVsockClient).toHaveBeenCalledWith( - '/tmp/awf/firecracker-jailer/firecracker/guest/root/run/awf-vsock.socket', + '/tmp/awf-fc-sockets-test/run/awf-vsock.socket', 52, 1, ); @@ -429,6 +431,58 @@ describe('FirecrackerManager', () => { expect(order).toEqual(['extract']); }); + it('retries while the booting guest has not started listening on vsock', async () => { + const firstClient = { + connect: jest.fn().mockRejectedValue( + Object.assign(new Error('connect ECONNREFUSED /tmp/awf-vsock.socket'), { + code: 'ECONNREFUSED', + }), + ), + destroy: jest.fn(), + } as unknown as FirecrackerVsockClient; + const readyClient = { + connect: jest.fn().mockResolvedValue(undefined), + shutdown: jest.fn().mockResolvedValue(undefined), + destroy: jest.fn(), + } as unknown as FirecrackerVsockClient; + const workspace = { + prepare: jest.fn().mockResolvedValue({ + workspaceImagePath: '/tmp/prepared-workspace.ext4', + rootfsImagePath: '/tmp/prepared-rootfs.ext4', + imageBytes: 1024, + originalManifest: new Map(), + }), + extractAfterStop: jest.fn().mockResolvedValue(undefined), + cleanup: jest.fn().mockResolvedValue(undefined), + } as unknown as FirecrackerWorkspaceImage; + const deps = dependencies({ + createWorkspaceImage: jest.fn().mockReturnValue(workspace), + createVsockClient: jest.fn() + .mockReturnValueOnce(firstClient) + .mockReturnValueOnce(readyClient), + }); + const manager = new FirecrackerManager( + config({ apiTimeoutMs: 50 }), + '/tmp/awf', + deps, + 'guest-retry', + networkConfig(), + { + workspacePath: '/workspace', + homePath: '/home/runner', + supervisorBinaryPath: '/opt/supervisor', + supervisorSha256: 'a'.repeat(64), + }, + ); + await manager.start(); + + await manager.startInstance(); + + expect(firstClient.destroy).toHaveBeenCalledTimes(1); + expect(deps.createVsockClient).toHaveBeenCalledTimes(2); + expect(deps.sleep).toHaveBeenCalledWith(25); + }); + it('delegates guest cancellation, stdin, and resize only after readiness', async () => { const cold = new FirecrackerManager( config(), @@ -538,7 +592,10 @@ describe('FirecrackerManager', () => { expect(workspace.extractAfterStop).toHaveBeenCalledTimes(1); expect(lifecycle.cleanup).not.toHaveBeenCalled(); expect(workspace.cleanup).not.toHaveBeenCalled(); - expect(deps.rm).not.toHaveBeenCalled(); + expect(deps.rm).toHaveBeenCalledWith( + '/tmp/awf-fc-sockets-test', + { recursive: true, force: true }, + ); }); it('builds explicit supervisor boot networking without widening policy', () => { diff --git a/src/firecracker/manager.ts b/src/firecracker/manager.ts index 3fa2ebb77..6488ee17e 100644 --- a/src/firecracker/manager.ts +++ b/src/firecracker/manager.ts @@ -1,5 +1,6 @@ import { randomBytes } from 'crypto'; import { constants, promises as fs } from 'fs'; +import * as os from 'os'; import * as path from 'path'; import execa, { type ExecaChildProcess } from 'execa'; import { @@ -41,6 +42,7 @@ const WORKSPACE_JAIL_PATH = '/workspace.ext4'; const VSOCK_JAIL_PATH = `/run/${VSOCK_SOCKET_NAME}`; export const FIRECRACKER_GUEST_VSOCK_PORT = 52; const FIRECRACKER_GUEST_SHUTDOWN_GRACE_MS = 5_000; +const FIRECRACKER_GUEST_CONNECT_RETRY_MS = 25; export interface FirecrackerRunPaths { runId: string; @@ -67,6 +69,8 @@ export interface FirecrackerManagerDependencies { }, ): ExecaChildProcess; mkdir(directory: string, options: { recursive: true; mode: number }): Promise; + mkdtemp(prefix: string): Promise; + symlink(target: string, path: string, type: 'dir'): Promise; copyFile(source: string, destination: string, flags: number): Promise; chmod(filePath: string, mode: number): Promise; chown(filePath: string, uid: number, gid: number): Promise; @@ -117,6 +121,8 @@ const defaultDependencies: FirecrackerManagerDependencies = { preflight: runFirecrackerPreflight, launch: (command, args, options) => execa(command, args, options), mkdir: fs.mkdir, + mkdtemp: fs.mkdtemp, + symlink: fs.symlink, copyFile: fs.copyFile, chmod: fs.chmod, chown: fs.chown, @@ -213,6 +219,10 @@ export class FirecrackerManager { private workspace: FirecrackerWorkspaceImage | undefined; private guestClient: FirecrackerVsockClient | undefined; private networkPlan: FirecrackerNetworkPlan | undefined; + private diagnosticNetworkPlan: FirecrackerNetworkPlan | undefined; + private socketBridgeDirectory: string | undefined; + private apiSocketPath: string | undefined; + private vsockSocketPath: string | undefined; private instanceStarted = false; private readonly stdoutCapture = new BoundedOutputCapture(FIRECRACKER_CAPTURE_LIMIT_BYTES); private readonly stderrCapture = new BoundedOutputCapture(FIRECRACKER_CAPTURE_LIMIT_BYTES); @@ -253,6 +263,7 @@ export class FirecrackerManager { jailerGid: identity.gid, }); this.networkPlan = networkPlan; + this.diagnosticNetworkPlan = networkPlan; this.network = this.dependencies.createNetwork(networkPlan, artifacts.tools); await this.network.setup(); let rootfsSource = artifacts.rootfsPath; @@ -280,6 +291,19 @@ export class FirecrackerManager { recursive: true, mode: 0o700, }); + this.socketBridgeDirectory = await this.dependencies.mkdtemp( + path.join(os.tmpdir(), 'awf-fc-sockets-'), + ); + const socketBridgeRunDirectory = path.join(this.socketBridgeDirectory, 'run'); + await this.dependencies.symlink( + path.join(this.paths.jailRoot, 'run'), + socketBridgeRunDirectory, + 'dir', + ); + const apiSocketPath = path.join(socketBridgeRunDirectory, API_SOCKET_NAME); + const vsockSocketPath = path.join(socketBridgeRunDirectory, VSOCK_SOCKET_NAME); + this.apiSocketPath = apiSocketPath; + this.vsockSocketPath = vsockSocketPath; this.process = this.dependencies.launch( this.config.jailerBinary, @@ -307,7 +331,7 @@ export class FirecrackerManager { this.stderrCapture.append(chunk); }); - await this.waitForApiSocket(); + await this.waitForApiSocket(apiSocketPath); await this.stageArtifact(artifacts.kernelPath, this.paths.kernelPath, 0o400, identity); await this.stageArtifact(rootfsSource, this.paths.rootfsPath, 0o600, identity); if (workspaceSource) { @@ -315,7 +339,7 @@ export class FirecrackerManager { } this.client = this.dependencies.createClient( - this.paths.apiSocketPath, + apiSocketPath, this.config.apiTimeoutMs, ); await this.stageDiagnosticFile(this.paths.logPath, identity); @@ -379,12 +403,28 @@ export class FirecrackerManager { await this.client.instanceStart(); this.instanceStarted = true; if (this.guestConfig) { - this.guestClient = this.dependencies.createVsockClient( - this.paths.vsockSocketPath, - this.guestConfig.vsockPort ?? FIRECRACKER_GUEST_VSOCK_PORT, - this.config.apiTimeoutMs, + const attempts = Math.max( + 1, + Math.ceil(this.config.apiTimeoutMs / FIRECRACKER_GUEST_CONNECT_RETRY_MS), ); - await this.guestClient.connect(); + for (let attempt = 0; attempt < attempts; attempt += 1) { + const guestClient = this.dependencies.createVsockClient( + this.vsockSocketPath ?? this.paths.vsockSocketPath, + this.guestConfig.vsockPort ?? FIRECRACKER_GUEST_VSOCK_PORT, + this.config.apiTimeoutMs, + ); + try { + await guestClient.connect(); + this.guestClient = guestClient; + return; + } catch (error) { + guestClient.destroy(); + if (!isRetryableGuestConnectError(error) || attempt === attempts - 1) { + throw error; + } + await this.dependencies.sleep(FIRECRACKER_GUEST_CONNECT_RETRY_MS); + } + } } } @@ -490,6 +530,20 @@ export class FirecrackerManager { this.process = undefined; this.client = undefined; + if (this.socketBridgeDirectory) { + try { + await this.dependencies.rm( + this.socketBridgeDirectory, + { recursive: true, force: true }, + ); + this.socketBridgeDirectory = undefined; + this.apiSocketPath = undefined; + this.vsockSocketPath = undefined; + } catch (error) { + errors.push(error); + } + } + if (this.workspace && instanceWasStarted) { try { await this.workspace.extractAfterStop(this.paths.workspacePath); @@ -582,7 +636,7 @@ export class FirecrackerManager { ); await this.dependencies.writeFile( path.join(directory, 'network-plan.json'), - `${JSON.stringify(this.networkPlan ?? null, null, 2)}\n`, + `${JSON.stringify(this.networkPlan ?? this.diagnosticNetworkPlan ?? null, null, 2)}\n`, { mode: 0o600 }, ); await this.dependencies.writeFile( @@ -599,7 +653,7 @@ export class FirecrackerManager { ); } - private async waitForApiSocket(): Promise { + private async waitForApiSocket(apiSocketPath: string): Promise { const deadline = Date.now() + this.config.apiTimeoutMs; while (Date.now() < deadline) { if (this.process && (this.process.exitCode != null || this.process.signalCode != null)) { @@ -609,7 +663,7 @@ export class FirecrackerManager { ); } try { - await this.dependencies.access(this.paths.apiSocketPath); + await this.dependencies.access(apiSocketPath); return; } catch (error) { const code = (error as NodeJS.ErrnoException).code; @@ -619,7 +673,7 @@ export class FirecrackerManager { } throw new Error( `Firecracker API socket was not ready after ${this.config.apiTimeoutMs}ms: ` + - this.paths.apiSocketPath, + apiSocketPath, ); } @@ -680,6 +734,13 @@ function formatError(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function isRetryableGuestConnectError(error: unknown): boolean { + if ((error as NodeJS.ErrnoException)?.code === 'ECONNREFUSED') return true; + const message = formatError(error); + return message === 'Firecracker guest disconnected before readiness' || + /^Firecracker vsock CONNECT failed: ERR 111(?:\D|$)/.test(message); +} + class BoundedOutputCapture { private buffer = Buffer.alloc(0); diff --git a/src/firecracker/network.test.ts b/src/firecracker/network.test.ts index 49f666707..3d24d060a 100644 --- a/src/firecracker/network.test.ts +++ b/src/firecracker/network.test.ts @@ -7,6 +7,7 @@ import { type FirecrackerNetworkCommandOptions, type FirecrackerNetworkPlan, } from './network'; +import { promises as fs } from 'fs'; interface CommandCall { command: string; @@ -150,6 +151,11 @@ describe('Firecracker nftables policy', () => { expect(ruleset).toContain('udp dport 53 drop'); expect(ruleset).toContain('tcp dport 53 drop'); expect(ruleset).toContain('ct state established,related accept'); + expect(ruleset).toContain( + `iifname "${plan.namespaceVethName}" oifname "${plan.tapName}" ` + + `ip daddr ${plan.guestIp} ct state established,related accept`, + ); + expect(ruleset).not.toContain(`ether daddr ${plan.guestMac}`); expect(ruleset).toContain('ip daddr 172.30.0.10 tcp dport 3128'); for (let port = 10000; port <= 10004; port += 1) { expect(ruleset).toContain(`ip daddr 172.30.0.30 tcp dport ${port}`); @@ -212,14 +218,20 @@ describe('Firecracker network lifecycle', () => { expect(calls[11].args).toContain('net.ipv4.ip_forward=1'); expect(calls[12].args).toContain('net.ipv6.conf.all.disable_ipv6=1'); expect(calls[13].args).toContain('net.ipv6.conf.default.disable_ipv6=1'); + const nftRulesPath = calls[14].args[calls[14].args.length - 1]; expect(calls[14]).toEqual({ command: 'ip', - args: ['netns', 'exec', plan.namespaceName, 'nft', '-f', '-'], - options: { - reject: true, - input: generateFirecrackerNftRuleset(plan), - }, + args: [ + 'netns', + 'exec', + plan.namespaceName, + 'nft', + '-f', + expect.stringMatching(/awf-firecracker-nft-.*\/ruleset\.nft$/), + ], + options: { reject: true }, }); + await expect(fs.access(nftRulesPath)).rejects.toThrow(); expect(probe.verify).toHaveBeenCalledWith(plan); }); diff --git a/src/firecracker/network.ts b/src/firecracker/network.ts index 9c55ceeef..58f8d6142 100644 --- a/src/firecracker/network.ts +++ b/src/firecracker/network.ts @@ -1,5 +1,8 @@ import { createHash } from 'crypto'; import execa from 'execa'; +import { promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; import type { FirecrackerHostToolPaths } from './preflight'; import { AGENT_IP, @@ -219,11 +222,23 @@ export class FirecrackerNetworkManager implements FirecrackerNetworkLifecycle { this.plan.namespaceName, 'net.ipv6.conf.default.disable_ipv6=1', ); - await this.commands.nftInNamespace( - this.plan.namespaceName, - ['-f', '-'], - generateFirecrackerNftRuleset(this.plan), + const nftRulesDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'awf-firecracker-nft-'), ); + const nftRulesPath = path.join(nftRulesDirectory, 'ruleset.nft'); + try { + await fs.writeFile( + nftRulesPath, + generateFirecrackerNftRuleset(this.plan), + { encoding: 'utf8', flag: 'wx', mode: 0o600 }, + ); + await this.commands.nftInNamespace( + this.plan.namespaceName, + ['-f', nftRulesPath], + ); + } finally { + await fs.rm(nftRulesDirectory, { force: true, recursive: true }); + } await this.probe?.verify(this.plan); this.setupComplete = true; return this.plan; @@ -381,8 +396,7 @@ export function generateFirecrackerNftRuleset(plan: FirecrackerNetworkPlan): str ` iifname "${plan.tapName}" udp dport 53 drop`, ` iifname "${plan.tapName}" tcp dport 53 drop`, ` iifname "${plan.namespaceVethName}" oifname "${plan.tapName}" ` + - `ether daddr ${plan.guestMac} ip daddr ${plan.guestIp} ` + - 'ct state established,related accept', + `ip daddr ${plan.guestIp} ct state established,related accept`, ...allowRules, ' }', ' chain postrouting {', diff --git a/src/types/docker.ts b/src/types/docker.ts index f0c35fe38..bffe6cb86 100644 --- a/src/types/docker.ts +++ b/src/types/docker.ts @@ -430,11 +430,11 @@ interface DockerNetwork { * networks with specific subnets for avoiding conflicts with existing * Docker networks. * - * @example { config: [{ subnet: '172.30.0.0/24' }] } + * @example { config: [{ subnet: '172.30.0.0/24', gateway: '172.30.0.1' }] } */ ipam?: { /** Array of subnet configurations */ - config: Array<{ subnet: string }>; + config: Array<{ subnet: string; gateway?: string }>; }; /**