fix(ci): restore Firecracker artifact executable modes

.github/workflows/test-firecracker.yml

@@ -98,6 +98,13 @@ jobs:
           name: firecracker-test-x86_64
           path: ${{ runner.temp }}/firecracker-test-x86_64
 
+      - name: Restore executable artifact modes
+        run: |
+          chmod 755 \
+            "$RUNNER_TEMP/firecracker-test-x86_64/firecracker" \
+            "$RUNNER_TEMP/firecracker-test-x86_64/jailer" \
+            "$RUNNER_TEMP/firecracker-test-x86_64/awf-firecracker-supervisor"
+
       - name: Grant workflow user access to KVM
         run: |
           if [ -e /dev/kvm ]; then
@@ -133,15 +140,16 @@ jobs:
             while IFS= read -r -d '' file; do
               relative=${file#"$source_root/"}
               mkdir -p "$destination/$(dirname "$relative")"
-              cp "$file" "$destination/$relative"
+              sudo cp "$file" "$destination/$relative"
             done < <(
-              find "$source_root" -type f \
+              sudo find "$source_root" -type f \
                 \( -path '*/audit/*' \
                 -o -path '*/proxy-logs/*' \
                 -o -name 'stdout.log' \
                 -o -name 'stderr.log' \) \
                 -print0
             )
+            sudo chown -R "$(id -u):$(id -g)" "$destination"
           fi
           if grep -R --binary-files=without-match \
             -F 'awf-firecracker-real-secret-do-not-expose' \

docs/INTEGRATION-TESTS.md

@@ -210,9 +210,11 @@ supervisor — from pinned, SHA-256 verified sources. Attests provenance. Upload
 as a 7-day workflow artifact.
 
 **Live job** (`ubuntu-24.04`): Runs on a GitHub-hosted x64 runner, downloads the
-build artifact, verifies all five SHA-256 digests, and runs the live
-smoke/security suite. The preflight requires usable KVM and fails closed if
-`/dev/kvm` or another required host capability is unavailable.
+build artifact, restores executable modes stripped by artifact transport,
+grants the workflow user access to `/dev/kvm`, verifies all five SHA-256
+digests, and runs the live smoke/security suite. The preflight requires usable
+KVM and fails closed if `/dev/kvm` or another required host capability is
+unavailable.
 
 Live assertions (see `scripts/ci/firecracker-live-smoke.sh`):
 

docs/firecracker-integration.md

@@ -327,7 +327,7 @@ retry.
 |----------|---------|-------------|-------------|
 | Firecracker binary | `--firecracker-binary` | `--firecracker-binary-sha256` | Firecracker VMM binary, **must be v1.16.1** |
 | Jailer binary | `--firecracker-jailer-binary` | `--firecracker-jailer-sha256` | Jailer binary, same version as Firecracker binary |
-| Guest kernel | `--firecracker-kernel` | `--firecracker-kernel-sha256` | A KVM-compatible Linux bzImage |
+| Guest kernel | `--firecracker-kernel` | `--firecracker-kernel-sha256` | A KVM-compatible uncompressed Linux ELF kernel |
 | Guest rootfs | `--firecracker-rootfs` | `--firecracker-rootfs-sha256` | Ext4 base image; staged as a private writable copy per run |
 | Guest supervisor | `--firecracker-supervisor` | `--firecracker-supervisor-sha256` | AWF vsock supervisor binary |
 
@@ -371,7 +371,7 @@ This tarball contains:
 |------|-------------|
 | `firecracker` | Firecracker v1.16.1 binary (extracted from upstream release, SHA-256 verified) |
 | `jailer` | Jailer v1.16.1 binary |
-| `vmlinux.bin` | Linux 6.1.141 bzImage built from upstream source with a pinned kernel config |
+| `vmlinux.bin` | Uncompressed Linux 6.1.141 ELF kernel built from upstream source with a pinned kernel config |
 | `rootfs.ext4` | Minimal BusyBox + supervisor rootfs image |
 | `awf-firecracker-supervisor` | AWF guest supervisor binary |
 | `SHA256SUMS` | SHA-256 digests for all five files |
@@ -948,16 +948,18 @@ host capability.
 
 It:
 
-1. Downloads the `firecracker-test-x86_64` artifact from the build job
-2. Runs `scripts/ci/firecracker-host-preflight.sh` — verifies Linux, x86_64,
+1. Downloads the `firecracker-test-x86_64` artifact from the build job and
+   restores executable modes stripped by GitHub artifact transport
+2. Grants the workflow user access to the hosted runner's `/dev/kvm`
+3. Runs `scripts/ci/firecracker-host-preflight.sh` — verifies Linux, x86_64,
    `/dev/kvm`, required tools, Firecracker/jailer version strings, and all five
    SHA-256 digests via `sha256sum --check --strict SHA256SUMS`
-3. Installs NPM dependencies, builds the AWF distribution, and builds the
+4. Installs NPM dependencies, builds the AWF distribution, and builds the
    Squid and API proxy container images locally
-4. Runs `scripts/ci/firecracker-live-smoke.sh` — the live test suite
-5. Collects redacted diagnostics (audit, proxy-logs, stdout/stderr) and scans
+5. Runs `scripts/ci/firecracker-live-smoke.sh` — the live test suite
+6. Collects redacted diagnostics (audit, proxy-logs, stdout/stderr) and scans
    for the secret sentinel before uploading
-6. Enforces final residue cleanup of all `awffc-*` network namespaces
+7. Enforces final residue cleanup of all `awffc-*` network namespaces
 
 ### Live smoke test assertions
 

guest/firecracker-supervisor/runtime_linux.go

@@ -137,7 +137,7 @@ func mountWorkspace(config bootConfig) error {
 	if err := os.MkdirAll(config.WorkspaceMount, 0755); err != nil {
 		return fmt.Errorf("create workspace mount: %w", err)
 	}
-	if err := syscall.Mount(config.WorkspaceDevice, config.WorkspaceMount, "", 0, ""); err != nil {
+	if err := syscall.Mount(config.WorkspaceDevice, config.WorkspaceMount, "ext4", 0, ""); err != nil {
 		return fmt.Errorf("mount workspace: %w", err)
 	}
 	return nil

guest/firecracker/build-test-artifacts.sh

@@ -86,9 +86,9 @@ make -C "$BUILD/linux-${LINUX_VERSION}" \
   KBUILD_BUILD_USER=awf \
   KBUILD_BUILD_HOST=github \
   LOCALVERSION=-awf-firecracker \
-  bzImage
+  vmlinux
 install -m 0644 \
-  "$BUILD/linux-${LINUX_VERSION}/arch/x86/boot/bzImage" \
+  "$BUILD/linux-${LINUX_VERSION}/vmlinux" \
   "$OUTPUT/vmlinux.bin"
 
 busybox_tar="$BUILD/downloads/busybox-${BUSYBOX_VERSION}.tar.bz2"

guest/firecracker/verify-test-artifacts.sh

@@ -25,7 +25,7 @@ done
 
 "$ARTIFACT_DIR/firecracker" --version | grep -F '1.16.1'
 "$ARTIFACT_DIR/jailer" --version | grep -F '1.16.1'
-file "$ARTIFACT_DIR/vmlinux.bin" | grep -E 'Linux kernel|boot executable'
+file "$ARTIFACT_DIR/vmlinux.bin" | grep -F 'ELF 64-bit LSB executable'
 e2fsck -f -n "$ARTIFACT_DIR/rootfs.ext4"
 debugfs -R 'stat /sbin/awf-supervisor' "$ARTIFACT_DIR/rootfs.ext4" 2>&1 \
   | grep -F 'Type: regular'

scripts/ci/firecracker-host-preflight.sh

@@ -31,6 +31,12 @@ sudo -n true || fail "Passwordless sudo is required for jailer and netns setup."
 docker info >/dev/null || fail "A host-visible Docker Engine is required."
 docker compose version >/dev/null || fail "Docker Compose v2 is required."
 
+[ -x "$ARTIFACT_DIR/firecracker" ] \
+  || fail "The Firecracker artifact must be executable."
+[ -x "$ARTIFACT_DIR/jailer" ] \
+  || fail "The jailer artifact must be executable."
+[ -x "$ARTIFACT_DIR/awf-firecracker-supervisor" ] \
+  || fail "The guest supervisor artifact must be executable."
 "$ARTIFACT_DIR/firecracker" --version | grep -Fq '1.16.1' \
   || fail "Firecracker v1.16.1 is required."
 "$ARTIFACT_DIR/jailer" --version | grep -Fq '1.16.1' \

src/compose-generator.test.ts

@@ -274,6 +274,7 @@ describe('generateDockerCompose', () => {
         expect(result.networks['awf-net'].external).toBeUndefined();
         expect(result.networks['awf-net'].name).toBe('awf-net');
         expect(result.networks['awf-net'].ipam?.config?.[0]?.subnet).toBe(mockNetworkConfig.subnet);
+        expect(result.networks['awf-net'].ipam?.config?.[0]?.gateway).toBe('172.30.0.1');
         expect(result.networks['awf-ext'].driver).toBe('bridge');
       });
 

src/compose-network.ts

@@ -1,6 +1,10 @@
 import { DockerComposeConfig } from './types';
 import { TOPOLOGY_NETWORK_NAME } from './topology';
-import { EXTERNAL_BRIDGE_NAME, EMBEDDED_DNS_RESOLVER } from './config/network-policy';
+import {
+  EMBEDDED_DNS_RESOLVER,
+  EXTERNAL_BRIDGE_NAME,
+  NETWORK_GATEWAY,
+} from './config/network-policy';
 import { NetworkConfig } from './services/squid-service';
 
 interface BuildComposeNetworksParams {
@@ -52,7 +56,7 @@ export function buildComposeNetworks(params: BuildComposeNetworksParams): Docker
           name: TOPOLOGY_NETWORK_NAME,
           internal: true,
           ipam: {
-            config: [{ subnet: networkConfig.subnet }],
+            config: [{ subnet: networkConfig.subnet, gateway: NETWORK_GATEWAY }],
           },
         },
         [EXTERNAL_BRIDGE_NAME]: {

src/config/network-policy-validation.test.ts

@@ -19,6 +19,7 @@ const BASE_POLICY = {
     networkName: 'awf-net',
     externalBridgeName: 'awf-ext',
     subnet: '172.30.0.0/24',
+    gateway: '172.30.0.1',
     hosts: {
       squid: { ip: '172.30.0.10', role: 'egress-proxy', required: true, dualHomed: true },
       agent: { ip: '172.30.0.20', role: 'workload', required: true },
@@ -113,6 +114,12 @@ describe('network-policy validation errors', () => {
   });
 
   describe('assertIpv4', () => {
+    it('throws when the topology gateway is invalid', () => {
+      const policy = deepClone(BASE_POLICY);
+      policy.topology.gateway = 'not-an-ip';
+      expect(loadWithPolicy(policy)).toThrow('topology.gateway must be a valid IPv4 address');
+    });
+
     it('throws when IP has an octet > 255', () => {
       const policy = deepClone(BASE_POLICY);
       policy.topology.hosts.squid.ip = '172.30.0.300';

src/config/network-policy.test.ts

@@ -3,6 +3,7 @@ import {
   NETWORK_NAME,
   EXTERNAL_BRIDGE_NAME,
   NETWORK_SUBNET,
+  NETWORK_GATEWAY,
   SQUID_IP,
   AGENT_IP,
   API_PROXY_IP,
@@ -161,6 +162,7 @@ describe('network-policy', () => {
       expect(NETWORK_NAME).toBe('awf-net');
       expect(EXTERNAL_BRIDGE_NAME).toBe('awf-ext');
       expect(NETWORK_SUBNET).toBe('172.30.0.0/24');
+      expect(NETWORK_GATEWAY).toBe('172.30.0.1');
       expect(HOST_GATEWAY).toBe('172.30.0.1');
       expect(SQUID_IP).toBe('172.30.0.10');
       expect(AGENT_IP).toBe('172.30.0.20');

src/config/network-policy.ts

@@ -65,6 +65,7 @@ export interface NetworkPolicy {
     readonly networkName: string;
     readonly externalBridgeName: string;
     readonly subnet: string;
+    readonly gateway: string;
     readonly hosts: {
       readonly squid: NetworkHost;
       readonly agent: NetworkHost;
@@ -201,6 +202,7 @@ function parseTopology(value: unknown): NetworkPolicy['topology'] {
     networkName: assertString(t.networkName, 'topology.networkName'),
     externalBridgeName: assertString(t.externalBridgeName, 'topology.externalBridgeName'),
     subnet: assertCidr(t.subnet, 'topology.subnet'),
+    gateway: assertIpv4(t.gateway, 'topology.gateway'),
     hosts: parseHosts(t.hosts),
   };
 }
@@ -325,6 +327,9 @@ export const EXTERNAL_BRIDGE_NAME: string = networkPolicy.topology.externalBridg
 /** IPv4 subnet (CIDR) of the internal topology network, shared by all runtimes. */
 export const NETWORK_SUBNET: string = networkPolicy.topology.subnet;
 
+/** Fixed gateway of the internal Docker topology network. */
+export const NETWORK_GATEWAY: string = networkPolicy.topology.gateway;
+
 /** Fixed IP of the Squid egress proxy on the internal network. */
 export const SQUID_IP: string = networkPolicy.topology.hosts.squid.ip;
 

src/config/sandbox-network-policy.json

@@ -2,10 +2,11 @@
   "$comment": "Central declarative NETWORK policy shared by ALL runtimes. The PRIMARY, default path is network-isolation (topology) mode: the agent runs on an `internal` Docker network whose only egress is the dual-homed Squid proxy — NO iptables (host or container), no NET_ADMIN. In that mode egress restriction is STRUCTURAL (default-deny by routing): blocked ports/CIDRs have no route out, so they are denied by construction rather than by rules. The `legacyIptables` section below is consumed ONLY by the uncommon --legacy-security / --enable-host-access path (host iptables + the container setup-iptables.sh NAT script). Every value here is a logical Docker network address (subnet, static IP, port, network name) that the daemon resolves — it carries NO host-filesystem or host-network assumptions, so the policy is ARC/DinD safe and needs no --docker-host-path-prefix translation. Loaded and validated by src/config/network-policy.ts.",
 
   "topology": {
-    "$comment": "CORE — consumed by the default iptables-less path. Compose IPAM assigns these static IPs on the internal `networkName`; Squid is dual-homed onto `externalBridgeName` as the sole egress. `subnet` is the fixed subnet all runtimes use. `hostGateway` lives in `legacyIptables`, not here — the default path never touches the host network.",
+    "$comment": "CORE — consumed by the default iptables-less path. Compose IPAM assigns the fixed subnet, bridge gateway, and static service IPs on the internal `networkName`; Squid is dual-homed onto `externalBridgeName` as the sole egress. `gateway` is the internal Docker bridge address. The separate `hostGateway` in `legacyIptables` is consumed by host-network rules only.",
     "networkName": "awf-net",
     "externalBridgeName": "awf-ext",
     "subnet": "172.30.0.0/24",
+    "gateway": "172.30.0.1",
     "hosts": {
       "squid": { "ip": "172.30.0.10", "role": "egress-proxy", "required": true, "dualHomed": true },
       "agent": { "ip": "172.30.0.20", "role": "workload", "required": true },

src/firecracker-runtime-backend.test.ts

@@ -188,6 +188,20 @@ describe('Firecracker runtime backend', () => {
       'probe',
       'vm-stop',
     ]);
+    expect(manager.execute).toHaveBeenNthCalledWith(1, expect.objectContaining({
+      argv: [
+        '/bin/sh',
+        '-c',
+        expect.stringContaining('timeout 5 nc 172.30.0.10 3128'),
+      ],
+    }));
+    expect(manager.execute).toHaveBeenNthCalledWith(1, expect.objectContaining({
+      argv: [
+        '/bin/sh',
+        '-c',
+        expect.stringContaining('wget -q -T 5 -O /dev/null http://172.30.0.30:10000/reflect'),
+      ],
+    }));
     expect(manager.execute).toHaveBeenNthCalledWith(2, expect.objectContaining({
       argv: ['/bin/sh', '-lc', 'printf hello'],
       cwd: '/workspace',
@@ -271,6 +285,23 @@ describe('Firecracker runtime backend', () => {
     expect(manager.stop).toHaveBeenCalledTimes(1);
   });
 
+  it('collects startup diagnostics before tearing down a failed VM', async () => {
+    const { manager, deps } = harness();
+    manager.startInstance.mockRejectedValue(new Error('guest boot failed'));
+    manager.collectDiagnostics.mockImplementation(async () => {
+      expect(manager.stop).not.toHaveBeenCalled();
+    });
+    const backend = new FirecrackerRuntimeBackend(
+      config({ diagnosticLogs: true, auditDir: '/tmp/audit' }),
+      deps,
+    );
+
+    await expect(backend.start('/tmp/awf', ['github.com']))
+      .rejects.toThrow(/guest boot failed/);
+    expect(manager.collectDiagnostics).toHaveBeenCalledWith('/tmp/audit/firecracker');
+    expect(manager.stop).toHaveBeenCalledTimes(1);
+  });
+
   it('fails closed when manager readiness or startup cleanup is unavailable', async () => {
       const missingIp = harness();
       Reflect.set(missingIp.manager, 'guestIp', undefined);
@@ -401,6 +432,8 @@ describe('Firecracker runtime backend', () => {
     expect(environment.OPENAI_API_KEY).not.toBe(secret);
     expect(Object.values(environment)).not.toContain(secret);
     expect(environment.HTTP_PROXY).toBe('http://172.30.0.10:3128');
+    expect(environment.http_proxy).toBe(environment.HTTP_PROXY);
+    expect(environment.https_proxy).toBe(environment.HTTPS_PROXY);
     expect(environment.HOME).toBe('/workspace/.awf-home');
 
     expect(() => buildFirecrackerGuestEnvironment(

src/firecracker-runtime-backend.ts

@@ -1,4 +1,4 @@
-import type { Readable, Writable } from 'stream';
+import { PassThrough, type Readable, type Writable } from 'stream';
 import type { WorkflowDependencies } from './cli-workflow';
 import type { ExternalAgentRuntimeBackend } from './external-runtime-backend';
 import {
@@ -216,6 +216,15 @@ export class FirecrackerRuntimeBackend implements ExternalAgentRuntimeBackend {
       this.dependencies.logger.warn(
         `[firecracker] stage=${stage} status=failed: ${formatError(error)}`,
       );
+      if (this.config.diagnosticLogs && this.manager) {
+        try {
+          await this.collectDiagnostics();
+        } catch (diagnosticError) {
+          this.dependencies.logger.warn(
+            `[firecracker] startup diagnostic collection failed: ${formatError(diagnosticError)}`,
+          );
+        }
+      }
       try {
         await this.manager?.stop();
       } catch (cleanupError) {
@@ -373,23 +382,40 @@ export class FirecrackerRuntimeBackend implements ExternalAgentRuntimeBackend {
       throw new Error('Firecracker guest identity is not ready');
     }
     const squidProbe =
-      `curl --silent --show-error --max-time 5 --output /dev/null ` +
-      `http://${SQUID_IP}:3128/`;
+      `printf 'GET http://localhost/ HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n' ` +
+      `| timeout 5 nc ${SQUID_IP} 3128 | grep -q '^HTTP/'`;
     const apiProxyProbe = this.config.enableApiProxy
-      ? ` && curl --fail --silent --show-error --max-time 5 --noproxy '*' ` +
-        `--output /dev/null http://${API_PROXY_IP}:10000/reflect`
+      ? ` && wget -q -T 5 -O /dev/null http://${API_PROXY_IP}:10000/reflect`
       : '';
+    const probeOutput = new PassThrough();
+    const probeError = new PassThrough();
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+    probeOutput.on('data', (chunk: Buffer) => stdout.push(chunk));
+    probeError.on('data', (chunk: Buffer) => stderr.push(chunk));
     const result = await manager.execute({
       requestId: `probe-${process.pid}-${Date.now()}`,
-      argv: ['/bin/sh', '-c', `set -eu; ${squidProbe}${apiProxyProbe}`],
+      argv: [
+        '/bin/sh',
+        '-c',
+        `set -eu; { ${squidProbe}${apiProxyProbe}; } || ` +
+          `{ status=$?; ip address show; ip route show; exit "$status"; }`,
+      ],
       env: environment,
       cwd: FIRECRACKER_GUEST_WORKSPACE,
       ...identity,
       timeoutMs: FIRECRACKER_PROBE_TIMEOUT_MS,
+      stdout: probeOutput,
+      stderr: probeError,
     });
     if (result.exitCode !== 0) {
+      const diagnostics = Buffer.concat([...stdout, ...stderr])
+        .subarray(0, 4096)
+        .toString('utf8')
+        .trim();
       throw new Error(
-        `Firecracker guest connectivity probe failed with exit code ${result.exitCode}`,
+        `Firecracker guest connectivity probe failed with exit code ${result.exitCode}` +
+        (diagnostics ? `:\n${diagnostics}` : ''),
       );
     }
     this.dependencies.logger.info(
@@ -421,6 +447,8 @@ export function buildFirecrackerGuestEnvironment(
     HOME: FIRECRACKER_GUEST_HOME,
     PWD: FIRECRACKER_GUEST_WORKSPACE,
     AWF_WORKDIR: FIRECRACKER_GUEST_WORKSPACE,
+    http_proxy: `http://${infrastructure.squidIp}:3128`,
+    https_proxy: `http://${infrastructure.squidIp}:3128`,
     SQUID_PROXY_HOST: infrastructure.squidIp,
     HOSTNAME: 'awf-firecracker',
     AWF_RUNTIME: 'firecracker',