Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions docs/cloud-hypervisor-foundation.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,36 @@ direct internet, arbitrary TCP, direct DNS, and instance metadata access.
Guest proxy environment variables improve client compatibility, but the
namespace policy is the enforcement boundary.

### Untrusted guest output

Guest stdout and stderr cross a host-side presentation boundary before AWF writes
them to the runner log. A streaming byte filter neutralizes lines that begin,
after optional runner-recognized leading whitespace, with GitHub Actions workflow-command
syntax such as `::set-output::`, `::add-mask::`, or `::stop-commands::`. The
filter operates across VSOCK frame boundaries, preserves non-command and
non-UTF-8 bytes, and retains only constant-size command candidates rather than
complete lines. It also neutralizes the runner's legacy `##[...]` command form
wherever it appears in a line. Output writes continue to honor stream
backpressure.

AWF intentionally has no workflow-command allowlist for guest output. Unlike a
trusted host helper, the guest cannot prove that an informational annotation
such as `::error::` came from a trusted producer, so allowing any command name
would preserve an unnecessary runner-control channel.

Filtering applies only to the live runner-facing stdout and stderr streams.
Internal readiness probes retain their original bytes and semantics. Before
filtering, AWF also captures the exact raw guest streams in bounded 1 MiB tails.
Diagnostic collection writes these private files with mode `0600`:

- `guest-stdout.raw.log`
- `guest-stderr.raw.log`

They are stored alongside the other Cloud Hypervisor diagnostics under the
configured audit directory, or under the work-directory diagnostics path when
no audit directory is configured. This preserves forensic evidence without
allowing raw guest bytes to reach the GitHub Actions command parser.

## Guest and workspace

The guest boots a pinned PCI-capable Linux kernel and deterministic BusyBox
Expand Down
50 changes: 50 additions & 0 deletions src/cloud-hypervisor-runtime-backend.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ function harness(overrides: Partial<CloudHypervisorRuntimeBackendDependencies> =
writeStdin: jest.fn().mockResolvedValue(undefined),
endStdin: jest.fn().mockResolvedValue(undefined),
collectDiagnostics: jest.fn().mockResolvedValue(undefined),
collectGuestOutputAudit: jest.fn().mockResolvedValue(undefined),
stop: jest.fn(async (options?: { beforeCleanup?: () => Promise<void> }) => {
order.push('vm-stop');
await options?.beforeCleanup?.();
Expand Down Expand Up @@ -353,11 +354,60 @@ describe('Cloud Hypervisor runtime backend', () => {
uid: 1000,
gid: 1000,
timeoutMs: 60_000,
filterWorkflowCommands: true,
}));
expect(manager.execute).toHaveBeenNthCalledWith(
1,
expect.not.objectContaining({ filterWorkflowCommands: true }),
);
expect(manager.execute).toHaveBeenNthCalledWith(
2,
expect.not.objectContaining({ filterWorkflowCommands: true }),
);
expect(manager.writeStdin).toHaveBeenCalledWith(
Buffer.from('input'),
expect.stringMatching(/^agent-/),
);
expect(manager.collectGuestOutputAudit).not.toHaveBeenCalled();
});

it('persists bounded raw guest output when an audit directory is configured', async () => {
const { manager, deps, stdin } = harness();
const backend = createBackend(config({ auditDir: '/tmp/audit' }), deps);

await backend.start('/tmp/awf', ['github.com']);
const execution = backend.exec('/tmp/awf', ['github.com']);
stdin.end();
await expect(execution).resolves.toEqual({ exitCode: 23 });

expect(manager.collectGuestOutputAudit).toHaveBeenCalledWith(
'/tmp/audit/cloud-hypervisor',
);
});

it('persists raw guest output after a failed agent execution', async () => {
const { manager, deps, stdin } = harness();
manager.execute.mockReset().mockImplementationOnce(async (request) => ({
requestId: request.requestId,
exitCode: 0,
signal: null,
timedOut: false,
})).mockImplementationOnce(async (request) => ({
requestId: request.requestId,
exitCode: 0,
signal: null,
timedOut: false,
})).mockRejectedValueOnce(new Error('guest transport failed'));
const backend = createBackend(config({ auditDir: '/tmp/audit' }), deps);

await backend.start('/tmp/awf', ['github.com']);
const execution = backend.exec('/tmp/awf', ['github.com']);
stdin.end();
await expect(execution).rejects.toThrow('guest transport failed');

expect(manager.collectGuestOutputAudit).toHaveBeenCalledWith(
'/tmp/audit/cloud-hypervisor',
);
});

it('discovers and probes trusted topology peers before agent execution', async () => {
Expand Down
5 changes: 5 additions & 0 deletions src/cloud-hypervisor-runtime-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ interface CloudHypervisorManagerAdapter {
endStdin(requestId?: string): Promise<void>;
stop(options?: { preserve?: boolean; beforeCleanup?: () => Promise<void> }): Promise<void>;
collectDiagnostics(directory: string): Promise<void>;
collectGuestOutputAudit(directory: string): Promise<void>;
}

/** @internal Exposed only for unit tests — not part of the public API. */
Expand Down Expand Up @@ -400,6 +401,7 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend {
...(timeoutMs === undefined ? {} : { timeoutMs }),
stdout: this.dependencies.stdout,
stderr: this.dependencies.stderr,
filterWorkflowCommands: true,
});
this.activeExecution = { requestId, promise: execution };

Expand Down Expand Up @@ -438,6 +440,9 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend {
this.dependencies.stdin.off('end', onEnd);
await forwarding;
this.activeExecution = undefined;
if (this.config.auditDir) {
await manager.collectGuestOutputAudit(`${this.config.auditDir}/cloud-hypervisor`);
}
}
};

Expand Down
37 changes: 37 additions & 0 deletions src/cloud-hypervisor/diagnostics.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { constants, promises as fs } from 'fs';
import * as path from 'path';
import { Writable } from 'stream';
import type { ExecaChildProcess } from 'execa';
import type { MicrovmNetworkLifecycle, MicrovmNetworkPlan } from '../microvm/network';
import {
Expand Down Expand Up @@ -55,6 +56,15 @@ export class BoundedOutputCapture {
contents(): Buffer {
return this.buffer;
}

writable(): Writable {
return new Writable({
write: (chunk: Buffer, _encoding, callback) => {
this.append(chunk);
callback();
},
});
}
}

/**
Expand Down Expand Up @@ -151,6 +161,8 @@ export interface CloudHypervisorDiagnosticsContext {
config: CloudHypervisorOptions;
stdoutCapture: BoundedOutputCapture;
stderrCapture: BoundedOutputCapture;
guestStdoutCapture: BoundedOutputCapture;
guestStderrCapture: BoundedOutputCapture;
network: MicrovmNetworkLifecycle | undefined;
networkPlan: MicrovmNetworkPlan | undefined;
client: CloudHypervisorApiClient | undefined;
Expand All @@ -162,6 +174,25 @@ export interface CloudHypervisorDiagnosticsContext {
fsDevices: readonly VirtiofsdDevice[];
}

export async function writeGuestOutputAudit(
directory: string,
dependencies: Pick<CloudHypervisorManagerDependencies, 'mkdir' | 'writeFile'>,
stdoutCapture: BoundedOutputCapture,
stderrCapture: BoundedOutputCapture,
): Promise<void> {
await dependencies.mkdir(directory, { recursive: true, mode: 0o700 });
await dependencies.writeFile(
path.join(directory, 'guest-stdout.raw.log'),
stdoutCapture.contents(),
{ mode: 0o600 },
);
await dependencies.writeFile(
path.join(directory, 'guest-stderr.raw.log'),
stderrCapture.contents(),
{ mode: 0o600 },
);
}

export async function collectCloudHypervisorDiagnostics(
directory: string,
context: CloudHypervisorDiagnosticsContext,
Expand Down Expand Up @@ -198,6 +229,12 @@ export async function collectCloudHypervisorDiagnostics(
};
await writeBounded('launcher-stdout.log', context.stdoutCapture.contents());
await writeBounded('launcher-stderr.log', context.stderrCapture.contents());
await writeGuestOutputAudit(
directory,
dependencies,
context.guestStdoutCapture,
context.guestStderrCapture,
);
await copyBoundedDiagnostic(
dependencies,
paths.logPath,
Expand Down
24 changes: 24 additions & 0 deletions src/cloud-hypervisor/manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,30 @@ describe('CloudHypervisorManager', () => {
uid: 1000,
gid: 1000,
})).resolves.toEqual(expect.objectContaining({ exitCode: 0 }));
const forwardedRequest = (guestClient.execute as jest.Mock).mock.calls[0][0];
const rawGuestStdout = Buffer.concat([
Buffer.from('discarded prefix'),
Buffer.alloc(1024 * 1024, 0x7a),
]);
forwardedRequest.rawStdout.write(rawGuestStdout);
forwardedRequest.rawStderr.write(Buffer.from([0xff, 0x00, 0xfe]));
await manager.collectDiagnostics('/tmp/diagnostics');
expect(deps.writeFile).toHaveBeenCalledWith(
'/tmp/diagnostics/guest-stdout.raw.log',
Buffer.alloc(1024 * 1024, 0x7a),
{ mode: 0o600 },
);
expect(deps.writeFile).toHaveBeenCalledWith(
'/tmp/diagnostics/guest-stderr.raw.log',
Buffer.from([0xff, 0x00, 0xfe]),
{ mode: 0o600 },
);
await manager.collectGuestOutputAudit('/tmp/audit/cloud-hypervisor');
expect(deps.writeFile).toHaveBeenCalledWith(
'/tmp/audit/cloud-hypervisor/guest-stdout.raw.log',
Buffer.alloc(1024 * 1024, 0x7a),
{ mode: 0o600 },
);
await manager.stop();

expect(guestClient.shutdown).toHaveBeenCalledTimes(1);
Expand Down
22 changes: 21 additions & 1 deletion src/cloud-hypervisor/manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
BoundedOutputCapture,
collectCloudHypervisorDiagnostics,
readBoundedTail,
writeGuestOutputAudit,
} from './diagnostics';
import {
CloudHypervisorGuestChannel,
Expand Down Expand Up @@ -169,6 +170,10 @@ export class CloudHypervisorManager {
private lastVmCounters: CloudHypervisorVmCounters | undefined;
private readonly stdoutCapture = new BoundedOutputCapture(CLOUD_HYPERVISOR_CAPTURE_LIMIT_BYTES);
private readonly stderrCapture = new BoundedOutputCapture(CLOUD_HYPERVISOR_CAPTURE_LIMIT_BYTES);
private readonly guestStdoutCapture = new BoundedOutputCapture(CLOUD_HYPERVISOR_CAPTURE_LIMIT_BYTES);
private readonly guestStderrCapture = new BoundedOutputCapture(CLOUD_HYPERVISOR_CAPTURE_LIMIT_BYTES);
private readonly guestStdoutAudit = this.guestStdoutCapture.writable();
private readonly guestStderrAudit = this.guestStderrCapture.writable();

get guestIp(): string | undefined {
return this.networkPlan?.guestIp;
Expand Down Expand Up @@ -244,7 +249,11 @@ export class CloudHypervisorManager {
if (!this.guest) {
throw new Error('Cloud Hypervisor guest supervisor is not ready');
}
return this.guest.execute(request);
return this.guest.execute({
...request,
rawStdout: this.guestStdoutAudit,
rawStderr: this.guestStderrAudit,
});
}

cancel(reason = 'host cancellation', requestId?: string): Promise<void> {
Expand Down Expand Up @@ -315,6 +324,8 @@ export class CloudHypervisorManager {
config: this.config,
stdoutCapture: this.stdoutCapture,
stderrCapture: this.stderrCapture,
guestStdoutCapture: this.guestStdoutCapture,
guestStderrCapture: this.guestStderrCapture,
network: this.network,
networkPlan: this.networkPlan,
client: this.client,
Expand All @@ -324,4 +335,13 @@ export class CloudHypervisorManager {
fsDevices: this.fsDevices,
});
}

async collectGuestOutputAudit(directory: string): Promise<void> {
await writeGuestOutputAudit(
directory,
this.dependencies,
this.guestStdoutCapture,
this.guestStderrCapture,
);
}
}
Loading
Loading