diff --git a/docs/cloud-hypervisor-foundation.md b/docs/cloud-hypervisor-foundation.md index 7db53b0b5..048471fad 100644 --- a/docs/cloud-hypervisor-foundation.md +++ b/docs/cloud-hypervisor-foundation.md @@ -74,15 +74,20 @@ AWF performs these steps for each run: run directory. 6. Create a bounded cgroup v2 leaf and launch Cloud Hypervisor as the invoking non-root identity. -7. Start one sandboxed `virtiofsd` process for each validated export. -8. Create and boot the VM, connect to the guest supervisor over VSOCK, verify +7. After the API responds, verify the launched VMM's trusted host `/proc` and + cgroup state. AWF fails closed before `vm.create` if the PID identity, + executable, credentials, capabilities, `no_new_privs`, seccomp worker, + network namespace, cgroup membership, or resource limits differ from the + launch policy. +8. Start one sandboxed `virtiofsd` process for each validated export. +9. Create and boot the VM, connect to the guest supervisor over VSOCK, verify loopback plus the configured guest interface, address, and route, and probe each trusted infrastructure service with bounded retries. An exhausted retryable readiness failure recreates the VM at most twice before the agent command is dispatched. -9. Execute the agent command and propagate its exit code. Timeouts return +10. Execute the agent command and propagate its exit code. Timeouts return `124`. -10. Sync and unmount guest filesystems, stop the VM and VMM, reap `virtiofsd`, +11. Sync and unmount guest filesystems, stop the VM and VMM, reap `virtiofsd`, and remove network, cgroup, and run-directory resources. Cleanup is idempotent and aggregates errors so one cleanup failure does not @@ -120,6 +125,23 @@ shell. The process: - receives a minimal Landlock filesystem allowlist; and - belongs to a cgroup v2 leaf with explicit memory, CPU, and PID limits. +API socket readiness alone is not treated as proof of confinement. Before +creating any VM or starting `virtiofsd`, AWF reads the VMM's host `/proc` +records and cgroup files as root. It verifies the PID twice using the kernel +start-time field and executable symlink to reject process-exit and PID-reuse +races. Credentials and capability sets are checked against the launcher's +current policy for every observed thread, and both the `vmm` worker and +`http-server` API thread must be in seccomp filter mode. The verifier also +compares network namespace inode links, requires exclusive membership in the +per-run cgroup, and checks the exact memory, CPU, and PID limits computed by the +cgroup policy. + +Successful verification produces bounded structured evidence in +`confinement.json` alongside the other run diagnostics. The evidence records +the stable process identity, expected credentials and capabilities, relevant +seccomp thread IDs, namespace inode, and cgroup membership and limits; it does +not copy unbounded `/proc` content. + The private run directory is under `/run/awf-cloud-hypervisor///`. Its per-run leaf is accessible only to the selected non-root identity and root. @@ -468,7 +490,8 @@ sudo nft list ruleset Inspect preserved workspace data under `/microvm-images//` and VMM diagnostics under the run's -preserved log directory. +preserved log directory. `confinement.json` contains the production +post-launch verification evidence captured before `vm.create`. :::caution Preserved namespaces and processes continue consuming host resources. Remove diff --git a/src/cloud-hypervisor/confinement-verifier.test.ts b/src/cloud-hypervisor/confinement-verifier.test.ts new file mode 100644 index 000000000..5edb09896 --- /dev/null +++ b/src/cloud-hypervisor/confinement-verifier.test.ts @@ -0,0 +1,171 @@ +import { + verifyCloudHypervisorConfinement, + type CloudHypervisorConfinementVerifierDependencies, +} from './confinement-verifier'; +import type { CloudHypervisorLaunchConfinementPolicy } from './launcher'; + +const PID = 4242; +const CGROUP = '/sys/fs/cgroup/awf-cloud-hypervisor/run-1'; +const CAPABILITY_MASK = '0000000000002000'; + +function launchPolicy(): CloudHypervisorLaunchConfinementPolicy { + return { + supplementaryGroups: [978], + capabilities: { + inheritable: CAPABILITY_MASK, + permitted: CAPABILITY_MASK, + effective: CAPABILITY_MASK, + bounding: CAPABILITY_MASK, + ambient: CAPABILITY_MASK, + }, + noNewPrivs: 1, + }; +} + +function status(name: string, seccomp: number, taskId = PID): string { + return [ + `Name:\t${name}`, + `Pid:\t${taskId}`, + `Tgid:\t${PID}`, + 'Uid:\t1000\t1000\t1000\t1000', + 'Gid:\t1001\t1001\t1001\t1001', + 'Groups:\t978', + `CapInh:\t${CAPABILITY_MASK}`, + `CapPrm:\t${CAPABILITY_MASK}`, + `CapEff:\t${CAPABILITY_MASK}`, + `CapBnd:\t${CAPABILITY_MASK}`, + `CapAmb:\t${CAPABILITY_MASK}`, + 'NoNewPrivs:\t1', + `Seccomp:\t${seccomp}`, + '', + ].join('\n'); +} + +function procStat(startTime: string): string { + return `${PID} (cloud hypervisor) ${['S', ...Array(18).fill('0'), startTime, '0'].join(' ')}`; +} + +function dependencies(overrides: { + statReads?: string[]; + executable?: string; + workerStatus?: string; + cgroupProcs?: string; +} = {}): CloudHypervisorConfinementVerifierDependencies { + const statReads = [...(overrides.statReads ?? [procStat('98765'), procStat('98765')])]; + const files: Record = { + [`/proc/${PID}/task/${PID}/status`]: status('cloud-hypervis', 0), + [`/proc/${PID}/task/${PID + 1}/status`]: + overrides.workerStatus ?? status('vmm', 2, PID + 1), + [`/proc/${PID}/task/${PID + 2}/status`]: status('http-server', 2, PID + 2), + [`/proc/${PID}/cgroup`]: '0::/awf-cloud-hypervisor/run-1\n', + [`${CGROUP}/cgroup.procs`]: overrides.cgroupProcs ?? `${PID}\n`, + [`${CGROUP}/memory.max`]: '805306368\n', + [`${CGROUP}/cpu.max`]: '300000 100000\n', + [`${CGROUP}/pids.max`]: '256\n', + }; + return { + readFile: jest.fn(async (filePath) => { + if (filePath === `/proc/${PID}/stat`) { + const value = statReads.shift(); + if (!value) throw new Error('unexpected stat read'); + return value; + } + const taskStatMatch = filePath.match(new RegExp(`^/proc/${PID}/task/(\\d+)/stat$`)); + if (taskStatMatch) return procStat(String(99000 + Number(taskStatMatch[1]))); + const value = files[filePath]; + if (value === undefined) throw new Error(`unexpected read: ${filePath}`); + return value; + }), + readlink: jest.fn(async (filePath) => { + if (filePath === `/proc/${PID}/exe`) { + return overrides.executable ?? '/opt/cloud-hypervisor'; + } + if (filePath === `/proc/${PID}/ns/net`) { + return 'net:[4026533000]'; + } + throw new Error(`unexpected readlink: ${filePath}`); + }), + readdir: jest.fn().mockResolvedValue([String(PID + 2), String(PID + 1), String(PID)]), + realpath: jest.fn().mockResolvedValue('/opt/cloud-hypervisor'), + stat: jest.fn().mockResolvedValue({ ino: 4026533000n }), + }; +} + +function options() { + return { + pid: PID, + expectedExecutable: '/opt/cloud-hypervisor', + identity: { uid: 1000, gid: 1001 }, + launchPolicy: launchPolicy(), + networkNamespace: 'awfvm-test', + cgroupPath: CGROUP, + cgroupLimits: { + memoryMax: '805306368', + cpuMax: '300000 100000', + pidsMax: '256', + }, + }; +} + +describe('verifyCloudHypervisorConfinement', () => { + it('verifies stable process, thread, namespace, and cgroup state with policy-derived capabilities', async () => { + const result = await verifyCloudHypervisorConfinement(options(), dependencies()); + + expect(result).toEqual(expect.objectContaining({ + schemaVersion: 1, + process: { + pid: PID, + startTimeTicks: '98765', + executable: '/opt/cloud-hypervisor', + }, + identity: { + uid: 1000, + gid: 1001, + supplementaryGroups: [978], + }, + capabilities: expect.objectContaining({ effective: CAPABILITY_MASK }), + noNewPrivs: 1, + seccomp: { + mode: 2, + relevantThreadIds: [PID + 1, PID + 2], + observedThreadCount: 3, + }, + networkNamespace: { + name: 'awfvm-test', + inode: 'net:[4026533000]', + }, + cgroup: expect.objectContaining({ + path: CGROUP, + membership: '/awf-cloud-hypervisor/run-1', + }), + })); + }); + + it('fails closed when PID identity changes while evidence is collected', async () => { + await expect(verifyCloudHypervisorConfinement( + options(), + dependencies({ statReads: [procStat('98765'), procStat('98766')] }), + )).rejects.toThrow(/process identity or thread-set race/); + }); + + it('rejects a different executable even when the PID exists', async () => { + await expect(verifyCloudHypervisorConfinement( + options(), + dependencies({ executable: '/usr/bin/setpriv' }), + )).rejects.toThrow(/found executable/); + }); + + it('requires the Cloud Hypervisor vmm worker to have seccomp filter mode 2', async () => { + await expect(verifyCloudHypervisorConfinement( + options(), + dependencies({ workerStatus: status('vmm', 0, PID + 1) }), + )).rejects.toThrow(/does not have seccomp filter mode 2/); + }); + + it('requires exclusive membership in the configured bounded cgroup', async () => { + await expect(verifyCloudHypervisorConfinement( + options(), + dependencies({ cgroupProcs: `${PID}\n5000\n` }), + )).rejects.toThrow(/cgroup\.procs to contain only PID/); + }); +}); diff --git a/src/cloud-hypervisor/confinement-verifier.ts b/src/cloud-hypervisor/confinement-verifier.ts new file mode 100644 index 000000000..811a07812 --- /dev/null +++ b/src/cloud-hypervisor/confinement-verifier.ts @@ -0,0 +1,400 @@ +import { promises as fs } from 'fs'; +import * as path from 'path'; +import type { + CloudHypervisorCgroupLimits, + CloudHypervisorLaunchConfinementPolicy, +} from './launcher'; + +const CGROUP_ROOT = '/sys/fs/cgroup'; +const NETWORK_NAMESPACE_ROOT = '/run/netns'; +const MAX_VERIFIED_THREADS = 256; +const MAX_EVIDENCE_STRING_LENGTH = 4096; +const SECCOMP_RELEVANT_THREAD_NAMES = ['http-server', 'vmm'] as const; + +export interface CloudHypervisorConfinementEvidence { + readonly schemaVersion: 1; + readonly verifiedAt: string; + readonly process: { + readonly pid: number; + readonly startTimeTicks: string; + readonly executable: string; + }; + readonly identity: { + readonly uid: number; + readonly gid: number; + readonly supplementaryGroups: readonly number[]; + }; + readonly capabilities: CloudHypervisorLaunchConfinementPolicy['capabilities']; + readonly noNewPrivs: 1; + readonly seccomp: { + readonly mode: 2; + readonly relevantThreadIds: readonly number[]; + readonly observedThreadCount: number; + }; + readonly networkNamespace: { + readonly name: string; + readonly inode: string; + }; + readonly cgroup: { + readonly path: string; + readonly membership: string; + readonly limits: CloudHypervisorCgroupLimits; + }; +} + +export interface CloudHypervisorConfinementVerificationOptions { + readonly pid: number; + readonly expectedExecutable: string; + readonly identity: { readonly uid: number; readonly gid: number }; + readonly launchPolicy: CloudHypervisorLaunchConfinementPolicy; + readonly networkNamespace: string; + readonly cgroupPath: string; + readonly cgroupLimits: CloudHypervisorCgroupLimits; +} + +export interface CloudHypervisorConfinementVerifierDependencies { + readFile(filePath: string, encoding: BufferEncoding): Promise; + readlink(filePath: string): Promise; + readdir(directory: string): Promise; + realpath(filePath: string): Promise; + stat(filePath: string): Promise<{ ino: bigint }>; +} + +const defaultDependencies: CloudHypervisorConfinementVerifierDependencies = { + readFile: (filePath, encoding) => fs.readFile(filePath, encoding), + readlink: fs.readlink, + readdir: (directory) => fs.readdir(directory), + realpath: fs.realpath, + stat: (filePath) => fs.stat(filePath, { bigint: true }), +}; + +export async function verifyCloudHypervisorConfinement( + options: CloudHypervisorConfinementVerificationOptions, + dependencies: CloudHypervisorConfinementVerifierDependencies = defaultDependencies, +): Promise { + if (!Number.isSafeInteger(options.pid) || options.pid <= 1) { + throw new Error(`Cloud Hypervisor confinement verification received an invalid PID: ${options.pid}`); + } + const procDirectory = `/proc/${options.pid}`; + const expectedExecutable = await dependencies.realpath(options.expectedExecutable); + assertBoundedString(expectedExecutable, 'expected executable'); + + const initialStat = await dependencies.readFile(path.join(procDirectory, 'stat'), 'utf8'); + const initialStartTime = parseProcessStartTime(initialStat); + const executable = await dependencies.readlink(path.join(procDirectory, 'exe')); + assertBoundedString(executable, 'process executable'); + if (executable !== expectedExecutable) { + throw new Error( + `Cloud Hypervisor confinement verification found executable ${JSON.stringify(executable)}, ` + + `expected ${JSON.stringify(expectedExecutable)}`, + ); + } + + const taskDirectory = path.join(procDirectory, 'task'); + const taskIds = parseTaskIds(await dependencies.readdir(taskDirectory)); + const taskStartTimes = new Map(); + const relevantThreadIds: number[] = []; + const relevantThreadNames = new Set(); + for (const taskId of taskIds) { + taskStartTimes.set( + taskId, + parseProcessStartTime( + await dependencies.readFile(path.join(taskDirectory, String(taskId), 'stat'), 'utf8'), + ), + ); + const status = parseStatus( + await dependencies.readFile(path.join(taskDirectory, String(taskId), 'status'), 'utf8'), + ); + const name = verifyThreadStatus(status, taskId, options); + if (name !== undefined) { + relevantThreadIds.push(taskId); + relevantThreadNames.add(name); + } + } + const missingRelevantThreads = SECCOMP_RELEVANT_THREAD_NAMES.filter( + (name) => !relevantThreadNames.has(name), + ); + if (missingRelevantThreads.length > 0) { + throw new Error( + `Cloud Hypervisor confinement verification did not observe seccomp-relevant thread(s): ` + + missingRelevantThreads.join(', '), + ); + } + + const processCgroup = parseUnifiedCgroup( + await dependencies.readFile(path.join(procDirectory, 'cgroup'), 'utf8'), + ); + const expectedCgroupMembership = cgroupMembership(options.cgroupPath); + if (processCgroup !== expectedCgroupMembership) { + throw new Error( + `Cloud Hypervisor confinement verification found cgroup ${JSON.stringify(processCgroup)}, ` + + `expected ${JSON.stringify(expectedCgroupMembership)}`, + ); + } + const cgroupPids = parseNumericLines( + await dependencies.readFile(path.join(options.cgroupPath, 'cgroup.procs'), 'utf8'), + 'cgroup.procs', + ); + if (cgroupPids.length !== 1 || cgroupPids[0] !== options.pid) { + throw new Error( + `Cloud Hypervisor confinement verification expected cgroup.procs to contain only PID ` + + `${options.pid}, found ${cgroupPids.join(', ') || 'none'}`, + ); + } + const observedLimits: CloudHypervisorCgroupLimits = { + memoryMax: (await dependencies.readFile(path.join(options.cgroupPath, 'memory.max'), 'utf8')).trim(), + cpuMax: (await dependencies.readFile(path.join(options.cgroupPath, 'cpu.max'), 'utf8')).trim(), + pidsMax: (await dependencies.readFile(path.join(options.cgroupPath, 'pids.max'), 'utf8')).trim(), + }; + for (const key of ['memoryMax', 'cpuMax', 'pidsMax'] as const) { + if (observedLimits[key] !== options.cgroupLimits[key]) { + throw new Error( + `Cloud Hypervisor confinement verification found ${key}=${JSON.stringify(observedLimits[key])}, ` + + `expected ${JSON.stringify(options.cgroupLimits[key])}`, + ); + } + } + + if (!/^[A-Za-z0-9_.-]+$/.test(options.networkNamespace)) { + throw new Error(`Unsafe Cloud Hypervisor network namespace name: ${options.networkNamespace}`); + } + const processNamespace = await dependencies.readlink(path.join(procDirectory, 'ns', 'net')); + assertNamespaceLink(processNamespace, 'process network namespace'); + const processNamespaceInode = processNamespace.slice(5, -1); + const expectedNamespaceInode = ( + await dependencies.stat(path.join(NETWORK_NAMESPACE_ROOT, options.networkNamespace)) + ).ino.toString(); + if (processNamespaceInode !== expectedNamespaceInode) { + throw new Error( + `Cloud Hypervisor confinement verification found network namespace ${processNamespace}, ` + + `expected net:[${expectedNamespaceInode}] (${options.networkNamespace})`, + ); + } + + const finalTaskIds = parseTaskIds(await dependencies.readdir(taskDirectory)); + const finalStat = await dependencies.readFile(path.join(procDirectory, 'stat'), 'utf8'); + const finalStartTime = parseProcessStartTime(finalStat); + const finalExecutable = await dependencies.readlink(path.join(procDirectory, 'exe')); + const finalTaskStartTimes = new Map(); + for (const taskId of finalTaskIds) { + finalTaskStartTimes.set( + taskId, + parseProcessStartTime( + await dependencies.readFile(path.join(taskDirectory, String(taskId), 'stat'), 'utf8'), + ), + ); + } + if ( + finalStartTime !== initialStartTime || + finalExecutable !== executable || + finalTaskIds.join(',') !== taskIds.join(',') || + finalTaskIds.some((taskId) => finalTaskStartTimes.get(taskId) !== taskStartTimes.get(taskId)) + ) { + throw new Error( + 'Cloud Hypervisor confinement verification detected a process identity or thread-set race', + ); + } + + return { + schemaVersion: 1, + verifiedAt: new Date().toISOString(), + process: { + pid: options.pid, + startTimeTicks: initialStartTime, + executable, + }, + identity: { + uid: options.identity.uid, + gid: options.identity.gid, + supplementaryGroups: [...options.launchPolicy.supplementaryGroups], + }, + capabilities: { ...options.launchPolicy.capabilities }, + noNewPrivs: 1, + seccomp: { + mode: 2, + relevantThreadIds, + observedThreadCount: taskIds.length, + }, + networkNamespace: { + name: options.networkNamespace, + inode: processNamespace, + }, + cgroup: { + path: options.cgroupPath, + membership: processCgroup, + limits: observedLimits, + }, + }; +} + +function verifyThreadStatus( + status: Readonly>, + taskId: number, + options: CloudHypervisorConfinementVerificationOptions, +): string | undefined { + const observedPid = parseSingleNumericStatus(status, 'Pid', taskId); + const observedTgid = parseSingleNumericStatus(status, 'Tgid', taskId); + if (observedPid !== taskId || observedTgid !== options.pid) { + throw new Error( + `Cloud Hypervisor task ${taskId} reports Pid=${observedPid} and Tgid=${observedTgid}, ` + + `expected Pid=${taskId} and Tgid=${options.pid}`, + ); + } + assertIdentityField(status.Uid, options.identity.uid, 'UID', taskId); + assertIdentityField(status.Gid, options.identity.gid, 'GID', taskId); + const groups = parseNumericFields(requiredStatus(status, 'Groups', taskId), 'Groups'); + if (groups.join(',') !== options.launchPolicy.supplementaryGroups.join(',')) { + throw new Error( + `Cloud Hypervisor thread ${taskId} has supplementary groups ${groups.join(',') || 'none'}, ` + + `expected ${options.launchPolicy.supplementaryGroups.join(',') || 'none'}`, + ); + } + + function parseSingleNumericStatus( + status: Readonly>, + field: string, + taskId: number, + ): number { + const values = parseNumericFields(requiredStatus(status, field, taskId), field); + if (values.length !== 1) { + throw new Error(`Cloud Hypervisor thread ${taskId} has malformed ${field}`); + } + return values[0]; + } + const capabilityFields = { + CapInh: 'inheritable', + CapPrm: 'permitted', + CapEff: 'effective', + CapBnd: 'bounding', + CapAmb: 'ambient', + } as const; + for (const [statusField, policyField] of Object.entries(capabilityFields) as + [keyof typeof capabilityFields, keyof CloudHypervisorLaunchConfinementPolicy['capabilities']][]) { + const observed = requiredStatus(status, statusField, taskId).toLowerCase(); + if (observed !== options.launchPolicy.capabilities[policyField]) { + throw new Error( + `Cloud Hypervisor thread ${taskId} has ${statusField}=${observed}, ` + + `expected ${options.launchPolicy.capabilities[policyField]}`, + ); + } + } + if (requiredStatus(status, 'NoNewPrivs', taskId) !== String(options.launchPolicy.noNewPrivs)) { + throw new Error(`Cloud Hypervisor thread ${taskId} does not have NoNewPrivs enabled`); + } + const name = requiredStatus(status, 'Name', taskId); + if ((SECCOMP_RELEVANT_THREAD_NAMES as readonly string[]).includes(name)) { + if (requiredStatus(status, 'Seccomp', taskId) !== '2') { + throw new Error( + `Cloud Hypervisor ${name} thread ${taskId} does not have seccomp filter mode 2`, + ); + } + return name; + } + return undefined; +} + +function parseStatus(contents: string): Readonly> { + const result: Record = {}; + for (const line of contents.split('\n')) { + if (!line) continue; + const separator = line.indexOf(':'); + if (separator <= 0) throw new Error(`Malformed Cloud Hypervisor /proc status line: ${line}`); + const key = line.slice(0, separator); + if (result[key] !== undefined) throw new Error(`Duplicate Cloud Hypervisor /proc status field: ${key}`); + result[key] = line.slice(separator + 1).trim(); + } + return result; +} + +function requiredStatus( + status: Readonly>, + field: string, + taskId: number, +): string { + const value = status[field]; + if (value === undefined || value === '') { + throw new Error(`Cloud Hypervisor thread ${taskId} is missing /proc status field ${field}`); + } + return value; +} + +function assertIdentityField(value: string | undefined, expected: number, label: string, taskId: number): void { + const observed = parseNumericFields( + value ?? '', + `${label} for thread ${taskId}`, + ); + if (observed.length !== 4 || observed.some((entry) => entry !== expected)) { + throw new Error( + `Cloud Hypervisor thread ${taskId} has ${label} values ${observed.join(',') || 'none'}, ` + + `expected four instances of ${expected}`, + ); + } +} + +function parseNumericFields(value: string, label: string): number[] { + const fields = value.trim().split(/\s+/).filter(Boolean); + if (fields.some((field) => !/^\d+$/.test(field))) { + throw new Error(`Cloud Hypervisor confinement verification found malformed ${label}: ${value}`); + } + return fields.map(Number); +} + +function parseNumericLines(value: string, label: string): number[] { + return parseNumericFields(value.replace(/\n/g, ' '), label); +} + +function parseTaskIds(entries: readonly string[]): number[] { + const taskIds = entries + .filter((entry) => /^\d+$/.test(entry)) + .map(Number) + .sort((left, right) => left - right); + if (taskIds.length === 0) { + throw new Error('Cloud Hypervisor confinement verification found no process threads'); + } + if (taskIds.length > MAX_VERIFIED_THREADS) { + throw new Error( + `Cloud Hypervisor confinement verification found ${taskIds.length} threads, ` + + `exceeding the ${MAX_VERIFIED_THREADS} thread evidence bound`, + ); + } + return taskIds; +} + +function parseProcessStartTime(stat: string): string { + const commandEnd = stat.lastIndexOf(')'); + if (commandEnd < 2) throw new Error('Malformed Cloud Hypervisor /proc stat contents'); + const fieldsAfterCommand = stat.slice(commandEnd + 2).trim().split(/\s+/); + const startTime = fieldsAfterCommand[19]; + if (!startTime || !/^\d+$/.test(startTime) || startTime.length > 32) { + throw new Error('Malformed Cloud Hypervisor process start time'); + } + return startTime; +} + +function parseUnifiedCgroup(contents: string): string { + const lines = contents.trim().split('\n'); + if (lines.length !== 1 || !lines[0].startsWith('0::/')) { + throw new Error(`Cloud Hypervisor process is not exclusively in a unified cgroup: ${contents.trim()}`); + } + return lines[0].slice(3); +} + +function cgroupMembership(cgroupPath: string): string { + const relative = path.relative(CGROUP_ROOT, cgroupPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`Unsafe Cloud Hypervisor cgroup path: ${cgroupPath}`); + } + return `/${relative}`; +} + +function assertNamespaceLink(value: string, label: string): void { + if (!/^net:\[\d+\]$/.test(value)) { + throw new Error(`Malformed ${label}: ${JSON.stringify(value)}`); + } +} + +function assertBoundedString(value: string, label: string): void { + if (Buffer.byteLength(value) > MAX_EVIDENCE_STRING_LENGTH) { + throw new Error(`Cloud Hypervisor ${label} exceeds the structured evidence bound`); + } +} diff --git a/src/cloud-hypervisor/diagnostics.ts b/src/cloud-hypervisor/diagnostics.ts index d6df5fa93..fd6b3eedf 100644 --- a/src/cloud-hypervisor/diagnostics.ts +++ b/src/cloud-hypervisor/diagnostics.ts @@ -21,6 +21,7 @@ import { type CloudHypervisorRunPaths, } from './manager-types'; import type { VirtiofsdDevice } from './virtiofsd'; +import type { CloudHypervisorConfinementEvidence } from './confinement-verifier'; /** Reads at most `maxBytes` from the end of `filePath`. */ export async function readBoundedTail(filePath: string, maxBytes: number): Promise { @@ -160,6 +161,7 @@ export interface CloudHypervisorDiagnosticsContext { lastVmInfo: CloudHypervisorVmInfo | undefined; lastVmCounters: CloudHypervisorVmCounters | undefined; fsDevices: readonly VirtiofsdDevice[]; + confinementEvidence: CloudHypervisorConfinementEvidence | undefined; } export async function collectCloudHypervisorDiagnostics( @@ -262,4 +264,9 @@ export async function collectCloudHypervisorDiagnostics( }, null, 2)}\n`, { mode: 0o600 }, ); + await dependencies.writeFile( + path.join(directory, 'confinement.json'), + `${JSON.stringify(context.confinementEvidence ?? null, null, 2)}\n`, + { mode: 0o600 }, + ); } diff --git a/src/cloud-hypervisor/launcher.test.ts b/src/cloud-hypervisor/launcher.test.ts index 668c632b5..97b6acee2 100644 --- a/src/cloud-hypervisor/launcher.test.ts +++ b/src/cloud-hypervisor/launcher.test.ts @@ -37,6 +37,17 @@ describe('buildCloudHypervisorLaunchCommand', () => { '--seccomp', 'true', ]); expect(result.args).not.toContain('--clear-groups'); + expect(result.confinementPolicy).toEqual({ + supplementaryGroups: [978], + capabilities: { + inheritable: '0000000000000000', + permitted: '0000000000000000', + effective: '0000000000000000', + bounding: '0000000000000000', + ambient: '0000000000000000', + }, + noNewPrivs: 1, + }); expect(result.args.some((arg) => arg.includes('+net_admin'))).toBe(false); // No argument contains shell metacharacters that would matter if ever // interpolated; more importantly, args are a plain array (never joined diff --git a/src/cloud-hypervisor/launcher.ts b/src/cloud-hypervisor/launcher.ts index 67b54d018..ecb816cb4 100644 --- a/src/cloud-hypervisor/launcher.ts +++ b/src/cloud-hypervisor/launcher.ts @@ -70,6 +70,7 @@ export interface CloudHypervisorLaunchIdentity { export interface CloudHypervisorLaunchCommand { readonly command: string; readonly args: readonly string[]; + readonly confinementPolicy: CloudHypervisorLaunchConfinementPolicy; } export interface CloudHypervisorLaunchToolPaths { @@ -77,6 +78,43 @@ export interface CloudHypervisorLaunchToolPaths { readonly setpriv: string; } +export interface CloudHypervisorLaunchConfinementPolicy { + readonly supplementaryGroups: readonly number[]; + readonly capabilities: { + readonly inheritable: string; + readonly permitted: string; + readonly effective: string; + readonly bounding: string; + readonly ambient: string; + }; + readonly noNewPrivs: 1; +} + +const CLOUD_HYPERVISOR_ALLOWED_CAPABILITIES: readonly { + readonly setprivName: string; + readonly bit: number; +}[] = []; + +function buildCloudHypervisorConfinementPolicy( + kvmGid: number, +): CloudHypervisorLaunchConfinementPolicy { + const capabilityMask = CLOUD_HYPERVISOR_ALLOWED_CAPABILITIES + .reduce((mask, capability) => mask | (1n << BigInt(capability.bit)), 0n) + .toString(16) + .padStart(16, '0'); + return { + supplementaryGroups: [kvmGid], + capabilities: { + inheritable: capabilityMask, + permitted: capabilityMask, + effective: capabilityMask, + bounding: capabilityMask, + ambient: capabilityMask, + }, + noNewPrivs: 1, + }; +} + /** * Builds the argv AWF spawns to launch Cloud Hypervisor: join the prepared * network namespace, drop to the non-root operator identity with no @@ -121,6 +159,12 @@ export function buildCloudHypervisorLaunchCommand(options: { if (!path.isAbsolute(options.apiSocketPath)) { throw new Error(`Cloud Hypervisor API socket path must be absolute: ${options.apiSocketPath}`); } + const setprivCapabilities = CLOUD_HYPERVISOR_ALLOWED_CAPABILITIES + .map((capability) => `+${capability.setprivName}`) + .join(','); + const resetCapabilitySet = setprivCapabilities + ? `-all,${setprivCapabilities}` + : '-all'; return { command: options.tools.ip, @@ -134,9 +178,9 @@ export function buildCloudHypervisorLaunchCommand(options: { // also drop kvm access). `--groups=${options.kvmGid}`, '--no-new-privs', - '--inh-caps=-all', - '--bounding-set=-all', - '--ambient-caps=-all', + `--inh-caps=${resetCapabilitySet}`, + `--bounding-set=${resetCapabilitySet}`, + `--ambient-caps=${setprivCapabilities || '-all'}`, '--', options.cloudHypervisorBinary, '--api-socket', `path=${options.apiSocketPath}`, @@ -144,6 +188,7 @@ export function buildCloudHypervisorLaunchCommand(options: { '-v', '--seccomp', 'true', ], + confinementPolicy: buildCloudHypervisorConfinementPolicy(options.kvmGid), }; } @@ -184,6 +229,12 @@ export interface CloudHypervisorResourceLimits { readonly vcpuCount: number; } +export interface CloudHypervisorCgroupLimits { + readonly memoryMax: string; + readonly cpuMax: string; + readonly pidsMax: string; +} + /** Fixed VMM/guest-overhead headroom added on top of configured guest memory. */ const CGROUP_MEMORY_HEADROOM_MIB = 256; /** @@ -237,6 +288,18 @@ const defaultCgroupDependencies: CloudHypervisorCgroupDependencies = { sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), }; +export function computeCloudHypervisorCgroupLimits( + limits: CloudHypervisorResourceLimits, +): CloudHypervisorCgroupLimits { + const memoryMaxBytes = (limits.memoryMib + CGROUP_MEMORY_HEADROOM_MIB) * 1024 * 1024; + const cpuQuotaUs = limits.vcpuCount * CGROUP_V2_PERIOD_US + CGROUP_CPU_HEADROOM_QUOTA_US; + return { + memoryMax: String(memoryMaxBytes), + cpuMax: `${cpuQuotaUs} ${CGROUP_V2_PERIOD_US}`, + pidsMax: String(CGROUP_MAX_PIDS), + }; +} + /** * Places one Cloud Hypervisor run under an explicit memory/CPU/PID cgroup, * created before launch and assigned by PID immediately after spawn (moving @@ -271,14 +334,14 @@ export class CloudHypervisorCgroup { await this.dependencies.mkdir(this.cgroupPath); this.created = true; - const memoryMaxBytes = (this.limits.memoryMib + CGROUP_MEMORY_HEADROOM_MIB) * 1024 * 1024; - const cpuQuotaUs = this.limits.vcpuCount * CGROUP_V2_PERIOD_US + CGROUP_CPU_HEADROOM_QUOTA_US; - await this.dependencies.writeFile(path.join(this.cgroupPath, 'memory.max'), String(memoryMaxBytes)); - await this.dependencies.writeFile( - path.join(this.cgroupPath, 'cpu.max'), - `${cpuQuotaUs} ${CGROUP_V2_PERIOD_US}`, - ); - await this.dependencies.writeFile(path.join(this.cgroupPath, 'pids.max'), String(CGROUP_MAX_PIDS)); + const expected = this.expectedLimits(); + await this.dependencies.writeFile(path.join(this.cgroupPath, 'memory.max'), expected.memoryMax); + await this.dependencies.writeFile(path.join(this.cgroupPath, 'cpu.max'), expected.cpuMax); + await this.dependencies.writeFile(path.join(this.cgroupPath, 'pids.max'), expected.pidsMax); + } + + expectedLimits(): CloudHypervisorCgroupLimits { + return computeCloudHypervisorCgroupLimits(this.limits); } async assign(pid: number): Promise { diff --git a/src/cloud-hypervisor/manager-start.ts b/src/cloud-hypervisor/manager-start.ts index 2ebf98405..e5addd0f0 100644 --- a/src/cloud-hypervisor/manager-start.ts +++ b/src/cloud-hypervisor/manager-start.ts @@ -30,6 +30,7 @@ import { hasReadOnlyWorkspaceMountPlan } from './filesystem-write-enforcement'; import type { VirtiofsdManager, VirtiofsdDevice } from './virtiofsd'; import { buildCloudHypervisorVmConfig } from './vm-config-builder'; import type { BoundedOutputCapture } from './diagnostics'; +import type { CloudHypervisorConfinementEvidence } from './confinement-verifier'; export interface CloudHypervisorStartContext { config: CloudHypervisorOptions; @@ -46,6 +47,7 @@ export interface CloudHypervisorStartContext { setCgroup(cgroup: CloudHypervisorCgroup | undefined): void; setProcess(process: ExecaChildProcess | undefined): void; setClient(client: CloudHypervisorApiClient | undefined): void; + setConfinementEvidence(evidence: CloudHypervisorConfinementEvidence | undefined): void; setVirtiofsd(virtiofsd: VirtiofsdManager | undefined): void; setFsDevices(devices: VirtiofsdDevice[]): void; getFsDevices(): VirtiofsdDevice[]; @@ -139,6 +141,18 @@ export async function startCloudHypervisor( const client = dependencies.createClient(paths.apiSocketPath, config.apiTimeoutMs); context.setClient(client); await client.ping(); + if (child.pid === undefined) { + throw new Error('Cloud Hypervisor launcher did not report a PID for confinement verification'); + } + context.setConfinementEvidence(await dependencies.verifyConfinement({ + pid: child.pid, + expectedExecutable: config.cloudHypervisorBinary, + identity, + launchPolicy: launchCommand.confinementPolicy, + networkNamespace: networkPlan.namespaceName, + cgroupPath: paths.cgroupPath, + cgroupLimits: cgroup.expectedLimits(), + })); if (guestConfig) { const virtiofsd = dependencies.createVirtiofsdManager( artifacts.virtiofsdBinary, paths.runDirectory, paths.virtiofsdShareDirectory, diff --git a/src/cloud-hypervisor/manager-types.ts b/src/cloud-hypervisor/manager-types.ts index 5ebefe032..73b09514c 100644 --- a/src/cloud-hypervisor/manager-types.ts +++ b/src/cloud-hypervisor/manager-types.ts @@ -15,6 +15,7 @@ import type { CloudHypervisorDirectoryExport } from './exports'; import type { CloudHypervisorCgroup, CloudHypervisorResourceLimits } from './launcher'; import type { CloudHypervisorHostToolPaths, runCloudHypervisorPreflight } from './preflight'; import type { VirtiofsdManager, VirtiofsdMountEnforcement } from './virtiofsd'; +import type { verifyCloudHypervisorConfinement } from './confinement-verifier'; const API_SOCKET_NAME = 'api.socket'; const VSOCK_SOCKET_NAME = 'awf-vsock.socket'; @@ -94,6 +95,7 @@ export interface CloudHypervisorManagerDependencies { ): VirtiofsdManager; createVsockClient(socketPath: string, guestPort: number, timeoutMs: number): MicrovmVsockClient; createCgroup(cgroupPath: string, limits: CloudHypervisorResourceLimits): CloudHypervisorCgroup; + verifyConfinement: typeof verifyCloudHypervisorConfinement; resolveIdentity(): { uid: number; gid: number }; } diff --git a/src/cloud-hypervisor/manager.test.ts b/src/cloud-hypervisor/manager.test.ts index 7ce0e8cea..6ca4e7a55 100644 --- a/src/cloud-hypervisor/manager.test.ts +++ b/src/cloud-hypervisor/manager.test.ts @@ -117,6 +117,11 @@ function cgroupMock(): CloudHypervisorCgroup { cgroupPath: '/sys/fs/cgroup/awf-cloud-hypervisor/run', setup: jest.fn().mockResolvedValue(undefined), assign: jest.fn().mockResolvedValue(undefined), + expectedLimits: jest.fn().mockReturnValue({ + memoryMax: String(768 * 1024 * 1024), + cpuMax: '300000 100000', + pidsMax: '256', + }), cleanup: jest.fn().mockResolvedValue(undefined), } as unknown as CloudHypervisorCgroup; } @@ -161,6 +166,35 @@ function dependencies( createVirtiofsdManager: jest.fn(() => virtiofsdManagerMock()), createVsockClient: jest.fn(), createCgroup: jest.fn(() => cgroupMock()), + verifyConfinement: jest.fn().mockResolvedValue({ + schemaVersion: 1, + verifiedAt: '2026-08-31T00:00:00.000Z', + process: { + pid: 4242, + startTimeTicks: '123', + executable: '/opt/cloud-hypervisor', + }, + identity: { uid: 1000, gid: 1000, supplementaryGroups: [978] }, + capabilities: { + inheritable: '0000000000001000', + permitted: '0000000000001000', + effective: '0000000000001000', + bounding: '0000000000001000', + ambient: '0000000000001000', + }, + noNewPrivs: 1, + seccomp: { mode: 2, relevantThreadIds: [4243], observedThreadCount: 2 }, + networkNamespace: { name: 'awfvm-test', inode: 'net:[42]' }, + cgroup: { + path: '/sys/fs/cgroup/awf-cloud-hypervisor/run', + membership: '/awf-cloud-hypervisor/run', + limits: { + memoryMax: String(768 * 1024 * 1024), + cpuMax: '300000 100000', + pidsMax: '256', + }, + }, + }), resolveIdentity: jest.fn().mockReturnValue({ uid: 1000, gid: 1000 }), ...overrides, }; @@ -293,6 +327,20 @@ describe('CloudHypervisorManager', () => { const cgroup = (deps.createCgroup as jest.Mock).mock.results[0].value as CloudHypervisorCgroup; expect(cgroup.setup).toHaveBeenCalledTimes(1); expect(cgroup.assign).toHaveBeenCalledWith(4242); + expect(deps.verifyConfinement).toHaveBeenCalledWith(expect.objectContaining({ + pid: 4242, + expectedExecutable: '/opt/cloud-hypervisor', + identity: { uid: 1000, gid: 1000 }, + networkNamespace: expect.stringMatching(/^awfvm-/), + cgroupPath: expect.stringContaining('awf-cloud-hypervisor/run-1'), + cgroupLimits: { + memoryMax: String(768 * 1024 * 1024), + cpuMax: '300000 100000', + pidsMax: '256', + }, + })); + expect((deps.verifyConfinement as jest.Mock).mock.invocationCallOrder[0]) + .toBeLessThan((client.vmCreate as jest.Mock).mock.invocationCallOrder[0]); // Private run directory: ancestor levels stay traversable-only (0711, // root-owned); only the leaf is chowned to the non-root identity. expect(deps.mkdir).toHaveBeenCalledWith('/run/awf-cloud-hypervisor', { recursive: true, mode: 0o711 }); @@ -354,6 +402,32 @@ describe('CloudHypervisorManager', () => { expect(cgroup.cleanup).toHaveBeenCalledTimes(1); }); + it('fails closed and never creates a VM when runtime confinement verification fails', async () => { + const child = processMock(); + const deps = dependencies({ + launch: jest.fn().mockReturnValue(child), + verifyConfinement: jest.fn().mockRejectedValue( + new Error('Cloud Hypervisor CapEff does not match launch policy'), + ), + }); + const manager = new CloudHypervisorManager( + config(), + '/tmp/awf', + deps, + 'unconfined', + networkConfig(), + ); + + await expect(manager.start()).rejects.toThrow(/CapEff does not match launch policy/); + const client = (deps.createClient as jest.Mock).mock.results[0].value as CloudHypervisorApiClient; + expect(client.ping).toHaveBeenCalledTimes(1); + expect(client.vmCreate).not.toHaveBeenCalled(); + expect(child.kill).toHaveBeenCalledWith( + 'SIGTERM', + { forceKillAfterTimeout: 2_000 }, + ); + }); + it('refuses to launch without host-side network enforcement', async () => { const deps = dependencies(); const manager = new CloudHypervisorManager(config(), '/tmp/awf', deps, 'unsafe'); @@ -377,6 +451,11 @@ describe('CloudHypervisorManager', () => { cgroupPath: '/sys/fs/cgroup/awf-cloud-hypervisor/cleanup', setup: jest.fn().mockResolvedValue(undefined), assign: jest.fn().mockResolvedValue(undefined), + expectedLimits: jest.fn().mockReturnValue({ + memoryMax: String(768 * 1024 * 1024), + cpuMax: '300000 100000', + pidsMax: '256', + }), cleanup: jest.fn(async () => { order.push('cgroup'); }), @@ -1059,6 +1138,11 @@ describe('CloudHypervisorManager', () => { expect.stringContaining('rx_bytes'), { mode: 0o600 }, ); + expect(deps.writeFile).toHaveBeenCalledWith( + '/tmp/diagnostics/confinement.json', + expect.stringContaining('"schemaVersion": 1'), + { mode: 0o600 }, + ); }); it('snapshots vm.info/vm.counters before any shutdown attempt, so collectDiagnostics() via beforeCleanup still has real data', async () => { diff --git a/src/cloud-hypervisor/manager.ts b/src/cloud-hypervisor/manager.ts index b6d4b0f9b..a0b495367 100644 --- a/src/cloud-hypervisor/manager.ts +++ b/src/cloud-hypervisor/manager.ts @@ -44,6 +44,10 @@ import { } from './manager-types'; import { runCloudHypervisorPreflight } from './preflight'; import type { CloudHypervisorHostToolPaths } from './preflight'; +import { + verifyCloudHypervisorConfinement, + type CloudHypervisorConfinementEvidence, +} from './confinement-verifier'; import { startCloudHypervisor } from './manager-start'; import { stopCloudHypervisor } from './manager-stop'; import { VirtiofsdManager, type VirtiofsdDevice } from './virtiofsd'; @@ -103,6 +107,7 @@ const defaultDependencies: CloudHypervisorManagerDependencies = { writeTimeoutMs: timeoutMs, }), createCgroup: (cgroupPath, limits) => new CloudHypervisorCgroup(cgroupPath, limits), + verifyConfinement: verifyCloudHypervisorConfinement, resolveIdentity: resolveCloudHypervisorIdentity, }; @@ -161,6 +166,7 @@ export class CloudHypervisorManager { private guest: CloudHypervisorGuestChannel | undefined; private cgroup: CloudHypervisorCgroup | undefined; private networkPlan: MicrovmNetworkPlan | undefined; + private confinementEvidence: CloudHypervisorConfinementEvidence | undefined; private instanceStarted = false; // Snapshotted in stop(), before any shutdown attempt, since the API // socket becomes unresponsive once the process is asked to exit -- @@ -217,6 +223,7 @@ export class CloudHypervisorManager { setCgroup: (value) => { this.cgroup = value; }, setProcess: (value) => { this.process = value; }, setClient: (value) => { this.client = value; }, + setConfinementEvidence: (value) => { this.confinementEvidence = value; }, setVirtiofsd: (value) => { this.virtiofsd = value; }, setFsDevices: (value) => { this.fsDevices = value; }, getFsDevices: () => this.fsDevices, @@ -322,6 +329,7 @@ export class CloudHypervisorManager { lastVmInfo: this.lastVmInfo, lastVmCounters: this.lastVmCounters, fsDevices: this.fsDevices, + confinementEvidence: this.confinementEvidence, }); } }