diff --git a/docs/cloud-hypervisor-foundation.md b/docs/cloud-hypervisor-foundation.md index 00872600c..c621c3c31 100644 --- a/docs/cloud-hypervisor-foundation.md +++ b/docs/cloud-hypervisor-foundation.md @@ -72,8 +72,10 @@ AWF performs these steps for each run: policy. 5. Copy the rootfs, inject the guest supervisor, and stage files in a private run directory. -6. Create a bounded cgroup v2 leaf and launch Cloud Hypervisor as the invoking - non-root identity. +6. Allocate a random-named, dedicated per-run system account, grant its + host-assigned uid/gid temporary access only to KVM/TUN, the pre-created TAP, + staged files, and required sockets, then launch Cloud Hypervisor under that + identity in a bounded cgroup v2 leaf. 7. After the API responds, verify the launched VMM's trusted host `/proc` and cgroup state. AWF fails closed before `vm.create` if the PID identity, executable, credentials, capabilities, `no_new_privs`, seccomp worker, @@ -89,7 +91,8 @@ AWF performs these steps for each run: 10. Execute the agent command and propagate its exit code. Timeouts return `124`. 11. Sync and unmount guest filesystems, stop the VM and VMM, reap `virtiofsd`, - and remove network, cgroup, and run-directory resources. + remove network, cgroup, and run-directory resources, revoke device ACLs, + and delete the exact per-run account. Cleanup is idempotent and aggregates errors so one cleanup failure does not skip later cleanup steps. @@ -163,8 +166,11 @@ release or production use. AWF launches Cloud Hypervisor through `ip netns exec` and `setpriv` without a shell. The process: -- runs as the non-root identity recorded by `SUDO_UID` and `SUDO_GID`; -- keeps only the KVM supplementary group; +- runs as a random `awfvmm-` system account allocated for that run, + independently of `SUDO_UID` and `SUDO_GID`; +- has no home, login shell, or supplementary groups; +- receives temporary uid-specific ACLs for `/dev/kvm` and `/dev/net/tun`; +- owns only its run directory, staged VMM files and sockets, and TAP; - sets `no_new_privs`; - has empty inheritable, permitted, effective, bounding, and ambient capability sets; @@ -191,7 +197,15 @@ not copy unbounded `/proc` content. The private run directory is under `/run/awf-cloud-hypervisor///`. Its per-run leaf is accessible -only to the selected non-root identity and root. +only to the dedicated VMM identity and root. Account allocation and deletion +are serialized by an owner-token lock that validates both PID and process start +time before reclaiming stale state, preventing PID reuse from stealing a live +lock. Cleanup validates the exact account uid/gid before deletion and revokes +only that run's ACL entries, so concurrent sibling runs remain untouched. + +The guest command still uses the invoking workspace uid/gid. That guest +identity is carried separately through the supervisor protocol and is never +reused as the host VMM identity. ### virtiofsd confinement @@ -206,8 +220,8 @@ configuration, AWF verifies the live parent and worker through `/proc`: - parent and worker UIDs/GIDs match the reviewed root namespace identity; - every parent capability set is empty, while the worker effective and permitted masks equal the pinned minimal virtiofsd set, its inheritable and - ambient sets are empty, and its bounding set contains the capabilities - needed during sandbox setup (rendered non-acquirable after `NoNewPrivs`); + ambient sets are empty, and its host bounding set remains empty after + entering the user namespace; - the worker has `NoNewPrivs: 1` and seccomp filter mode `2`; - the worker mount, PID, and network namespaces differ from the host; - the worker root inode is the inode of the declared export, proving the diff --git a/scripts/ci/cloud-hypervisor-live-smoke.sh b/scripts/ci/cloud-hypervisor-live-smoke.sh index 4f901ef03..b57825e5f 100755 --- a/scripts/ci/cloud-hypervisor-live-smoke.sh +++ b/scripts/ci/cloud-hypervisor-live-smoke.sh @@ -665,6 +665,24 @@ done proc_uid=$(sudo stat -c %u "/proc/$vmm_pid" 2>/dev/null || echo "") [ -n "$proc_uid" ] || fail_security "could not stat /proc/$vmm_pid" [ "$proc_uid" != "0" ] || fail_security "Cloud Hypervisor process is running as root" +operator_uid=$(id -u) +[ "$proc_uid" != "$operator_uid" ] \ + || fail_security "Cloud Hypervisor process reused the invoking operator uid $operator_uid" +vmm_passwd=$(getent passwd "$proc_uid" || true) +[ -n "$vmm_passwd" ] || fail_security "no passwd entry for Cloud Hypervisor uid $proc_uid" +vmm_name=$(printf '%s' "$vmm_passwd" | cut -d: -f1) +vmm_gid=$(printf '%s' "$vmm_passwd" | cut -d: -f4) +[ "$(printf '%s' "$vmm_passwd" | cut -d: -f6)" = "/nonexistent" ] \ + || fail_security "Cloud Hypervisor account has an unexpected home directory" +[ "$(printf '%s' "$vmm_passwd" | cut -d: -f7)" = "/usr/sbin/nologin" ] \ + || fail_security "Cloud Hypervisor account has an interactive shell" +printf '%s' "$vmm_name" | grep -Eq '^awfvmm-[0-9a-f]{20}$' \ + || fail_security "Cloud Hypervisor account name is not a random per-run name: $vmm_name" +[ "$(id -G "$vmm_name")" = "$vmm_gid" ] \ + || fail_security "Cloud Hypervisor account inherited supplementary groups" +getfacl --absolute-names --numeric /dev/kvm 2>/dev/null \ + | grep -q "^user:$proc_uid:rw-" \ + || fail_security "Cloud Hypervisor uid lacks its scoped /dev/kvm ACL" # Every capability set is empty. In particular, a zero CapBnd prevents the # non-root VMM from regaining CAP_NET_ADMIN after exec. @@ -858,6 +876,14 @@ node -e ' # `ip netns exec`, not a bare `ip link show`. sudo ip netns exec "$expected_namespace" ip link show "$expected_tap" >/dev/null 2>&1 \ || fail_security "expected TAP interface $expected_tap not found in namespace $expected_namespace" +tap_details=$(sudo ip netns exec "$expected_namespace" \ + ip -details tuntap show dev "$expected_tap" 2>/dev/null || true) +tap_line=$(printf '%s\n' "$tap_details" | grep "^${expected_tap}: " || true) +[ -n "$tap_line" ] || fail_security "expected TAP details were not found: $tap_details" +printf '%s' "$tap_line" | grep -Eq "(^|[[:space:]])user[[:space:]]+$proc_uid([[:space:]]|$)" \ + || fail_security "TAP is not owned by the per-run VMM uid $proc_uid: $tap_details" +printf '%s' "$tap_line" | grep -Eq "(^|[[:space:]])group[[:space:]]+$vmm_gid([[:space:]]|$)" \ + || fail_security "TAP is not owned by the per-run VMM gid $vmm_gid: $tap_details" kill -TERM "$sec_pid" set +e @@ -869,5 +895,14 @@ set -e exit 1 } assert_no_residue +if getent passwd "$vmm_name" >/dev/null; then + fail_security "per-run Cloud Hypervisor account remains after cancellation: $vmm_name" +fi +if getfacl --absolute-names --numeric /dev/kvm 2>/dev/null | grep -q "^user:$proc_uid:"; then + fail_security "per-run /dev/kvm ACL remains after cancellation for uid $proc_uid" +fi +if getfacl --absolute-names --numeric /dev/net/tun 2>/dev/null | grep -q "^user:$proc_uid:"; then + fail_security "per-run /dev/net/tun ACL remains after cancellation for uid $proc_uid" +fi echo "Cloud Hypervisor live smoke/security suite passed." diff --git a/src/cloud-hypervisor-runtime-backend.test.ts b/src/cloud-hypervisor-runtime-backend.test.ts index c99903d5c..ea68ca6cc 100644 --- a/src/cloud-hypervisor-runtime-backend.test.ts +++ b/src/cloud-hypervisor-runtime-backend.test.ts @@ -99,6 +99,10 @@ const preflightResult = { cgroupVersion: 2 as const, kvmGid: 978, tools: { + getfacl: '/usr/bin/getfacl', + getent: '/usr/bin/getent', + groupdel: '/usr/sbin/groupdel', + id: '/usr/bin/id', ip: '/usr/bin/ip', nft: '/usr/sbin/nft', sysctl: '/usr/sbin/sysctl', @@ -110,6 +114,9 @@ const preflightResult = { mount: '/usr/bin/mount', umount: '/usr/bin/umount', setpriv: '/usr/bin/setpriv', + setfacl: '/usr/bin/setfacl', + useradd: '/usr/sbin/useradd', + userdel: '/usr/sbin/userdel', }, }; @@ -243,6 +250,8 @@ describe('Cloud Hypervisor runtime backend', () => { infrastructure(), [{ tag: 'workspace', source: '/workspace', target: '/workspace', mode: 'rw' }], { uid: 1000, gid: 1000 }, + undefined, + preflightResult, )).toBeDefined(); expect(createCloudHypervisorRuntimeBackend(config(), startInfrastructure)) .toEqual(expect.objectContaining({ runtime: 'cloud-hypervisor' })); @@ -302,6 +311,7 @@ describe('Cloud Hypervisor runtime backend', () => { }, ], }, + preflightResult, ); expect(deps.logger.info).toHaveBeenCalledWith( '[cloud-hypervisor] stage=filesystem-write-policy boundary /workspace=ro ' + @@ -325,7 +335,8 @@ describe('Cloud Hypervisor runtime backend', () => { { tag: 'workspace', source: '/workspace-host', target: '/workspace', mode: 'rw' }, ]); expect(call[5]).toBeUndefined(); - expect(call).toHaveLength(6); + expect(call[6]).toBe(preflightResult); + expect(call).toHaveLength(7); expect(deps.logger.info).not.toHaveBeenCalledWith( expect.stringContaining('stage=filesystem-write-policy'), ); diff --git a/src/cloud-hypervisor-runtime-backend.ts b/src/cloud-hypervisor-runtime-backend.ts index 6335a83bf..abb5b9dff 100644 --- a/src/cloud-hypervisor-runtime-backend.ts +++ b/src/cloud-hypervisor-runtime-backend.ts @@ -113,7 +113,8 @@ export interface CloudHypervisorRuntimeBackendDependencies { infrastructure: MicrovmInfrastructureSnapshot, exports: readonly CloudHypervisorDirectoryExport[], identity: { uid: number; gid: number }, - mountEnforcement?: VirtiofsdMountEnforcement, + mountEnforcement: VirtiofsdMountEnforcement | undefined, + verifiedArtifacts: CloudHypervisorPreflightResult, ): CloudHypervisorManagerAdapter; resolveExports(mountPolicy: CloudHypervisorOptions['mountPolicy']): Promise; identity(): { uid: number; gid: number }; @@ -133,7 +134,15 @@ function defaultDependencies( preflight: runCloudHypervisorPreflight, resolveInfrastructure: (enableApiProxy, ipPath, topologyPeerNames) => resolveMicrovmInfrastructure(enableApiProxy, undefined, ipPath, topologyPeerNames), - createManager: (config, workDir, infrastructure, exports, identity, mountEnforcement) => + createManager: ( + config, + workDir, + infrastructure, + exports, + identity, + mountEnforcement, + verifiedArtifacts, + ) => new CloudHypervisorManager( config, workDir, @@ -156,6 +165,7 @@ function defaultDependencies( supervisorSha256: config.sha256!.supervisor!, identity, }, + verifiedArtifacts, ), resolveExports: (mountPolicy) => resolveCloudHypervisorExports( process.env, @@ -316,6 +326,7 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend { exports, this.identity, mountEnforcement, + this.preflightResult!, ); try { stage = 'vmm-configuration'; diff --git a/src/cloud-hypervisor/confinement-verifier.test.ts b/src/cloud-hypervisor/confinement-verifier.test.ts index 5edb09896..31a65834a 100644 --- a/src/cloud-hypervisor/confinement-verifier.test.ts +++ b/src/cloud-hypervisor/confinement-verifier.test.ts @@ -22,14 +22,14 @@ function launchPolicy(): CloudHypervisorLaunchConfinementPolicy { }; } -function status(name: string, seccomp: number, taskId = PID): string { +function status(name: string, seccomp: number, taskId = PID, groups = '978'): string { return [ `Name:\t${name}`, `Pid:\t${taskId}`, `Tgid:\t${PID}`, 'Uid:\t1000\t1000\t1000\t1000', 'Gid:\t1001\t1001\t1001\t1001', - 'Groups:\t978', + `Groups:\t${groups}`, `CapInh:\t${CAPABILITY_MASK}`, `CapPrm:\t${CAPABILITY_MASK}`, `CapEff:\t${CAPABILITY_MASK}`, @@ -50,13 +50,14 @@ function dependencies(overrides: { executable?: string; workerStatus?: string; cgroupProcs?: string; + groups?: string; } = {}): CloudHypervisorConfinementVerifierDependencies { const statReads = [...(overrides.statReads ?? [procStat('98765'), procStat('98765')])]; const files: Record = { - [`/proc/${PID}/task/${PID}/status`]: status('cloud-hypervis', 0), + [`/proc/${PID}/task/${PID}/status`]: status('cloud-hypervis', 0, PID, overrides.groups), [`/proc/${PID}/task/${PID + 1}/status`]: - overrides.workerStatus ?? status('vmm', 2, PID + 1), - [`/proc/${PID}/task/${PID + 2}/status`]: status('http-server', 2, PID + 2), + overrides.workerStatus ?? status('vmm', 2, PID + 1, overrides.groups), + [`/proc/${PID}/task/${PID + 2}/status`]: status('http-server', 2, PID + 2, overrides.groups), [`/proc/${PID}/cgroup`]: '0::/awf-cloud-hypervisor/run-1\n', [`${CGROUP}/cgroup.procs`]: overrides.cgroupProcs ?? `${PID}\n`, [`${CGROUP}/memory.max`]: '805306368\n', @@ -141,6 +142,21 @@ describe('verifyCloudHypervisorConfinement', () => { })); }); + it('accepts an empty Groups field when no supplementary groups are expected', async () => { + const verificationOptions = options(); + verificationOptions.launchPolicy = { + ...verificationOptions.launchPolicy, + supplementaryGroups: [], + }; + + const result = await verifyCloudHypervisorConfinement( + verificationOptions, + dependencies({ groups: '' }), + ); + + expect(result.identity.supplementaryGroups).toEqual([]); + }); + it('fails closed when PID identity changes while evidence is collected', async () => { await expect(verifyCloudHypervisorConfinement( options(), diff --git a/src/cloud-hypervisor/confinement-verifier.ts b/src/cloud-hypervisor/confinement-verifier.ts index 811a07812..0f0336861 100644 --- a/src/cloud-hypervisor/confinement-verifier.ts +++ b/src/cloud-hypervisor/confinement-verifier.ts @@ -242,7 +242,11 @@ function verifyThreadStatus( } assertIdentityField(status.Uid, options.identity.uid, 'UID', taskId); assertIdentityField(status.Gid, options.identity.gid, 'GID', taskId); - const groups = parseNumericFields(requiredStatus(status, 'Groups', taskId), 'Groups'); + const groupStatus = status.Groups; + if (groupStatus === undefined) { + throw new Error(`Cloud Hypervisor thread ${taskId} is missing /proc status field Groups`); + } + const groups = parseNumericFields(groupStatus, 'Groups'); if (groups.join(',') !== options.launchPolicy.supplementaryGroups.join(',')) { throw new Error( `Cloud Hypervisor thread ${taskId} has supplementary groups ${groups.join(',') || 'none'}, ` + diff --git a/src/cloud-hypervisor/launcher.test.ts b/src/cloud-hypervisor/launcher.test.ts index 97b6acee2..db3821815 100644 --- a/src/cloud-hypervisor/launcher.test.ts +++ b/src/cloud-hypervisor/launcher.test.ts @@ -10,13 +10,12 @@ describe('buildCloudHypervisorLaunchCommand', () => { tools: { ip: '/usr/sbin/ip', setpriv: '/usr/bin/setpriv' }, namespaceName: 'awfch-abc123', identity: { uid: 1000, gid: 1000 }, - kvmGid: 978, cloudHypervisorBinary: '/opt/cloud-hypervisor', apiSocketPath: '/run/awf/api.socket', logFilePath: '/run/awf/cloud-hypervisor.log', }; - it('joins the namespace, retains only the kvm group, empties all capability sets, and execs Cloud Hypervisor with no shell', () => { + it('joins the namespace, drops privileges and groups, empties capabilities, then execs without a shell', () => { const result = buildCloudHypervisorLaunchCommand(baseOptions); expect(result.command).toBe('/usr/sbin/ip'); expect(result.args).toEqual([ @@ -24,7 +23,7 @@ describe('buildCloudHypervisorLaunchCommand', () => { '/usr/bin/setpriv', '--reuid=1000', '--regid=1000', - '--groups=978', + '--clear-groups', '--no-new-privs', '--inh-caps=-all', '--bounding-set=-all', @@ -36,9 +35,9 @@ describe('buildCloudHypervisorLaunchCommand', () => { '-v', '--seccomp', 'true', ]); - expect(result.args).not.toContain('--clear-groups'); + expect(result.args).toContain('--clear-groups'); expect(result.confinementPolicy).toEqual({ - supplementaryGroups: [978], + supplementaryGroups: [], capabilities: { inheritable: '0000000000000000', permitted: '0000000000000000', @@ -59,17 +58,12 @@ describe('buildCloudHypervisorLaunchCommand', () => { ['unsafe namespace name', { namespaceName: '../etc' }, /Unsafe Cloud Hypervisor network namespace name/], ['zero uid', { identity: { uid: 0, gid: 1000 } }, /uid must be a positive integer/], ['negative gid', { identity: { uid: 1000, gid: -1 } }, /gid must be a positive integer/], - ['negative kvm gid', { kvmGid: -1 }, /\/dev\/kvm group id must be a non-negative integer/], ['relative binary path', { cloudHypervisorBinary: 'cloud-hypervisor' }, /binary path must be absolute/], ['relative socket path', { apiSocketPath: 'api.socket' }, /API socket path must be absolute/], ])('rejects %s', (_label, overrides, error) => { expect(() => buildCloudHypervisorLaunchCommand({ ...baseOptions, ...overrides })) .toThrow(error); }); - - it('accepts a kvm gid of 0 (root-owned /dev/kvm on unusual hosts)', () => { - expect(() => buildCloudHypervisorLaunchCommand({ ...baseOptions, kvmGid: 0 })).not.toThrow(); - }); }); describe('computeCloudHypervisorLandlockRules', () => { diff --git a/src/cloud-hypervisor/launcher.ts b/src/cloud-hypervisor/launcher.ts index ecb816cb4..7ac8a66f6 100644 --- a/src/cloud-hypervisor/launcher.ts +++ b/src/cloud-hypervisor/launcher.ts @@ -16,12 +16,10 @@ import type { CloudHypervisorLandlockRule } from './api-client'; * per-run namespace {@link https://man7.org/linux/man-pages/man8/ip-netns.8.html} * without an intermediate fork, so the resulting process keeps the PID * the host process observes. - * 2. **Privilege drop** — `setpriv --reuid --regid --groups= - * --no-new-privs --inh-caps=-all --bounding-set=-all - * --ambient-caps=-all` execs the Cloud Hypervisor binary as the non-root - * operator uid/gid with empty capability sets and `no_new_privs` set, - * before any guest code runs. The sole supplementary group grants - * `/dev/kvm` access without a process capability. + * 2. **Privilege drop** — `setpriv --reuid --regid --clear-groups + * --no-new-privs --inh-caps=-all --bounding-set=-all` execs the Cloud + * Hypervisor binary as a dedicated per-run system uid/gid with no + * supplementary groups and `no_new_privs` set, before any guest code runs. * 3. **Filesystem confinement** — Cloud Hypervisor has no chroot of its * own, and jailer's userspace chroot+pivot_root cannot be replicated * for a foreign static binary without reimplementing jailer itself. @@ -95,15 +93,13 @@ const CLOUD_HYPERVISOR_ALLOWED_CAPABILITIES: readonly { readonly bit: number; }[] = []; -function buildCloudHypervisorConfinementPolicy( - kvmGid: number, -): CloudHypervisorLaunchConfinementPolicy { +function buildCloudHypervisorConfinementPolicy(): CloudHypervisorLaunchConfinementPolicy { const capabilityMask = CLOUD_HYPERVISOR_ALLOWED_CAPABILITIES .reduce((mask, capability) => mask | (1n << BigInt(capability.bit)), 0n) .toString(16) .padStart(16, '0'); return { - supplementaryGroups: [kvmGid], + supplementaryGroups: [], capabilities: { inheritable: capabilityMask, permitted: capabilityMask, @@ -117,18 +113,14 @@ function buildCloudHypervisorConfinementPolicy( /** * Builds the argv AWF spawns to launch Cloud Hypervisor: join the prepared - * network namespace, drop to the non-root operator identity with no + * network namespace, drop to the dedicated per-run VMM identity with no * capabilities, then exec the pinned Cloud Hypervisor binary with only its API * socket configured; the VM itself is created and booted afterwards over that * socket. * - * The launched process retains exactly one supplementary group: the group - * that owns `/dev/kvm` (resolved by preflight). A blanket `--clear-groups` - * would also drop that membership, and since the documented supported - * setup relies on kvm-group access for the non-root operator identity - * (see docs/cloud-hypervisor-foundation.md), that would make every real - * launch fail with EACCES opening `/dev/kvm` even though preflight (which - * runs as root) passed. + * The launched process has no supplementary groups. Temporary per-run ACLs + * grant its uid access to `/dev/kvm` and `/dev/net/tun`, avoiding membership + * in a persistent host group. * * The network manager creates, configures, and brings up the TAP before this * process starts, with the target uid/gid recorded as its owner. Cloud @@ -142,7 +134,6 @@ export function buildCloudHypervisorLaunchCommand(options: { readonly tools: CloudHypervisorLaunchToolPaths; readonly namespaceName: string; readonly identity: CloudHypervisorLaunchIdentity; - readonly kvmGid: number; readonly cloudHypervisorBinary: string; readonly apiSocketPath: string; readonly logFilePath: string; @@ -150,9 +141,6 @@ export function buildCloudHypervisorLaunchCommand(options: { assertSafeNamespaceName(options.namespaceName); assertPositiveIdentity(options.identity.uid, 'uid'); assertPositiveIdentity(options.identity.gid, 'gid'); - if (!Number.isSafeInteger(options.kvmGid) || options.kvmGid < 0) { - throw new Error(`Cloud Hypervisor launch /dev/kvm group id must be a non-negative integer: ${options.kvmGid}`); - } if (!path.isAbsolute(options.cloudHypervisorBinary)) { throw new Error(`Cloud Hypervisor binary path must be absolute: ${options.cloudHypervisorBinary}`); } @@ -173,10 +161,7 @@ export function buildCloudHypervisorLaunchCommand(options: { options.tools.setpriv, `--reuid=${options.identity.uid}`, `--regid=${options.identity.gid}`, - // Replaces the operator's full supplementary group list with only - // the /dev/kvm-owning group, instead of --clear-groups (which would - // also drop kvm access). - `--groups=${options.kvmGid}`, + '--clear-groups', '--no-new-privs', `--inh-caps=${resetCapabilitySet}`, `--bounding-set=${resetCapabilitySet}`, @@ -188,7 +173,7 @@ export function buildCloudHypervisorLaunchCommand(options: { '-v', '--seccomp', 'true', ], - confinementPolicy: buildCloudHypervisorConfinementPolicy(options.kvmGid), + confinementPolicy: buildCloudHypervisorConfinementPolicy(), }; } diff --git a/src/cloud-hypervisor/manager-start.ts b/src/cloud-hypervisor/manager-start.ts index 38d25834d..92ec91ebd 100644 --- a/src/cloud-hypervisor/manager-start.ts +++ b/src/cloud-hypervisor/manager-start.ts @@ -31,6 +31,8 @@ import type { VirtiofsdManager, VirtiofsdDevice } from './virtiofsd'; import { buildCloudHypervisorVmConfig } from './vm-config-builder'; import type { BoundedOutputCapture } from './diagnostics'; import type { CloudHypervisorConfinementEvidence } from './confinement-verifier'; +import type { CloudHypervisorVmmIdentityManager } from './vmm-identity'; +import type { CloudHypervisorPreflightResult } from './preflight'; export interface CloudHypervisorStartContext { config: CloudHypervisorOptions; @@ -39,12 +41,14 @@ export interface CloudHypervisorStartContext { paths: CloudHypervisorRunPaths; networkConfig?: CloudHypervisorManagerNetworkConfig; guestConfig?: CloudHypervisorManagerGuestConfig; + verifiedArtifacts?: CloudHypervisorPreflightResult; stdoutCapture: BoundedOutputCapture; stderrCapture: BoundedOutputCapture; setNetworkPlan(plan: MicrovmNetworkPlan | undefined): void; setNetwork(network: MicrovmNetworkLifecycle | undefined): void; setRootfsPreparer(preparer: MicrovmRootfsPreparer | undefined): void; setCgroup(cgroup: CloudHypervisorCgroup | undefined): void; + setVmmIdentity(identity: CloudHypervisorVmmIdentityManager | undefined): void; setProcess(process: ExecaChildProcess | undefined): void; setClient(client: CloudHypervisorApiClient | undefined): void; setConfinementEvidence(evidence: CloudHypervisorConfinementEvidence | undefined): void; @@ -58,7 +62,7 @@ export async function startCloudHypervisor( context: CloudHypervisorStartContext, ): Promise { const { - config, workDir, dependencies, paths, networkConfig, guestConfig, + config, workDir, dependencies, paths, networkConfig, guestConfig, verifiedArtifacts, } = context; if (!networkConfig) { throw new Error( @@ -68,8 +72,20 @@ export async function startCloudHypervisor( let startupError: unknown; try { - const artifacts = await dependencies.preflight(config); - const identity = guestConfig?.identity ?? dependencies.resolveIdentity(); + const artifacts = verifiedArtifacts ?? await dependencies.preflight(config); + const guestIdentity = guestConfig?.identity ?? dependencies.resolveIdentity(); + const vmmIdentityManager = dependencies.createVmmIdentity(paths.runId, { + getfacl: artifacts.tools.getfacl, + getent: artifacts.tools.getent, + groupdel: artifacts.tools.groupdel, + id: artifacts.tools.id, + ip: artifacts.tools.ip, + setfacl: artifacts.tools.setfacl, + useradd: artifacts.tools.useradd, + userdel: artifacts.tools.userdel, + }); + context.setVmmIdentity(vmmIdentityManager); + const identity = await vmmIdentityManager.allocate(); const reservation = await dependencies.reserveNetwork(paths.runId, { ...networkConfig, tapOwnerUid: identity.uid, @@ -115,12 +131,24 @@ export async function startCloudHypervisor( await stageArtifact(dependencies, rootfsSource, paths.rootfsPath, 0o600, identity); await stageDiagnosticFile(dependencies, paths.logPath, identity); await stageDiagnosticFile(dependencies, paths.serialLogPath, identity); + await vmmIdentityManager.validateOwnedPaths([ + paths.runDirectory, + paths.kernelPath, + paths.rootfsPath, + paths.logPath, + paths.serialLogPath, + ]); + await vmmIdentityManager.validateTapOwnership( + artifacts.tools.ip, + networkPlan.namespaceName, + networkPlan.tapName, + ); + await vmmIdentityManager.grantDeviceAccess(); const launchCommand = buildCloudHypervisorLaunchCommand({ tools: { ip: artifacts.tools.ip, setpriv: artifacts.tools.setpriv }, namespaceName: networkPlan.namespaceName, identity, - kvmGid: artifacts.kvmGid, cloudHypervisorBinary: config.cloudHypervisorBinary, apiSocketPath: paths.apiSocketPath, logFilePath: paths.logPath, @@ -139,6 +167,7 @@ export async function startCloudHypervisor( if (child.pid !== undefined) await cgroup.assign(child.pid); await waitForApiSocket(dependencies, paths, config.apiTimeoutMs, child); + await vmmIdentityManager.validateOwnedPaths([paths.apiSocketPath]); const client = dependencies.createClient(paths.apiSocketPath, config.apiTimeoutMs); context.setClient(client); await client.ping(); @@ -168,9 +197,15 @@ export async function startCloudHypervisor( context.setFsDevices(virtiofsd.getDiagnosticDevices()); throw error; } + await vmmIdentityManager.validateOwnedPaths( + context.getFsDevices().map((device) => device.socketPath), + ); } await client.vmCreate(buildCloudHypervisorVmConfig({ - config, paths, networkPlan, ...(guestConfig ? { guestConfig } : {}), + config, + paths, + networkPlan, + ...(guestConfig ? { guestConfig: { ...guestConfig, identity: guestIdentity } } : {}), fsDevices: context.getFsDevices(), })); return client; diff --git a/src/cloud-hypervisor/manager-stop.ts b/src/cloud-hypervisor/manager-stop.ts index 0b420c59e..b4c4a92a0 100644 --- a/src/cloud-hypervisor/manager-stop.ts +++ b/src/cloud-hypervisor/manager-stop.ts @@ -12,6 +12,7 @@ import type { CloudHypervisorCgroup } from './launcher'; import type { MicrovmRootfsPreparer } from '../microvm/rootfs'; import type { VirtiofsdManager, VirtiofsdDevice } from './virtiofsd'; import type { CloudHypervisorGuestChannel } from './guest-execution'; +import type { CloudHypervisorVmmIdentityManager } from './vmm-identity'; const SHUTDOWN_GRACE_MS = 5_000; @@ -28,6 +29,7 @@ export interface CloudHypervisorStopContext { fsDevices: VirtiofsdDevice[]; guest?: CloudHypervisorGuestChannel; cgroup?: CloudHypervisorCgroup; + vmmIdentity?: CloudHypervisorVmmIdentityManager; instanceStarted: boolean; lastVmInfo?: CloudHypervisorVmInfo; lastVmCounters?: CloudHypervisorVmCounters; @@ -42,6 +44,7 @@ export interface CloudHypervisorStopContext { setFsDevices(devices: VirtiofsdDevice[]): void; setGuest(guest: CloudHypervisorGuestChannel | undefined): void; setCgroup(cgroup: CloudHypervisorCgroup | undefined): void; + setVmmIdentity(identity: CloudHypervisorVmmIdentityManager | undefined): void; setInstanceStarted(started: boolean): void; setLastVmInfo(info: CloudHypervisorVmInfo | undefined): void; setLastVmCounters(counters: CloudHypervisorVmCounters | undefined): void; @@ -49,6 +52,7 @@ export interface CloudHypervisorStopContext { export async function stopCloudHypervisor(context: CloudHypervisorStopContext): Promise { const errors: unknown[] = []; + let identityResourcesRemoved = true; const instanceWasStarted = context.instanceStarted; if (context.client && instanceWasStarted) { try { context.setLastVmInfo(await context.client.vmInfo()); } catch { context.setLastVmInfo(undefined); } @@ -121,7 +125,7 @@ export async function stopCloudHypervisor(context: CloudHypervisorStopContext): return; } try { await context.network?.cleanup(); context.setNetwork(undefined); context.setNetworkPlan(undefined); } - catch (error) { errors.push(error); } + catch (error) { identityResourcesRemoved = false; errors.push(error); } try { await context.cgroup?.cleanup(); } catch (error) { errors.push(error); } context.setCgroup(undefined); if (!instanceWasStarted || terminationConfirmed) { @@ -130,7 +134,19 @@ export async function stopCloudHypervisor(context: CloudHypervisorStopContext): path.join(context.paths.runBaseDir, path.basename(context.config.cloudHypervisorBinary), context.paths.runId), { recursive: true, force: true }, ); - } catch (error) { errors.push(error); } + } catch (error) { identityResourcesRemoved = false; errors.push(error); } + } + if (identityResourcesRemoved) { + try { + await context.vmmIdentity?.cleanup(); + context.setVmmIdentity(undefined); + } catch (error) { + errors.push(error); + } + } else if (context.vmmIdentity) { + errors.push(new Error( + 'Cloud Hypervisor VMM identity retained because owned run resources could not be fully removed', + )); } throwCleanupErrors(errors, 'Cloud Hypervisor cleanup failed: '); } diff --git a/src/cloud-hypervisor/manager-types.ts b/src/cloud-hypervisor/manager-types.ts index 8c33525c3..1e063feaf 100644 --- a/src/cloud-hypervisor/manager-types.ts +++ b/src/cloud-hypervisor/manager-types.ts @@ -18,6 +18,10 @@ import type { CloudHypervisorCgroup, CloudHypervisorResourceLimits } from './lau import type { CloudHypervisorHostToolPaths, runCloudHypervisorPreflight } from './preflight'; import type { VirtiofsdManager, VirtiofsdMountEnforcement } from './virtiofsd'; import type { verifyCloudHypervisorConfinement } from './confinement-verifier'; +import type { + CloudHypervisorVmmIdentityManager, + CloudHypervisorVmmIdentityToolPaths, +} from './vmm-identity'; const API_SOCKET_NAME = 'api.socket'; const VSOCK_SOCKET_NAME = 'awf-vsock.socket'; @@ -107,6 +111,10 @@ export interface CloudHypervisorManagerDependencies { createVsockClient(socketPath: string, guestPort: number, timeoutMs: number): MicrovmVsockClient; createCgroup(cgroupPath: string, limits: CloudHypervisorResourceLimits): CloudHypervisorCgroup; verifyConfinement: typeof verifyCloudHypervisorConfinement; + createVmmIdentity( + runId: string, + tools: CloudHypervisorVmmIdentityToolPaths, + ): CloudHypervisorVmmIdentityManager; resolveIdentity(): { uid: number; gid: number }; } diff --git a/src/cloud-hypervisor/manager.test.ts b/src/cloud-hypervisor/manager.test.ts index 5759fd048..8e34883ff 100644 --- a/src/cloud-hypervisor/manager.test.ts +++ b/src/cloud-hypervisor/manager.test.ts @@ -22,8 +22,13 @@ import { type CloudHypervisorManagerNetworkConfig, } from './manager'; import type { CloudHypervisorHostToolPaths } from './preflight'; +import type { CloudHypervisorVmmIdentityManager } from './vmm-identity'; const hostTools: CloudHypervisorHostToolPaths = { + getfacl: '/usr/bin/getfacl', + getent: '/usr/bin/getent', + groupdel: '/usr/sbin/groupdel', + id: '/usr/bin/id', ip: '/usr/bin/ip', nft: '/usr/sbin/nft', sysctl: '/usr/sbin/sysctl', @@ -35,6 +40,9 @@ const hostTools: CloudHypervisorHostToolPaths = { mount: '/usr/bin/mount', umount: '/usr/bin/umount', setpriv: '/usr/bin/setpriv', + setfacl: '/usr/bin/setfacl', + useradd: '/usr/sbin/useradd', + userdel: '/usr/sbin/userdel', }; const exportsConfig = [ @@ -135,6 +143,16 @@ function cgroupMock(): CloudHypervisorCgroup { } as unknown as CloudHypervisorCgroup; } +function vmmIdentityMock(): CloudHypervisorVmmIdentityManager { + return { + allocate: jest.fn().mockResolvedValue({ name: 'awfvmm-test', uid: 2001, gid: 2002 }), + grantDeviceAccess: jest.fn().mockResolvedValue(undefined), + validateOwnedPaths: jest.fn().mockResolvedValue(undefined), + validateTapOwnership: jest.fn().mockResolvedValue(undefined), + cleanup: jest.fn().mockResolvedValue(undefined), + } as unknown as CloudHypervisorVmmIdentityManager; +} + function dependencies( overrides: Partial = {}, ): CloudHypervisorManagerDependencies { @@ -208,6 +226,7 @@ function dependencies( }, }, }), + createVmmIdentity: jest.fn(() => vmmIdentityMock()), resolveIdentity: jest.fn().mockReturnValue({ uid: 1000, gid: 1000 }), ...overrides, }; @@ -301,9 +320,9 @@ describe('CloudHypervisorManager', () => { expect.arrayContaining([ 'netns', 'exec', expect.stringMatching(/^awfvm-/), '/usr/bin/setpriv', - '--reuid=1000', - '--regid=1000', - '--groups=978', + '--reuid=2001', + '--regid=2002', + '--clear-groups', ]), expect.objectContaining({ reject: false, @@ -347,7 +366,7 @@ describe('CloudHypervisorManager', () => { expect(deps.verifyConfinement).toHaveBeenCalledWith(expect.objectContaining({ pid: 4242, expectedExecutable: '/opt/cloud-hypervisor', - identity: { uid: 1000, gid: 1000 }, + identity: expect.objectContaining({ uid: 2001, gid: 2002 }), networkNamespace: expect.stringMatching(/^awfvm-/), cgroupPath: expect.stringContaining('awf-cloud-hypervisor/run-1'), cgroupLimits: { @@ -370,14 +389,14 @@ describe('CloudHypervisorManager', () => { ); expect(deps.chown).toHaveBeenCalledWith( '/run/awf-cloud-hypervisor/cloud-hypervisor/run-1', - 1000, - 1000, + 2001, + 2002, ); expect(deps.createNetwork).toHaveBeenCalledWith( expect.objectContaining({ infrastructureBridge: 'awfbr0', - tapOwnerUid: 1000, - tapOwnerGid: 1000, + tapOwnerUid: 2001, + tapOwnerGid: 2002, tapVnetHdr: true, }), hostTools, @@ -386,6 +405,15 @@ describe('CloudHypervisorManager', () => { const lifecycle = (deps.createNetwork as jest.Mock).mock.results[0] .value as MicrovmNetworkLifecycle; expect(lifecycle.setup).toHaveBeenCalledTimes(1); + const vmmIdentity = (deps.createVmmIdentity as jest.Mock).mock.results[0] + .value as CloudHypervisorVmmIdentityManager; + expect(vmmIdentity.allocate).toHaveBeenCalledTimes(1); + expect(vmmIdentity.validateTapOwnership).toHaveBeenCalledWith( + '/usr/bin/ip', + expect.stringMatching(/^awfvm-/), + expect.stringMatching(/^vmt/), + ); + expect(vmmIdentity.grantDeviceAccess).toHaveBeenCalledTimes(1); }); it('terminates the partial process and removes its run directory on readiness failure', async () => { @@ -418,6 +446,9 @@ describe('CloudHypervisorManager', () => { expect(lifecycle.cleanup).toHaveBeenCalledTimes(1); const cgroup = (deps.createCgroup as jest.Mock).mock.results[0].value as CloudHypervisorCgroup; expect(cgroup.cleanup).toHaveBeenCalledTimes(1); + const vmmIdentity = (deps.createVmmIdentity as jest.Mock).mock.results[0] + .value as CloudHypervisorVmmIdentityManager; + expect(vmmIdentity.cleanup).toHaveBeenCalledTimes(1); }); it('fails closed and never creates a VM when runtime confinement verification fails', async () => { @@ -481,6 +512,12 @@ describe('CloudHypervisorManager', () => { rm: jest.fn(async () => { order.push('run-directory'); }), + createVmmIdentity: jest.fn(() => ({ + ...vmmIdentityMock(), + cleanup: jest.fn(async () => { + order.push('vmm-identity'); + }), + } as unknown as CloudHypervisorVmmIdentityManager)), }); const manager = new CloudHypervisorManager( config(), @@ -493,7 +530,7 @@ describe('CloudHypervisorManager', () => { await manager.start(); await manager.stop(); - expect(order).toEqual(['network', 'cgroup', 'run-directory']); + expect(order).toEqual(['network', 'cgroup', 'run-directory', 'vmm-identity']); }); it('configures one rootfs disk and virtio-fs devices, then stops daemons after the VMM', async () => { @@ -573,8 +610,25 @@ describe('CloudHypervisorManager', () => { expect(vmConfig.landlock_rules).not.toEqual(expect.arrayContaining([ expect.objectContaining({ path: '/workspace' }), ])); + const vmmIdentity = (deps.createVmmIdentity as jest.Mock).mock.results[0] + .value as CloudHypervisorVmmIdentityManager; + expect(vmmIdentity.validateOwnedPaths).not.toHaveBeenCalledWith([ + '/run/awf-cloud-hypervisor/cloud-hypervisor/guest/awf-vsock.socket', + ]); await manager.startInstance(); expect(client.vmBoot).toHaveBeenCalledTimes(1); + expect(vmmIdentity.validateOwnedPaths).toHaveBeenCalledWith([ + '/run/awf-cloud-hypervisor/cloud-hypervisor/guest/awf-vsock.socket', + ]); + const ownershipValidationOrder = (vmmIdentity.validateOwnedPaths as jest.Mock) + .mock.invocationCallOrder; + const vsockOwnershipValidationOrder = + ownershipValidationOrder[ownershipValidationOrder.length - 1]; + expect(vsockOwnershipValidationOrder).toBeGreaterThan( + (client.vmBoot as jest.Mock).mock.invocationCallOrder[0], + ); + expect((deps.createVsockClient as jest.Mock).mock.invocationCallOrder[0]) + .toBeGreaterThan(vsockOwnershipValidationOrder); expect(deps.createVsockClient).toHaveBeenCalledWith( expect.stringContaining('/run/awf-cloud-hypervisor/cloud-hypervisor/guest/awf-vsock.socket'), 52, @@ -1005,6 +1059,9 @@ describe('CloudHypervisorManager', () => { expect.stringContaining('/run/awf-cloud-hypervisor/'), { recursive: true, force: true }, ); + const vmmIdentity = (deps.createVmmIdentity as jest.Mock).mock.results[0] + .value as CloudHypervisorVmmIdentityManager; + expect(vmmIdentity.cleanup).toHaveBeenCalledTimes(1); }); it('builds explicit supervisor boot cmdline with PCI-required root/interface naming', () => { diff --git a/src/cloud-hypervisor/manager.ts b/src/cloud-hypervisor/manager.ts index 1128c77c2..3f780d118 100644 --- a/src/cloud-hypervisor/manager.ts +++ b/src/cloud-hypervisor/manager.ts @@ -45,7 +45,10 @@ import { type CloudHypervisorRunPaths, } from './manager-types'; import { runCloudHypervisorPreflight } from './preflight'; -import type { CloudHypervisorHostToolPaths } from './preflight'; +import type { + CloudHypervisorHostToolPaths, + CloudHypervisorPreflightResult, +} from './preflight'; import { verifyCloudHypervisorConfinement, type CloudHypervisorConfinementEvidence, @@ -53,6 +56,7 @@ import { import { startCloudHypervisor } from './manager-start'; import { stopCloudHypervisor } from './manager-stop'; import { VirtiofsdManager, type VirtiofsdDevice } from './virtiofsd'; +import { CloudHypervisorVmmIdentityManager } from './vmm-identity'; export { CLOUD_HYPERVISOR_GUEST_VSOCK_PORT, @@ -113,6 +117,7 @@ const defaultDependencies: CloudHypervisorManagerDependencies = { }), createCgroup: (cgroupPath, limits) => new CloudHypervisorCgroup(cgroupPath, limits), verifyConfinement: verifyCloudHypervisorConfinement, + createVmmIdentity: (runId, tools) => new CloudHypervisorVmmIdentityManager(runId, tools), resolveIdentity: resolveCloudHypervisorIdentity, }; @@ -170,6 +175,7 @@ export class CloudHypervisorManager { private fsDevices: VirtiofsdDevice[] = []; private guest: CloudHypervisorGuestChannel | undefined; private cgroup: CloudHypervisorCgroup | undefined; + private vmmIdentity: CloudHypervisorVmmIdentityManager | undefined; private networkPlan: MicrovmNetworkPlan | undefined; private confinementEvidence: CloudHypervisorConfinementEvidence | undefined; private instanceStarted = false; @@ -212,6 +218,7 @@ export class CloudHypervisorManager { runId?: string, private readonly networkConfig?: CloudHypervisorManagerNetworkConfig, private readonly guestConfig?: CloudHypervisorManagerGuestConfig, + private readonly verifiedArtifacts?: CloudHypervisorPreflightResult, ) { this.paths = createCloudHypervisorRunPaths(config.cloudHypervisorBinary, runId); } @@ -224,12 +231,14 @@ export class CloudHypervisorManager { paths: this.paths, networkConfig: this.networkConfig, guestConfig: this.guestConfig, + verifiedArtifacts: this.verifiedArtifacts, stdoutCapture: this.stdoutCapture, stderrCapture: this.stderrCapture, setNetworkPlan: (value) => { this.networkPlan = value; }, setNetwork: (value) => { this.network = value; }, setRootfsPreparer: (value) => { this.rootfsPreparer = value; }, setCgroup: (value) => { this.cgroup = value; }, + setVmmIdentity: (value) => { this.vmmIdentity = value; }, setProcess: (value) => { this.process = value; }, setClient: (value) => { this.client = value; }, setConfinementEvidence: (value) => { this.confinementEvidence = value; }, @@ -245,6 +254,10 @@ export class CloudHypervisorManager { await this.client.vmBoot(); this.instanceStarted = true; if (this.guestConfig) { + if (!this.vmmIdentity) { + throw new Error('Cloud Hypervisor VMM identity is not configured'); + } + await this.vmmIdentity.validateOwnedPaths([this.paths.vsockSocketPath]); this.guest = await CloudHypervisorGuestChannel.connect( this.dependencies, this.paths.vsockSocketPath, @@ -309,6 +322,7 @@ export class CloudHypervisorManager { fsDevices: this.fsDevices, guest: this.guest, cgroup: this.cgroup, + vmmIdentity: this.vmmIdentity, instanceStarted: this.instanceStarted, lastVmInfo: this.lastVmInfo, lastVmCounters: this.lastVmCounters, @@ -322,6 +336,7 @@ export class CloudHypervisorManager { setFsDevices: (value) => { this.fsDevices = value; }, setGuest: (value) => { this.guest = value; }, setCgroup: (value) => { this.cgroup = value; }, + setVmmIdentity: (value) => { this.vmmIdentity = value; }, setInstanceStarted: (value) => { this.instanceStarted = value; }, setLastVmInfo: (value) => { this.lastVmInfo = value; }, setLastVmCounters: (value) => { this.lastVmCounters = value; }, diff --git a/src/cloud-hypervisor/preflight.test.ts b/src/cloud-hypervisor/preflight.test.ts index 3f77e758a..ea16ac499 100644 --- a/src/cloud-hypervisor/preflight.test.ts +++ b/src/cloud-hypervisor/preflight.test.ts @@ -268,7 +268,7 @@ describe('Cloud Hypervisor preflight (foundation only)', () => { constants.R_OK | constants.W_OK, ); expect(deps.sha256).toHaveBeenCalledTimes(5); - expect(deps.assertToolAvailable).toHaveBeenCalledTimes(12); + expect(deps.assertToolAvailable).toHaveBeenCalledTimes(20); expect(deps.assertDockerInfrastructure).toHaveBeenCalledWith('/usr/bin/docker'); expect(deps.verifyManifestAttestation).toHaveBeenCalledWith( '/usr/bin/gh', @@ -276,6 +276,10 @@ describe('Cloud Hypervisor preflight (foundation only)', () => { '/snapshot/manifest.sigstore.jsonl', ); expect(result.tools).toEqual({ + getfacl: '/usr/bin/getfacl', + getent: '/usr/bin/getent', + groupdel: '/usr/bin/groupdel', + id: '/usr/bin/id', ip: '/usr/bin/ip', nft: '/usr/bin/nft', sysctl: '/usr/bin/sysctl', @@ -287,6 +291,9 @@ describe('Cloud Hypervisor preflight (foundation only)', () => { mount: '/usr/bin/mount', umount: '/usr/bin/umount', setpriv: '/usr/bin/setpriv', + setfacl: '/usr/bin/setfacl', + useradd: '/usr/bin/useradd', + userdel: '/usr/bin/userdel', }); }); diff --git a/src/cloud-hypervisor/preflight.ts b/src/cloud-hypervisor/preflight.ts index 33c1559d7..d258a5cd4 100644 --- a/src/cloud-hypervisor/preflight.ts +++ b/src/cloud-hypervisor/preflight.ts @@ -62,6 +62,10 @@ export interface CloudHypervisorPreflightDependencies { } export type CloudHypervisorHostToolPaths = Readonly<{ + getfacl: string; + getent: string; + groupdel: string; + id: string; ip: string; nft: string; sysctl: string; @@ -80,6 +84,9 @@ export type CloudHypervisorHostToolPaths = Readonly<{ * this for Cloud Hypervisor). See `src/cloud-hypervisor/launcher.ts`. */ setpriv: string; + setfacl: string; + useradd: string; + userdel: string; }>; export interface CloudHypervisorArtifactSnapshotSources { @@ -99,7 +106,8 @@ export interface CloudHypervisorArtifactSnapshot extends CloudHypervisorArtifact const CLOUD_HYPERVISOR_ARTIFACT_SNAPSHOT_ROOT = '/run/awf-cloud-hypervisor/trusted-artifacts'; const CLOUD_HYPERVISOR_HOST_TOOLS: (keyof CloudHypervisorHostToolPaths)[] = [ - 'ip', 'nft', 'sysctl', 'flock', 'mke2fs', 'debugfs', 'e2fsck', 'rsync', 'mount', 'umount', 'setpriv', + 'getent', 'getfacl', 'groupdel', 'id', 'ip', 'nft', 'sysctl', 'flock', 'mke2fs', 'debugfs', 'e2fsck', + 'rsync', 'mount', 'umount', 'setfacl', 'setpriv', 'useradd', 'userdel', ]; const defaultDependencies: CloudHypervisorPreflightDependencies = { diff --git a/src/cloud-hypervisor/virtiofsd-sandbox.ts b/src/cloud-hypervisor/virtiofsd-sandbox.ts index c47b4da50..34ded42cf 100644 --- a/src/cloud-hypervisor/virtiofsd-sandbox.ts +++ b/src/cloud-hypervisor/virtiofsd-sandbox.ts @@ -3,9 +3,7 @@ import * as path from 'path'; const WORKER_READY_TIMEOUT_MS = 5_000; const WORKER_READY_INTERVAL_MS = 50; const REVIEWED_WORKER_CAPABILITIES = '00000000880000db'; -const REVIEWED_WORKER_CAPABILITY_BITS = BigInt(`0x${REVIEWED_WORKER_CAPABILITIES}`); const ZERO_CAPABILITIES = /^0+$/; -const CAPABILITY_MASK = /^[0-9a-f]{16}$/; const REQUIRED_NAMESPACES = ['mnt', 'pid', 'net'] as const; const CAPABILITY_FIELDS = ['CapInh', 'CapPrm', 'CapEff', 'CapBnd', 'CapAmb'] as const; const CGROUP_ROOT = '/sys/fs/cgroup'; @@ -141,13 +139,8 @@ export async function verifyVirtiofsdSandbox( ) { throw new Error('virtiofsd worker capabilities differ from the reviewed sandbox set'); } - const workerBounding = worker.capabilities.CapBnd ?? ''; - if ( - !CAPABILITY_MASK.test(workerBounding) || - (BigInt(`0x${workerBounding}`) & REVIEWED_WORKER_CAPABILITY_BITS) !== - REVIEWED_WORKER_CAPABILITY_BITS - ) { - throw new Error('virtiofsd worker bounding set excludes reviewed runtime capabilities'); + if (!ZERO_CAPABILITIES.test(worker.capabilities.CapBnd ?? '')) { + throw new Error('virtiofsd worker bounding capability set is not empty'); } if (worker.noNewPrivs !== 1 || worker.seccomp !== 2) { throw new Error('virtiofsd worker is missing NoNewPrivs or seccomp filtering'); diff --git a/src/cloud-hypervisor/virtiofsd.test.ts b/src/cloud-hypervisor/virtiofsd.test.ts index 8cd2c2247..243b1d6b9 100644 --- a/src/cloud-hypervisor/virtiofsd.test.ts +++ b/src/cloud-hypervisor/virtiofsd.test.ts @@ -74,7 +74,7 @@ function dependencies( `CapInh:\t${zero}`, `CapPrm:\t${isWorker ? reviewed : zero}`, `CapEff:\t${isWorker ? reviewed : zero}`, - `CapBnd:\t${isWorker ? reviewed : zero}`, + `CapBnd:\t${zero}`, `CapAmb:\t${zero}`, `NoNewPrivs:\t${isWorker ? 1 : 0}`, `Seccomp:\t${isWorker ? 2 : 0}`, @@ -351,11 +351,11 @@ describe('VirtiofsdManager', () => { deps.readFile = jest.fn(async (filePath: string, encoding: BufferEncoding) => { const contents = await readFile(filePath, encoding); return filePath === '/proc/1100/status' - ? contents.replace('CapBnd:\t00000000880000db', 'CapBnd:\t0000000000000000') + ? contents.replace('CapBnd:\t0000000000000000', 'CapBnd:\t00000000880000db') : contents; }); }, - error: /bounding set excludes reviewed runtime capabilities/, + error: /bounding capability set is not empty/, }, { name: 'worker NoNewPrivs', diff --git a/src/cloud-hypervisor/vmm-identity.test.ts b/src/cloud-hypervisor/vmm-identity.test.ts new file mode 100644 index 000000000..3401bcdd5 --- /dev/null +++ b/src/cloud-hypervisor/vmm-identity.test.ts @@ -0,0 +1,861 @@ +import { + CloudHypervisorVmmIdentityManager, + cloudHypervisorVmmIdentityTestHelpers, + createAccountName, + type CloudHypervisorVmmIdentityDependencies, + type CloudHypervisorVmmIdentityToolPaths, +} from './vmm-identity'; + +const tools: CloudHypervisorVmmIdentityToolPaths = { + getfacl: '/usr/bin/getfacl', + getent: '/usr/bin/getent', + groupdel: '/usr/sbin/groupdel', + id: '/usr/bin/id', + ip: '/usr/bin/ip', + setfacl: '/usr/bin/setfacl', + useradd: '/usr/sbin/useradd', + userdel: '/usr/sbin/userdel', +}; + +function dependencies(overrides: Partial = {}) { + let lockExists = false; + let accountExists = false; + let groupExists = false; + let accountName = ''; + let ownerContents = ''; + const aclPaths = new Set(); + const run = jest.fn(async (command: string, args: readonly string[]) => { + if (command === tools.useradd) { + accountExists = true; + groupExists = true; + accountName = args[args.length - 1]; + return { stdout: '', stderr: '' }; + } + if (command === tools.userdel) { + accountExists = false; + groupExists = false; + return { stdout: '', stderr: '' }; + } + if (command === tools.groupdel) { + groupExists = false; + return { stdout: '', stderr: '' }; + } + if (command === tools.setfacl) { + const devicePath = args[2]; + if (args[0] === '--modify') aclPaths.add(devicePath); + else aclPaths.delete(devicePath); + return { stdout: '', stderr: '' }; + } + if (command === tools.getfacl) { + return { + stdout: aclPaths.has(args[2]) ? 'user:23001:rw-\n' : '', + stderr: '', + }; + } + if (command === tools.getent) { + if (args[0] === 'group') { + if (!groupExists) throw Object.assign(new Error('missing group'), { exitCode: 2 }); + return { stdout: `${accountName}:x:23002:\n`, stderr: '' }; + } + return { + stdout: `${accountName}:x:23001:23002:AWF Cloud Hypervisor:/nonexistent:/usr/sbin/nologin\n`, + stderr: '', + }; + } + if (command === tools.id) { + if (!accountExists) throw Object.assign(new Error('missing account'), { exitCode: 1 }); + if (args[0] === '-u') return { stdout: '23001\n', stderr: '' }; + if (args[0] === '-g') return { stdout: '23002\n', stderr: '' }; + return { stdout: '23002\n', stderr: '' }; + } + if (command === tools.ip) { + return { stdout: 'vmt123: tap persist user 23001 group 23002\n', stderr: '' }; + } + throw new Error(`unexpected command: ${command}`); + }); + const deps: CloudHypervisorVmmIdentityDependencies = { + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + rm: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) lockExists = false; + }), + rmdir: jest.fn().mockResolvedValue(undefined), + lstat: jest.fn().mockResolvedValue({ uid: 23001, gid: 23002, ino: 1, mtimeMs: 0 }), + run, + sleep: jest.fn().mockResolvedValue(undefined), + pid: 1234, + processStartTime: jest.fn().mockResolvedValue('99'), + ...overrides, + }; + return { deps, run }; +} + +describe('CloudHypervisorVmmIdentityManager', () => { + it('creates a no-login system account, validates resources, grants ACLs, and removes exact state', async () => { + const { deps, run } = dependencies(); + const manager = new CloudHypervisorVmmIdentityManager('run-1', tools, deps); + + const identity = await manager.allocate(); + expect(identity).toEqual({ + name: expect.stringMatching(/^awfvmm-[a-f0-9]{20}$/), + uid: 23001, + gid: 23002, + }); + expect(run).toHaveBeenCalledWith(tools.useradd, expect.arrayContaining([ + '--system', + '--user-group', + '--no-create-home', + '--home-dir', '/nonexistent', + '--shell', '/usr/sbin/nologin', + ])); + + await manager.validateOwnedPaths(['/run/awf/kernel', '/run/awf/rootfs']); + await manager.validateTapOwnership(tools.ip, 'awfvm-123', 'vmt123'); + await manager.grantDeviceAccess(); + expect(run).toHaveBeenCalledWith( + tools.setfacl, + ['--modify', 'user:23001:rw', '/dev/kvm'], + ); + expect(run).toHaveBeenCalledWith( + tools.setfacl, + ['--modify', 'user:23001:rw', '/dev/net/tun'], + ); + + await manager.cleanup(); + expect(run).toHaveBeenCalledWith(tools.userdel, [identity.name]); + expect(run).not.toHaveBeenCalledWith(tools.groupdel, [identity.name]); + expect(run).toHaveBeenCalledWith( + tools.setfacl, + ['--remove', 'user:23001', '/dev/kvm'], + ); + expect(run).toHaveBeenCalledWith( + tools.setfacl, + ['--remove', 'user:23001', '/dev/net/tun'], + ); + }); + + it('serializes concurrent allocation calls on one manager', async () => { + let releaseUseradd!: () => void; + const useraddGate = new Promise((resolve) => { + releaseUseradd = resolve; + }); + const base = dependencies({ + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), + }); + const originalRun = base.deps.run; + base.deps.run = jest.fn(async (command, args) => { + if (command === tools.useradd) await useraddGate; + return originalRun(command, args); + }); + const manager = new CloudHypervisorVmmIdentityManager('concurrent', tools, base.deps); + + const first = manager.allocate(); + const second = manager.allocate(); + releaseUseradd(); + + const [firstIdentity, secondIdentity] = await Promise.all([first, second]); + expect(secondIdentity).toEqual(firstIdentity); + expect((base.deps.run as jest.Mock).mock.calls.filter( + ([command]) => command === tools.useradd, + )).toHaveLength(1); + }); + + it('rejects inherited supplementary groups and rolls back the account', async () => { + const base = dependencies(); + const originalRun = base.deps.run; + base.deps.run = jest.fn(async (command, args) => { + const result = await originalRun(command, args); + if (command === tools.id && args[0] === '-G') return { stdout: '23002 27\n', stderr: '' }; + return result; + }); + + const manager = new CloudHypervisorVmmIdentityManager('run-2', tools, base.deps); + + await expect(manager.allocate()).rejects.toThrow(/inherited supplementary groups/); + expect(base.deps.run).toHaveBeenCalledWith( + tools.userdel, + [expect.stringMatching(/^awfvmm-/)], + ); + }); + + it('preserves provisional state and reports a failed allocation rollback', async () => { + const base = dependencies(); + const originalRun = base.deps.run; + let failRollback = true; + base.deps.run = jest.fn(async (command, args) => { + if (command === tools.userdel && failRollback) throw new Error('userdel failed'); + const result = await originalRun(command, args); + if (command === tools.id && args[0] === '-G') return { stdout: '23002 27\n', stderr: '' }; + return result; + }); + const manager = new CloudHypervisorVmmIdentityManager('rollback', tools, base.deps); + + await expect(manager.allocate()).rejects.toThrow( + /account allocation failed.*inherited supplementary groups.*rollback also failed.*userdel failed/, + ); + await expect(manager.allocate()).rejects.toThrow(/account cleanup is still pending/); + failRollback = false; + await expect(manager.cleanup()).resolves.toBeUndefined(); + expect((base.deps.run as jest.Mock).mock.calls.filter( + ([command]) => command === tools.userdel, + )).toHaveLength(2); + }); + + it('rejects mismatched staged path and TAP ownership', async () => { + const base = dependencies(); + const manager = new CloudHypervisorVmmIdentityManager('ownership', tools, base.deps); + await manager.allocate(); + base.deps.lstat = jest.fn().mockResolvedValue({ uid: 999, gid: 998 }); + await expect(manager.validateOwnedPaths(['/run/awf/rootfs'])) + .rejects.toThrow(/path ownership mismatch/); + const originalRun = base.deps.run; + base.deps.run = jest.fn(async (command, args) => { + if (command === tools.ip) { + return { stdout: 'other: tap persist user 23001 group 23002\n', stderr: '' }; + } + return originalRun(command, args); + }); + await expect(manager.validateTapOwnership(tools.ip, 'awfvm-123', 'vmt123')) + .rejects.toThrow(/was not found/); + }); + + it('rolls back accounts with unsafe passwd state', async () => { + const base = dependencies(); + const originalRun = base.deps.run; + base.deps.run = jest.fn(async (command, args) => { + if (command === tools.getent && args[0] === 'passwd') { + return { stdout: `${args[1]}:x:23001:23002:AWF:/home/unsafe:/bin/bash\n`, stderr: '' }; + } + return originalRun(command, args); + }); + const manager = new CloudHypervisorVmmIdentityManager('unsafe-passwd', tools, base.deps); + + await expect(manager.allocate()).rejects.toThrow(/unsafe passwd state/); + expect(base.deps.run).toHaveBeenCalledWith( + tools.userdel, + [expect.stringMatching(/^awfvmm-/)], + ); + }); + + it('fails closed when a device ACL grant or removal cannot be verified', async () => { + const grantBase = dependencies(); + const grantOriginalRun = grantBase.deps.run; + grantBase.deps.run = jest.fn(async (command, args) => { + const result = await grantOriginalRun(command, args); + if (command === tools.getfacl) return { stdout: '', stderr: '' }; + return result; + }); + const grantManager = new CloudHypervisorVmmIdentityManager('acl-grant', tools, grantBase.deps); + await grantManager.allocate(); + await expect(grantManager.grantDeviceAccess()).rejects.toThrow(/ACL validation failed/); + + const removalBase = dependencies(); + const removalManager = new CloudHypervisorVmmIdentityManager( + 'acl-removal', + tools, + removalBase.deps, + ); + const identity = await removalManager.allocate(); + await removalManager.grantDeviceAccess(); + const removalOriginalRun = removalBase.deps.run; + removalBase.deps.run = jest.fn(async (command, args) => { + if (command === tools.setfacl && args[0] === '--remove') { + return { stdout: '', stderr: '' }; + } + return removalOriginalRun(command, args); + }); + + await expect(removalManager.cleanup()).rejects.toThrow(/ACL removal validation failed/); + expect(removalBase.deps.run).not.toHaveBeenCalledWith(tools.userdel, [identity.name]); + }); + + it('refuses to delete an account whose uid/gid changed before cleanup', async () => { + const base = dependencies(); + const manager = new CloudHypervisorVmmIdentityManager('reused-account', tools, base.deps); + await manager.allocate(); + await manager.grantDeviceAccess(); + (base.deps.run as jest.Mock).mockClear(); + const originalRun = base.deps.run; + base.deps.run = jest.fn(async (command, args) => { + if (command === tools.id && args[0] === '-u') return { stdout: '24001\n', stderr: '' }; + if (command === tools.id && args[0] === '-g') return { stdout: '24002\n', stderr: '' }; + if (command === tools.id && args[0] === '-G') return { stdout: '24002\n', stderr: '' }; + if (command === tools.getent && args[0] === 'passwd') { + return { + stdout: `${args[1]}:x:24001:24002:AWF:/nonexistent:/usr/sbin/nologin\n`, + stderr: '', + }; + } + return originalRun(command, args); + }); + + await expect(manager.cleanup()).rejects.toThrow(/Refusing to remove reused/); + expect(base.deps.run).not.toHaveBeenCalledWith( + tools.setfacl, + expect.arrayContaining(['--remove']), + ); + expect(base.deps.run).not.toHaveBeenCalledWith( + tools.userdel, + [expect.stringMatching(/^awfvmm-/)], + ); + }); + + it('removes a residual private group when userdel leaves it behind', async () => { + const base = dependencies(); + const originalRun = base.deps.run; + base.deps.run = jest.fn(async (command, args) => { + if (command === tools.userdel) return { stdout: '', stderr: '' }; + return originalRun(command, args); + }); + const manager = new CloudHypervisorVmmIdentityManager('residual-group', tools, base.deps); + const identity = await manager.allocate(); + + await manager.cleanup(); + expect(base.deps.run).toHaveBeenCalledWith(tools.groupdel, [identity.name]); + }); + + it('rejects invalid allocator identity output and missing lock-owner process metadata', async () => { + const invalidBase = dependencies(); + const invalidOriginalRun = invalidBase.deps.run; + invalidBase.deps.run = jest.fn(async (command, args) => { + const result = await invalidOriginalRun(command, args); + if (command === tools.id && args[0] === '-u') return { stdout: '0\n', stderr: '' }; + return result; + }); + const invalidManager = new CloudHypervisorVmmIdentityManager( + 'invalid-identity', + tools, + invalidBase.deps, + ); + await expect(invalidManager.allocate()).rejects.toThrow(/invalid uid/); + + const missingStart = dependencies({ + processStartTime: jest.fn().mockResolvedValue(undefined), + }); + const missingStartManager = new CloudHypervisorVmmIdentityManager( + 'missing-start', + tools, + missingStart.deps, + ); + await expect(missingStartManager.allocate()).rejects.toThrow(/Cannot determine AWF process start time/); + }); + + it('treats a missing process as a stale lock owner', async () => { + await expect(cloudHypervisorVmmIdentityTestHelpers.readProcessStartTime(2_147_483_647)) + .resolves.toBeUndefined(); + }); + + it('strictly validates lock owners and positive numeric account identifiers', () => { + const valid = { pid: 1234, startTime: '99', nonce: 'a'.repeat(32) }; + expect(cloudHypervisorVmmIdentityTestHelpers.isLockOwner(valid)).toBe(true); + for (const invalid of [ + { ...valid, pid: 0 }, + { ...valid, pid: Number.MAX_SAFE_INTEGER + 1 }, + { ...valid, startTime: 99 }, + { ...valid, startTime: 'not-numeric' }, + { ...valid, nonce: 123 }, + { ...valid, nonce: 'not-a-nonce' }, + ]) { + expect(cloudHypervisorVmmIdentityTestHelpers.isLockOwner(invalid as never)).toBe(false); + } + expect(cloudHypervisorVmmIdentityTestHelpers.isSameLockOwner(valid, valid)).toBe(true); + expect(cloudHypervisorVmmIdentityTestHelpers.isSameLockOwner( + valid, + { ...valid, nonce: 'b'.repeat(32) }, + )).toBe(false); + expect(cloudHypervisorVmmIdentityTestHelpers.isSameLockOwner( + valid, + { ...valid, pid: 4321 }, + )).toBe(false); + expect(cloudHypervisorVmmIdentityTestHelpers.isSameLockOwner( + valid, + { ...valid, startTime: '100' }, + )).toBe(false); + expect(cloudHypervisorVmmIdentityTestHelpers.isSameLockOwner( + valid, + { ...valid, nonce: 'invalid' }, + )).toBe(false); + expect(cloudHypervisorVmmIdentityTestHelpers.parsePositiveInteger('42\n', 'uid')).toBe(42); + expect(() => cloudHypervisorVmmIdentityTestHelpers.parsePositiveInteger( + '9007199254740992', + 'uid', + )).toThrow(/unsafe uid/); + expect(cloudHypervisorVmmIdentityTestHelpers.formatError(new Error('failure'))).toBe('failure'); + expect(cloudHypervisorVmmIdentityTestHelpers.formatError('failure')).toBe('failure'); + const processFields = Array.from({ length: 20 }, (_, index) => String(index)); + processFields[19] = '777'; + expect(cloudHypervisorVmmIdentityTestHelpers.parseProcessStatStartTime( + `123 (command with spaces) ${processFields.join(' ')}`, + )).toBe('777'); + }); + + it('runs account tools with the hardened default command executor', async () => { + await expect(cloudHypervisorVmmIdentityTestHelpers.defaultRun('/usr/bin/true', [])) + .resolves.toEqual({ stdout: '', stderr: '' }); + await expect(cloudHypervisorVmmIdentityTestHelpers.defaultRun('/usr/bin/false', [])) + .rejects.toThrow(/exited with code 1/); + await expect(cloudHypervisorVmmIdentityTestHelpers.defaultSleep(0)).resolves.toBeUndefined(); + await expect(cloudHypervisorVmmIdentityTestHelpers.readProcessStartTime( + 1234, + jest.fn().mockResolvedValue( + `1234 (command) ${Array.from({ length: 20 }, (_, index) => + index === 19 ? '888' : String(index)).join(' ')}`, + ), + )).resolves.toBe('888'); + await expect(cloudHypervisorVmmIdentityTestHelpers.readProcessStartTime( + 1234, + jest.fn().mockRejectedValue(Object.assign(new Error('denied'), { code: 'EACCES' })), + )).rejects.toThrow('denied'); + }); + + it('rejects a pre-existing generated account and malformed TAP ownership', async () => { + const existing = dependencies(); + existing.deps.run = jest.fn(async (command, args) => { + if (command === tools.id && args[0] === '-u') return { stdout: '23001\n', stderr: '' }; + return existing.run(command, args); + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'existing-account', + tools, + existing.deps, + ).allocate()).rejects.toThrow(/account already exists/); + + const tapBase = dependencies(); + const tapManager = new CloudHypervisorVmmIdentityManager('tap-fields', tools, tapBase.deps); + await tapManager.allocate(); + for (const tapLine of [ + 'vmt123: tap persist group 23002', + 'vmt123: tap persist user 24001 group 23002', + 'vmt123: tap persist user 23001', + 'vmt123: tap persist user 23001 group 24002', + ]) { + tapBase.deps.run = jest.fn(async (command, args) => { + if (command === tools.ip) return { stdout: `${tapLine}\n`, stderr: '' }; + return tapBase.run(command, args); + }); + await expect(tapManager.validateTapOwnership(tools.ip, 'awfvm-123', 'vmt123')) + .rejects.toThrow(/TAP ownership mismatch/); + } + }); + + it('waits for a live account-lock owner instead of reclaiming it', async () => { + let lockExists = true; + const owner = { pid: 4321, startTime: '77', nonce: 'a'.repeat(32) }; + let ownerContents = `${JSON.stringify(owner)}\n`; + const base = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + lstat: jest.fn().mockResolvedValue({ uid: 0, gid: 0, ino: 42, mtimeMs: 0 }), + processStartTime: jest.fn(async (pid) => pid === owner.pid ? owner.startTime : '99'), + sleep: jest.fn(async () => { + lockExists = false; + }), + }); + const manager = new CloudHypervisorVmmIdentityManager('live-lock', tools, base.deps); + + await expect(manager.allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + expect(base.deps.rm).toHaveBeenCalledTimes(1); + }); + + it('waits for a fresh incomplete account lock', async () => { + let lockExists = true; + let ownerContents = '{'; + const base = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + lstat: jest.fn().mockResolvedValue({ + uid: 0, + gid: 0, + ino: 42, + mtimeMs: Date.now(), + }), + sleep: jest.fn(async () => { + lockExists = false; + }), + }); + const manager = new CloudHypervisorVmmIdentityManager('fresh-lock', tools, base.deps); + + await expect(manager.allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + }); + + it('backs off when another process owns stale-lock reclamation', async () => { + let lockExists = true; + let reaperExists = true; + const owner = { pid: 4321, startTime: '77', nonce: 'a'.repeat(32) }; + let ownerContents = `${JSON.stringify(owner)}\n`; + const base = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.reaper')) { + if (reaperExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + reaperExists = true; + } else if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + lstat: jest.fn(async (filePath) => ({ + uid: 0, + gid: 0, + ino: 42, + mtimeMs: filePath.endsWith('.reaper') ? 0 : Date.now(), + })), + processStartTime: jest.fn(async (pid) => pid === 4321 ? 'stale' : '99'), + rmdir: jest.fn(async () => { + reaperExists = false; + }), + sleep: jest.fn(async () => { + lockExists = false; + }), + }); + const manager = new CloudHypervisorVmmIdentityManager('reaper-lock', tools, base.deps); + + await expect(manager.allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + expect(base.deps.rmdir).toHaveBeenCalled(); + }); + + it('does not reap a lock whose inode changes after claiming the reaper', async () => { + let lockExists = true; + let ownerContents = `${JSON.stringify({ + pid: 4321, + startTime: '77', + nonce: 'a'.repeat(32), + })}\n`; + let lockStatsReads = 0; + const base = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + lstat: jest.fn(async () => ({ + uid: 0, + gid: 0, + ino: ++lockStatsReads === 1 ? 42 : 43, + mtimeMs: 0, + })), + processStartTime: jest.fn(async (pid) => pid === 4321 ? 'stale' : '99'), + sleep: jest.fn(async () => { + lockExists = false; + }), + }); + const manager = new CloudHypervisorVmmIdentityManager('replaced-lock', tools, base.deps); + + await expect(manager.allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + expect(base.deps.rm).toHaveBeenCalledTimes(1); + }); + + it('revalidates stale-lock liveness and freshness after claiming the reaper', async () => { + const owner = { pid: 4321, startTime: '77', nonce: 'a'.repeat(32) }; + let lockExists = true; + let ownerContents = `${JSON.stringify(owner)}\n`; + let ownerChecks = 0; + const becameLive = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + lstat: jest.fn().mockResolvedValue({ uid: 0, gid: 0, ino: 42, mtimeMs: 0 }), + processStartTime: jest.fn(async (pid) => { + if (pid !== owner.pid) return '99'; + return ownerChecks++ === 0 ? 'stale' : owner.startTime; + }), + sleep: jest.fn(async () => { + lockExists = false; + }), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'became-live-lock', + tools, + becameLive.deps, + ).allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + + lockExists = true; + ownerContents = ''; + let statsReads = 0; + const becameFresh = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + lstat: jest.fn(async () => ({ + uid: 0, + gid: 0, + ino: 42, + mtimeMs: statsReads++ === 0 ? 0 : Date.now(), + })), + sleep: jest.fn(async () => { + lockExists = false; + }), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'became-fresh-lock', + tools, + becameFresh.deps, + ).allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + }); + + it('surfaces unexpected stale-lock metadata errors', async () => { + const unreadable = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + } + }), + lstat: jest.fn().mockResolvedValue({ uid: 0, gid: 0, ino: 42, mtimeMs: 0 }), + readFile: jest.fn(async () => { + throw Object.assign(new Error('denied'), { code: 'EACCES' }); + }), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'unreadable-lock', + tools, + unreadable.deps, + ).allocate()).rejects.toThrow('denied'); + + const reaperDenied = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory === '/run/awf-cloud-hypervisor') return; + throw Object.assign( + new Error('denied'), + { code: directory.endsWith('.reaper') ? 'EACCES' : 'EEXIST' }, + ); + }), + lstat: jest.fn().mockResolvedValue({ uid: 0, gid: 0, ino: 42, mtimeMs: 0 }), + readFile: jest.fn(async () => ''), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'reaper-denied', + tools, + reaperDenied.deps, + ).allocate()).rejects.toThrow('denied'); + }); + + it('handles missing and unexpected lock-stat failures', async () => { + const now = jest.spyOn(Date, 'now') + .mockReturnValueOnce(0) + .mockReturnValue(10_001); + const missing = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + } + }), + lstat: jest.fn(async () => { + throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + }), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'missing-lock', + tools, + missing.deps, + ).allocate()).rejects.toThrow(/Timed out waiting/); + now.mockRestore(); + + const denied = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.account-lock')) { + throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + } + }), + lstat: jest.fn(async () => { + throw Object.assign(new Error('denied'), { code: 'EACCES' }); + }), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'lock-stat-denied', + tools, + denied.deps, + ).allocate()).rejects.toThrow('denied'); + }); + + it('backs off when a competing reaper disappears', async () => { + let lockExists = true; + const owner = { pid: 4321, startTime: '77', nonce: 'a'.repeat(32) }; + let ownerContents = `${JSON.stringify(owner)}\n`; + const base = dependencies({ + mkdir: jest.fn(async (directory) => { + if (directory.endsWith('.reaper')) { + throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + } + if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }), + writeFile: jest.fn(async (_filePath, contents) => { + ownerContents = contents; + }), + readFile: jest.fn(async () => ownerContents), + lstat: jest.fn(async (filePath) => { + if (filePath.endsWith('.reaper')) { + throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + } + return { uid: 0, gid: 0, ino: 42, mtimeMs: 0 }; + }), + processStartTime: jest.fn(async (pid) => pid === owner.pid ? 'stale' : '99'), + sleep: jest.fn(async () => { + lockExists = false; + }), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'vanished-reaper', + tools, + base.deps, + ).allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + }); + + it('detects identity replacement while waiting to grant device access', async () => { + const base = dependencies(); + const manager = new CloudHypervisorVmmIdentityManager('changed-identity', tools, base.deps); + await manager.allocate(); + let lockAttempt = 0; + base.deps.mkdir = jest.fn(async (directory) => { + if (directory.endsWith('.account-lock') && lockAttempt++ === 0) { + throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + } + }); + base.deps.lstat = jest.fn(async () => { + throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + }); + base.deps.sleep = jest.fn(async () => { + Reflect.set(manager, 'identity', undefined); + }); + + await expect(manager.grantDeviceAccess()).rejects.toThrow(/identity changed/); + }); + + it('rejects removal when acquired lock ownership changes', async () => { + let ownerContents = ''; + const base = dependencies({ + writeFile: jest.fn(async (_filePath, contents) => { + const owner = JSON.parse(contents) as { pid: number; startTime: string; nonce: string }; + ownerContents = JSON.stringify({ ...owner, nonce: 'b'.repeat(32) }); + }), + readFile: jest.fn(async () => ownerContents), + }); + await expect(new CloudHypervisorVmmIdentityManager( + 'changed-owner', + tools, + base.deps, + ).allocate()).rejects.toThrow(/lock ownership changed unexpectedly/); + }); + + it('does not release the uid when a device ACL cannot be removed', async () => { + const base = dependencies(); + const manager = new CloudHypervisorVmmIdentityManager('run-3', tools, base.deps); + const identity = await manager.allocate(); + await manager.grantDeviceAccess(); + const originalRun = base.deps.run; + base.deps.run = jest.fn(async (command, args) => { + if ( + command === tools.setfacl && + args[0] === '--remove' && + args[2] === '/dev/kvm' + ) { + throw new Error('ACL removal failed'); + } + return originalRun(command, args); + }); + + await expect(manager.cleanup()).rejects.toThrow(/ACL cleanup failed/); + expect(base.deps.run).not.toHaveBeenCalledWith(tools.userdel, [identity.name]); + }); + + it('recovers a stale zero-byte lock without racing a replacement owner', async () => { + const base = dependencies(); + let lockExists = true; + let reaperExists = false; + let ownerContents = ''; + base.deps.mkdir = jest.fn(async (directory) => { + if (directory.endsWith('.reaper')) { + if (reaperExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + reaperExists = true; + } else if (directory.endsWith('.account-lock')) { + if (lockExists) throw Object.assign(new Error('exists'), { code: 'EEXIST' }); + lockExists = true; + } + }); + base.deps.writeFile = jest.fn(async (filePath, contents) => { + if (filePath.endsWith('owner.json')) ownerContents = contents; + }); + base.deps.readFile = jest.fn(async () => ownerContents); + base.deps.lstat = jest.fn(async (filePath) => { + if (filePath.endsWith('.reaper') && !reaperExists) { + throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + } + if (!lockExists) throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + return { uid: 0, gid: 0, ino: 42, mtimeMs: 0 }; + }); + base.deps.rm = jest.fn(async (filePath) => { + if (filePath.endsWith('.account-lock')) { + lockExists = false; + reaperExists = false; + } + }); + base.deps.rmdir = jest.fn(async () => { + if (!reaperExists) throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + reaperExists = false; + }); + const manager = new CloudHypervisorVmmIdentityManager('run-4', tools, base.deps); + + await expect(manager.allocate()).resolves.toMatchObject({ uid: 23001, gid: 23002 }); + expect(base.deps.rm).toHaveBeenCalledWith( + '/run/awf-cloud-hypervisor/.account-lock', + { recursive: true, force: true }, + ); + }); + + it('uses random non-PID account names for repeated allocations', () => { + expect(new CloudHypervisorVmmIdentityManager('default-dependencies', tools)).toBeDefined(); + const first = createAccountName(); + const second = createAccountName(); + expect(first).toMatch(/^awfvmm-[a-f0-9]{20}$/); + expect(second).not.toBe(first); + expect(first).not.toContain(String(process.pid)); + }); +}); diff --git a/src/cloud-hypervisor/vmm-identity.ts b/src/cloud-hypervisor/vmm-identity.ts new file mode 100644 index 000000000..8a6c20699 --- /dev/null +++ b/src/cloud-hypervisor/vmm-identity.ts @@ -0,0 +1,512 @@ +import { randomBytes } from 'crypto'; +import { promises as fs } from 'fs'; +import * as path from 'path'; +import execa from 'execa'; + +const ACCOUNT_PREFIX = 'awfvmm-'; +const ACCOUNT_LOCK_DIRECTORY = '/run/awf-cloud-hypervisor/.account-lock'; +const ACCOUNT_REAPER_DIRECTORY = path.join(ACCOUNT_LOCK_DIRECTORY, '.reaper'); +const ACCOUNT_LOCK_RETRY_MS = 25; +const ACCOUNT_LOCK_TIMEOUT_MS = 10_000; +const INCOMPLETE_LOCK_STALE_MS = 1_000; +const VMM_DEVICE_PATHS = ['/dev/kvm', '/dev/net/tun'] as const; + +export interface CloudHypervisorVmmIdentity { + readonly name: string; + readonly uid: number; + readonly gid: number; +} + +export interface CloudHypervisorVmmIdentityToolPaths { + readonly getfacl: string; + readonly groupdel: string; + readonly getent: string; + readonly id: string; + readonly ip: string; + readonly setfacl: string; + readonly useradd: string; + readonly userdel: string; +} + +export interface CloudHypervisorVmmIdentityDependencies { + mkdir(directory: string, options?: { recursive?: boolean; mode?: number }): Promise; + writeFile(filePath: string, contents: string, options?: { flag?: string; mode?: number }): Promise; + readFile(filePath: string, encoding: 'utf8'): Promise; + rm(filePath: string, options: { recursive: true; force: true }): Promise; + rmdir(directory: string): Promise; + lstat(filePath: string): Promise<{ + uid: number; + gid: number; + ino?: number; + mtimeMs?: number; + }>; + run(command: string, args: readonly string[]): Promise<{ stdout: string; stderr: string }>; + sleep(milliseconds: number): Promise; + pid: number; + processStartTime(pid: number): Promise; +} + +const defaultDependencies: CloudHypervisorVmmIdentityDependencies = { + mkdir: fs.mkdir, + writeFile: fs.writeFile, + readFile: fs.readFile, + rm: fs.rm, + rmdir: fs.rmdir, + lstat: fs.lstat, + run: async (command, args) => { + const result = await execa(command, [...args], { + reject: false, + stdio: ['ignore', 'pipe', 'pipe'], + env: { PATH: '/usr/sbin:/usr/bin:/sbin:/bin' }, + extendEnv: false, + }); + if (result.exitCode !== 0) { + throw new Error( + `${command} ${args.join(' ')} exited with code ${result.exitCode}: ` + + `${result.stderr.trim() || result.stdout.trim()}`, + ); + } + return { stdout: result.stdout, stderr: result.stderr }; + }, + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), + pid: process.pid, + processStartTime: readProcessStartTime, +}; + +interface LockOwner { + readonly pid: number; + readonly startTime: string; + readonly nonce: string; +} + +export class CloudHypervisorVmmIdentityManager { + private identity: CloudHypervisorVmmIdentity | undefined; + private provisionalAccountName: string | undefined; + private readonly aclPaths = new Set(); + + constructor( + private readonly runId: string, + private readonly tools: CloudHypervisorVmmIdentityToolPaths, + private readonly dependencies: CloudHypervisorVmmIdentityDependencies = defaultDependencies, + ) {} + + async allocate(): Promise { + if (this.identity) return this.identity; + return this.withAccountLock(async () => { + if (this.identity) return this.identity; + if (this.provisionalAccountName) { + throw new Error( + `Cloud Hypervisor VMM account cleanup is still pending: ${this.provisionalAccountName}`, + ); + } + const name = createAccountName(); + if (await this.accountExists(name)) { + throw new Error(`Cloud Hypervisor VMM account already exists: ${name}`); + } + try { + await this.dependencies.run(this.tools.useradd, [ + '--system', + '--user-group', + '--no-create-home', + '--home-dir', '/nonexistent', + '--shell', '/usr/sbin/nologin', + '--comment', `AWF Cloud Hypervisor ${this.runId}`, + name, + ]); + this.provisionalAccountName = name; + const identity = await this.resolveAndValidateAccount(name); + this.identity = identity; + this.provisionalAccountName = undefined; + return identity; + } catch (error) { + if (await this.accountExists(name)) { + this.provisionalAccountName = name; + } + if (this.provisionalAccountName) { + try { + await this.removeAccountState(name); + this.provisionalAccountName = undefined; + } catch (rollbackError) { + throw new Error( + `Cloud Hypervisor VMM account allocation failed: ${formatError(error)}; ` + + `rollback also failed: ${formatError(rollbackError)}`, + ); + } + } + throw error; + } + }); + } + + async grantDeviceAccess(): Promise { + const identity = this.requireIdentity(); + await this.withAccountLock(async () => { + if (this.identity !== identity) { + throw new Error('Cloud Hypervisor VMM identity changed before device ACL grant'); + } + for (const devicePath of VMM_DEVICE_PATHS) { + await this.dependencies.run(this.tools.setfacl, [ + '--modify', `user:${identity.uid}:rw`, devicePath, + ]); + this.aclPaths.add(devicePath); + const { stdout } = await this.dependencies.run(this.tools.getfacl, [ + '--absolute-names', '--numeric', devicePath, + ]); + if (!stdout.split(/\r?\n/).includes(`user:${identity.uid}:rw-`)) { + throw new Error(`Cloud Hypervisor VMM ACL validation failed for ${devicePath}`); + } + } + }); + } + + async validateOwnedPaths(paths: readonly string[]): Promise { + const identity = this.requireIdentity(); + for (const ownedPath of paths) { + const stats = await this.dependencies.lstat(ownedPath); + if (stats.uid !== identity.uid || stats.gid !== identity.gid) { + throw new Error( + `Cloud Hypervisor VMM path ownership mismatch for ${ownedPath}: ` + + `expected ${identity.uid}:${identity.gid}, got ${stats.uid}:${stats.gid}`, + ); + } + } + } + + async validateTapOwnership(ipPath: string, namespaceName: string, tapName: string): Promise { + const identity = this.requireIdentity(); + const { stdout } = await this.dependencies.run(ipPath, [ + 'netns', 'exec', namespaceName, + ipPath, '-details', 'tuntap', 'show', 'dev', tapName, + ]); + const tapLine = stdout.split(/\r?\n/).find((line) => line.startsWith(`${tapName}:`)); + if (!tapLine) { + throw new Error(`Cloud Hypervisor TAP ${namespaceName}/${tapName} was not found`); + } + const fields = tapLine.trim().split(/\s+/); + const userIndex = fields.indexOf('user'); + const groupIndex = fields.indexOf('group'); + if ( + userIndex < 0 || + fields[userIndex + 1] !== String(identity.uid) || + groupIndex < 0 || + fields[groupIndex + 1] !== String(identity.gid) + ) { + throw new Error( + `Cloud Hypervisor TAP ownership mismatch for ${namespaceName}/${tapName}`, + ); + } + } + + async cleanup(): Promise { + const identity = this.identity; + const provisionalAccountName = this.provisionalAccountName; + if (!identity && !provisionalAccountName) return; + await this.withAccountLock(async () => { + if (identity && this.identity !== identity) return; + if (!identity && provisionalAccountName) { + if (this.provisionalAccountName !== provisionalAccountName) return; + await this.removeAccountState(provisionalAccountName); + this.provisionalAccountName = undefined; + return; + } + if (!identity) return; + const current = await this.resolveAndValidateAccount(identity.name); + if (current.uid !== identity.uid || current.gid !== identity.gid) { + throw new Error( + `Refusing to remove reused Cloud Hypervisor VMM account ${identity.name}`, + ); + } + const errors: unknown[] = []; + for (const devicePath of [...this.aclPaths].reverse()) { + try { + await this.dependencies.run(this.tools.setfacl, [ + '--remove', `user:${identity.uid}`, devicePath, + ]); + const { stdout } = await this.dependencies.run(this.tools.getfacl, [ + '--absolute-names', '--numeric', devicePath, + ]); + if (stdout.split(/\r?\n/).some((line) => line.startsWith(`user:${identity.uid}:`))) { + throw new Error(`Cloud Hypervisor VMM ACL removal validation failed for ${devicePath}`); + } + this.aclPaths.delete(devicePath); + } catch (error) { + errors.push(error); + } + } + if (errors.length > 0 || this.aclPaths.size > 0) { + throw new Error( + `Cloud Hypervisor VMM ACL cleanup failed: ${errors.map(formatError).join('; ')}`, + ); + } + await this.removeAccountState(identity.name); + this.identity = undefined; + }); + } + + private async removeAccountState(name: string): Promise { + if (await this.accountExists(name)) { + await this.dependencies.run(this.tools.userdel, [name]); + } + if (await this.groupExists(name)) { + await this.dependencies.run(this.tools.groupdel, [name]); + } + } + + private async groupExists(name: string): Promise { + try { + await this.dependencies.run(this.tools.getent, ['group', name]); + return true; + } catch { + return false; + } + } + + private async resolveAndValidateAccount(name: string): Promise { + const [ + { stdout: uidText }, + { stdout: gidText }, + { stdout: groupsText }, + { stdout: passwdText }, + ] = await Promise.all([ + this.dependencies.run(this.tools.id, ['-u', name]), + this.dependencies.run(this.tools.id, ['-g', name]), + this.dependencies.run(this.tools.id, ['-G', name]), + this.dependencies.run(this.tools.getent, ['passwd', name]), + ]); + const uid = parsePositiveInteger(uidText, 'uid'); + const gid = parsePositiveInteger(gidText, 'gid'); + const groups = groupsText.trim().split(/\s+/).filter(Boolean).map((value) => + parsePositiveInteger(value, 'supplementary group')); + if (groups.length !== 1 || groups[0] !== gid) { + throw new Error( + `Cloud Hypervisor VMM account ${name} inherited supplementary groups: ${groups.join(' ')}`, + ); + } + const passwd = passwdText.trim().split(':'); + if ( + passwd.length !== 7 || + passwd[0] !== name || + passwd[2] !== String(uid) || + passwd[3] !== String(gid) || + passwd[5] !== '/nonexistent' || + passwd[6] !== '/usr/sbin/nologin' + ) { + throw new Error(`Cloud Hypervisor VMM account ${name} has unsafe passwd state`); + } + return { name, uid, gid }; + } + + private async accountExists(name: string): Promise { + try { + await this.dependencies.run(this.tools.id, ['-u', name]); + return true; + } catch { + return false; + } + } + + + private requireIdentity(): CloudHypervisorVmmIdentity { + if (!this.identity) throw new Error('Cloud Hypervisor VMM identity has not been allocated'); + return this.identity; + } + + private async withAccountLock(operation: () => Promise): Promise { + const parent = path.dirname(ACCOUNT_LOCK_DIRECTORY); + await this.dependencies.mkdir(parent, { recursive: true, mode: 0o711 }); + const startTime = await this.dependencies.processStartTime(this.dependencies.pid); + if (!startTime) throw new Error('Cannot determine AWF process start time for VMM account lock'); + const owner: LockOwner = { + pid: this.dependencies.pid, + startTime, + nonce: randomBytes(16).toString('hex'), + }; + const deadline = Date.now() + ACCOUNT_LOCK_TIMEOUT_MS; + for (;;) { + let acquired = false; + try { + await this.dependencies.mkdir(ACCOUNT_LOCK_DIRECTORY, { mode: 0o700 }); + acquired = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + if (acquired) { + try { + await this.dependencies.writeFile( + path.join(ACCOUNT_LOCK_DIRECTORY, 'owner.json'), + `${JSON.stringify(owner)}\n`, + { flag: 'wx', mode: 0o600 }, + ); + return await operation(); + } finally { + await this.removeOwnedLock(owner); + } + } + await this.reclaimStaleLock(); + if (Date.now() >= deadline) { + throw new Error('Timed out waiting for the Cloud Hypervisor VMM account lock'); + } + await this.dependencies.sleep(ACCOUNT_LOCK_RETRY_MS); + } + } + + private async reclaimStaleLock(): Promise { + let initialStats: Awaited>; + try { + initialStats = await this.dependencies.lstat(ACCOUNT_LOCK_DIRECTORY); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + let owner = await this.readLockOwner(); + if ( + !owner && + (initialStats.mtimeMs === undefined || + Date.now() - initialStats.mtimeMs < INCOMPLETE_LOCK_STALE_MS) + ) { + return; + } + if (owner) { + const liveStartTime = await this.dependencies.processStartTime(owner.pid); + if (liveStartTime === owner.startTime) return; + } + if (!await this.tryClaimReaper(initialStats)) return; + try { + const currentStats = await this.dependencies.lstat(ACCOUNT_LOCK_DIRECTORY); + if ( + initialStats.ino === undefined || + currentStats.ino === undefined || + currentStats.ino !== initialStats.ino + ) return; + owner = await this.readLockOwner(); + if (owner) { + const liveStartTime = await this.dependencies.processStartTime(owner.pid); + if (liveStartTime === owner.startTime) return; + } else if ( + initialStats.mtimeMs === undefined || + Date.now() - initialStats.mtimeMs < INCOMPLETE_LOCK_STALE_MS + ) { + return; + } + await this.dependencies.rm(ACCOUNT_LOCK_DIRECTORY, { recursive: true, force: true }); + } finally { + await this.dependencies.rmdir(ACCOUNT_REAPER_DIRECTORY).catch((error) => { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + }); + } + } + + private async readLockOwner(): Promise { + try { + const parsed = JSON.parse( + await this.dependencies.readFile(path.join(ACCOUNT_LOCK_DIRECTORY, 'owner.json'), 'utf8'), + ) as LockOwner; + return isLockOwner(parsed) ? parsed : undefined; + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code === 'ENOENT' || + error instanceof SyntaxError + ) return undefined; + throw error; + } + } + + private async tryClaimReaper( + lockStats: Awaited>, + ): Promise { + try { + await this.dependencies.mkdir(ACCOUNT_REAPER_DIRECTORY, { mode: 0o700 }); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; + } + try { + const reaperStats = await this.dependencies.lstat(ACCOUNT_REAPER_DIRECTORY); + if ( + lockStats.ino !== undefined && + reaperStats.mtimeMs !== undefined && + Date.now() - reaperStats.mtimeMs >= INCOMPLETE_LOCK_STALE_MS + ) { + await this.dependencies.rmdir(ACCOUNT_REAPER_DIRECTORY); + } + } catch (reaperError) { + if ((reaperError as NodeJS.ErrnoException).code !== 'ENOENT') throw reaperError; + } + return false; + } + } + + private async removeOwnedLock(owner: LockOwner): Promise { + const current = JSON.parse( + await this.dependencies.readFile(path.join(ACCOUNT_LOCK_DIRECTORY, 'owner.json'), 'utf8'), + ) as LockOwner; + if (!isSameLockOwner(owner, current)) { + throw new Error('Cloud Hypervisor VMM account lock ownership changed unexpectedly'); + } + await this.dependencies.rm(ACCOUNT_LOCK_DIRECTORY, { recursive: true, force: true }); + } +} + +export function createAccountName(): string { + return `${ACCOUNT_PREFIX}${randomBytes(10).toString('hex')}`; +} + +/** @internal Exposed only for focused lock-owner tests. */ +export const cloudHypervisorVmmIdentityTestHelpers = { + defaultRun: defaultDependencies.run, + defaultSleep: defaultDependencies.sleep, + formatError, + isLockOwner, + isSameLockOwner, + parseProcessStatStartTime, + parsePositiveInteger, + readProcessStartTime, +}; + +async function readProcessStartTime( + pid: number, + readFile: typeof fs.readFile = fs.readFile, +): Promise { + try { + const stat = await readFile(`/proc/${pid}/stat`, 'utf8'); + return parseProcessStatStartTime(stat); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} + +function parseProcessStatStartTime(stat: string): string | undefined { + const close = stat.lastIndexOf(')'); + const fields = stat.slice(close + 2).split(' '); + return fields[19]; +} + +function parsePositiveInteger(value: string, label: string): number { + const trimmed = value.trim(); + if (!/^[1-9]\d*$/.test(trimmed)) { + throw new Error(`Cloud Hypervisor VMM account returned an invalid ${label}: ${trimmed}`); + } + const parsed = Number(trimmed); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`Cloud Hypervisor VMM account returned an unsafe ${label}: ${trimmed}`); + } + return parsed; +} + +function isLockOwner(value: LockOwner): boolean { + return Number.isSafeInteger(value?.pid) && value.pid > 0 && + typeof value.startTime === 'string' && /^\d+$/.test(value.startTime) && + typeof value.nonce === 'string' && /^[a-f0-9]{32}$/.test(value.nonce); +} + +function isSameLockOwner(left: LockOwner, right: LockOwner): boolean { + return isLockOwner(left) && isLockOwner(right) && + left.pid === right.pid && left.startTime === right.startTime && left.nonce === right.nonce; +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +}