diff --git a/src/cloud-hypervisor/manager-start.ts b/src/cloud-hypervisor/manager-start.ts new file mode 100644 index 000000000..2ebf98405 --- /dev/null +++ b/src/cloud-hypervisor/manager-start.ts @@ -0,0 +1,168 @@ +import * as path from 'path'; +import type { ExecaChildProcess } from 'execa'; +import type { CloudHypervisorOptions } from '../types/runtime-options'; +import type { MicrovmRootfsPreparer } from '../microvm/rootfs'; +import { + createMicrovmNetworkPlan, + type MicrovmNetworkLifecycle, + type MicrovmNetworkPlan, +} from '../microvm/network'; +import type { CloudHypervisorApiClient } from './api-client'; +import { + prepareRunDirectory, + stageArtifact, + stageDiagnosticFile, + waitForApiSocket, +} from './diagnostics'; +import { + CloudHypervisorCgroup, + buildCloudHypervisorLaunchCommand, +} from './launcher'; +import { + formatError, + type CloudHypervisorManagerDependencies, + type CloudHypervisorManagerGuestConfig, + type CloudHypervisorManagerNetworkConfig, + type CloudHypervisorRunPaths, +} from './manager-types'; +import { validateCloudHypervisorExports } from './exports'; +import { hasReadOnlyWorkspaceMountPlan } from './filesystem-write-enforcement'; +import type { VirtiofsdManager, VirtiofsdDevice } from './virtiofsd'; +import { buildCloudHypervisorVmConfig } from './vm-config-builder'; +import type { BoundedOutputCapture } from './diagnostics'; + +export interface CloudHypervisorStartContext { + config: CloudHypervisorOptions; + workDir: string; + dependencies: CloudHypervisorManagerDependencies; + paths: CloudHypervisorRunPaths; + networkConfig?: CloudHypervisorManagerNetworkConfig; + guestConfig?: CloudHypervisorManagerGuestConfig; + stdoutCapture: BoundedOutputCapture; + stderrCapture: BoundedOutputCapture; + setNetworkPlan(plan: MicrovmNetworkPlan | undefined): void; + setNetwork(network: MicrovmNetworkLifecycle | undefined): void; + setRootfsPreparer(preparer: MicrovmRootfsPreparer | undefined): void; + setCgroup(cgroup: CloudHypervisorCgroup | undefined): void; + setProcess(process: ExecaChildProcess | undefined): void; + setClient(client: CloudHypervisorApiClient | undefined): void; + setVirtiofsd(virtiofsd: VirtiofsdManager | undefined): void; + setFsDevices(devices: VirtiofsdDevice[]): void; + getFsDevices(): VirtiofsdDevice[]; + stop(): Promise; +} + +export async function startCloudHypervisor( + context: CloudHypervisorStartContext, +): Promise { + const { + config, workDir, dependencies, paths, networkConfig, guestConfig, + } = context; + if (!networkConfig) { + throw new Error( + 'Cloud Hypervisor network configuration is required; refusing to launch an unfiltered microVM', + ); + } + + let startupError: unknown; + try { + const artifacts = await dependencies.preflight(config); + const identity = guestConfig?.identity ?? dependencies.resolveIdentity(); + const networkPlan = createMicrovmNetworkPlan(paths.runId, { + ...networkConfig, + tapOwnerUid: identity.uid, + tapOwnerGid: identity.gid, + tapVnetHdr: true, + }); + context.setNetworkPlan(networkPlan); + const network = dependencies.createNetwork(networkPlan, artifacts.tools); + context.setNetwork(network); + await network.setup(); + let rootfsSource = artifacts.rootfsPath; + if (guestConfig) { + validateCloudHypervisorExports(guestConfig.exports, { + allowReadOnlyWorkspace: hasReadOnlyWorkspaceMountPlan(guestConfig.mountEnforcement), + }); + const rootfsPreparationDirectory = path.join( + workDir, 'cloud-hypervisor-rootfs', paths.runId, + ); + const rootfsPreparer = dependencies.createRootfsPreparer({ + runDirectory: rootfsPreparationDirectory, + baseRootfsPath: artifacts.rootfsPath, + supervisorBinaryPath: guestConfig.supervisorBinaryPath, + supervisorSha256: guestConfig.supervisorSha256, + supervisorGuestPath: '/usr/sbin/awf-supervisor', + hostAliases: { + ...(networkConfig.apiProxyIp ? { 'api-proxy': networkConfig.apiProxyIp } : {}), + ...(networkConfig.hostAliases ?? {}), + }, + }, artifacts.tools); + context.setRootfsPreparer(rootfsPreparer); + rootfsSource = await rootfsPreparer.prepare(); + } + + await prepareRunDirectory(dependencies, paths, identity); + const cgroup = dependencies.createCgroup( + paths.cgroupPath, + { memoryMib: config.memoryMib, vcpuCount: config.vcpuCount }, + ); + context.setCgroup(cgroup); + await cgroup.setup(); + await stageArtifact(dependencies, artifacts.kernelPath, paths.kernelPath, 0o400, identity); + await stageArtifact(dependencies, rootfsSource, paths.rootfsPath, 0o600, identity); + await stageDiagnosticFile(dependencies, paths.logPath, identity); + await stageDiagnosticFile(dependencies, paths.serialLogPath, identity); + + const launchCommand = buildCloudHypervisorLaunchCommand({ + tools: { ip: artifacts.tools.ip, setpriv: artifacts.tools.setpriv }, + namespaceName: networkPlan.namespaceName, + identity, + kvmGid: artifacts.kvmGid, + cloudHypervisorBinary: config.cloudHypervisorBinary, + apiSocketPath: paths.apiSocketPath, + logFilePath: paths.logPath, + }); + const child = dependencies.launch(launchCommand.command, [...launchCommand.args], { + reject: false, + stdio: ['ignore', 'pipe', 'pipe'], + // Cloud Hypervisor directly processes untrusted guest/device input, so + // its environment must not expose the host's provider credentials. + extendEnv: false, + env: { PATH: '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' }, + }); + context.setProcess(child); + child.stdout?.on('data', (chunk: Buffer | string) => context.stdoutCapture.append(chunk)); + child.stderr?.on('data', (chunk: Buffer | string) => context.stderrCapture.append(chunk)); + if (child.pid !== undefined) await cgroup.assign(child.pid); + + await waitForApiSocket(dependencies, paths, config.apiTimeoutMs, child); + const client = dependencies.createClient(paths.apiSocketPath, config.apiTimeoutMs); + context.setClient(client); + await client.ping(); + if (guestConfig) { + const virtiofsd = dependencies.createVirtiofsdManager( + artifacts.virtiofsdBinary, paths.runDirectory, paths.virtiofsdShareDirectory, + identity, cgroup, { mount: artifacts.tools.mount, umount: artifacts.tools.umount }, + ); + context.setVirtiofsd(virtiofsd); + context.setFsDevices(await virtiofsd.start(guestConfig.exports, guestConfig.mountEnforcement)); + } + await client.vmCreate(buildCloudHypervisorVmConfig({ + config, paths, networkPlan, ...(guestConfig ? { guestConfig } : {}), + fsDevices: context.getFsDevices(), + })); + return client; + } catch (error) { + startupError = error; + } + + try { + await context.stop(); + } catch (cleanupError) { + throw new Error( + `Cloud Hypervisor startup failed: ${formatError(startupError)}; ` + + `partial-start cleanup also failed: ${formatError(cleanupError)}`, + ); + } + throw startupError; +} diff --git a/src/cloud-hypervisor/manager-stop.ts b/src/cloud-hypervisor/manager-stop.ts new file mode 100644 index 000000000..0b420c59e --- /dev/null +++ b/src/cloud-hypervisor/manager-stop.ts @@ -0,0 +1,153 @@ +import * as path from 'path'; +import type { ExecaChildProcess } from 'execa'; +import type { CloudHypervisorOptions } from '../types/runtime-options'; +import type { MicrovmNetworkLifecycle, MicrovmNetworkPlan } from '../microvm/network'; +import type { CloudHypervisorApiClient, CloudHypervisorVmCounters, CloudHypervisorVmInfo } from './api-client'; +import { + formatError, + type CloudHypervisorManagerDependencies, + type CloudHypervisorRunPaths, +} from './manager-types'; +import type { CloudHypervisorCgroup } from './launcher'; +import type { MicrovmRootfsPreparer } from '../microvm/rootfs'; +import type { VirtiofsdManager, VirtiofsdDevice } from './virtiofsd'; +import type { CloudHypervisorGuestChannel } from './guest-execution'; + +const SHUTDOWN_GRACE_MS = 5_000; + +export interface CloudHypervisorStopContext { + config: CloudHypervisorOptions; + dependencies: CloudHypervisorManagerDependencies; + paths: CloudHypervisorRunPaths; + process?: ExecaChildProcess; + client?: CloudHypervisorApiClient; + network?: MicrovmNetworkLifecycle; + networkPlan?: MicrovmNetworkPlan; + rootfsPreparer?: MicrovmRootfsPreparer; + virtiofsd?: VirtiofsdManager; + fsDevices: VirtiofsdDevice[]; + guest?: CloudHypervisorGuestChannel; + cgroup?: CloudHypervisorCgroup; + instanceStarted: boolean; + lastVmInfo?: CloudHypervisorVmInfo; + lastVmCounters?: CloudHypervisorVmCounters; + preserve?: boolean; + beforeCleanup?: () => Promise; + setProcess(process: ExecaChildProcess | undefined): void; + setClient(client: CloudHypervisorApiClient | undefined): void; + setNetwork(network: MicrovmNetworkLifecycle | undefined): void; + setNetworkPlan(plan: MicrovmNetworkPlan | undefined): void; + setRootfsPreparer(preparer: MicrovmRootfsPreparer | undefined): void; + setVirtiofsd(virtiofsd: VirtiofsdManager | undefined): void; + setFsDevices(devices: VirtiofsdDevice[]): void; + setGuest(guest: CloudHypervisorGuestChannel | undefined): void; + setCgroup(cgroup: CloudHypervisorCgroup | undefined): void; + setInstanceStarted(started: boolean): void; + setLastVmInfo(info: CloudHypervisorVmInfo | undefined): void; + setLastVmCounters(counters: CloudHypervisorVmCounters | undefined): void; +} + +export async function stopCloudHypervisor(context: CloudHypervisorStopContext): Promise { + const errors: unknown[] = []; + const instanceWasStarted = context.instanceStarted; + if (context.client && instanceWasStarted) { + try { context.setLastVmInfo(await context.client.vmInfo()); } catch { context.setLastVmInfo(undefined); } + try { context.setLastVmCounters(await context.client.vmCounters()); } catch { context.setLastVmCounters(undefined); } + } + let guestShutdownAcknowledged = false; + if (context.guest) { + const outcome = await context.guest.shutdown(); + guestShutdownAcknowledged = outcome.acknowledged; + if (outcome.error !== undefined) errors.push(outcome.error); + } + context.setGuest(undefined); + if (context.client && instanceWasStarted && guestShutdownAcknowledged) { + try { await context.client.vmShutdown(); } catch { /* process termination is authoritative */ } + } + if (context.client) { + try { await context.client.vmmShutdown(); } catch { /* process termination is authoritative */ } + } + + let terminationConfirmed = !context.process || + context.process.exitCode !== null || context.process.signalCode !== null; + if (context.process && context.process.exitCode === null && context.process.signalCode === null) { + const child = context.process; + try { + terminationConfirmed = await waitForProcessExit(child, context.dependencies, SHUTDOWN_GRACE_MS); + if (!child.killed && child.exitCode === null && child.signalCode === null) { + child.kill('SIGTERM', { forceKillAfterTimeout: 2_000 }); + } + if (!terminationConfirmed) { + await child; + if (child.exitCode === null && child.signalCode === null) { + throw new Error('Cloud Hypervisor process termination was not confirmed'); + } + } + terminationConfirmed = true; + } catch (error) { + terminationConfirmed = child.exitCode !== null || child.signalCode !== null; + errors.push(error); + } + } + if (!terminationConfirmed && context.process) { + if (errors.length === 0) errors.push(new Error('Cloud Hypervisor process termination was not confirmed')); + try { await context.virtiofsd?.stop(); context.setVirtiofsd(undefined); context.setFsDevices([]); } + catch (error) { errors.push(error); } + throw new Error(`Cloud Hypervisor cleanup stopped before network/run-directory removal: ${errors.map(formatError).join('; ')}`); + } + context.setProcess(undefined); + context.setClient(undefined); + let virtiofsdTerminationConfirmed = true; + try { await context.virtiofsd?.stop(); context.setVirtiofsd(undefined); } + catch (error) { virtiofsdTerminationConfirmed = false; errors.push(error); } + if (context.beforeCleanup) { + try { await context.beforeCleanup(); } catch (error) { errors.push(error); } + } + if (!virtiofsdTerminationConfirmed) { + throw new Error(`Cloud Hypervisor cleanup stopped before cgroup/run-directory removal: ${errors.map(formatError).join('; ')}`); + } + context.setFsDevices([]); + context.setInstanceStarted(false); + if (context.rootfsPreparer) { + try { + await context.dependencies.rm(path.dirname(context.rootfsPreparer.rootfsImagePath), { recursive: true, force: true }); + } catch (error) { errors.push(error); } + } + context.setRootfsPreparer(undefined); + if (context.preserve) { + try { await context.cgroup?.cleanup(); } catch (error) { errors.push(error); } + context.setCgroup(undefined); + throwCleanupErrors(errors, 'Cloud Hypervisor preservation failed: '); + return; + } + try { await context.network?.cleanup(); context.setNetwork(undefined); context.setNetworkPlan(undefined); } + catch (error) { errors.push(error); } + try { await context.cgroup?.cleanup(); } catch (error) { errors.push(error); } + context.setCgroup(undefined); + if (!instanceWasStarted || terminationConfirmed) { + try { + await context.dependencies.rm( + path.join(context.paths.runBaseDir, path.basename(context.config.cloudHypervisorBinary), context.paths.runId), + { recursive: true, force: true }, + ); + } catch (error) { errors.push(error); } + } + throwCleanupErrors(errors, 'Cloud Hypervisor cleanup failed: '); +} + +async function waitForProcessExit( + child: ExecaChildProcess, + dependencies: CloudHypervisorManagerDependencies, + timeoutMs: number, +): Promise { + for (let attempt = 0; attempt < Math.max(1, Math.ceil(timeoutMs / 25)); attempt += 1) { + if (child.exitCode !== null || child.signalCode !== null) return true; + await dependencies.sleep(25); + } + return child.exitCode !== null || child.signalCode !== null; +} + +function throwCleanupErrors(errors: unknown[], prefix: string): void { + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new Error(`${prefix}${errors.map(formatError).join('; ')}`); +} diff --git a/src/cloud-hypervisor/manager.ts b/src/cloud-hypervisor/manager.ts index b5d7d5420..b6d4b0f9b 100644 --- a/src/cloud-hypervisor/manager.ts +++ b/src/cloud-hypervisor/manager.ts @@ -1,12 +1,10 @@ import { promises as fs } from 'fs'; -import * as path from 'path'; import execa, { type ExecaChildProcess } from 'execa'; import type { CloudHypervisorOptions } from '../types/runtime-options'; import { getSafeHostGid, getSafeHostUid } from '../host-identity'; import { LinuxNetworkCommands, MicrovmNetworkManager, - createMicrovmNetworkPlan, type MicrovmNetworkLifecycle, type MicrovmNetworkPlan, } from '../microvm/network'; @@ -26,24 +24,18 @@ import { import { BoundedOutputCapture, collectCloudHypervisorDiagnostics, - prepareRunDirectory, readBoundedTail, - stageArtifact, - stageDiagnosticFile, - waitForApiSocket, } from './diagnostics'; import { CloudHypervisorGuestChannel, } from './guest-execution'; import { CloudHypervisorCgroup, - buildCloudHypervisorLaunchCommand, } from './launcher'; import { CLOUD_HYPERVISOR_CAPTURE_LIMIT_BYTES, CLOUD_HYPERVISOR_GUEST_VSOCK_PORT, createCloudHypervisorRunPaths, - formatError, type CloudHypervisorIdentity, type CloudHypervisorManagerDependencies, type CloudHypervisorManagerGuestConfig, @@ -52,14 +44,9 @@ import { } from './manager-types'; import { runCloudHypervisorPreflight } from './preflight'; import type { CloudHypervisorHostToolPaths } from './preflight'; -import { - validateCloudHypervisorExports, -} from './exports'; -import { hasReadOnlyWorkspaceMountPlan } from './filesystem-write-enforcement'; +import { startCloudHypervisor } from './manager-start'; +import { stopCloudHypervisor } from './manager-stop'; import { VirtiofsdManager, type VirtiofsdDevice } from './virtiofsd'; -import { - buildCloudHypervisorVmConfig, -} from './vm-config-builder'; export { CLOUD_HYPERVISOR_GUEST_VSOCK_PORT, @@ -76,8 +63,6 @@ export { encodeVirtiofsBootArg, } from './vm-config-builder'; -const CLOUD_HYPERVISOR_GUEST_SHUTDOWN_GRACE_MS = 5_000; -const CLOUD_HYPERVISOR_GUEST_SUPERVISOR = '/usr/sbin/awf-supervisor'; const defaultDependencies: CloudHypervisorManagerDependencies = { preflight: runCloudHypervisorPreflight, @@ -217,164 +202,26 @@ export class CloudHypervisorManager { } async start(): Promise { - if (!this.networkConfig) { - throw new Error( - 'Cloud Hypervisor network configuration is required; refusing to launch an unfiltered microVM', - ); - } - - let startupError: unknown; - try { - const artifacts = await this.dependencies.preflight(this.config); - const identity = this.guestConfig?.identity ?? this.dependencies.resolveIdentity(); - const networkPlan = createMicrovmNetworkPlan(this.paths.runId, { - ...this.networkConfig, - tapOwnerUid: identity.uid, - tapOwnerGid: identity.gid, - // Cloud Hypervisor's own tap handling (Tap::open_named() in - // net_util/src/tap.rs) always re-opens the tap with - // IFF_VNET_HDR requested; the tap must be *created* with that - // feature available or the host and Cloud Hypervisor disagree - // on frame layout for the host-to-guest direction, and guest - // connectivity checks silently time out even though the - // guest's own outbound traffic (and the host-side veth/nft - // layer) works normally. Discovered via live-KVM validation: - // tap RX=10 packets (guest-to-host, unaffected) vs. TX=1 packet - // (host-to-guest, effectively stalled) despite response - // packets already having arrived on the host-side veth. - // This backend requires IFF_VNET_HDR, so it opts in explicitly. - tapVnetHdr: true, - }); - this.networkPlan = networkPlan; - this.network = this.dependencies.createNetwork(networkPlan, artifacts.tools); - await this.network.setup(); - let rootfsSource = artifacts.rootfsPath; - if (this.guestConfig) { - validateCloudHypervisorExports(this.guestConfig.exports, { - allowReadOnlyWorkspace: hasReadOnlyWorkspaceMountPlan( - this.guestConfig.mountEnforcement, - ), - }); - const rootfsPreparationDirectory = path.join( - this.workDir, - 'cloud-hypervisor-rootfs', - this.paths.runId, - ); - this.rootfsPreparer = this.dependencies.createRootfsPreparer({ - runDirectory: rootfsPreparationDirectory, - baseRootfsPath: artifacts.rootfsPath, - supervisorBinaryPath: this.guestConfig.supervisorBinaryPath, - supervisorSha256: this.guestConfig.supervisorSha256, - supervisorGuestPath: CLOUD_HYPERVISOR_GUEST_SUPERVISOR, - hostAliases: { - ...(this.networkConfig.apiProxyIp - ? { 'api-proxy': this.networkConfig.apiProxyIp } - : {}), - ...(this.networkConfig.hostAliases ?? {}), - }, - }, artifacts.tools); - rootfsSource = await this.rootfsPreparer.prepare(); - } - - await prepareRunDirectory(this.dependencies, this.paths, identity); - - this.cgroup = this.dependencies.createCgroup( - this.paths.cgroupPath, - { memoryMib: this.config.memoryMib, vcpuCount: this.config.vcpuCount }, - ); - await this.cgroup.setup(); - - await stageArtifact( - this.dependencies, artifacts.kernelPath, this.paths.kernelPath, 0o400, identity, - ); - await stageArtifact( - this.dependencies, rootfsSource, this.paths.rootfsPath, 0o600, identity, - ); - await stageDiagnosticFile(this.dependencies, this.paths.logPath, identity); - await stageDiagnosticFile(this.dependencies, this.paths.serialLogPath, identity); - - const launchCommand = buildCloudHypervisorLaunchCommand({ - tools: { ip: artifacts.tools.ip, setpriv: artifacts.tools.setpriv }, - namespaceName: networkPlan.namespaceName, - identity, - kvmGid: artifacts.kvmGid, - cloudHypervisorBinary: this.config.cloudHypervisorBinary, - apiSocketPath: this.paths.apiSocketPath, - logFilePath: this.paths.logPath, - }); - this.process = this.dependencies.launch( - launchCommand.command, - [...launchCommand.args], - { - reject: false, - stdio: ['ignore', 'pipe', 'pipe'], - // Explicit minimal environment: the launched process must never - // inherit AWF's host environment (provider/GitHub credentials - // the guest environment deliberately excludes). Cloud Hypervisor - // directly processes untrusted guest/device input, so a VMM - // compromise reading `process.env` would bypass the API-proxy - // credential isolation boundary entirely. `extendEnv: false` - // stops execa from merging this back with `process.env`. - extendEnv: false, - env: buildLauncherEnvironment(), - }, - ); - this.process.stdout?.on('data', (chunk: Buffer | string) => { - this.stdoutCapture.append(chunk); - }); - this.process.stderr?.on('data', (chunk: Buffer | string) => { - this.stderrCapture.append(chunk); - }); - if (this.process.pid !== undefined) { - await this.cgroup.assign(this.process.pid); - } - - await waitForApiSocket( - this.dependencies, - this.paths, - this.config.apiTimeoutMs, - this.process, - ); - this.client = this.dependencies.createClient( - this.paths.apiSocketPath, - this.config.apiTimeoutMs, - ); - await this.client.ping(); - if (this.guestConfig) { - this.virtiofsd = this.dependencies.createVirtiofsdManager( - artifacts.virtiofsdBinary, - this.paths.runDirectory, - this.paths.virtiofsdShareDirectory, - identity, - this.cgroup, - { mount: artifacts.tools.mount, umount: artifacts.tools.umount }, - ); - this.fsDevices = await this.virtiofsd.start( - this.guestConfig.exports, - this.guestConfig.mountEnforcement, - ); - } - await this.client.vmCreate(buildCloudHypervisorVmConfig({ - config: this.config, - paths: this.paths, - networkPlan, - ...(this.guestConfig ? { guestConfig: this.guestConfig } : {}), - fsDevices: this.fsDevices, - })); - return this.client; - } catch (error) { - startupError = error; - } - - try { - await this.stop(); - } catch (cleanupError) { - throw new Error( - `Cloud Hypervisor startup failed: ${formatError(startupError)}; ` + - `partial-start cleanup also failed: ${formatError(cleanupError)}`, - ); - } - throw startupError; + return startCloudHypervisor({ + config: this.config, + workDir: this.workDir, + dependencies: this.dependencies, + paths: this.paths, + networkConfig: this.networkConfig, + guestConfig: this.guestConfig, + stdoutCapture: this.stdoutCapture, + stderrCapture: this.stderrCapture, + setNetworkPlan: (value) => { this.networkPlan = value; }, + setNetwork: (value) => { this.network = value; }, + setRootfsPreparer: (value) => { this.rootfsPreparer = value; }, + setCgroup: (value) => { this.cgroup = value; }, + setProcess: (value) => { this.process = value; }, + setClient: (value) => { this.client = value; }, + setVirtiofsd: (value) => { this.virtiofsd = value; }, + setFsDevices: (value) => { this.fsDevices = value; }, + getFsDevices: () => this.fsDevices, + stop: () => this.stop(), + }); } async startInstance(): Promise { @@ -429,214 +276,36 @@ export class CloudHypervisorManager { } async stop(options: { preserve?: boolean; beforeCleanup?: () => Promise } = {}): Promise { - const errors: unknown[] = []; - const instanceWasStarted = this.instanceStarted; - // vm.info/vm.counters require the Cloud Hypervisor API socket to - // still be responsive, which is only true *before* vmm.shutdown()/ - // process termination below -- the opposite ordering constraint from - // serial console capture (which needs the process already exited to - // guarantee flushed output; see the beforeCleanup comment further - // down). Snapshot both here, before any shutdown attempt, so - // collectDiagnostics() (invoked later, via beforeCleanup, after the - // process has already exited) has a real, non-null snapshot to write - // instead of failing silently against an already-closed socket. - if (this.client && instanceWasStarted) { - try { - this.lastVmInfo = await this.client.vmInfo(); - } catch { - this.lastVmInfo = undefined; - } - try { - this.lastVmCounters = await this.client.vmCounters(); - } catch { - this.lastVmCounters = undefined; - } - } - let guestShutdownAcknowledged = false; - if (this.guest) { - const outcome = await this.guest.shutdown(); - guestShutdownAcknowledged = outcome.acknowledged; - if (outcome.error !== undefined) errors.push(outcome.error); - } - this.guest = undefined; - - if (this.client && instanceWasStarted && guestShutdownAcknowledged) { - try { - await this.client.vmShutdown(); - } catch { - // The process-level termination below remains authoritative; a - // failed graceful vm.shutdown just means we fall through to SIGTERM. - } - } - if (this.client) { - try { - await this.client.vmmShutdown(); - } catch { - // Same as above: SIGTERM/SIGKILL below is authoritative. - } - } - - let terminationConfirmed = !this.process || - this.process.exitCode !== null || - this.process.signalCode !== null; - if ( - this.process && - this.process.exitCode === null && - this.process.signalCode === null - ) { - const child = this.process; - try { - terminationConfirmed = await this.waitForProcessExit( - child, - CLOUD_HYPERVISOR_GUEST_SHUTDOWN_GRACE_MS, - ); - if (!child.killed) { - if (child.exitCode === null && child.signalCode === null) { - child.kill('SIGTERM', { forceKillAfterTimeout: 2_000 }); - } - } - if (!terminationConfirmed) { - await child; - if (child.exitCode === null && child.signalCode === null) { - throw new Error('Cloud Hypervisor process termination was not confirmed'); - } - } - terminationConfirmed = true; - } catch (error) { - terminationConfirmed = child.exitCode !== null || child.signalCode !== null; - errors.push(error); - } - } - if (!terminationConfirmed && this.process) { - if (errors.length === 0) { - errors.push(new Error('Cloud Hypervisor process termination was not confirmed')); - } - try { - await this.virtiofsd?.stop(); - this.virtiofsd = undefined; - this.fsDevices = []; - } catch (error) { - errors.push(error); - } - throw new Error( - `Cloud Hypervisor cleanup stopped before network/run-directory removal: ` + - `${errors.map(formatError).join('; ')}`, - ); - } - this.process = undefined; - this.client = undefined; - - let virtiofsdTerminationConfirmed = true; - try { - await this.virtiofsd?.stop(); - this.virtiofsd = undefined; - } catch (error) { - virtiofsdTerminationConfirmed = false; - errors.push(error); - } - - // Run any caller-supplied diagnostics collection now: the Cloud - // Hypervisor process is confirmed terminated (so any buffered guest - // serial console / log output has been flushed by process exit), but - // the run directory containing those files has not been removed yet - // (that happens below). Collecting diagnostics any earlier (e.g. - // before vmm.shutdown()/process termination above) can observe a - // still-empty serial console log, since Cloud Hypervisor does not - // guarantee flushing it before the process actually exits. - if (options.beforeCleanup) { - try { - await options.beforeCleanup(); - } catch (error) { - errors.push(error); - } - } - if (!virtiofsdTerminationConfirmed) { - throw new Error( - `Cloud Hypervisor cleanup stopped before cgroup/run-directory removal: ` + - `${errors.map(formatError).join('; ')}`, - ); - } - this.fsDevices = []; - - this.instanceStarted = false; - - if (this.rootfsPreparer) { - try { - await this.dependencies.rm( - path.dirname(this.rootfsPreparer.rootfsImagePath), - { recursive: true, force: true }, - ); - } catch (error) { - errors.push(error); - } - } - this.rootfsPreparer = undefined; - - if (options.preserve) { - try { - await this.cgroup?.cleanup(); - } catch (error) { - errors.push(error); - } - this.cgroup = undefined; - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) { - throw new Error( - `Cloud Hypervisor preservation failed: ${errors.map(formatError).join('; ')}`, - ); - } - return; - } - - try { - await this.network?.cleanup(); - this.network = undefined; - this.networkPlan = undefined; - } catch (error) { - errors.push(error); - } - - try { - await this.cgroup?.cleanup(); - } catch (error) { - errors.push(error); - } - this.cgroup = undefined; - - if (!instanceWasStarted || terminationConfirmed) { - try { - await this.dependencies.rm( - path.join( - this.paths.runBaseDir, - path.basename(this.config.cloudHypervisorBinary), - this.paths.runId, - ), - { recursive: true, force: true }, - ); - } catch (error) { - errors.push(error); - } - } - - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) { - throw new Error( - `Cloud Hypervisor cleanup failed: ${errors.map(formatError).join('; ')}`, - ); - } - } - - private async waitForProcessExit( - child: ExecaChildProcess, - timeoutMs: number, - ): Promise { - const pollIntervalMs = 25; - const attempts = Math.max(1, Math.ceil(timeoutMs / pollIntervalMs)); - for (let attempt = 0; attempt < attempts; attempt += 1) { - if (child.exitCode !== null || child.signalCode !== null) return true; - await this.dependencies.sleep(pollIntervalMs); - } - return child.exitCode !== null || child.signalCode !== null; + return stopCloudHypervisor({ + config: this.config, + dependencies: this.dependencies, + paths: this.paths, + process: this.process, + client: this.client, + network: this.network, + networkPlan: this.networkPlan, + rootfsPreparer: this.rootfsPreparer, + virtiofsd: this.virtiofsd, + fsDevices: this.fsDevices, + guest: this.guest, + cgroup: this.cgroup, + instanceStarted: this.instanceStarted, + lastVmInfo: this.lastVmInfo, + lastVmCounters: this.lastVmCounters, + ...options, + setProcess: (value) => { this.process = value; }, + setClient: (value) => { this.client = value; }, + setNetwork: (value) => { this.network = value; }, + setNetworkPlan: (value) => { this.networkPlan = value; }, + setRootfsPreparer: (value) => { this.rootfsPreparer = value; }, + setVirtiofsd: (value) => { this.virtiofsd = value; }, + setFsDevices: (value) => { this.fsDevices = value; }, + setGuest: (value) => { this.guest = value; }, + setCgroup: (value) => { this.cgroup = value; }, + setInstanceStarted: (value) => { this.instanceStarted = value; }, + setLastVmInfo: (value) => { this.lastVmInfo = value; }, + setLastVmCounters: (value) => { this.lastVmCounters = value; }, + }); } async collectDiagnostics(directory: string): Promise { @@ -656,18 +325,3 @@ export class CloudHypervisorManager { }); } } - -/** - * Explicit, minimal environment for the launched `ip netns exec ... setpriv - * ... cloud-hypervisor` process. Deliberately does **not** include - * `process.env` — Cloud Hypervisor directly parses untrusted guest/device - * input, so a VMM compromise reading its own inherited environment could - * read provider/GitHub credentials and bypass the API-proxy credential - * isolation boundary. Callers must also pass `extendEnv: false` to execa; - * otherwise execa merges this object back into `process.env`. - */ -function buildLauncherEnvironment(): NodeJS.ProcessEnv { - return { - PATH: '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }; -}