Split Cloud Hypervisor manager lifecycle into focused modules

src/cloud-hypervisor/manager-start.ts

@@ -0,0 +1,168 @@
+import * as path from 'path';
+import type { ExecaChildProcess } from 'execa';
+import type { CloudHypervisorOptions } from '../types/runtime-options';
+import type { MicrovmRootfsPreparer } from '../microvm/rootfs';
+import {
+  createMicrovmNetworkPlan,
+  type MicrovmNetworkLifecycle,
+  type MicrovmNetworkPlan,
+} from '../microvm/network';
+import type { CloudHypervisorApiClient } from './api-client';
+import {
+  prepareRunDirectory,
+  stageArtifact,
+  stageDiagnosticFile,
+  waitForApiSocket,
+} from './diagnostics';
+import {
+  CloudHypervisorCgroup,
+  buildCloudHypervisorLaunchCommand,
+} from './launcher';
+import {
+  formatError,
+  type CloudHypervisorManagerDependencies,
+  type CloudHypervisorManagerGuestConfig,
+  type CloudHypervisorManagerNetworkConfig,
+  type CloudHypervisorRunPaths,
+} from './manager-types';
+import { validateCloudHypervisorExports } from './exports';
+import { hasReadOnlyWorkspaceMountPlan } from './filesystem-write-enforcement';
+import type { VirtiofsdManager, VirtiofsdDevice } from './virtiofsd';
+import { buildCloudHypervisorVmConfig } from './vm-config-builder';
+import type { BoundedOutputCapture } from './diagnostics';
+
+export interface CloudHypervisorStartContext {
+  config: CloudHypervisorOptions;
+  workDir: string;
+  dependencies: CloudHypervisorManagerDependencies;
+  paths: CloudHypervisorRunPaths;
+  networkConfig?: CloudHypervisorManagerNetworkConfig;
+  guestConfig?: CloudHypervisorManagerGuestConfig;
+  stdoutCapture: BoundedOutputCapture;
+  stderrCapture: BoundedOutputCapture;
+  setNetworkPlan(plan: MicrovmNetworkPlan | undefined): void;
+  setNetwork(network: MicrovmNetworkLifecycle | undefined): void;
+  setRootfsPreparer(preparer: MicrovmRootfsPreparer | undefined): void;
+  setCgroup(cgroup: CloudHypervisorCgroup | undefined): void;
+  setProcess(process: ExecaChildProcess<string> | undefined): void;
+  setClient(client: CloudHypervisorApiClient | undefined): void;
+  setVirtiofsd(virtiofsd: VirtiofsdManager | undefined): void;
+  setFsDevices(devices: VirtiofsdDevice[]): void;
+  getFsDevices(): VirtiofsdDevice[];
+  stop(): Promise<void>;
+}
+
+export async function startCloudHypervisor(
+  context: CloudHypervisorStartContext,
+): Promise<CloudHypervisorApiClient> {
+  const {
+    config, workDir, dependencies, paths, networkConfig, guestConfig,
+  } = context;
+  if (!networkConfig) {
+    throw new Error(
+      'Cloud Hypervisor network configuration is required; refusing to launch an unfiltered microVM',
+    );
+  }
+
+  let startupError: unknown;
+  try {
+    const artifacts = await dependencies.preflight(config);
+    const identity = guestConfig?.identity ?? dependencies.resolveIdentity();
+    const networkPlan = createMicrovmNetworkPlan(paths.runId, {
+      ...networkConfig,
+      tapOwnerUid: identity.uid,
+      tapOwnerGid: identity.gid,
+      tapVnetHdr: true,
+    });
+    context.setNetworkPlan(networkPlan);
+    const network = dependencies.createNetwork(networkPlan, artifacts.tools);
+    context.setNetwork(network);
+    await network.setup();
+    let rootfsSource = artifacts.rootfsPath;
+    if (guestConfig) {
+      validateCloudHypervisorExports(guestConfig.exports, {
+        allowReadOnlyWorkspace: hasReadOnlyWorkspaceMountPlan(guestConfig.mountEnforcement),
+      });
+      const rootfsPreparationDirectory = path.join(
+        workDir, 'cloud-hypervisor-rootfs', paths.runId,
+      );
+      const rootfsPreparer = dependencies.createRootfsPreparer({
+        runDirectory: rootfsPreparationDirectory,
+        baseRootfsPath: artifacts.rootfsPath,
+        supervisorBinaryPath: guestConfig.supervisorBinaryPath,
+        supervisorSha256: guestConfig.supervisorSha256,
+        supervisorGuestPath: '/usr/sbin/awf-supervisor',
+        hostAliases: {
+          ...(networkConfig.apiProxyIp ? { 'api-proxy': networkConfig.apiProxyIp } : {}),
+          ...(networkConfig.hostAliases ?? {}),
+        },
+      }, artifacts.tools);
+      context.setRootfsPreparer(rootfsPreparer);
+      rootfsSource = await rootfsPreparer.prepare();
+    }
+
+    await prepareRunDirectory(dependencies, paths, identity);
+    const cgroup = dependencies.createCgroup(
+      paths.cgroupPath,
+      { memoryMib: config.memoryMib, vcpuCount: config.vcpuCount },
+    );
+    context.setCgroup(cgroup);
+    await cgroup.setup();
+    await stageArtifact(dependencies, artifacts.kernelPath, paths.kernelPath, 0o400, identity);
+    await stageArtifact(dependencies, rootfsSource, paths.rootfsPath, 0o600, identity);
+    await stageDiagnosticFile(dependencies, paths.logPath, identity);
+    await stageDiagnosticFile(dependencies, paths.serialLogPath, identity);
+
+    const launchCommand = buildCloudHypervisorLaunchCommand({
+      tools: { ip: artifacts.tools.ip, setpriv: artifacts.tools.setpriv },
+      namespaceName: networkPlan.namespaceName,
+      identity,
+      kvmGid: artifacts.kvmGid,
+      cloudHypervisorBinary: config.cloudHypervisorBinary,
+      apiSocketPath: paths.apiSocketPath,
+      logFilePath: paths.logPath,
+    });
+    const child = dependencies.launch(launchCommand.command, [...launchCommand.args], {
+      reject: false,
+      stdio: ['ignore', 'pipe', 'pipe'],
+      // Cloud Hypervisor directly processes untrusted guest/device input, so
+      // its environment must not expose the host's provider credentials.
+      extendEnv: false,
+      env: { PATH: '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' },
+    });
+    context.setProcess(child);
+    child.stdout?.on('data', (chunk: Buffer | string) => context.stdoutCapture.append(chunk));
+    child.stderr?.on('data', (chunk: Buffer | string) => context.stderrCapture.append(chunk));
+    if (child.pid !== undefined) await cgroup.assign(child.pid);
+
+    await waitForApiSocket(dependencies, paths, config.apiTimeoutMs, child);
+    const client = dependencies.createClient(paths.apiSocketPath, config.apiTimeoutMs);
+    context.setClient(client);
+    await client.ping();
+    if (guestConfig) {
+      const virtiofsd = dependencies.createVirtiofsdManager(
+        artifacts.virtiofsdBinary, paths.runDirectory, paths.virtiofsdShareDirectory,
+        identity, cgroup, { mount: artifacts.tools.mount, umount: artifacts.tools.umount },
+      );
+      context.setVirtiofsd(virtiofsd);
+      context.setFsDevices(await virtiofsd.start(guestConfig.exports, guestConfig.mountEnforcement));
+    }
+    await client.vmCreate(buildCloudHypervisorVmConfig({
+      config, paths, networkPlan, ...(guestConfig ? { guestConfig } : {}),
+      fsDevices: context.getFsDevices(),
+    }));
+    return client;
+  } catch (error) {
+    startupError = error;
+  }
+
+  try {
+    await context.stop();
+  } catch (cleanupError) {
+    throw new Error(
+      `Cloud Hypervisor startup failed: ${formatError(startupError)}; ` +
+      `partial-start cleanup also failed: ${formatError(cleanupError)}`,
+    );
+  }
+  throw startupError;
+}

src/cloud-hypervisor/manager-stop.ts

@@ -0,0 +1,153 @@
+import * as path from 'path';
+import type { ExecaChildProcess } from 'execa';
+import type { CloudHypervisorOptions } from '../types/runtime-options';
+import type { MicrovmNetworkLifecycle, MicrovmNetworkPlan } from '../microvm/network';
+import type { CloudHypervisorApiClient, CloudHypervisorVmCounters, CloudHypervisorVmInfo } from './api-client';
+import {
+  formatError,
+  type CloudHypervisorManagerDependencies,
+  type CloudHypervisorRunPaths,
+} from './manager-types';
+import type { CloudHypervisorCgroup } from './launcher';
+import type { MicrovmRootfsPreparer } from '../microvm/rootfs';
+import type { VirtiofsdManager, VirtiofsdDevice } from './virtiofsd';
+import type { CloudHypervisorGuestChannel } from './guest-execution';
+
+const SHUTDOWN_GRACE_MS = 5_000;
+
+export interface CloudHypervisorStopContext {
+  config: CloudHypervisorOptions;
+  dependencies: CloudHypervisorManagerDependencies;
+  paths: CloudHypervisorRunPaths;
+  process?: ExecaChildProcess<string>;
+  client?: CloudHypervisorApiClient;
+  network?: MicrovmNetworkLifecycle;
+  networkPlan?: MicrovmNetworkPlan;
+  rootfsPreparer?: MicrovmRootfsPreparer;
+  virtiofsd?: VirtiofsdManager;
+  fsDevices: VirtiofsdDevice[];
+  guest?: CloudHypervisorGuestChannel;
+  cgroup?: CloudHypervisorCgroup;
+  instanceStarted: boolean;
+  lastVmInfo?: CloudHypervisorVmInfo;
+  lastVmCounters?: CloudHypervisorVmCounters;
+  preserve?: boolean;
+  beforeCleanup?: () => Promise<void>;
+  setProcess(process: ExecaChildProcess<string> | undefined): void;
+  setClient(client: CloudHypervisorApiClient | undefined): void;
+  setNetwork(network: MicrovmNetworkLifecycle | undefined): void;
+  setNetworkPlan(plan: MicrovmNetworkPlan | undefined): void;
+  setRootfsPreparer(preparer: MicrovmRootfsPreparer | undefined): void;
+  setVirtiofsd(virtiofsd: VirtiofsdManager | undefined): void;
+  setFsDevices(devices: VirtiofsdDevice[]): void;
+  setGuest(guest: CloudHypervisorGuestChannel | undefined): void;
+  setCgroup(cgroup: CloudHypervisorCgroup | undefined): void;
+  setInstanceStarted(started: boolean): void;
+  setLastVmInfo(info: CloudHypervisorVmInfo | undefined): void;
+  setLastVmCounters(counters: CloudHypervisorVmCounters | undefined): void;
+}
+
+export async function stopCloudHypervisor(context: CloudHypervisorStopContext): Promise<void> {
+  const errors: unknown[] = [];
+  const instanceWasStarted = context.instanceStarted;
+  if (context.client && instanceWasStarted) {
+    try { context.setLastVmInfo(await context.client.vmInfo()); } catch { context.setLastVmInfo(undefined); }
+    try { context.setLastVmCounters(await context.client.vmCounters()); } catch { context.setLastVmCounters(undefined); }
+  }
+  let guestShutdownAcknowledged = false;
+  if (context.guest) {
+    const outcome = await context.guest.shutdown();
+    guestShutdownAcknowledged = outcome.acknowledged;
+    if (outcome.error !== undefined) errors.push(outcome.error);
+  }
+  context.setGuest(undefined);
+  if (context.client && instanceWasStarted && guestShutdownAcknowledged) {
+    try { await context.client.vmShutdown(); } catch { /* process termination is authoritative */ }
+  }
+  if (context.client) {
+    try { await context.client.vmmShutdown(); } catch { /* process termination is authoritative */ }
+  }
+
+  let terminationConfirmed = !context.process ||
+    context.process.exitCode !== null || context.process.signalCode !== null;
+  if (context.process && context.process.exitCode === null && context.process.signalCode === null) {
+    const child = context.process;
+    try {
+      terminationConfirmed = await waitForProcessExit(child, context.dependencies, SHUTDOWN_GRACE_MS);
+      if (!child.killed && child.exitCode === null && child.signalCode === null) {
+        child.kill('SIGTERM', { forceKillAfterTimeout: 2_000 });
+      }
+      if (!terminationConfirmed) {
+        await child;
+        if (child.exitCode === null && child.signalCode === null) {
+          throw new Error('Cloud Hypervisor process termination was not confirmed');
+        }
+      }
+      terminationConfirmed = true;
+    } catch (error) {
+      terminationConfirmed = child.exitCode !== null || child.signalCode !== null;
+      errors.push(error);
+    }
+  }
+  if (!terminationConfirmed && context.process) {
+    if (errors.length === 0) errors.push(new Error('Cloud Hypervisor process termination was not confirmed'));
+    try { await context.virtiofsd?.stop(); context.setVirtiofsd(undefined); context.setFsDevices([]); }
+    catch (error) { errors.push(error); }
+    throw new Error(`Cloud Hypervisor cleanup stopped before network/run-directory removal: ${errors.map(formatError).join('; ')}`);
+  }
+  context.setProcess(undefined);
+  context.setClient(undefined);
+  let virtiofsdTerminationConfirmed = true;
+  try { await context.virtiofsd?.stop(); context.setVirtiofsd(undefined); }
+  catch (error) { virtiofsdTerminationConfirmed = false; errors.push(error); }
+  if (context.beforeCleanup) {
+    try { await context.beforeCleanup(); } catch (error) { errors.push(error); }
+  }
+  if (!virtiofsdTerminationConfirmed) {
+    throw new Error(`Cloud Hypervisor cleanup stopped before cgroup/run-directory removal: ${errors.map(formatError).join('; ')}`);
+  }
+  context.setFsDevices([]);
+  context.setInstanceStarted(false);
+  if (context.rootfsPreparer) {
+    try {
+      await context.dependencies.rm(path.dirname(context.rootfsPreparer.rootfsImagePath), { recursive: true, force: true });
+    } catch (error) { errors.push(error); }
+  }
+  context.setRootfsPreparer(undefined);
+  if (context.preserve) {
+    try { await context.cgroup?.cleanup(); } catch (error) { errors.push(error); }
+    context.setCgroup(undefined);
+    throwCleanupErrors(errors, 'Cloud Hypervisor preservation failed: ');
+    return;
+  }
+  try { await context.network?.cleanup(); context.setNetwork(undefined); context.setNetworkPlan(undefined); }
+  catch (error) { errors.push(error); }
+  try { await context.cgroup?.cleanup(); } catch (error) { errors.push(error); }
+  context.setCgroup(undefined);
+  if (!instanceWasStarted || terminationConfirmed) {
+    try {
+      await context.dependencies.rm(
+        path.join(context.paths.runBaseDir, path.basename(context.config.cloudHypervisorBinary), context.paths.runId),
+        { recursive: true, force: true },
+      );
+    } catch (error) { errors.push(error); }
+  }
+  throwCleanupErrors(errors, 'Cloud Hypervisor cleanup failed: ');
+}
+
+async function waitForProcessExit(
+  child: ExecaChildProcess<string>,
+  dependencies: CloudHypervisorManagerDependencies,
+  timeoutMs: number,
+): Promise<boolean> {
+  for (let attempt = 0; attempt < Math.max(1, Math.ceil(timeoutMs / 25)); attempt += 1) {
+    if (child.exitCode !== null || child.signalCode !== null) return true;
+    await dependencies.sleep(25);
+  }
+  return child.exitCode !== null || child.signalCode !== null;
+}
+
+function throwCleanupErrors(errors: unknown[], prefix: string): void {
+  if (errors.length === 1) throw errors[0];
+  if (errors.length > 1) throw new Error(`${prefix}${errors.map(formatError).join('; ')}`);
+}