diff --git a/scripts/managed-bootstrap-trampoline.sh b/scripts/managed-bootstrap-trampoline.sh index ba21039c3c7..b0319b0f387 100644 --- a/scripts/managed-bootstrap-trampoline.sh +++ b/scripts/managed-bootstrap-trampoline.sh @@ -90,17 +90,26 @@ fi || fail "request file path is not the fixed bootstrap path" _nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs" +_nemoclaw_request_directory="${_nemoclaw_request%/*}" +_nemoclaw_request_basename="${_nemoclaw_request##*/}" +_nemoclaw_claim="${_nemoclaw_request_directory}/.${_nemoclaw_request_basename}.nemoclaw-claim/request" if [ ! -f "$_nemoclaw_runtime" ] || [ -L "$_nemoclaw_runtime" ]; then fail "managed startup runtime is missing" fi -if [ -L "$_nemoclaw_request" ]; then - fail "bootstrap request path is a symbolic link" -fi -if [ -e "$_nemoclaw_request" ]; then - if [ ! -f "$_nemoclaw_request" ] \ - || [ "$(/usr/bin/stat -c '%u:%g:%a:%h' "$_nemoclaw_request" 9<&-)" != "0:0:400:1" ]; then - fail "bootstrap request failed root ownership validation" - fi +/usr/bin/env -i \ + HOME="/root" \ + LANG="C.UTF-8" \ + LC_ALL="C.UTF-8" \ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION="1" \ + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + /usr/local/bin/node "$_nemoclaw_runtime" \ + --recover-bootstrap-claim \ + --agent "$_nemoclaw_agent" \ + --profile-fingerprint "$_nemoclaw_fingerprint" \ + --bootstrap-identity "$_nemoclaw_bootstrap_identity" \ + 9<&- +if [ -e "$_nemoclaw_request" ] || [ -L "$_nemoclaw_request" ] \ + || [ -e "$_nemoclaw_claim" ] || [ -L "$_nemoclaw_claim" ]; then /usr/bin/env -i \ HOME="/root" \ LANG="C.UTF-8" \ @@ -113,10 +122,6 @@ if [ -e "$_nemoclaw_request" ]; then --profile-fingerprint "$_nemoclaw_fingerprint" \ --bootstrap-identity "$_nemoclaw_bootstrap_identity" \ 9<&- - /usr/bin/rm -f -- "$_nemoclaw_request" 9<&- - if [ -e "$_nemoclaw_request" ] || [ -L "$_nemoclaw_request" ]; then - fail "bootstrap runtime did not consume its request" - fi fi /usr/bin/env -i \ HOME="/root" \ diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index fafd40611d3..789b0d9dfba 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -186,15 +186,51 @@ does not select a driver-specific bootstrap implementation. The native entrypoint source is intentionally not compiled into production artifacts, and neither image-owned source is installed or selected in a runtime -image yet. Production onboarding imports only the provider-neutral create -contract; no activation path or registered provider imports or selects the -driver-specific Docker candidate. The current image definitions do not package -`nemoclaw-managed-startup-hold`, -`managed-startup-image-runtime.cjs`, or the shared-state bootstrap modes consumed -by the adapter. Later persistence and qualification slices must compile and -verify the freestanding entrypoint for amd64 and arm64 in every agent image, add -the image-runtime prerequisites, and provide the canonical durable authority -store. The remaining integration and qualification work is tracked in +image yet. The dormant managed-bootstrap image runtime composes the neutral +managed-startup APIs with modes that consume the protected bootstrap envelope, +bind shared-state authority to the exact attempt, publish an identity-bound +completion, and authenticate that completion together with the ordinary startup +handoff. The runtime retains the protected envelope through application and +completion publication so the same attempt can retry after interruption; +it atomically moves the authenticated inode into a root-private, same-filesystem +claim before application. The canonical request is the producer-visible fixed +bootstrap request path. If that path was replaced between authentication and +rename, the runtime exclusively hard-links the displaced request back to the +canonical path without overwriting a later request, then removes the private +candidate. Restart recovery restores a protected, parseable displaced request +when a crash leaves it private immediately after rename, and reconciles a crash +between the later link and unlink steps only when the canonical and private +paths are protected two-link aliases of the same inode. A second replacement makes restoration fail closed while +preserving both the latest canonical request and the displaced private file. +The private claim remains the sole retry authority after an application or +completion-write failure; restart recovery resumes it without moving, deleting, +or overwriting a newer canonical request. Success removes only the authenticated +private claim. This protocol assumes the OCI writable layer supports same-device +atomic rename and hard links, the producer writes only the canonical request +path, one bootstrap consumer owns that path at a time, and container uid 0 is +trusted. An unsupported cross-device rename or hard link fails closed before +application and leaves request data intact; the protocol does not claim +protection from a hostile root process that can mutate the private mode-0700 +namespace. The trampoline only sequences the authoritative Node +recovery, apply, and verification modes; claim ownership and state transitions +remain in that runtime. Bootstrap completion verification adds the +bootstrap-identity receipt and then delegates the shared startup completion and +environment checks. The dependency direction is one-way: this +managed-bootstrap composition imports managed-startup, while managed-startup +does not import managed-bootstrap. +Production onboarding imports only the provider-neutral create contract; no +activation path or registered provider imports or selects the driver-specific +Docker candidate. The current image definitions still do not package +`nemoclaw-managed-startup-hold`, `nemoclaw-managed-bootstrap`, or +`managed-startup-image-runtime.cjs`; no production provider can invoke these +modes yet. Later persistence and qualification slices must compile and verify +the freestanding entrypoint for amd64 and arm64 in every agent image, add those +artifacts and the image-runtime prerequisites, and provide the canonical durable +authority store. Future image packaging must compile the composed +`managed-bootstrap/image-runtime.ts` entrypoint as +`managed-startup-image-runtime.cjs`; packaging the standalone managed-startup +entrypoint does not provide the bootstrap modes. The remaining integration and +qualification work is tracked in [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744). Until that complete boundary passes protected E2E, every production runtime provider keeps bootstrap unsupported. diff --git a/src/lib/onboard/managed-bootstrap/image-runtime.ts b/src/lib/onboard/managed-bootstrap/image-runtime.ts new file mode 100644 index 00000000000..80ffdfa84bc --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/image-runtime.ts @@ -0,0 +1,566 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { + applyManagedStartupRootRequest, + atomicWriteRootFile, + MANAGED_STARTUP_COMPLETION_FILE, + MANAGED_STARTUP_RUNTIME_ENV_FILE, + ManagedStartupImageRuntimeError, + type ManagedStartupRootApplyResult, + main as mainManagedStartupImageRuntime, + readStableRegularFileSnapshot, + verifyManagedStartupImageCompletion, +} from "../managed-startup/image-runtime"; +import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_COMPLETION_MAX_BYTES, + MANAGED_BOOTSTRAP_ENVELOPE_MAX_BYTES, + MANAGED_BOOTSTRAP_REQUEST_FILE, + type ManagedBootstrapImageCompletion, + parseManagedBootstrapEnvelope, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapImageCompletion, +} from "./envelope"; + +const SHA256_RE = /^[a-f0-9]{64}$/u; +type Environment = Record; + +export interface ManagedBootstrapImageRuntimeExpected { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; +} + +interface ManagedBootstrapEnvelopeSnapshot { + readonly bootstrapIdentity: string; + readonly bytes: Buffer; + readonly request: ManagedStartupRootApplyRequest; + readonly stat: fs.BigIntStats; +} + +export interface ManagedBootstrapEnvelopeClaimPaths { + readonly directory: string; + readonly file: string; + readonly requestFile: string; +} + +interface ManagedBootstrapEnvelopeClaim extends ManagedBootstrapEnvelopeClaimPaths { + readonly bytes: Buffer; + readonly request: ManagedStartupRootApplyRequest; + readonly stat: fs.BigIntStats; +} + +function fail(message: string): never { + throw new ManagedStartupImageRuntimeError(message); +} + +function requireRoot(): void { + if (process.geteuid?.() !== 0) { + fail("managed bootstrap image runtime requires container effective uid 0"); + } +} + +function exactAgent(value: string): ManagedStartupAgent { + if (!MANAGED_STARTUP_AGENTS.includes(value as ManagedStartupAgent)) { + fail(`unsupported agent ${JSON.stringify(value)}`); + } + return value as ManagedStartupAgent; +} + +function readExpected(argv: readonly string[]): ManagedBootstrapImageRuntimeExpected { + if (argv.length !== 7) { + fail( + "usage: managed-startup-image-runtime [--recover-bootstrap-claim|--apply-bootstrap-file|--verify-bootstrap-completion] --agent --profile-fingerprint --bootstrap-identity ", + ); + } + const valueAfter = (flag: string): string => { + const index = argv.indexOf(flag); + if (index < 0 || index + 1 >= argv.length) + fail(`managed bootstrap ${flag} argument is missing`); + return argv[index + 1] as string; + }; + const profileFingerprint = valueAfter("--profile-fingerprint"); + const bootstrapIdentity = valueAfter("--bootstrap-identity"); + if (!SHA256_RE.test(profileFingerprint) || !SHA256_RE.test(bootstrapIdentity)) { + fail("managed bootstrap image runtime identities must encode 32 lowercase-hex bytes"); + } + return { + agent: exactAgent(valueAfter("--agent")), + profileFingerprint, + bootstrapIdentity, + }; +} + +function readProtectedManagedBootstrapEnvelopeSnapshot( + requestFile: string = MANAGED_BOOTSTRAP_REQUEST_FILE, +): ManagedBootstrapEnvelopeSnapshot { + requireRoot(); + const { bytes, stat } = readStableRegularFileSnapshot( + requestFile, + MANAGED_BOOTSTRAP_ENVELOPE_MAX_BYTES, + ); + if (!isProtectedManagedBootstrapFile(stat)) { + fail("managed bootstrap envelope must be root:root mode 0400 with one link"); + } + const envelope = parseManagedBootstrapEnvelope(bytes.toString("utf8")); + return { + bootstrapIdentity: envelope.bootstrapIdentity, + bytes, + request: envelope.rootApplyRequest, + stat, + }; +} + +function managedBootstrapEnvelopeMatchesExpected( + snapshot: ManagedBootstrapEnvelopeSnapshot, + expected: ManagedBootstrapImageRuntimeExpected, +): boolean { + return ( + snapshot.bootstrapIdentity === expected.bootstrapIdentity && + snapshot.request.agent === expected.agent && + snapshot.request.profileFingerprint === expected.profileFingerprint + ); +} + +function readManagedBootstrapEnvelopeSnapshot( + expected: ManagedBootstrapImageRuntimeExpected, + requestFile: string = MANAGED_BOOTSTRAP_REQUEST_FILE, +): ManagedBootstrapEnvelopeSnapshot { + const snapshot = readProtectedManagedBootstrapEnvelopeSnapshot(requestFile); + if (!managedBootstrapEnvelopeMatchesExpected(snapshot, expected)) { + fail("managed bootstrap envelope identity does not match the replacement"); + } + return snapshot; +} + +export function readManagedBootstrapEnvelope( + expected: ManagedBootstrapImageRuntimeExpected, + requestFile: string = MANAGED_BOOTSTRAP_REQUEST_FILE, +): ManagedStartupRootApplyRequest { + return readManagedBootstrapEnvelopeSnapshot(expected, requestFile).request; +} + +export function managedBootstrapEnvelopeClaimPaths( + requestFile: string = MANAGED_BOOTSTRAP_REQUEST_FILE, +): ManagedBootstrapEnvelopeClaimPaths { + if (!path.isAbsolute(requestFile)) fail("managed bootstrap request path must be absolute"); + const directory = path.join( + path.dirname(requestFile), + `.${path.basename(requestFile)}.nemoclaw-claim`, + ); + return { directory, file: path.join(directory, "request"), requestFile }; +} + +function lstatManagedBootstrapPath(target: string): fs.BigIntStats | null { + try { + return fs.lstatSync(target, { bigint: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + fail(`could not inspect managed bootstrap path ${target}`); + } +} + +function sameStableManagedBootstrapFile(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return sameClaimedManagedBootstrapFile(left, right) && left.ctimeNs === right.ctimeNs; +} + +function sameClaimedManagedBootstrapFile(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs + ); +} + +function isProtectedManagedBootstrapFile(stat: fs.BigIntStats, expectedLinks = 1n): boolean { + return ( + stat.isFile() && + stat.nlink === expectedLinks && + stat.uid === 0n && + stat.gid === 0n && + Number(stat.mode & 0o777n) === 0o400 + ); +} + +function requirePrivateManagedBootstrapClaimDirectory(directory: string): void { + const stat = lstatManagedBootstrapPath(directory); + if ( + stat === null || + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.uid !== 0n || + stat.gid !== 0n || + Number(stat.mode & 0o777n) !== 0o700 + ) { + fail("managed bootstrap claim directory must be root:root mode 0700"); + } +} + +function removeManagedBootstrapClaimDirectory(directory: string): void { + try { + fs.rmdirSync(directory); + } catch { + fail("could not remove managed bootstrap claim directory"); + } +} + +function requireSafeManagedBootstrapClaimParent(directory: string): void { + const parent = lstatManagedBootstrapPath(path.dirname(directory)); + if ( + parent === null || + !parent.isDirectory() || + parent.isSymbolicLink() || + parent.uid !== 0n || + parent.gid !== 0n || + Number(parent.mode & 0o022n) !== 0 + ) { + fail("managed bootstrap claim parent must be a protected root-owned directory"); + } +} + +function managedBootstrapClaimEntries(directory: string): string[] { + let entries: string[]; + try { + entries = fs.readdirSync(directory); + } catch { + fail("could not inspect managed bootstrap claim directory contents"); + } + return entries.sort(); +} + +function ensurePrivateManagedBootstrapClaimDirectory(directory: string): void { + requireSafeManagedBootstrapClaimParent(directory); + const current = lstatManagedBootstrapPath(directory); + if (current !== null) { + requirePrivateManagedBootstrapClaimDirectory(directory); + return; + } + try { + fs.mkdirSync(directory, { mode: 0o700 }); + fs.chownSync(directory, 0, 0); + fs.chmodSync(directory, 0o700); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + fail("could not create private managed bootstrap envelope claim"); + } + } + requirePrivateManagedBootstrapClaimDirectory(directory); +} + +interface OpenManagedBootstrapEnvelopeSnapshot extends ManagedBootstrapEnvelopeSnapshot { + readonly descriptor: number; +} + +function openManagedBootstrapEnvelopeSnapshot( + expected: ManagedBootstrapImageRuntimeExpected, + target: string, +): OpenManagedBootstrapEnvelopeSnapshot { + requireRoot(); + if (typeof fs.constants.O_NOFOLLOW !== "number") { + fail("O_NOFOLLOW is unavailable for managed bootstrap envelope reads"); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | nonblock); + } catch { + fail(`refusing unsafe or unreadable managed bootstrap envelope ${target}`); + } + + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !isProtectedManagedBootstrapFile(before) || + before.size < 1n || + before.size > BigInt(MANAGED_BOOTSTRAP_ENVELOPE_MAX_BYTES) + ) { + fail("managed bootstrap envelope must be a bounded root:root mode 0400 file with one link"); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const bytesRead = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) break; + offset += bytesRead; + } + const overflow = Buffer.alloc(1); + const overflowBytes = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.length || + overflowBytes !== 0 || + !sameStableManagedBootstrapFile(before, after) + ) { + fail("managed bootstrap envelope changed while it was authenticated"); + } + const envelope = parseManagedBootstrapEnvelope(bytes.toString("utf8")); + const snapshot = { + bootstrapIdentity: envelope.bootstrapIdentity, + bytes, + request: envelope.rootApplyRequest, + stat: after, + }; + if (!managedBootstrapEnvelopeMatchesExpected(snapshot, expected)) { + fail("managed bootstrap envelope identity does not match the replacement"); + } + return { ...snapshot, descriptor }; + } catch (error) { + try { + fs.closeSync(descriptor); + } catch { + // Preserve the authentication failure. + } + throw error; + } +} + +function restoreUnclaimedManagedBootstrapEnvelope(paths: ManagedBootstrapEnvelopeClaimPaths): void { + try { + fs.linkSync(paths.file, paths.requestFile); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + fail( + "canonical managed bootstrap request was replaced again; the displaced request remains in the private claim", + ); + } + fail("could not restore replacement managed bootstrap envelope to its canonical path"); + } + try { + fs.unlinkSync(paths.file); + } catch { + fail("could not remove restored managed bootstrap envelope from the private claim"); + } + removeManagedBootstrapClaimDirectory(paths.directory); +} + +function reconcileInterruptedManagedBootstrapRestoration( + paths: ManagedBootstrapEnvelopeClaimPaths, +): boolean { + const canonical = lstatManagedBootstrapPath(paths.requestFile); + const privateClaim = lstatManagedBootstrapPath(paths.file); + if ( + canonical === null || + privateClaim === null || + !isProtectedManagedBootstrapFile(canonical, 2n) || + !isProtectedManagedBootstrapFile(privateClaim, 2n) || + !sameStableManagedBootstrapFile(canonical, privateClaim) + ) { + return false; + } + try { + fs.unlinkSync(paths.file); + } catch { + fail("could not reconcile interrupted managed bootstrap envelope restoration"); + } + removeManagedBootstrapClaimDirectory(paths.directory); + return true; +} + +function claimManagedBootstrapEnvelope( + expected: ManagedBootstrapImageRuntimeExpected, + requestFile: string, +): ManagedBootstrapEnvelopeClaim { + const paths = managedBootstrapEnvelopeClaimPaths(requestFile); + ensurePrivateManagedBootstrapClaimDirectory(paths.directory); + const entries = managedBootstrapClaimEntries(paths.directory); + if (entries.length === 1 && entries[0] === path.basename(paths.file)) { + const resumed = readManagedBootstrapEnvelopeSnapshot(expected, paths.file); + return { ...paths, ...resumed }; + } + if (entries.length !== 0) { + fail("managed bootstrap claim directory contains unexpected entries"); + } + + const opened = openManagedBootstrapEnvelopeSnapshot(expected, requestFile); + let claimed: ManagedBootstrapEnvelopeSnapshot; + try { + try { + fs.renameSync(requestFile, paths.file); + } catch { + fail("could not atomically claim managed bootstrap envelope"); + } + try { + const descriptorAfterRename = fs.fstatSync(opened.descriptor, { bigint: true }); + claimed = readManagedBootstrapEnvelopeSnapshot(expected, paths.file); + if ( + !sameClaimedManagedBootstrapFile(opened.stat, descriptorAfterRename) || + !sameStableManagedBootstrapFile(descriptorAfterRename, claimed.stat) || + !opened.bytes.equals(claimed.bytes) + ) { + fail("managed bootstrap envelope changed before its atomic claim"); + } + } catch { + restoreUnclaimedManagedBootstrapEnvelope(paths); + fail("managed bootstrap envelope changed before its atomic claim"); + } + } finally { + try { + fs.closeSync(opened.descriptor); + } catch { + fail("could not close authenticated managed bootstrap envelope"); + } + } + return { ...paths, ...claimed }; +} + +export function recoverManagedBootstrapEnvelopeClaim( + expected: ManagedBootstrapImageRuntimeExpected, + requestFile: string = MANAGED_BOOTSTRAP_REQUEST_FILE, +): boolean { + requireRoot(); + const paths = managedBootstrapEnvelopeClaimPaths(requestFile); + requireSafeManagedBootstrapClaimParent(paths.directory); + const directoryStat = lstatManagedBootstrapPath(paths.directory); + if (directoryStat === null) return false; + requirePrivateManagedBootstrapClaimDirectory(paths.directory); + const entries = managedBootstrapClaimEntries(paths.directory); + if (entries.length === 0) { + if (lstatManagedBootstrapPath(requestFile) !== null) return true; + removeManagedBootstrapClaimDirectory(paths.directory); + return false; + } + if (entries.length !== 1 || entries[0] !== path.basename(paths.file)) { + fail("managed bootstrap claim directory contains unexpected entries"); + } + if (reconcileInterruptedManagedBootstrapRestoration(paths)) return true; + const claim = readProtectedManagedBootstrapEnvelopeSnapshot(paths.file); + if (managedBootstrapEnvelopeMatchesExpected(claim, expected)) return true; + if (lstatManagedBootstrapPath(requestFile) !== null) { + fail("managed bootstrap envelope identity does not match the replacement"); + } + restoreUnclaimedManagedBootstrapEnvelope(paths); + return true; +} + +function consumeManagedBootstrapEnvelopeClaim( + expected: ManagedBootstrapImageRuntimeExpected, + claim: ManagedBootstrapEnvelopeClaim, +): void { + requireSafeManagedBootstrapClaimParent(claim.directory); + requirePrivateManagedBootstrapClaimDirectory(claim.directory); + const current = readManagedBootstrapEnvelopeSnapshot(expected, claim.file); + if ( + !sameStableManagedBootstrapFile(current.stat, claim.stat) || + !current.bytes.equals(claim.bytes) + ) { + fail("managed bootstrap envelope claim changed before completion cleanup"); + } + try { + fs.unlinkSync(claim.file); + } catch { + fail("could not consume managed bootstrap envelope claim"); + } + removeManagedBootstrapClaimDirectory(claim.directory); +} + +export async function applyManagedBootstrapEnvelope( + expected: ManagedBootstrapImageRuntimeExpected, + env: Environment = process.env, + requestFile: string = MANAGED_BOOTSTRAP_REQUEST_FILE, + completionFile: string = MANAGED_BOOTSTRAP_COMPLETION_FILE, +): Promise { + const claim = claimManagedBootstrapEnvelope(expected, requestFile); + const result = await applyManagedStartupRootRequest(claim.request, env, { + bootstrapIdentity: expected.bootstrapIdentity, + }); + atomicWriteRootFile( + completionFile, + serializeManagedBootstrapImageCompletion({ + agent: result.agent, + bootstrapIdentity: expected.bootstrapIdentity, + profileFingerprint: result.fingerprint, + transactionPending: result.transactionPending, + }), + 0o444, + ); + consumeManagedBootstrapEnvelopeClaim(expected, claim); + return result; +} + +export function verifyManagedBootstrapImageCompletion( + expected: ManagedBootstrapImageRuntimeExpected, + completionFile: string = MANAGED_BOOTSTRAP_COMPLETION_FILE, + startupCompletionFile: string = MANAGED_STARTUP_COMPLETION_FILE, + runtimeEnvironmentFile: string = MANAGED_STARTUP_RUNTIME_ENV_FILE, +): ManagedBootstrapImageCompletion { + requireRoot(); + const { bytes, stat } = readStableRegularFileSnapshot( + completionFile, + MANAGED_BOOTSTRAP_COMPLETION_MAX_BYTES, + ); + if ( + stat.nlink !== 1n || + stat.uid !== 0n || + stat.gid !== 0n || + Number(stat.mode & 0o777n) !== 0o444 + ) { + fail("managed bootstrap completion must be root:root mode 0444 with one link"); + } + const completion = parseManagedBootstrapImageCompletion(bytes.toString("utf8")); + if ( + completion.agent !== expected.agent || + completion.profileFingerprint !== expected.profileFingerprint || + completion.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail("managed bootstrap completion identity does not match the replacement"); + } + verifyManagedStartupImageCompletion( + expected.agent, + expected.profileFingerprint, + startupCompletionFile, + runtimeEnvironmentFile, + ); + return completion; +} + +export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { + if (argv[0] === "--recover-bootstrap-claim") { + const expected = readExpected(argv); + const pending = recoverManagedBootstrapEnvelopeClaim(expected); + console.log( + pending + ? `[managed-startup] found pending ${expected.agent} bootstrap request claim` + : `[managed-startup] no pending ${expected.agent} bootstrap request claim`, + ); + return; + } + if (argv[0] === "--apply-bootstrap-file") { + const expected = readExpected(argv); + const result = await applyManagedBootstrapEnvelope(expected); + console.log( + result.transactionPending + ? `[managed-startup] applied ${result.agent} profile ${result.fingerprint}; transaction pending` + : `[managed-startup] ${result.agent} profile ${result.fingerprint} was already complete`, + ); + return; + } + if (argv[0] === "--verify-bootstrap-completion") { + const expected = readExpected(argv); + const completion = verifyManagedBootstrapImageCompletion(expected); + console.log( + `[managed-startup] verified ${expected.agent} profile ${expected.profileFingerprint} bootstrap ${expected.bootstrapIdentity}${ + completion.transactionPending ? "; transaction pending" : "" + }`, + ); + return; + } + await mainManagedStartupImageRuntime(argv); +} + +if (typeof require !== "undefined" && typeof module !== "undefined" && require.main === module) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 78873418467..1c6378642e4 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -26,6 +26,16 @@ export { serializeManagedBootstrapEnvelope, serializeManagedBootstrapImageCompletion, } from "./envelope"; +export { + applyManagedBootstrapEnvelope, + type ManagedBootstrapEnvelopeClaimPaths, + type ManagedBootstrapImageRuntimeExpected, + main as mainManagedBootstrapImageRuntime, + managedBootstrapEnvelopeClaimPaths, + readManagedBootstrapEnvelope, + recoverManagedBootstrapEnvelopeClaim, + verifyManagedBootstrapImageCompletion, +} from "./image-runtime"; export type { ManagedBootstrapRuntimeCreateLifecycle, ManagedBootstrapRuntimePatch, diff --git a/src/lib/onboard/managed-startup-bootstrap-image-runtime.test.ts b/src/lib/onboard/managed-startup-bootstrap-image-runtime.test.ts new file mode 100644 index 00000000000..fca52bdf656 --- /dev/null +++ b/src/lib/onboard/managed-startup-bootstrap-image-runtime.test.ts @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + serializeManagedBootstrapEnvelope, + serializeManagedBootstrapImageCompletion, +} from "./managed-bootstrap/envelope"; +import { + readManagedBootstrapEnvelope, + verifyManagedBootstrapImageCompletion, +} from "./managed-bootstrap/image-runtime"; +import { + MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + serializeManagedStartupCompletionMarker, +} from "./managed-startup/image-runtime"; +import { + encodeManagedStartupProfile, + fingerprintManagedStartupProfile, +} from "./managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; + +const temporaryDirectories: string[] = []; +const BOOTSTRAP_IDENTITY = "b".repeat(64); + +function temporaryDirectory(): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bootstrap-image-runtime-")); + temporaryDirectories.push(directory); + return directory; +} + +function mockRootFileOwnership(): void { + const realFstatSync = fs.fstatSync.bind(fs); + const rootOwnership = new Map([ + ["uid", 0n], + ["gid", 0n], + ]); + const rootOwned = (stat: fs.BigIntStats): fs.BigIntStats => + new Proxy(stat, { + get(inner, property) { + const value = rootOwnership.has(property) + ? rootOwnership.get(property) + : (Reflect.get(inner, property, inner) as unknown); + return typeof value === "function" ? value.bind(inner) : value; + }, + }); + vi.spyOn(process, "geteuid").mockReturnValue(0); + vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number, options: { bigint: true }) => + rootOwned(realFstatSync(descriptor, options))) as typeof fs.fstatSync); +} + +function writeProtectedFile(target: string, contents: string, mode: number): void { + fs.writeFileSync(target, contents, { encoding: "utf8", mode: 0o600 }); + fs.chmodSync(target, mode); +} + +function requestFixture() { + const profile = managedStartupE2eProfile("openclaw"); + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile: encodeManagedStartupProfile(profile), + }); + return { + expected: { + agent: profile.agent, + profileFingerprint: rootApplyRequest.profileFingerprint, + bootstrapIdentity: BOOTSTRAP_IDENTITY, + } as const, + rootApplyRequest, + }; +} + +function completionFixture(directory: string) { + const profile = managedStartupE2eProfile("openclaw"); + const profileFingerprint = fingerprintManagedStartupProfile(profile); + const runtimeEnvironment = "export NEMOCLAW_MANAGED_STARTUP_APPLIED='1'\n"; + const completionFile = path.join(directory, "managed-bootstrap-completion.json"); + const startupCompletionFile = path.join(directory, "managed-startup-complete.json"); + const runtimeEnvironmentFile = path.join(directory, "managed-startup-runtime.env"); + writeProtectedFile(runtimeEnvironmentFile, runtimeEnvironment, 0o444); + writeProtectedFile( + startupCompletionFile, + serializeManagedStartupCompletionMarker({ + schemaVersion: MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + agent: profile.agent, + profileFingerprint, + runtimeEnvironmentSha256: createHash("sha256").update(runtimeEnvironment).digest("hex"), + corporateCaMerged: false, + }), + 0o444, + ); + writeProtectedFile( + completionFile, + serializeManagedBootstrapImageCompletion({ + agent: profile.agent, + bootstrapIdentity: BOOTSTRAP_IDENTITY, + profileFingerprint, + transactionPending: true, + }), + 0o444, + ); + return { + completionFile, + expected: { + agent: profile.agent, + profileFingerprint, + bootstrapIdentity: BOOTSTRAP_IDENTITY, + } as const, + runtimeEnvironmentFile, + startupCompletionFile, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("managed bootstrap image runtime", () => { + it("validates one exact root-owned bootstrap envelope without ending retry authority", () => { + const directory = temporaryDirectory(); + const requestFile = path.join(directory, "request.json"); + const fixture = requestFixture(); + writeProtectedFile( + requestFile, + serializeManagedBootstrapEnvelope({ + bootstrapIdentity: BOOTSTRAP_IDENTITY, + rootApplyRequest: fixture.rootApplyRequest, + }), + 0o400, + ); + mockRootFileOwnership(); + + expect(readManagedBootstrapEnvelope(fixture.expected, requestFile)).toEqual( + fixture.rootApplyRequest, + ); + expect(fs.existsSync(requestFile)).toBe(true); + }); + + it.each([ + ["wrong replacement identity", "c".repeat(64), 0o400, /identity does not match/u], + ["writable request", BOOTSTRAP_IDENTITY, 0o600, /root:root mode 0400/u], + ] as const)("rejects a %s without consuming the request", (_label, identity, mode, message) => { + const directory = temporaryDirectory(); + const requestFile = path.join(directory, "request.json"); + const fixture = requestFixture(); + writeProtectedFile( + requestFile, + serializeManagedBootstrapEnvelope({ + bootstrapIdentity: BOOTSTRAP_IDENTITY, + rootApplyRequest: fixture.rootApplyRequest, + }), + mode, + ); + mockRootFileOwnership(); + + expect(() => + readManagedBootstrapEnvelope( + { ...fixture.expected, bootstrapIdentity: identity }, + requestFile, + ), + ).toThrow(message); + expect(fs.existsSync(requestFile)).toBe(true); + }); + + it("authenticates the bootstrap marker and nested startup handoff together", () => { + const fixture = completionFixture(temporaryDirectory()); + mockRootFileOwnership(); + + expect( + verifyManagedBootstrapImageCompletion( + fixture.expected, + fixture.completionFile, + fixture.startupCompletionFile, + fixture.runtimeEnvironmentFile, + ), + ).toMatchObject({ + ...fixture.expected, + transactionPending: true, + }); + }); + + it.each([ + ["another bootstrap identity", "c".repeat(64), 0o444, /identity does not match/u], + ["a writable bootstrap marker", BOOTSTRAP_IDENTITY, 0o640, /root:root mode 0444/u], + ] as const)("rejects %s", (_label, bootstrapIdentity, mode, message) => { + const fixture = completionFixture(temporaryDirectory()); + fs.chmodSync(fixture.completionFile, mode); + mockRootFileOwnership(); + + expect(() => + verifyManagedBootstrapImageCompletion( + { ...fixture.expected, bootstrapIdentity }, + fixture.completionFile, + fixture.startupCompletionFile, + fixture.runtimeEnvironmentFile, + ), + ).toThrow(message); + }); +}); diff --git a/src/lib/onboard/managed-startup-image-runtime-handoff.test.ts b/src/lib/onboard/managed-startup-image-runtime-handoff.test.ts new file mode 100644 index 00000000000..8ea626a39f1 --- /dev/null +++ b/src/lib/onboard/managed-startup-image-runtime-handoff.test.ts @@ -0,0 +1,521 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, X509Certificate } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { mapManagedStartupProfileToAgentEnvironment } from "./managed-startup/agent-environment"; +import { + applyManagedStartupCommandEnvironmentPlan, + buildManagedStartupImageActionPlan, + MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + MANAGED_STARTUP_MERGED_CA_FILE, + normalizeHermesManagedConfigDescriptor, + readStableRegularFile, + serializeManagedStartupCompletionMarker, + serializeManagedStartupRuntimeEnvironment, + verifyManagedStartupImageCompletion, +} from "./managed-startup/image-runtime"; +import { + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupProfile, + validateManagedStartupProfile, +} from "./managed-startup/profile"; + +const PROXY_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; +const OPENCLAW_APPLICATION_RUNTIME_NAMES = [ + "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", + "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", + "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", + "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", +] as const; + +describe("managed startup image runtime handoff and descriptor integrity", () => { + let temporaryDirectoryPath = ""; + + beforeEach(() => { + temporaryDirectoryPath = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-startup-")); + }); + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(temporaryDirectoryPath, { force: true, recursive: true }); + }); + + function temporaryDirectory(): string { + return temporaryDirectoryPath; + } + + function mockDescriptorOwnership(uid: bigint, gid: bigint): void { + const realFstatSync = fs.fstatSync.bind(fs); + const realLstatSync = fs.lstatSync.bind(fs); + const ownership = new Map([ + ["uid", uid], + ["gid", gid], + ]); + const owned = (stat: fs.BigIntStats): fs.BigIntStats => + new Proxy(stat, { + get(inner, property) { + const value = ownership.has(property) + ? ownership.get(property) + : (Reflect.get(inner, property, inner) as unknown); + return typeof value === "function" ? value.bind(inner) : value; + }, + }); + vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number, options: { bigint: true }) => + owned(realFstatSync(descriptor, options))) as typeof fs.fstatSync); + vi.spyOn(fs, "lstatSync").mockImplementation(((file: fs.PathLike, options: { bigint: true }) => + owned(realLstatSync(file, options))) as typeof fs.lstatSync); + } + + function writeCompletionFixture( + profile: ManagedStartupProfile, + corporateCaMerged = false, + ): { + readonly agent: ManagedStartupAgent; + readonly completionFile: string; + readonly fingerprint: string; + readonly runtimeEnvironmentFile: string; + } { + const mapped = mapManagedStartupProfileToAgentEnvironment(profile); + const runtimeEnvironment = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + corporateCaMerged, + mapped.configurationEnvironment, + ); + const fingerprint = fingerprintManagedStartupProfile(profile); + const completionFile = path.join(temporaryDirectory(), "managed-startup-complete.json"); + const runtimeEnvironmentFile = path.join(temporaryDirectory(), "managed-startup-runtime.env"); + fs.writeFileSync(runtimeEnvironmentFile, runtimeEnvironment, { mode: 0o444 }); + fs.chmodSync(runtimeEnvironmentFile, 0o444); + fs.writeFileSync( + completionFile, + serializeManagedStartupCompletionMarker({ + schemaVersion: MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + agent: profile.agent, + profileFingerprint: fingerprint, + runtimeEnvironmentSha256: createHash("sha256") + .update(runtimeEnvironment, "utf8") + .digest("hex"), + corporateCaMerged, + }), + { mode: 0o444 }, + ); + fs.chmodSync(completionFile, 0o444); + return { + agent: profile.agent, + completionFile, + fingerprint, + runtimeEnvironmentFile, + }; + } + it.each( + MANAGED_STARTUP_AGENTS, + )("maps the complete %s profile into the reviewed image command contract", (agent) => { + const mapped = mapManagedStartupProfileToAgentEnvironment(managedStartupE2eProfile(agent)); + const plan = buildManagedStartupImageActionPlan({ + agent: mapped.agent, + actions: mapped.actions, + }); + + expect(plan.map(({ action }) => action)).toEqual( + agent === "langchain-deepagents-code" + ? ["generate-agent-config"] + : ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"], + ); + expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("provides valid same-profile and changed-profile fixtures for %s recreation checks", (agent) => { + const initial = validateManagedStartupProfile(managedStartupE2eProfile(agent)); + const same = validateManagedStartupProfile(managedStartupE2eProfile(agent)); + const changed = validateManagedStartupProfile(managedStartupE2eProfile(agent, true)); + + expect(fingerprintManagedStartupProfile(same)).toBe(fingerprintManagedStartupProfile(initial)); + expect(fingerprintManagedStartupProfile(changed)).not.toBe( + fingerprintManagedStartupProfile(initial), + ); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("accepts the root completion marker and exact runtime handoff for %s", (agent) => { + const fixture = writeCompletionFixture(managedStartupE2eProfile(agent)); + mockDescriptorOwnership(0n, 0n); + expect( + verifyManagedStartupImageCompletion( + agent, + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toEqual({ agent, fingerprint: fixture.fingerprint }); + }); + + it("rejects a changed profile against the root completion fingerprint", () => { + const initial = writeCompletionFixture(managedStartupE2eProfile("openclaw")); + const changedProfile = managedStartupE2eProfile("openclaw", true); + mockDescriptorOwnership(0n, 0n); + expect(() => + verifyManagedStartupImageCompletion( + "openclaw", + fingerprintManagedStartupProfile(changedProfile), + initial.completionFile, + initial.runtimeEnvironmentFile, + ), + ).toThrow(/completion marker does not match the requested profile/u); + }); + + it("rejects runtime handoff drift after a matching completion", () => { + const fixture = writeCompletionFixture(managedStartupE2eProfile("hermes")); + mockDescriptorOwnership(0n, 0n); + fs.chmodSync(fixture.runtimeEnvironmentFile, 0o644); + fs.appendFileSync(fixture.runtimeEnvironmentFile, "export NEMOCLAW_MODEL='tampered/model'\n"); + fs.chmodSync(fixture.runtimeEnvironmentFile, 0o444); + + expect(() => + verifyManagedStartupImageCompletion( + "hermes", + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toThrow(/runtime environment digest mismatch/u); + }); + + it("accepts merged CA paths without putting the CA payload in the readable handoff", () => { + const fixture = writeCompletionFixture( + managedStartupE2eProfile("langchain-deepagents-code", false, true), + true, + ); + mockDescriptorOwnership(0n, 0n); + expect( + verifyManagedStartupImageCompletion( + "langchain-deepagents-code", + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toEqual({ + agent: "langchain-deepagents-code", + fingerprint: fixture.fingerprint, + }); + expect(fs.readFileSync(fixture.runtimeEnvironmentFile, "utf8")).not.toContain( + "NEMOCLAW_CORPORATE_CA_B64", + ); + }); + + it("binds the real corporate-CA fixture into every agent profile by exact digest", () => { + expect(() => new X509Certificate(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM)).not.toThrow(); + const digest = createHash("sha256").update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).digest("hex"); + + for (const agent of MANAGED_STARTUP_AGENTS) { + expect(managedStartupE2eProfile(agent, false, true).corporateCa.bundleSha256).toBe(digest); + } + }); + + it("writes a deterministic root-sourced runtime environment without profile transport", () => { + const applicationRuntime = { + exportEnvironment: { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "0.25", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + }, + unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], + }; + const script = serializeManagedStartupRuntimeEnvironment( + { + NEMOCLAW_MODEL: "model-with-'quote", + NEMOCLAW_OBSERVABILITY: "0", + }, + true, + { + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_MODEL: "model-with-'quote", + }, + applicationRuntime, + ); + + expect(script).toContain("unset NEMOCLAW_INFERENCE_BASE_URL"); + expect(script).toContain("unset NEMOCLAW_MINIMAL_BOOTSTRAP"); + expect(script).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS='0.25'"); + expect(script).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS='3'"); + expect(script).toContain("export NEMOCLAW_MANAGED_STARTUP_APPLIED='1'"); + expect(script).toContain("export NEMOCLAW_MODEL='model-with-'\"'\"'quote'"); + expect(script).toContain(`export SSL_CERT_FILE='${MANAGED_STARTUP_MERGED_CA_FILE}'`); + expect(script).toContain("export _NEMOCLAW_CORPORATE_CA_MERGED='1'"); + expect(script).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + expect(script).not.toContain("NEMOCLAW_CORPORATE_CA_B64"); + expect(script.endsWith("\n")).toBe(true); + expect( + serializeManagedStartupRuntimeEnvironment( + { + NEMOCLAW_MODEL: "model-with-'quote", + NEMOCLAW_OBSERVABILITY: "0", + }, + true, + { + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_MODEL: "model-with-'quote", + }, + applicationRuntime, + ), + ).toBe(script); + }); + + it("validates runtime plans while removing launch-only exports and unsets from child commands", () => { + const ambient = { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "stale", + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + PRESERVED: "yes", + }; + const applied = applyManagedStartupCommandEnvironmentPlan(ambient, { + exportEnvironment: { NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3" }, + unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], + }); + + expect(applied).toEqual({ + PRESERVED: "yes", + }); + expect(ambient).toHaveProperty("NEMOCLAW_MINIMAL_BOOTSTRAP", "1"); + expect(() => + applyManagedStartupCommandEnvironmentPlan(ambient, { + exportEnvironment: { NEMOCLAW_MINIMAL_BOOTSTRAP: "1" }, + unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], + }), + ).toThrow(/both export and unset NEMOCLAW_MINIMAL_BOOTSTRAP/u); + expect(ambient).toHaveProperty("NEMOCLAW_MINIMAL_BOOTSTRAP", "1"); + }); + + it.each([ + "hermes", + "langchain-deepagents-code", + ] as const)("removes OpenClaw launch controls and cleanup obligations from %s children and runtime", (agent) => { + const mapped = mapManagedStartupProfileToAgentEnvironment(managedStartupE2eProfile(agent), { + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "invalid-for-this-agent", + }); + const ambient = { + ...Object.fromEntries(OPENCLAW_APPLICATION_RUNTIME_NAMES.map((name) => [name, "ambient"])), + NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + PRESERVED: "yes", + }; + const child = applyManagedStartupCommandEnvironmentPlan(ambient, mapped.applicationRuntime); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + mapped.applicationRuntime, + ); + + expect(child).toEqual({ PRESERVED: "yes" }); + for (const name of [ + ...OPENCLAW_APPLICATION_RUNTIME_NAMES, + "NEMOCLAW_DASHBOARD_BIND", + "NEMOCLAW_MINIMAL_BOOTSTRAP", + ]) { + expect(script).toContain(`unset ${name}`); + expect(script).not.toContain(`export ${name}=`); + } + }); + + it("rejects a serialized runtime export that conflicts with an explicit unset", () => { + expect(() => + serializeManagedStartupRuntimeEnvironment( + { NEMOCLAW_MINIMAL_BOOTSTRAP: "1" }, + false, + {}, + { exportEnvironment: {}, unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"] }, + ), + ).toThrow(/runtime environment cannot both export and unset NEMOCLAW_MINIMAL_BOOTSTRAP/u); + }); + + it.each([ + [ + { exportEnvironment: { "BAD-NAME": "value" }, unsetEnvironment: [] }, + /invalid application runtime environment key/u, + ], + [ + { exportEnvironment: { VALID_NAME: "line 1\nline 2" }, unsetEnvironment: [] }, + /must be single-line text/u, + ], + [ + { exportEnvironment: {}, unsetEnvironment: ["DUPLICATE", "DUPLICATE"] }, + /duplicate application runtime unset/u, + ], + ])("rejects a malformed application runtime plan before command mutation", (plan, message) => { + const ambient = { PRESERVED: "yes" }; + expect(() => applyManagedStartupCommandEnvironmentPlan(ambient, plan)).toThrow(message); + expect(ambient).toEqual({ PRESERVED: "yes" }); + }); + + it.each(["openclaw", "hermes"] as const)("preserves launch-only proxy env for %s", (agent) => { + const mapped = mapManagedStartupProfileToAgentEnvironment( + managedStartupE2eProfile(agent, false, false, true), + ); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + ); + for (const name of PROXY_ENV_NAMES) { + expect(script).not.toMatch(new RegExp(`(?:export|unset) ${name}(?:=|$)`, "mu")); + } + }); + + it("clears launch-only proxy env when DCode pins managed routing", () => { + const mapped = mapManagedStartupProfileToAgentEnvironment( + managedStartupE2eProfile("langchain-deepagents-code", false, false, true), + ); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + ); + for (const name of PROXY_ENV_NAMES) { + expect(script).toContain(`unset ${name}`); + } + }); + + it("rejects multiline runtime values before producing a sourceable file", () => { + expect(() => + serializeManagedStartupRuntimeEnvironment({ NEMOCLAW_MODEL: "bad\nvalue" }, false), + ).toThrow(/single-line/u); + }); + + it("refuses a symlink instead of opening its target", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "target"); + const link = path.join(directory, "link"); + fs.writeFileSync(target, "trusted\n"); + fs.symlinkSync(target, link); + + expect(() => readStableRegularFile(link, 1024)).toThrow(/unsafe or unreadable/u); + }); + + it("rejects descriptor metadata drift after a bounded read", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "material"); + fs.writeFileSync(target, "trusted\n", { mode: 0o600 }); + const realReadSync = fs.readSync.bind(fs); + vi.spyOn(fs, "readSync") + .mockImplementationOnce((( + descriptor: number, + buffer: NodeJS.ArrayBufferView, + offset: number, + length: number, + position: number | null, + ) => { + const bytesRead = realReadSync(descriptor, buffer, offset, length, position); + fs.chmodSync(target, 0o644); + return bytesRead; + }) as typeof fs.readSync) + .mockImplementation(realReadSync as typeof fs.readSync); + + expect(() => readStableRegularFile(target, 1024)).toThrow(/changed while it was read/u); + }); + + it("normalizes mutable sandbox-owned Hermes config descriptors to mode 0640", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + mockDescriptorOwnership(501n, 20n); + + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }); + + expect(fs.readFileSync(target, "utf8")).toBe("model: managed\n"); + expect(fs.statSync(target).mode & 0o777).toBe(0o640); + }); + + it("preserves a root-owned shields-up Hermes descriptor without chmod", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, ".env"); + fs.writeFileSync(target, "OPENAI_API_KEY=managed\n", { mode: 0o444 }); + mockDescriptorOwnership(0n, 0n); + const chmod = vi.spyOn(fs, "fchmodSync"); + + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }); + + expect(chmod).not.toHaveBeenCalled(); + expect(fs.readFileSync(target, "utf8")).toBe("OPENAI_API_KEY=managed\n"); + }); + + it.each([0o440, 0o644, 0o660])("fails closed on unexpected mutable Hermes mode %s", (mode) => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode }); + fs.chmodSync(target, mode); + mockDescriptorOwnership(501n, 20n); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/unexpected Hermes managed config descriptor/u); + expect(fs.statSync(target).mode & 0o777).toBe(mode); + }); + + it("fails closed on an unexpected Hermes descriptor owner", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + mockDescriptorOwnership(502n, 21n); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/unexpected Hermes managed config descriptor/u); + expect(fs.statSync(target).mode & 0o777).toBe(0o600); + }); + + it("detects a path replacement while normalizing through the trusted descriptor", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + const displaced = path.join(directory, "displaced.yaml"); + const replacement = path.join(directory, "replacement.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + fs.writeFileSync(replacement, "model: replaced\n", { mode: 0o640 }); + mockDescriptorOwnership(501n, 20n); + const realFchmodSync = fs.fchmodSync.bind(fs); + vi.spyOn(fs, "fchmodSync").mockImplementation((descriptor, mode) => { + realFchmodSync(descriptor, mode); + fs.renameSync(target, displaced); + fs.renameSync(replacement, target); + }); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/changed during normalization/u); + expect(fs.readFileSync(target, "utf8")).toBe("model: replaced\n"); + }); +}); diff --git a/src/lib/onboard/managed-startup-image-runtime.test.ts b/src/lib/onboard/managed-startup-image-runtime.test.ts index 1e15462a2c9..22a47e04ee9 100644 --- a/src/lib/onboard/managed-startup-image-runtime.test.ts +++ b/src/lib/onboard/managed-startup-image-runtime.test.ts @@ -1,9 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash, X509Certificate } from "node:crypto"; import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -12,26 +10,34 @@ const coordinatorMock = vi.hoisted(() => ({ })); vi.mock("./managed-startup/coordinator", () => coordinatorMock); +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import { - MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, - managedStartupE2eProfile, -} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; -import { mapManagedStartupProfileToAgentEnvironment } from "./managed-startup/agent-environment"; + mockRootReplayFilesystem, + observeMatchingLink, + observeMatchingRename, + observeMatchingRenameTarget, + observeMatchingUnlink, +} from "../../../test/helpers/managed-startup-root-replay-filesystem"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION, + MANAGED_BOOTSTRAP_REQUEST_FILE, + parseManagedBootstrapImageCompletion, + serializeManagedBootstrapEnvelope, +} from "./managed-bootstrap/envelope"; +import { + applyManagedBootstrapEnvelope, + type ManagedBootstrapImageRuntimeExpected, + main as mainManagedBootstrapImageRuntime, + managedBootstrapEnvelopeClaimPaths, + recoverManagedBootstrapEnvelopeClaim, +} from "./managed-bootstrap/image-runtime"; import { - applyManagedStartupCommandEnvironmentPlan, applyManagedStartupImageProfile, applyManagedStartupRootRequest, buildManagedStartupImageActionPlan, - MANAGED_STARTUP_COMPLETION_FILE, - MANAGED_STARTUP_MERGED_CA_FILE, MANAGED_STARTUP_PROFILE_ENV, - MANAGED_STARTUP_RUNTIME_ENV_FILE, type ManagedStartupImageActionPlanInput, - normalizeHermesManagedConfigDescriptor, - readStableRegularFile, - serializeManagedStartupCompletionMarker, - serializeManagedStartupRuntimeEnvironment, - verifyManagedStartupImageCompletion, } from "./managed-startup/image-runtime"; import { encodeManagedStartupProfile, @@ -40,9 +46,9 @@ import { type ManagedStartupAgent, type ManagedStartupDashboard, type ManagedStartupProfile, - validateManagedStartupProfile, } from "./managed-startup/profile"; import { createManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; +import * as sharedStateTransaction from "./managed-startup/shared-state-transaction"; function dashboard(agent: ManagedStartupAgent): ManagedStartupDashboard { switch (agent) { @@ -320,170 +326,14 @@ describe("buildManagedStartupImageActionPlan", () => { }); }); -const PROXY_ENV_NAMES = [ - "HTTP_PROXY", - "HTTPS_PROXY", - "NO_PROXY", - "http_proxy", - "https_proxy", - "no_proxy", -] as const; -const OPENCLAW_APPLICATION_RUNTIME_NAMES = [ - "NEMOCLAW_AUTO_PAIR_DEADLINE_SECS", - "NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS", - "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS", - "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS", - "NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS", - "NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS", -] as const; - describe("managed startup image runtime", () => { - let temporaryDirectoryPath = ""; - beforeEach(() => { coordinatorMock.coordinateManagedStartupApplication.mockReset(); - temporaryDirectoryPath = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-startup-")); }); afterEach(() => { vi.restoreAllMocks(); - fs.rmSync(temporaryDirectoryPath, { force: true, recursive: true }); }); - function temporaryDirectory(): string { - return temporaryDirectoryPath; - } - - function mockDescriptorOwnership(uid: bigint, gid: bigint): void { - const realFstatSync = fs.fstatSync.bind(fs); - const realLstatSync = fs.lstatSync.bind(fs); - const ownership = new Map([ - ["uid", uid], - ["gid", gid], - ]); - const owned = (stat: fs.BigIntStats): fs.BigIntStats => - new Proxy(stat, { - get(inner, property) { - const value = ownership.has(property) - ? ownership.get(property) - : (Reflect.get(inner, property, inner) as unknown); - return typeof value === "function" ? value.bind(inner) : value; - }, - }); - vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number, options: { bigint: true }) => - owned(realFstatSync(descriptor, options))) as typeof fs.fstatSync); - vi.spyOn(fs, "lstatSync").mockImplementation(((file: fs.PathLike, options: { bigint: true }) => - owned(realLstatSync(file, options))) as typeof fs.lstatSync); - } - - function mockRootReplayFilesystem(runtimeWrites: string[]): void { - const directories = new Set([ - "/", - "/run", - "/run/nemoclaw", - "/var", - "/var/lib", - "/var/lib/nemoclaw", - ]); - const files = new Map(); - const descriptorTargets = new Map(); - const pendingFiles = new Map(); - let nextDescriptor = 91; - const stat = (kind: "directory" | "file", mode: number) => - ({ - gid: 0, - isDirectory: () => kind === "directory", - isFile: () => kind === "file", - isSymbolicLink: () => false, - mode, - nlink: 1, - uid: 0, - }) as fs.Stats; - const bigFileStat = (bytes: Buffer) => - ({ - ctimeNs: 1n, - dev: 1n, - gid: 0n, - ino: 2n, - isFile: () => true, - mode: 0o100444n, - mtimeNs: 1n, - nlink: 1n, - size: BigInt(bytes.length), - uid: 0n, - }) as fs.BigIntStats; - const missing = (): never => { - throw Object.assign(new Error("missing"), { code: "ENOENT" }); - }; - const allocateDescriptor = (resolved: string): number => { - const descriptor = nextDescriptor; - nextDescriptor += 1; - descriptorTargets.set(descriptor, resolved); - return descriptor; - }; - - vi.spyOn(process, "geteuid").mockReturnValue(0); - vi.spyOn(fs, "lstatSync").mockImplementation(((target: fs.PathLike) => { - const resolved = String(target); - return directories.has(resolved) - ? stat("directory", 0o755) - : files.has(resolved) - ? stat("file", 0o444) - : missing(); - }) as typeof fs.lstatSync); - vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); - vi.spyOn(fs, "chownSync").mockImplementation(() => undefined); - vi.spyOn(fs, "chmodSync").mockImplementation(() => undefined); - vi.spyOn(fs, "existsSync").mockReturnValue(false); - vi.spyOn(fs, "openSync").mockImplementation(((target: fs.PathLike) => { - const resolved = String(target); - return (resolved === MANAGED_STARTUP_RUNTIME_ENV_FILE || - resolved === MANAGED_STARTUP_COMPLETION_FILE) && - !files.has(resolved) - ? missing() - : allocateDescriptor(resolved); - }) as typeof fs.openSync); - vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number) => { - const target = descriptorTargets.get(descriptor); - const bytes = target === undefined ? undefined : files.get(target); - return bytes === undefined ? missing() : bigFileStat(bytes); - }) as typeof fs.fstatSync); - vi.spyOn(fs, "readSync").mockImplementation((( - descriptor: number, - buffer: NodeJS.ArrayBufferView, - offset: number, - length: number, - position: number | null, - ) => { - const target = descriptorTargets.get(descriptor); - const bytes = (target === undefined ? undefined : files.get(target)) ?? missing(); - const start = position ?? 0; - const count = Math.min(length, Math.max(0, bytes.length - start)); - bytes.copy(buffer as Buffer, offset, start, start + count); - return count; - }) as typeof fs.readSync); - vi.spyOn(fs, "fchownSync").mockImplementation(() => undefined); - vi.spyOn(fs, "writeFileSync").mockImplementation(((target: fs.PathOrFileDescriptor, value) => { - const resolved = - (typeof target === "number" ? descriptorTargets.get(target) : undefined) ?? missing(); - pendingFiles.set( - resolved, - Buffer.isBuffer(value) ? Buffer.from(value) : Buffer.from(String(value), "utf8"), - ); - }) as typeof fs.writeFileSync); - vi.spyOn(fs, "fchmodSync").mockImplementation(() => undefined); - vi.spyOn(fs, "fsyncSync").mockImplementation(() => undefined); - vi.spyOn(fs, "closeSync").mockImplementation(() => undefined); - vi.spyOn(fs, "renameSync").mockImplementation((source, target) => { - const pending = pendingFiles.get(String(source)) ?? missing(); - files.set(String(target), pending); - pendingFiles.delete(String(source)); - runtimeWrites.push( - ...(String(target) === MANAGED_STARTUP_RUNTIME_ENV_FILE ? [pending.toString("utf8")] : []), - ); - }); - vi.spyOn(fs, "unlinkSync").mockImplementation(missing); - } - it("rejects invalid OpenClaw launch controls before filesystem or coordinator mutation", async () => { const profile = managedStartupE2eProfile("openclaw"); const request = createManagedStartupRootApplyRequest({ @@ -572,437 +422,870 @@ describe("managed startup image runtime", () => { expect(coordinatorMock.coordinateManagedStartupApplication).toHaveBeenCalledTimes(2); }); - function writeCompletionFixture( - profile: ManagedStartupProfile, - corporateCaMerged = false, - ): { - readonly agent: ManagedStartupAgent; - readonly completionFile: string; - readonly fingerprint: string; - readonly runtimeEnvironmentFile: string; - } { - const mapped = mapManagedStartupProfileToAgentEnvironment(profile); - const runtimeEnvironment = serializeManagedStartupRuntimeEnvironment( - mapped.runtimeEnvironment, - corporateCaMerged, - mapped.configurationEnvironment, - ); + it("publishes bootstrap completion only after application and preserves attempt identity", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); const fingerprint = fingerprintManagedStartupProfile(profile); - const completionFile = path.join(temporaryDirectory(), "managed-startup-complete.json"); - const runtimeEnvironmentFile = path.join(temporaryDirectory(), "managed-startup-runtime.env"); - fs.writeFileSync(runtimeEnvironmentFile, runtimeEnvironment, { mode: 0o444 }); - fs.chmodSync(runtimeEnvironmentFile, 0o444); - fs.writeFileSync( - completionFile, - serializeManagedStartupCompletionMarker({ - schemaVersion: 1, - agent: profile.agent, - profileFingerprint: fingerprint, - runtimeEnvironmentSha256: createHash("sha256") - .update(runtimeEnvironment, "utf8") - .digest("hex"), - corporateCaMerged, - }), - { mode: 0o444 }, - ); - fs.chmodSync(completionFile, 0o444); - return { + const bootstrapIdentity = "b".repeat(64); + const requestFile = MANAGED_BOOTSTRAP_REQUEST_FILE; + const completionFile = MANAGED_BOOTSTRAP_COMPLETION_FILE; + const rootApplyRequest = createManagedStartupRootApplyRequest({ agent: profile.agent, - completionFile, - fingerprint, - runtimeEnvironmentFile, - }; - } - it.each( - MANAGED_STARTUP_AGENTS, - )("maps the complete %s profile into the reviewed image command contract", (agent) => { - const mapped = mapManagedStartupProfileToAgentEnvironment(managedStartupE2eProfile(agent)); - const plan = buildManagedStartupImageActionPlan({ - agent: mapped.agent, - actions: mapped.actions, + encodedProfile, }); - - expect(plan.map(({ action }) => action)).toEqual( - agent === "langchain-deepagents-code" - ? ["generate-agent-config"] - : ["messaging-runtime-setup", "generate-agent-config", "messaging-post-agent-install"], + const filesystem = mockRootReplayFilesystem( + [], + new Map([ + [ + requestFile, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest, + }), + mode: 0o400, + }, + ], + ]), ); - expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); - }); + const beginTransaction = vi + .spyOn(sharedStateTransaction, "beginManagedStartupSharedStateTransaction") + .mockReturnValue(true); + coordinatorMock.coordinateManagedStartupApplication.mockImplementation(async () => { + expect(filesystem.hasFile(completionFile)).toBe(false); + return { + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, + }; + }); - it.each( - MANAGED_STARTUP_AGENTS, - )("provides valid same-profile and changed-profile fixtures for %s recreation checks", (agent) => { - const initial = validateManagedStartupProfile(managedStartupE2eProfile(agent)); - const same = validateManagedStartupProfile(managedStartupE2eProfile(agent)); - const changed = validateManagedStartupProfile(managedStartupE2eProfile(agent, true)); + await expect( + applyManagedBootstrapEnvelope( + { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }, + {}, + requestFile, + completionFile, + ), + ).resolves.toMatchObject({ fingerprint, transactionPending: true }); - expect(fingerprintManagedStartupProfile(same)).toBe(fingerprintManagedStartupProfile(initial)); - expect(fingerprintManagedStartupProfile(changed)).not.toBe( - fingerprintManagedStartupProfile(initial), + expect(filesystem.hasFile(requestFile)).toBe(false); + expect(beginTransaction).toHaveBeenCalledWith( + expect.objectContaining({ agent: profile.agent }), + { bootstrapIdentity }, ); - }); - - it.each( - MANAGED_STARTUP_AGENTS, - )("accepts the root completion marker and exact runtime handoff for %s", (agent) => { - const fixture = writeCompletionFixture(managedStartupE2eProfile(agent)); - mockDescriptorOwnership(0n, 0n); expect( - verifyManagedStartupImageCompletion( - agent, - fixture.fingerprint, - fixture.completionFile, - fixture.runtimeEnvironmentFile, - ), - ).toEqual({ agent, fingerprint: fixture.fingerprint }); - }); - - it("rejects a changed profile against the root completion fingerprint", () => { - const initial = writeCompletionFixture(managedStartupE2eProfile("openclaw")); - const changedProfile = managedStartupE2eProfile("openclaw", true); - mockDescriptorOwnership(0n, 0n); - expect(() => - verifyManagedStartupImageCompletion( - "openclaw", - fingerprintManagedStartupProfile(changedProfile), - initial.completionFile, - initial.runtimeEnvironmentFile, + fingerprintManagedStartupProfile( + beginTransaction.mock.calls[0]?.[0] as ManagedStartupProfile, ), - ).toThrow(/completion marker does not match the requested profile/u); - }); - - it("rejects runtime handoff drift after a matching completion", () => { - const fixture = writeCompletionFixture(managedStartupE2eProfile("hermes")); - mockDescriptorOwnership(0n, 0n); - fs.chmodSync(fixture.runtimeEnvironmentFile, 0o644); - fs.appendFileSync(fixture.runtimeEnvironmentFile, "export NEMOCLAW_MODEL='tampered/model'\n"); - fs.chmodSync(fixture.runtimeEnvironmentFile, 0o444); - - expect(() => - verifyManagedStartupImageCompletion( - "hermes", - fixture.fingerprint, - fixture.completionFile, - fixture.runtimeEnvironmentFile, - ), - ).toThrow(/runtime environment digest mismatch/u); + ).toBe(fingerprint); + expect(parseManagedBootstrapImageCompletion(filesystem.readFile(completionFile) ?? "")).toEqual( + { + schemaVersion: MANAGED_BOOTSTRAP_ENVELOPE_SCHEMA_VERSION, + agent: profile.agent, + bootstrapIdentity, + profileFingerprint: fingerprint, + transactionPending: true, + }, + ); }); - it("accepts merged CA paths without putting the CA payload in the readable handoff", () => { - const fixture = writeCompletionFixture( - managedStartupE2eProfile("langchain-deepagents-code", false, true), + it("applies and verifies bootstrap completion through the CLI modes", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }); + mockRootReplayFilesystem( + [], + new Map([ + [ + MANAGED_BOOTSTRAP_REQUEST_FILE, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest, + }), + mode: 0o400, + }, + ], + ]), + ); + vi.spyOn(sharedStateTransaction, "beginManagedStartupSharedStateTransaction").mockReturnValue( true, ); - mockDescriptorOwnership(0n, 0n); - expect( - verifyManagedStartupImageCompletion( - "langchain-deepagents-code", - fixture.fingerprint, - fixture.completionFile, - fixture.runtimeEnvironmentFile, - ), - ).toEqual({ - agent: "langchain-deepagents-code", - fingerprint: fixture.fingerprint, + coordinatorMock.coordinateManagedStartupApplication.mockResolvedValue({ + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, }); - expect(fs.readFileSync(fixture.runtimeEnvironmentFile, "utf8")).not.toContain( - "NEMOCLAW_CORPORATE_CA_B64", + vi.spyOn( + sharedStateTransaction, + "getManagedStartupSharedStateTransactionStatus", + ).mockReturnValue("pending"); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const cliArguments = [ + "--agent", + profile.agent, + "--profile-fingerprint", + fingerprint, + "--bootstrap-identity", + bootstrapIdentity, + ]; + + await mainManagedBootstrapImageRuntime(["--apply-bootstrap-file", ...cliArguments]); + expect(log).toHaveBeenLastCalledWith( + `[managed-startup] applied ${profile.agent} profile ${fingerprint}; transaction pending`, + ); + await mainManagedBootstrapImageRuntime(["--verify-bootstrap-completion", ...cliArguments]); + expect(log).toHaveBeenLastCalledWith( + `[managed-startup] verified ${profile.agent} profile ${fingerprint} bootstrap ${bootstrapIdentity}; transaction pending`, ); }); - it("binds the real corporate-CA fixture into every agent profile by exact digest", () => { - expect(() => new X509Certificate(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM)).not.toThrow(); - const digest = createHash("sha256").update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).digest("hex"); + it.each([ + ["unsafe metadata", 0o444, "b".repeat(64), /mode 0400/u], + ["mismatched identity", 0o400, "c".repeat(64), /identity does not match/u], + ])("rejects bootstrap envelope %s without consuming the canonical request", async (_label, mode, envelopeIdentity, error) => { + const profile = managedStartupE2eProfile("openclaw"); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const request = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: envelopeIdentity, + rootApplyRequest: createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile: encodeManagedStartupProfile(profile), + }), + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([[MANAGED_BOOTSTRAP_REQUEST_FILE, { contents: request, mode }]]), + ); - for (const agent of MANAGED_STARTUP_AGENTS) { - expect(managedStartupE2eProfile(agent, false, true).corporateCa.bundleSha256).toBe(digest); - } + await expect(applyManagedBootstrapEnvelope(expected, {})).rejects.toThrow(error); + expect(filesystem.readFile(MANAGED_BOOTSTRAP_REQUEST_FILE)).toBe(request); + expect(filesystem.hasFile(managedBootstrapEnvelopeClaimPaths().file)).toBe(false); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); }); - it("writes a deterministic root-sourced runtime environment without profile transport", () => { - const applicationRuntime = { - exportEnvironment: { - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "0.25", - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", - }, - unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], - }; - const script = serializeManagedStartupRuntimeEnvironment( - { - NEMOCLAW_MODEL: "model-with-'quote", - NEMOCLAW_OBSERVABILITY: "0", - }, + it("retains the exact bootstrap request after failure and consumes it after a successful retry", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const requestFile = MANAGED_BOOTSTRAP_REQUEST_FILE; + const completionFile = MANAGED_BOOTSTRAP_COMPLETION_FILE; + const filesystem = mockRootReplayFilesystem( + [], + new Map([ + [ + requestFile, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest: createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }), + }), + mode: 0o400, + }, + ], + ]), + ); + vi.spyOn(sharedStateTransaction, "beginManagedStartupSharedStateTransaction").mockReturnValue( true, - { - NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", - NEMOCLAW_MODEL: "model-with-'quote", - }, - applicationRuntime, - ); - - expect(script).toContain("unset NEMOCLAW_INFERENCE_BASE_URL"); - expect(script).toContain("unset NEMOCLAW_MINIMAL_BOOTSTRAP"); - expect(script).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS='0.25'"); - expect(script).toContain("export NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS='3'"); - expect(script).toContain("export NEMOCLAW_MANAGED_STARTUP_APPLIED='1'"); - expect(script).toContain("export NEMOCLAW_MODEL='model-with-'\"'\"'quote'"); - expect(script).toContain(`export SSL_CERT_FILE='${MANAGED_STARTUP_MERGED_CA_FILE}'`); - expect(script).toContain("export _NEMOCLAW_CORPORATE_CA_MERGED='1'"); - expect(script).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); - expect(script).not.toContain("NEMOCLAW_CORPORATE_CA_B64"); - expect(script.endsWith("\n")).toBe(true); - expect( - serializeManagedStartupRuntimeEnvironment( - { - NEMOCLAW_MODEL: "model-with-'quote", - NEMOCLAW_OBSERVABILITY: "0", - }, - true, - { - NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", - NEMOCLAW_MODEL: "model-with-'quote", + ); + coordinatorMock.coordinateManagedStartupApplication + .mockRejectedValueOnce(new Error("application failed")) + .mockResolvedValue({ + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, }, - applicationRuntime, - ), - ).toBe(script); - }); + }); - it("validates runtime plans while removing launch-only exports and unsets from child commands", () => { - const ambient = { - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "stale", - NEMOCLAW_MINIMAL_BOOTSTRAP: "1", - PRESERVED: "yes", - }; - const applied = applyManagedStartupCommandEnvironmentPlan(ambient, { - exportEnvironment: { NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3" }, - unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], - }); + await expect( + applyManagedBootstrapEnvelope( + { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }, + {}, + requestFile, + completionFile, + ), + ).rejects.toThrow("application failed"); + const claim = managedBootstrapEnvelopeClaimPaths(requestFile); + expect(filesystem.hasFile(requestFile)).toBe(false); + expect(filesystem.hasFile(claim.file)).toBe(true); + fs.chmodSync(path.dirname(claim.directory), 0o777); + expect(() => recoverManagedBootstrapEnvelopeClaim(expected, requestFile)).toThrow( + "claim parent must be a protected root-owned directory", + ); + fs.chmodSync(path.dirname(claim.directory), 0o755); + expect( + recoverManagedBootstrapEnvelopeClaim( + { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }, + requestFile, + ), + ).toBe(true); + expect(filesystem.hasFile(completionFile)).toBe(false); - expect(applied).toEqual({ - PRESERVED: "yes", + await expect( + applyManagedBootstrapEnvelope( + { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }, + {}, + requestFile, + completionFile, + ), + ).resolves.toMatchObject({ fingerprint, transactionPending: true }); + expect(filesystem.hasFile(requestFile)).toBe(false); + expect(filesystem.hasFile(claim.file)).toBe(false); + expect( + parseManagedBootstrapImageCompletion(filesystem.readFile(completionFile) ?? ""), + ).toMatchObject({ + agent: profile.agent, + bootstrapIdentity, + profileFingerprint: fingerprint, + transactionPending: true, }); - expect(ambient).toHaveProperty("NEMOCLAW_MINIMAL_BOOTSTRAP", "1"); - expect(() => - applyManagedStartupCommandEnvironmentPlan(ambient, { - exportEnvironment: { NEMOCLAW_MINIMAL_BOOTSTRAP: "1" }, - unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"], - }), - ).toThrow(/both export and unset NEMOCLAW_MINIMAL_BOOTSTRAP/u); - expect(ambient).toHaveProperty("NEMOCLAW_MINIMAL_BOOTSTRAP", "1"); + expect(coordinatorMock.coordinateManagedStartupApplication).toHaveBeenCalledTimes(2); }); it.each([ - "hermes", - "langchain-deepagents-code", - ] as const)("removes OpenClaw launch controls and cleanup obligations from %s children and runtime", (agent) => { - const mapped = mapManagedStartupProfileToAgentEnvironment(managedStartupE2eProfile(agent), { - NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "invalid-for-this-agent", - }); - const ambient = { - ...Object.fromEntries(OPENCLAW_APPLICATION_RUNTIME_NAMES.map((name) => [name, "ambient"])), - NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", - NEMOCLAW_MINIMAL_BOOTSTRAP: "1", - PRESERVED: "yes", - }; - const child = applyManagedStartupCommandEnvironmentPlan(ambient, mapped.applicationRuntime); - const script = serializeManagedStartupRuntimeEnvironment( - mapped.runtimeEnvironment, - false, - mapped.configurationEnvironment, - mapped.applicationRuntime, - ); - - expect(child).toEqual({ PRESERVED: "yes" }); - for (const name of [ - ...OPENCLAW_APPLICATION_RUNTIME_NAMES, - "NEMOCLAW_DASHBOARD_BIND", - "NEMOCLAW_MINIMAL_BOOTSTRAP", - ]) { - expect(script).toContain(`unset ${name}`); - expect(script).not.toContain(`export ${name}=`); - } - }); + { + label: "default", + requestFile: MANAGED_BOOTSTRAP_REQUEST_FILE, + completionFile: MANAGED_BOOTSTRAP_COMPLETION_FILE, + apply: (expected: ManagedBootstrapImageRuntimeExpected) => + applyManagedBootstrapEnvelope(expected, {}), + }, + { + label: "injected", + requestFile: "/run/nemoclaw/injected-managed-bootstrap-request.json", + completionFile: "/run/nemoclaw/injected-managed-bootstrap-completion.json", + apply: (expected: ManagedBootstrapImageRuntimeExpected) => + applyManagedBootstrapEnvelope( + expected, + {}, + "/run/nemoclaw/injected-managed-bootstrap-request.json", + "/run/nemoclaw/injected-managed-bootstrap-completion.json", + ), + }, + ] satisfies ReadonlyArray<{ + readonly label: string; + readonly requestFile: string; + readonly completionFile: string; + readonly apply: (expected: ManagedBootstrapImageRuntimeExpected) => Promise; + }>)("preserves a newly staged $label request after claiming the exact attempt", async ({ + requestFile, + completionFile, + apply, + }) => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }); + const replacement = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: "c".repeat(64), + rootApplyRequest, + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([ + [ + requestFile, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest, + }), + mode: 0o400, + }, + ], + ]), + ); + vi.spyOn(sharedStateTransaction, "beginManagedStartupSharedStateTransaction").mockReturnValue( + true, + ); + coordinatorMock.coordinateManagedStartupApplication.mockImplementation(async () => { + filesystem.writeFile(requestFile, replacement, 0o400); + return { + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, + }; + }); - it("rejects a serialized runtime export that conflicts with an explicit unset", () => { - expect(() => - serializeManagedStartupRuntimeEnvironment( - { NEMOCLAW_MINIMAL_BOOTSTRAP: "1" }, - false, - {}, - { exportEnvironment: {}, unsetEnvironment: ["NEMOCLAW_MINIMAL_BOOTSTRAP"] }, - ), - ).toThrow(/runtime environment cannot both export and unset NEMOCLAW_MINIMAL_BOOTSTRAP/u); + await expect(apply(expected)).resolves.toMatchObject({ + fingerprint, + transactionPending: true, + }); + expect(filesystem.readFile(requestFile)).toBe(replacement); + expect( + parseManagedBootstrapImageCompletion(filesystem.readFile(completionFile) ?? ""), + ).toMatchObject({ bootstrapIdentity, profileFingerprint: fingerprint }); + expect(coordinatorMock.coordinateManagedStartupApplication).toHaveBeenCalledOnce(); }); it.each([ - [ - { exportEnvironment: { "BAD-NAME": "value" }, unsetEnvironment: [] }, - /invalid application runtime environment key/u, - ], - [ - { exportEnvironment: { VALID_NAME: "line 1\nline 2" }, unsetEnvironment: [] }, - /must be single-line text/u, - ], - [ - { exportEnvironment: {}, unsetEnvironment: ["DUPLICATE", "DUPLICATE"] }, - /duplicate application runtime unset/u, - ], - ])("rejects a malformed application runtime plan before command mutation", (plan, message) => { - const ambient = { PRESERVED: "yes" }; - expect(() => applyManagedStartupCommandEnvironmentPlan(ambient, plan)).toThrow(message); - expect(ambient).toEqual({ PRESERVED: "yes" }); - }); - - it.each(["openclaw", "hermes"] as const)("preserves launch-only proxy env for %s", (agent) => { - const mapped = mapManagedStartupProfileToAgentEnvironment( - managedStartupE2eProfile(agent, false, false, true), + { + label: "default", + requestFile: MANAGED_BOOTSTRAP_REQUEST_FILE, + completionFile: MANAGED_BOOTSTRAP_COMPLETION_FILE, + }, + { + label: "injected", + requestFile: "/run/nemoclaw/preclaim-managed-bootstrap-request.json", + completionFile: "/run/nemoclaw/preclaim-managed-bootstrap-completion.json", + }, + ])("preserves a pre-claim $label replacement without publishing completion", async ({ + requestFile, + completionFile, + }) => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }); + const replacement = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: "c".repeat(64), + rootApplyRequest, + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([ + [ + requestFile, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest, + }), + mode: 0o400, + }, + ], + ]), ); - const script = serializeManagedStartupRuntimeEnvironment( - mapped.runtimeEnvironment, - false, - mapped.configurationEnvironment, + const claim = managedBootstrapEnvelopeClaimPaths(requestFile); + filesystem.beforeRename( + observeMatchingRename(requestFile, claim.file, () => { + filesystem.writeFile(requestFile, replacement, 0o400); + }), ); - for (const name of PROXY_ENV_NAMES) { - expect(script).not.toMatch(new RegExp(`(?:export|unset) ${name}(?:=|$)`, "mu")); - } + + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).rejects.toThrow(/changed before its atomic claim/u); + expect(filesystem.readFile(requestFile)).toBe(replacement); + expect(filesystem.hasFile(claim.file)).toBe(false); + expect(filesystem.hasFile(completionFile)).toBe(false); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); + expect(recoverManagedBootstrapEnvelopeClaim(expected, requestFile)).toBe(false); }); - it("clears launch-only proxy env when DCode pins managed routing", () => { - const mapped = mapManagedStartupProfileToAgentEnvironment( - managedStartupE2eProfile("langchain-deepagents-code", false, false, true), + it.each([ + { + label: "default", + requestFile: MANAGED_BOOTSTRAP_REQUEST_FILE, + completionFile: MANAGED_BOOTSTRAP_COMPLETION_FILE, + }, + { + label: "injected", + requestFile: "/run/nemoclaw/post-rename-managed-bootstrap-request.json", + completionFile: "/run/nemoclaw/post-rename-managed-bootstrap-completion.json", + }, + ])("restores a $label replacement after interruption immediately after claim rename", async ({ + requestFile, + completionFile, + }) => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }); + const replacement = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: "c".repeat(64), + rootApplyRequest, + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([ + [ + requestFile, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest, + }), + mode: 0o400, + }, + ], + ]), + ); + const claim = managedBootstrapEnvelopeClaimPaths(requestFile); + filesystem.beforeRename( + observeMatchingRename(requestFile, claim.file, () => { + filesystem.writeFile(requestFile, replacement, 0o400); + }), ); - const script = serializeManagedStartupRuntimeEnvironment( - mapped.runtimeEnvironment, - false, - mapped.configurationEnvironment, + filesystem.afterRename( + observeMatchingRename(requestFile, claim.file, () => { + throw new Error("claim process interrupted"); + }), ); - for (const name of PROXY_ENV_NAMES) { - expect(script).toContain(`unset ${name}`); - } - }); - it("rejects multiline runtime values before producing a sourceable file", () => { - expect(() => - serializeManagedStartupRuntimeEnvironment({ NEMOCLAW_MODEL: "bad\nvalue" }, false), - ).toThrow(/single-line/u); + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).rejects.toThrow(/could not atomically claim managed bootstrap envelope/u); + expect(filesystem.hasFile(requestFile)).toBe(false); + expect(filesystem.readFile(claim.file)).toBe(replacement); + expect(filesystem.hasFile(completionFile)).toBe(false); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); + + filesystem.afterRename(null); + expect(recoverManagedBootstrapEnvelopeClaim(expected, requestFile)).toBe(true); + expect(filesystem.readFile(requestFile)).toBe(replacement); + expect(filesystem.linkCount(requestFile)).toBe(1n); + expect(filesystem.hasFile(claim.file)).toBe(false); + expect(filesystem.hasFile(completionFile)).toBe(false); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); }); - it("refuses a symlink instead of opening its target", () => { - const directory = temporaryDirectory(); - const target = path.join(directory, "target"); - const link = path.join(directory, "link"); - fs.writeFileSync(target, "trusted\n"); - fs.symlinkSync(target, link); + it.each([ + { + label: "default", + requestFile: MANAGED_BOOTSTRAP_REQUEST_FILE, + completionFile: MANAGED_BOOTSTRAP_COMPLETION_FILE, + }, + { + label: "injected", + requestFile: "/run/nemoclaw/interrupted-restore-managed-bootstrap-request.json", + completionFile: "/run/nemoclaw/interrupted-restore-managed-bootstrap-completion.json", + }, + ])("reconciles an interrupted $label pre-claim replacement restoration", async ({ + requestFile, + completionFile, + }) => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const replacementIdentity = "c".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const replacementExpected = { + agent: profile.agent, + profileFingerprint: fingerprint, + bootstrapIdentity: replacementIdentity, + }; + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }); + const replacement = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: replacementIdentity, + rootApplyRequest, + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([ + [ + requestFile, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest, + }), + mode: 0o400, + }, + ], + ]), + ); + const claim = managedBootstrapEnvelopeClaimPaths(requestFile); + let interruptRestoration = true; + filesystem.beforeRename( + observeMatchingRename(requestFile, claim.file, () => { + filesystem.writeFile(requestFile, replacement, 0o400); + }), + ); + filesystem.beforeUnlink( + observeMatchingUnlink( + claim.file, + () => { + throw new Error("restoration cleanup interrupted"); + }, + () => interruptRestoration, + ), + ); - expect(() => readStableRegularFile(link, 1024)).toThrow(/unsafe or unreadable/u); - }); + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).rejects.toThrow(/could not remove restored managed bootstrap envelope/u); + expect(filesystem.readFile(requestFile)).toBe(replacement); + expect(filesystem.readFile(claim.file)).toBe(replacement); + expect(filesystem.linkCount(requestFile)).toBe(2n); + expect(filesystem.hasFile(completionFile)).toBe(false); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); - it("rejects descriptor metadata drift after a bounded read", () => { - const directory = temporaryDirectory(); - const target = path.join(directory, "material"); - fs.writeFileSync(target, "trusted\n", { mode: 0o600 }); - const realReadSync = fs.readSync.bind(fs); - vi.spyOn(fs, "readSync") - .mockImplementationOnce((( - descriptor: number, - buffer: NodeJS.ArrayBufferView, - offset: number, - length: number, - position: number | null, - ) => { - const bytesRead = realReadSync(descriptor, buffer, offset, length, position); - fs.chmodSync(target, 0o644); - return bytesRead; - }) as typeof fs.readSync) - .mockImplementation(realReadSync as typeof fs.readSync); - - expect(() => readStableRegularFile(target, 1024)).toThrow(/changed while it was read/u); + interruptRestoration = false; + expect(recoverManagedBootstrapEnvelopeClaim(replacementExpected, requestFile)).toBe(true); + expect(filesystem.readFile(requestFile)).toBe(replacement); + expect(filesystem.linkCount(requestFile)).toBe(1n); + expect(filesystem.hasFile(claim.file)).toBe(false); + expect(filesystem.hasFile(completionFile)).toBe(false); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); }); - it("normalizes mutable sandbox-owned Hermes config descriptors to mode 0640", () => { - const directory = temporaryDirectory(); - const target = path.join(directory, "config.yaml"); - fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); - mockDescriptorOwnership(501n, 20n); - - normalizeHermesManagedConfigDescriptor(target, { - uid: 501, - gid: 20, + it("fails closed without clobbering a second replacement during claim restoration", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const displacedIdentity = "c".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const displacedExpected = { + agent: profile.agent, + profileFingerprint: fingerprint, + bootstrapIdentity: displacedIdentity, + }; + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }); + const displaced = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: displacedIdentity, + rootApplyRequest, }); + const latest = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: "d".repeat(64), + rootApplyRequest, + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([ + [ + MANAGED_BOOTSTRAP_REQUEST_FILE, + { + contents: serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest, + }), + mode: 0o400, + }, + ], + ]), + ); + const claim = managedBootstrapEnvelopeClaimPaths(); + filesystem.beforeRename( + observeMatchingRename(MANAGED_BOOTSTRAP_REQUEST_FILE, claim.file, () => { + filesystem.writeFile(MANAGED_BOOTSTRAP_REQUEST_FILE, displaced, 0o400); + }), + ); + filesystem.beforeLink( + observeMatchingLink(claim.file, MANAGED_BOOTSTRAP_REQUEST_FILE, () => { + filesystem.writeFile(MANAGED_BOOTSTRAP_REQUEST_FILE, latest, 0o400); + }), + ); - expect(fs.readFileSync(target, "utf8")).toBe("model: managed\n"); - expect(fs.statSync(target).mode & 0o777).toBe(0o640); + await expect(applyManagedBootstrapEnvelope(expected, {})).rejects.toThrow( + /canonical managed bootstrap request was replaced again/u, + ); + expect(filesystem.readFile(MANAGED_BOOTSTRAP_REQUEST_FILE)).toBe(latest); + expect(filesystem.readFile(claim.file)).toBe(displaced); + expect(filesystem.linkCount(MANAGED_BOOTSTRAP_REQUEST_FILE)).toBe(1n); + expect(filesystem.linkCount(claim.file)).toBe(1n); + expect(filesystem.hasFile(MANAGED_BOOTSTRAP_COMPLETION_FILE)).toBe(false); + expect(coordinatorMock.coordinateManagedStartupApplication).not.toHaveBeenCalled(); + expect(recoverManagedBootstrapEnvelopeClaim(displacedExpected)).toBe(true); + expect(filesystem.readFile(MANAGED_BOOTSTRAP_REQUEST_FILE)).toBe(latest); + expect(filesystem.readFile(claim.file)).toBe(displaced); }); - it("preserves a root-owned shields-up Hermes descriptor without chmod", () => { - const directory = temporaryDirectory(); - const target = path.join(directory, ".env"); - fs.writeFileSync(target, "OPENAI_API_KEY=managed\n", { mode: 0o444 }); - mockDescriptorOwnership(0n, 0n); - const chmod = vi.spyOn(fs, "fchmodSync"); - - normalizeHermesManagedConfigDescriptor(target, { - uid: 501, - gid: 20, + it.each([ + { + label: "default", + requestFile: MANAGED_BOOTSTRAP_REQUEST_FILE, + completionFile: MANAGED_BOOTSTRAP_COMPLETION_FILE, + }, + { + label: "injected", + requestFile: "/run/nemoclaw/failure-managed-bootstrap-request.json", + completionFile: "/run/nemoclaw/failure-managed-bootstrap-completion.json", + }, + ])("keeps the $label private claim across completion failure and a new canonical request", async ({ + requestFile, + completionFile, + }) => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, + }); + const original = serializeManagedBootstrapEnvelope({ bootstrapIdentity, rootApplyRequest }); + const replacement = serializeManagedBootstrapEnvelope({ + bootstrapIdentity: "c".repeat(64), + rootApplyRequest, + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([[requestFile, { contents: original, mode: 0o400 }]]), + ); + const claim = managedBootstrapEnvelopeClaimPaths(requestFile); + vi.spyOn(sharedStateTransaction, "beginManagedStartupSharedStateTransaction").mockReturnValue( + true, + ); + coordinatorMock.coordinateManagedStartupApplication.mockResolvedValue({ + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, }); + filesystem.beforeRename( + observeMatchingRenameTarget(completionFile, () => { + filesystem.writeFile(requestFile, replacement, 0o400); + throw new Error("completion publication interrupted"); + }), + ); - expect(chmod).not.toHaveBeenCalled(); - expect(fs.readFileSync(target, "utf8")).toBe("OPENAI_API_KEY=managed\n"); + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).rejects.toThrow("could not atomically write"); + expect(filesystem.readFile(claim.file)).toBe(original); + expect(filesystem.readFile(requestFile)).toBe(replacement); + expect(filesystem.hasFile(completionFile)).toBe(false); + + filesystem.beforeRename(null); + vi.spyOn( + sharedStateTransaction, + "getManagedStartupSharedStateTransactionStatus", + ).mockReturnValue("pending"); + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).resolves.toMatchObject({ fingerprint, transactionPending: true }); + expect(filesystem.hasFile(claim.file)).toBe(false); + expect(filesystem.readFile(requestFile)).toBe(replacement); + expect( + parseManagedBootstrapImageCompletion(filesystem.readFile(completionFile) ?? ""), + ).toMatchObject({ bootstrapIdentity, profileFingerprint: fingerprint }); }); - it.each([0o440, 0o644, 0o660])("fails closed on unexpected mutable Hermes mode %s", (mode) => { - const directory = temporaryDirectory(); - const target = path.join(directory, "config.yaml"); - fs.writeFileSync(target, "model: managed\n", { mode }); - fs.chmodSync(target, mode); - mockDescriptorOwnership(501n, 20n); - - expect(() => - normalizeHermesManagedConfigDescriptor(target, { - uid: 501, - gid: 20, + it("recovers an empty claim directory and a completion-published claim cleanup interruption", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + const expected = { agent: profile.agent, profileFingerprint: fingerprint, bootstrapIdentity }; + const requestFile = "/run/nemoclaw/crash-managed-bootstrap-request.json"; + const completionFile = "/run/nemoclaw/crash-managed-bootstrap-completion.json"; + const original = serializeManagedBootstrapEnvelope({ + bootstrapIdentity, + rootApplyRequest: createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile, }), - ).toThrow(/unexpected Hermes managed config descriptor/u); - expect(fs.statSync(target).mode & 0o777).toBe(mode); + }); + const filesystem = mockRootReplayFilesystem( + [], + new Map([[requestFile, { contents: original, mode: 0o400 }]]), + ); + const claim = managedBootstrapEnvelopeClaimPaths(requestFile); + let interruptClaim = true; + filesystem.beforeRename( + observeMatchingRename( + requestFile, + claim.file, + () => { + throw new Error("claim interrupted"); + }, + () => interruptClaim, + ), + ); + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).rejects.toThrow("could not atomically claim"); + expect(recoverManagedBootstrapEnvelopeClaim(expected, requestFile)).toBe(true); + + interruptClaim = false; + vi.spyOn(sharedStateTransaction, "beginManagedStartupSharedStateTransaction").mockReturnValue( + true, + ); + coordinatorMock.coordinateManagedStartupApplication.mockResolvedValue({ + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, + }); + let interruptCleanup = true; + filesystem.beforeUnlink( + observeMatchingUnlink( + claim.file, + () => { + throw new Error("claim cleanup interrupted"); + }, + () => interruptCleanup, + ), + ); + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).rejects.toThrow("could not consume managed bootstrap envelope claim"); + expect(filesystem.hasFile(completionFile)).toBe(true); + expect(filesystem.hasFile(claim.file)).toBe(true); + + interruptCleanup = false; + vi.spyOn( + sharedStateTransaction, + "getManagedStartupSharedStateTransactionStatus", + ).mockReturnValue("pending"); + await expect( + applyManagedBootstrapEnvelope(expected, {}, requestFile, completionFile), + ).resolves.toMatchObject({ fingerprint, transactionPending: true }); + expect(filesystem.hasFile(claim.file)).toBe(false); + expect(filesystem.hasFile(completionFile)).toBe(true); }); - it("fails closed on an unexpected Hermes descriptor owner", () => { - const directory = temporaryDirectory(); - const target = path.join(directory, "config.yaml"); - fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); - mockDescriptorOwnership(502n, 21n); + it.each([ + ["pending", true], + ["committed", false], + ] as const)("binds a completed profile replay to its %s bootstrap transaction", async (status, transactionPending) => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + const bootstrapIdentity = "b".repeat(64); + mockRootReplayFilesystem([]); + coordinatorMock.coordinateManagedStartupApplication.mockResolvedValue({ + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, + }); + await applyManagedStartupImageProfile("openclaw", { + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + [MANAGED_STARTUP_PROFILE_ENV]: encodedProfile, + }); + const statusProbe = vi + .spyOn(sharedStateTransaction, "getManagedStartupSharedStateTransactionStatus") + .mockReturnValue(status); + + const result = await applyManagedStartupRootRequest( + createManagedStartupRootApplyRequest({ agent: profile.agent, encodedProfile }), + {}, + { bootstrapIdentity }, + ); - expect(() => - normalizeHermesManagedConfigDescriptor(target, { - uid: 501, - gid: 20, - }), - ).toThrow(/unexpected Hermes managed config descriptor/u); - expect(fs.statSync(target).mode & 0o777).toBe(0o600); + expect(result).toMatchObject({ fingerprint, transactionPending }); + expect(statusProbe).toHaveBeenCalledWith({ + agent: "openclaw", + profileFingerprint: fingerprint, + bootstrapIdentity, + }); }); - it("detects a path replacement while normalizing through the trusted descriptor", () => { - const directory = temporaryDirectory(); - const target = path.join(directory, "config.yaml"); - const displaced = path.join(directory, "displaced.yaml"); - const replacement = path.join(directory, "replacement.yaml"); - fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); - fs.writeFileSync(replacement, "model: replaced\n", { mode: 0o640 }); - mockDescriptorOwnership(501n, 20n); - const realFchmodSync = fs.fchmodSync.bind(fs); - vi.spyOn(fs, "fchmodSync").mockImplementation((descriptor, mode) => { - realFchmodSync(descriptor, mode); - fs.renameSync(target, displaced); - fs.renameSync(replacement, target); + it("rejects a completed profile that has no authority for the bootstrap attempt", async () => { + const profile = managedStartupE2eProfile("openclaw"); + const encodedProfile = encodeManagedStartupProfile(profile); + const fingerprint = fingerprintManagedStartupProfile(profile); + mockRootReplayFilesystem([]); + coordinatorMock.coordinateManagedStartupApplication.mockResolvedValue({ + adapterApplied: false, + application: { + status: "committed", + stateDirectory: "/var/lib/nemoclaw/managed-startup", + generationDirectory: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}`, + profilePath: `/var/lib/nemoclaw/managed-startup/generation-${fingerprint}/profile.json`, + corporateCaPath: null, + fingerprint, + expectedAgent: "openclaw", + profile, + }, + }); + await applyManagedStartupImageProfile("openclaw", { + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + [MANAGED_STARTUP_PROFILE_ENV]: encodedProfile, }); + vi.spyOn( + sharedStateTransaction, + "getManagedStartupSharedStateTransactionStatus", + ).mockReturnValue("none"); - expect(() => - normalizeHermesManagedConfigDescriptor(target, { - uid: 501, - gid: 20, - }), - ).toThrow(/changed during normalization/u); - expect(fs.readFileSync(target, "utf8")).toBe("model: replaced\n"); + await expect( + applyManagedStartupRootRequest( + createManagedStartupRootApplyRequest({ agent: profile.agent, encodedProfile }), + {}, + { bootstrapIdentity: "b".repeat(64) }, + ), + ).rejects.toThrow(/no shared-state authority/u); + expect(coordinatorMock.coordinateManagedStartupApplication).toHaveBeenCalledTimes(1); }); }); diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index 898783c7f30..54e170fb564 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -62,7 +62,7 @@ const HERMES_MANAGED_CONFIG_FILES = [ ] as const; const FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const SHA256_RE = /^[a-f0-9]{64}$/u; -const MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION = 1; +export const MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION = 1; const MAX_MANAGED_STARTUP_COMPLETION_BYTES = 4096; const MAX_MANAGED_STARTUP_RUNTIME_ENVIRONMENT_BYTES = 512 * 1024; @@ -135,6 +135,11 @@ export interface ManagedStartupRootApplyResult extends ManagedStartupImageApplyR readonly transactionPending: boolean; } +export interface ManagedStartupRootApplyOptions { + /** One-attempt identity for managed bootstrap; null keeps the direct root-apply contract. */ + readonly bootstrapIdentity?: string | null; +} + export interface ManagedStartupCompletionMarker { readonly schemaVersion: typeof MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION; readonly agent: ManagedStartupAgent; @@ -337,7 +342,7 @@ function requireSafeExistingRootTarget(target: string): void { } } -function atomicWriteRootFile(target: string, contents: string | Buffer, mode: number): void { +export function atomicWriteRootFile(target: string, contents: string | Buffer, mode: number): void { const parent = path.dirname(target); const parentStat = fs.lstatSync(parent); if ( @@ -739,7 +744,7 @@ function sealOpenClawConfiguration( runInternalSandboxAction("write-openclaw-hash", configurationEnvironment, applicationRuntime); } -interface StableRegularFile { +export interface StableRegularFile { readonly bytes: Buffer; readonly stat: fs.BigIntStats; } @@ -763,7 +768,7 @@ function sameStableFileMetadata(left: fs.BigIntStats, right: fs.BigIntStats): bo ); } -function readStableRegularFileSnapshot(target: string, maxBytes: number): StableRegularFile { +export function readStableRegularFileSnapshot(target: string, maxBytes: number): StableRegularFile { if (typeof fs.constants.O_NOFOLLOW !== "number") { fail("O_NOFOLLOW is unavailable for managed startup file reads"); } @@ -1421,6 +1426,7 @@ function completionAlreadyPublished(request: ManagedStartupRootApplyRequest): bo export async function applyManagedStartupRootRequest( request: ManagedStartupRootApplyRequest, env: Environment = process.env, + options: ManagedStartupRootApplyOptions = {}, ): Promise { requireRoot(); const profile = decodeManagedStartupProfile(request.encodedProfile); @@ -1445,12 +1451,27 @@ export async function applyManagedStartupRootRequest( // these non-fingerprinted application-runtime values. mapManagedStartupProfileToAgentEnvironment(profile, imageEnvironment); const alreadyPublished = completionAlreadyPublished(request); + const bootstrapIdentity = options.bootstrapIdentity ?? null; + const transactionStatus = + alreadyPublished && bootstrapIdentity !== null + ? getManagedStartupSharedStateTransactionStatus({ + agent: request.agent, + profileFingerprint: request.profileFingerprint, + bootstrapIdentity, + }) + : null; + if (transactionStatus === "none") { + fail("completed startup profile has no shared-state authority for this bootstrap attempt"); + } if (!alreadyPublished) { ensureRootOwnedDirectory(ROOT_STATE_PARENT); - beginManagedStartupSharedStateTransaction(profile); + beginManagedStartupSharedStateTransaction(profile, { bootstrapIdentity }); } const result = await applyManagedStartupImageProfile(request.agent, imageEnvironment); - return { ...result, transactionPending: !alreadyPublished }; + return { + ...result, + transactionPending: !alreadyPublished || transactionStatus === "pending", + }; } function readBoundedRootApplyStdin(): string { diff --git a/test/helpers/managed-startup-root-replay-filesystem.ts b/test/helpers/managed-startup-root-replay-filesystem.ts new file mode 100644 index 00000000000..603ec7046dc --- /dev/null +++ b/test/helpers/managed-startup-root-replay-filesystem.ts @@ -0,0 +1,429 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { vi } from "vitest"; + +import { MANAGED_STARTUP_RUNTIME_ENV_FILE } from "../../src/lib/onboard/managed-startup/image-runtime"; + +type RenameObserver = (source: string, target: string) => void; +type UnlinkObserver = (target: string) => void; +type ObserverEffect = () => void; +type ObserverGate = () => boolean; + +const alwaysObserve: ObserverGate = () => true; + +export function observeMatchingRename( + expectedSource: string, + expectedTarget: string, + effect: ObserverEffect, + enabled: ObserverGate = alwaysObserve, +): RenameObserver { + return (source, target) => { + if (enabled() && source === expectedSource && target === expectedTarget) effect(); + }; +} + +export function observeMatchingLink( + expectedSource: string, + expectedTarget: string, + effect: ObserverEffect, +): RenameObserver { + return observeMatchingRename(expectedSource, expectedTarget, effect); +} + +export function observeMatchingRenameTarget( + expectedTarget: string, + effect: ObserverEffect, +): RenameObserver { + return (_source, target) => { + if (target === expectedTarget) effect(); + }; +} + +export function observeMatchingUnlink( + expectedTarget: string, + effect: ObserverEffect, + enabled: ObserverGate = alwaysObserve, +): UnlinkObserver { + return (target) => { + if (enabled() && target === expectedTarget) effect(); + }; +} + +export function mockRootReplayFilesystem( + runtimeWrites: string[], + seededFiles: ReadonlyMap< + string, + { readonly contents: string | Buffer; readonly mode: number } + > = new Map(), +): { + readonly beforeRename: (callback: ((source: string, target: string) => void) | null) => void; + readonly afterRename: (callback: ((source: string, target: string) => void) | null) => void; + readonly beforeLink: (callback: ((source: string, target: string) => void) | null) => void; + readonly beforeUnlink: (callback: ((target: string) => void) | null) => void; + readonly hasFile: (target: string) => boolean; + readonly linkCount: (target: string) => bigint; + readonly readFile: (target: string) => string | null; + readonly writeFile: (target: string, contents: string | Buffer, mode: number) => void; +} { + const directories = new Set([ + "/", + "/run", + "/run/nemoclaw", + "/var", + "/var/lib", + "/var/lib/nemoclaw", + ]); + const files: Map = new Map( + [...seededFiles].map(([target, file]) => [ + target, + Buffer.isBuffer(file.contents) + ? Buffer.from(file.contents) + : Buffer.from(file.contents, "utf8"), + ]), + ); + const directoryModes = new Map([...directories].map((target) => [target, 0o755])); + const fileModes = new Map([...seededFiles].map(([target, file]) => [target, file.mode])); + let nextFileInode = 2n; + const fileInodes = new Map(); + const fileCtimes = new Map(); + for (const target of files.keys()) { + fileInodes.set(target, nextFileInode); + fileCtimes.set(target, 1n); + nextFileInode += 1n; + } + const descriptorTargets = new Map(); + const descriptorSnapshots = new Map< + number, + { + readonly bytes: Buffer; + readonly ctimeNs: bigint; + readonly ino: bigint; + readonly mode: number; + } + >(); + const pendingFiles = new Map(); + const pendingModes = new Map(); + let linkObserver: ((source: string, target: string) => void) | null = null; + let renameObserver: ((source: string, target: string) => void) | null = null; + let afterRenameObserver: ((source: string, target: string) => void) | null = null; + let unlinkObserver: ((target: string) => void) | null = null; + let nextDescriptor = 91; + const fileLinkCount = (ino: bigint): bigint => + BigInt([...fileInodes.values()].filter((candidate) => candidate === ino).length); + const bumpFileCtime = (ino: bigint): void => { + const currentCtimes = [ + ...[...fileCtimes].flatMap(([target, ctimeNs]) => + fileInodes.get(target) === ino ? [ctimeNs] : [], + ), + ...[...descriptorSnapshots.values()].flatMap((snapshot) => + snapshot.ino === ino ? [snapshot.ctimeNs] : [], + ), + ]; + const nextCtime = + currentCtimes.reduce((latest, ctimeNs) => (ctimeNs > latest ? ctimeNs : latest), 0n) + 1n; + for (const [target, targetInode] of fileInodes) { + if (targetInode === ino) fileCtimes.set(target, nextCtime); + } + for (const [descriptor, snapshot] of descriptorSnapshots) { + if (snapshot.ino === ino) + descriptorSnapshots.set(descriptor, { ...snapshot, ctimeNs: nextCtime }); + } + }; + const stat = (kind: "directory" | "file", mode: number) => + ({ + gid: 0, + isDirectory: () => kind === "directory", + isFile: () => kind === "file", + isSymbolicLink: () => false, + mode, + nlink: 1, + uid: 0, + }) as fs.Stats; + const bigDirectoryStat = (target: string) => + ({ + ctimeNs: 1n, + dev: 1n, + gid: 0n, + ino: 1n, + isDirectory: () => true, + isFile: () => false, + isSymbolicLink: () => false, + mode: BigInt(0o040000 | (directoryModes.get(target) ?? 0o755)), + mtimeNs: 1n, + nlink: 1n, + size: 0n, + uid: 0n, + }) as fs.BigIntStats; + const bigFileStat = (bytes: Buffer, mode: number, ino: bigint, ctimeNs: bigint, nlink: bigint) => + ({ + ctimeNs, + dev: 1n, + gid: 0n, + ino, + isDirectory: () => false, + isFile: () => true, + isSymbolicLink: () => false, + mode: BigInt(0o100000 | mode), + mtimeNs: 1n, + nlink, + size: BigInt(bytes.length), + uid: 0n, + }) as fs.BigIntStats; + const missing = (): never => { + throw Object.assign(new Error("missing"), { code: "ENOENT" }); + }; + const allocateDescriptor = (resolved: string, mode = 0o600): number => { + const descriptor = nextDescriptor; + nextDescriptor += 1; + descriptorTargets.set(descriptor, resolved); + pendingModes.set(resolved, mode); + return descriptor; + }; + const deleteExistingFile = (resolved: string): void => { + void (files.get(resolved) ?? missing()); + const inode = fileInodes.get(resolved) ?? missing(); + files.delete(resolved); + fileInodes.delete(resolved); + fileCtimes.delete(resolved); + fileModes.delete(resolved); + bumpFileCtime(inode); + }; + + vi.spyOn(process, "geteuid").mockReturnValue(0); + vi.spyOn(fs, "lstatSync").mockImplementation((( + target: fs.PathLike, + options?: { bigint?: boolean }, + ) => { + const resolved = String(target); + const bytes = files.get(resolved); + const mode = fileModes.get(resolved) ?? 0o444; + return directories.has(resolved) + ? options?.bigint + ? bigDirectoryStat(resolved) + : stat("directory", directoryModes.get(resolved) ?? 0o755) + : bytes === undefined + ? missing() + : options?.bigint + ? bigFileStat( + bytes, + mode, + fileInodes.get(resolved) ?? missing(), + fileCtimes.get(resolved) ?? missing(), + fileLinkCount(fileInodes.get(resolved) ?? missing()), + ) + : stat("file", mode); + }) as typeof fs.lstatSync); + vi.spyOn(fs, "mkdirSync").mockImplementation((( + target: fs.PathLike, + options?: { mode?: number }, + ) => { + const resolved = String(target); + if (directories.has(resolved) || files.has(resolved)) { + throw Object.assign(new Error("exists"), { code: "EEXIST" }); + } + directories.add(resolved); + directoryModes.set(resolved, options?.mode ?? 0o777); + return undefined; + }) as typeof fs.mkdirSync); + vi.spyOn(fs, "chownSync").mockImplementation(() => undefined); + vi.spyOn(fs, "chmodSync").mockImplementation(((target: fs.PathLike, mode: fs.Mode) => { + const resolved = String(target); + const numeric = typeof mode === "number" ? mode : Number.parseInt(mode, 8); + if (directories.has(resolved)) directoryModes.set(resolved, numeric); + else if (files.has(resolved)) fileModes.set(resolved, numeric); + else missing(); + }) as typeof fs.chmodSync); + vi.spyOn(fs, "existsSync").mockReturnValue(false); + vi.spyOn(fs, "openSync").mockImplementation(((target: fs.PathLike, flags, mode) => { + const resolved = String(target); + const creates = + typeof flags === "number" ? (flags & fs.constants.O_CREAT) !== 0 : /[awx]/u.test(flags); + if (!creates && !files.has(resolved) && !directories.has(resolved)) missing(); + const descriptor = allocateDescriptor(resolved, typeof mode === "number" ? mode : 0o600); + const bytes = files.get(resolved); + if (bytes !== undefined) { + descriptorSnapshots.set(descriptor, { + bytes: Buffer.from(bytes), + ctimeNs: fileCtimes.get(resolved) ?? missing(), + ino: fileInodes.get(resolved) ?? missing(), + mode: fileModes.get(resolved) ?? 0o444, + }); + } + return descriptor; + }) as typeof fs.openSync); + vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number) => { + const snapshot = descriptorSnapshots.get(descriptor); + if (snapshot !== undefined) { + return bigFileStat( + snapshot.bytes, + snapshot.mode, + snapshot.ino, + snapshot.ctimeNs, + fileLinkCount(snapshot.ino), + ); + } + const target = descriptorTargets.get(descriptor); + const bytes = target === undefined ? undefined : files.get(target); + return bytes === undefined + ? missing() + : bigFileStat( + bytes, + fileModes.get(target as string) ?? 0o444, + fileInodes.get(target as string) ?? missing(), + fileCtimes.get(target as string) ?? missing(), + fileLinkCount(fileInodes.get(target as string) ?? missing()), + ); + }) as typeof fs.fstatSync); + vi.spyOn(fs, "readSync").mockImplementation((( + descriptor: number, + buffer: NodeJS.ArrayBufferView, + offset: number, + length: number, + position: number | null, + ) => { + const target = descriptorTargets.get(descriptor); + const bytes = + descriptorSnapshots.get(descriptor)?.bytes ?? + (target === undefined ? undefined : files.get(target)) ?? + missing(); + const start = position ?? 0; + const count = Math.min(length, Math.max(0, bytes.length - start)); + bytes.copy(buffer as Buffer, offset, start, start + count); + return count; + }) as typeof fs.readSync); + vi.spyOn(fs, "fchownSync").mockImplementation(() => undefined); + vi.spyOn(fs, "writeFileSync").mockImplementation(((target: fs.PathOrFileDescriptor, value) => { + const resolved = + (typeof target === "number" ? descriptorTargets.get(target) : undefined) ?? missing(); + pendingFiles.set( + resolved, + Buffer.isBuffer(value) ? Buffer.from(value) : Buffer.from(String(value), "utf8"), + ); + }) as typeof fs.writeFileSync); + vi.spyOn(fs, "fchmodSync").mockImplementation((descriptor, mode) => { + const target = descriptorTargets.get(descriptor) ?? missing(); + pendingModes.set(target, typeof mode === "number" ? mode : Number.parseInt(mode, 8)); + }); + vi.spyOn(fs, "fsyncSync").mockImplementation(() => undefined); + vi.spyOn(fs, "closeSync").mockImplementation((descriptor) => { + descriptorSnapshots.delete(descriptor); + descriptorTargets.delete(descriptor); + }); + vi.spyOn(fs, "linkSync").mockImplementation(((existingPath, newPath) => { + const resolvedSource = String(existingPath); + const resolvedTarget = String(newPath); + linkObserver?.(resolvedSource, resolvedTarget); + if (files.has(resolvedTarget) || directories.has(resolvedTarget)) { + throw Object.assign(new Error("exists"), { code: "EEXIST" }); + } + const sourceInode = fileInodes.get(resolvedSource) ?? missing(); + files.set(resolvedTarget, files.get(resolvedSource) ?? missing()); + fileInodes.set(resolvedTarget, sourceInode); + fileCtimes.set(resolvedTarget, fileCtimes.get(resolvedSource) ?? missing()); + fileModes.set(resolvedTarget, fileModes.get(resolvedSource) ?? missing()); + bumpFileCtime(sourceInode); + }) as typeof fs.linkSync); + vi.spyOn(fs, "renameSync").mockImplementation((source, target) => { + const resolvedSource = String(source); + const resolvedTarget = String(target); + renameObserver?.(resolvedSource, resolvedTarget); + const pending = pendingFiles.get(resolvedSource); + if (pending !== undefined) { + if (files.has(resolvedTarget)) deleteExistingFile(resolvedTarget); + files.set(resolvedTarget, pending); + fileInodes.set(resolvedTarget, nextFileInode); + fileCtimes.set(resolvedTarget, 1n); + nextFileInode += 1n; + fileModes.set(resolvedTarget, pendingModes.get(resolvedSource) ?? 0o444); + pendingFiles.delete(resolvedSource); + pendingModes.delete(resolvedSource); + } else { + const sourceBytes = files.get(resolvedSource) ?? missing(); + const sourceInode = fileInodes.get(resolvedSource) ?? missing(); + const targetInode = fileInodes.get(resolvedTarget); + if (targetInode === sourceInode) { + afterRenameObserver?.(resolvedSource, resolvedTarget); + return; + } + const sourceMode = fileModes.get(resolvedSource) ?? missing(); + if (files.has(resolvedTarget)) deleteExistingFile(resolvedTarget); + files.set(resolvedTarget, sourceBytes); + fileInodes.set(resolvedTarget, sourceInode); + fileModes.set(resolvedTarget, sourceMode); + fileCtimes.set(resolvedTarget, fileCtimes.get(resolvedSource) ?? missing()); + files.delete(resolvedSource); + fileInodes.delete(resolvedSource); + fileModes.delete(resolvedSource); + fileCtimes.delete(resolvedSource); + bumpFileCtime(sourceInode); + } + runtimeWrites.push( + ...(resolvedTarget === MANAGED_STARTUP_RUNTIME_ENV_FILE + ? [(files.get(resolvedTarget) ?? missing()).toString("utf8")] + : []), + ); + afterRenameObserver?.(resolvedSource, resolvedTarget); + }); + vi.spyOn(fs, "unlinkSync").mockImplementation(((target: fs.PathLike) => { + const resolved = String(target); + unlinkObserver?.(resolved); + const removedPendingFile = pendingFiles.delete(resolved); + pendingModes.delete(resolved); + if (removedPendingFile) return; + deleteExistingFile(resolved); + }) as typeof fs.unlinkSync); + vi.spyOn(fs, "readdirSync").mockImplementation(((target: fs.PathLike) => { + const resolved = String(target); + if (!directories.has(resolved)) return missing(); + const prefix = `${resolved}/`; + return [...files.keys(), ...directories] + .filter((entry) => entry.startsWith(prefix)) + .map((entry) => entry.slice(prefix.length)) + .filter((entry) => entry.length > 0 && !entry.includes("/")); + }) as typeof fs.readdirSync); + vi.spyOn(fs, "rmdirSync").mockImplementation(((target: fs.PathLike) => { + const resolved = String(target); + if (!directories.has(resolved)) return missing(); + const prefix = `${resolved}/`; + if ( + [...files.keys(), ...directories].some( + (entry) => entry !== resolved && entry.startsWith(prefix), + ) + ) { + throw Object.assign(new Error("not empty"), { code: "ENOTEMPTY" }); + } + directories.delete(resolved); + directoryModes.delete(resolved); + }) as typeof fs.rmdirSync); + + return { + afterRename: (callback) => { + afterRenameObserver = callback; + }, + beforeLink: (callback) => { + linkObserver = callback; + }, + beforeRename: (callback) => { + renameObserver = callback; + }, + beforeUnlink: (callback) => { + unlinkObserver = callback; + }, + hasFile: (target) => files.has(target), + linkCount: (target) => fileLinkCount(fileInodes.get(target) ?? missing()), + readFile: (target) => files.get(target)?.toString("utf8") ?? null, + writeFile: (target, contents, mode) => { + if (files.has(target)) deleteExistingFile(target); + files.set( + target, + Buffer.isBuffer(contents) ? Buffer.from(contents) : Buffer.from(contents, "utf8"), + ); + fileInodes.set(target, nextFileInode); + fileCtimes.set(target, 1n); + nextFileInode += 1n; + fileModes.set(target, mode); + }, + }; +} diff --git a/test/managed-bootstrap-trampoline.test.ts b/test/managed-bootstrap-trampoline.test.ts index 00e806e4d6d..5da02532562 100644 --- a/test/managed-bootstrap-trampoline.test.ts +++ b/test/managed-bootstrap-trampoline.test.ts @@ -603,10 +603,12 @@ exec /usr/bin/env -i NEMOCLAW_MANAGED_BOOTSTRAP_RESUME=1 ${JSON.stringify( it.each( MANAGED_STARTUP_AGENTS, - )("consumes the protected %s request before exact supervisor exec and drops bootstrap variables", (agent) => { + )("consumes the protected %s request or recovered claim before exact supervisor exec and drops bootstrap variables", (agent) => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bootstrap-trampoline-")); try { const request = path.join(directory, "request.json"); + const claimDirectory = path.join(directory, ".request.json.nemoclaw-claim"); + const claim = path.join(claimDirectory, "request"); const completion = path.join(directory, "completion"); const runtime = path.join(directory, "runtime.cjs"); const sandbox = path.join(directory, "sandbox"); @@ -647,7 +649,8 @@ test ! -e /proc/self/fd/9 printf 'node:%s:home=%s:path=%s:lang=%s:capability=%s\\n' "$*" "$HOME" "$PATH" "$LANG" "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" >>${JSON.stringify(trace)} case "$*" in *--apply-bootstrap-file*) - /bin/rm -f ${JSON.stringify(request)} + /bin/rm -f ${JSON.stringify(request)} ${JSON.stringify(claim)} + if test -d ${JSON.stringify(claimDirectory)}; then /bin/rmdir ${JSON.stringify(claimDirectory)}; fi printf '%s\\n' '${agent}:${"a".repeat(64)}:${"b".repeat(64)}' >${JSON.stringify(completion)} ;; *--verify-bootstrap-completion*) @@ -662,6 +665,7 @@ esac set -e test ! -e /proc/self/fd/9 test ! -e "$REQUEST" +test ! -e "$CLAIM" test "$#" -eq 3 test "$1" = "supervise" test "$2" = "two words" @@ -722,6 +726,7 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab ]; const environment = { REQUEST: request, + CLAIM: claim, TRACE: trace, BASH_ENV: path.join(directory, "bash-env"), "BASH_FUNC_attacker%%": `() { /usr/bin/touch ${attackerFunction}; }`, @@ -745,6 +750,7 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab expect(fs.existsSync(loader.earlyTrace)).toBe(false); expect(fs.existsSync(loader.afterTrace)).toBe(true); expect(fs.readFileSync(trace, "utf8").trim().split("\n")).toEqual([ + `node:${runtime} --recover-bootstrap-claim --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, `node:${runtime} --apply-bootstrap-file --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, `node:${runtime} --verify-bootstrap-completion --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, "startup after validation", @@ -753,10 +759,24 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab execFileSync(entrypoint, argv, { env: environment }); let lines = fs.readFileSync(trace, "utf8").trim().split("\n"); + expect(lines.filter((line) => line.includes("--recover-bootstrap-claim"))).toHaveLength(2); expect(lines.filter((line) => line.includes("--apply-bootstrap-file"))).toHaveLength(1); expect(lines.filter((line) => line.startsWith("supervisor:"))).toHaveLength(2); expect(lines.filter((line) => line === "startup after validation")).toHaveLength(2); + fs.rmSync(completion); + fs.mkdirSync(claimDirectory, { mode: 0o700 }); + fs.writeFileSync(claim, "{}\n", { mode: 0o400 }); + execFileSync(entrypoint, argv, { env: environment }); + expect(fs.existsSync(request)).toBe(false); + expect(fs.existsSync(claim)).toBe(false); + expect(fs.existsSync(claimDirectory)).toBe(false); + lines = fs.readFileSync(trace, "utf8").trim().split("\n"); + expect(lines.filter((line) => line.includes("--recover-bootstrap-claim"))).toHaveLength(3); + expect(lines.filter((line) => line.includes("--apply-bootstrap-file"))).toHaveLength(2); + expect(lines.filter((line) => line.startsWith("supervisor:"))).toHaveLength(3); + expect(lines.filter((line) => line === "startup after validation")).toHaveLength(3); + fs.writeFileSync(completion, `${agent}:${fingerprint}:${"c".repeat(64)}\n`); const tamperedRestart = spawnSync(entrypoint, argv, { encoding: "utf8", @@ -764,9 +784,9 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab }); expect(tamperedRestart.status).not.toBe(0); lines = fs.readFileSync(trace, "utf8").trim().split("\n"); - expect(lines.filter((line) => line.includes("--apply-bootstrap-file"))).toHaveLength(1); - expect(lines.filter((line) => line.startsWith("supervisor:"))).toHaveLength(2); - expect(lines.filter((line) => line === "startup after validation")).toHaveLength(2); + expect(lines.filter((line) => line.includes("--apply-bootstrap-file"))).toHaveLength(2); + expect(lines.filter((line) => line.startsWith("supervisor:"))).toHaveLength(3); + expect(lines.filter((line) => line === "startup after validation")).toHaveLength(3); } finally { fs.rmSync(directory, { force: true, recursive: true }); } diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index d4f3f74de5b..adb34e42d82 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -141,6 +141,7 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/managed-bootstrap/docker-test-fixture.ts", "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", + "src/lib/onboard/managed-bootstrap/image-runtime.ts", "src/lib/onboard/managed-bootstrap/index.ts", "src/lib/onboard/managed-bootstrap/runtime-create.ts", ]);