From 8bf16d217c989640d523ace346957716c4136492 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 05:44:44 -0700 Subject: [PATCH 1/4] fix(onboard): claim bootstrap terminal outcomes Reconstruct the net #8077 terminal-outcome slice on current main. Signed-off-by: Aaron Erickson (cherry picked from commit ce6f313e9f7705aa9374c0184710ca995dafdfa8) --- .../docker-gpu-local-inference.test.ts | 21 ++++ src/lib/onboard/docker-gpu-local-inference.ts | 2 +- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 29 ++++- src/lib/onboard/docker-gpu-sandbox-create.ts | 41 ++++--- ...ker-startup-command-sandbox-create.test.ts | 2 +- src/lib/onboard/managed-bootstrap/README.md | 9 +- .../managed-bootstrap/docker-runtime.test.ts | 106 ++++++++++++++++++ .../managed-bootstrap/docker-runtime.ts | 26 ++--- .../managed-bootstrap/runtime-create.test.ts | 46 ++++++++ .../managed-bootstrap/runtime-create.ts | 36 ++++++ src/lib/onboard/sandbox-create-launch.ts | 8 +- 11 files changed, 280 insertions(+), 46 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-runtime.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/runtime-create.test.ts diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 3f0f6180ec4..7e0bd73b992 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -425,6 +425,27 @@ describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => { expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled(); }); + + it("treats a failed commit as terminal without attempting rollback", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(async () => { + throw new Error("durable commit acknowledgement failed"); + }), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await expect( + verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() }, + }, + runtimePatch, + ), + ).rejects.toThrow("durable commit acknowledgement failed"); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + }); }); describe("printDockerGpuSandboxInferenceVerificationFailure", () => { diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 496ee830448..f0999099539 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -512,7 +512,6 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( ): Promise { try { verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); - await runtimePatch.commitAfterReady(); } catch (error) { const failure = error instanceof Error ? error : new Error(String(error)); try { @@ -524,4 +523,5 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( } throw failure; } + await runtimePatch.commitAfterReady(); } diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 6ddea611aa9..cf614528c94 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -159,7 +159,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(onPatchFailureExit).not.toHaveBeenCalled(); - await patch.commitAfterReady(); + await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup"); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( @@ -177,6 +177,33 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { ); }); + it("rejects an early commit after rolling back before supervisor reconnect", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + + await expect(patch.commitAfterReady()).rejects.toThrow( + "cannot commit before the recreated OpenShell supervisor reconnects", + ); + expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledOnce(); + }); + it("rolls back to the backup container and surfaces rolledBack=true diagnostics when supervisorReady=false", () => { const deps = makeDeps(); const result = deferredCreateResult(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 52c15d0b5be..cd7e9aa8af6 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -380,18 +380,15 @@ export function createDockerGpuSandboxCreatePatch( "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", ); const rollbackError = await rollbackAfterFailure(); - onPatchFailureExit( - options.sandboxName, - rollbackError - ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) - : error, - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, - ); - return; + const failure = rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error; + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + throw failure; } if (cutoverFinalization) { if (cutoverFinalizationOutcome !== "commit") { @@ -429,7 +426,7 @@ export function createDockerGpuSandboxCreatePatch( rolledBack: rollbackError === null, }, }); - return; + throw failure; } } const finalizeOutcome = result @@ -437,16 +434,16 @@ export function createDockerGpuSandboxCreatePatch( : null; cutoverFinalized = true; if (!finalizeOutcome || finalizeOutcome.backupRemoved) return; - onPatchFailureExit( - options.sandboxName, - new Error("Managed startup passed Ready, but its rollback backup could not be removed."), - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: failureContext(), - }, + const failure = new Error( + "Managed startup passed Ready, but its rollback backup could not be removed.", ); + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: failureContext(), + }); + throw failure; })(); cutoverFinalization = finalization; cutoverFinalizationOutcome = "commit"; diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index ec0ad3cb306..8ccc7295e87 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -233,7 +233,7 @@ describe("Docker startup-command sandbox creation", () => { rollback, }); - await patch.commitAfterReady(); + await expect(patch.commitAfterReady()).rejects.toThrow("receipt validation failed"); expect(events).toEqual(["commit", "rollback", "exit"]); expect(onPatchFailureExit).toHaveBeenCalledWith( diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 186d8347a4c..b737ab6cb37 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -85,9 +85,12 @@ and sandbox ID and then enter the destructive cutover. Post-cutover rollback publishes `rollback-authorized` before exact replacement deletion; pre-cutover staged cleanup removes only the exact prepared replacement without that journal transition. Commit publishes `shared-state-committed` before exact backup -deletion. Cleanup is bound to full runtime IDs. Its private state root retains -versioned, identity-addressed transaction records containing the provider and -sandbox identities, plan and profile +deletion. Cleanup is bound to full runtime IDs. Commit or rollback is claimed +synchronously before asynchronous finalization begins. Repeated calls for the +claimed outcome share its one pending result, while the opposite outcome remains +invalid even if acknowledgement of the first finalization is lost. Its private +state root retains versioned, identity-addressed transaction records containing +the provider and sandbox identities, plan and profile fingerprints, exact original and replacement IDs, rollback target, and phase. Exact commit and cleanup receipts are durable terminal records, so adapter recreation does not depend on process-local transaction sets or tombstone maps. diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts new file mode 100644 index 00000000000..e9b40f6340b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const adapterMocks = vi.hoisted(() => ({ + activate: vi.fn(), + finalize: vi.fn(), + prepare: vi.fn(), +})); + +vi.mock("./adapter", async (importOriginal) => ({ + ...(await importOriginal()), + activateManagedBootstrapSequence: adapterMocks.activate, + finalizeManagedBootstrapSequence: adapterMocks.finalize, + prepareManagedBootstrapSequence: adapterMocks.prepare, +})); + +import type { + ManagedBootstrapActivatedTransaction, + ManagedBootstrapAdapter, + ManagedBootstrapPreparedTransaction, +} from "./adapter"; +import { createDockerManagedBootstrapSurface } from "./docker-runtime"; +import { authority, IDENTITY, NEW_ID, OLD_ID } from "./docker-test-fixture"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("Docker managed-bootstrap lifecycle composition", () => { + it("does not finalize rollback after a claimed commit loses acknowledgement", async () => { + const seed = authority("openclaw"); + const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction; + const activated = Object.freeze({ + snapshot: { runtimeId: OLD_ID }, + replacement: { replacementRuntimeId: NEW_ID }, + }) as ManagedBootstrapActivatedTransaction; + adapterMocks.prepare.mockImplementation(async (_adapter, input) => { + await input.create.launch({ + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + bootstrapIdentity: IDENTITY, + }); + return prepared; + }); + adapterMocks.activate.mockResolvedValue(activated); + adapterMocks.finalize.mockRejectedValue(new Error("commit acknowledgement lost")); + const onPatchFailure = vi.fn((error: unknown): never => { + throw error; + }); + const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({ + providerId: "docker", + bootstrapIdentity: IDENTITY, + request: seed.request, + image: seed.plan.image, + agentIdentity: seed.plan.agentIdentity, + intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, + expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, + launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + authorityStore: { + recordPreparedAuthority: vi.fn(), + }, + adapterOverride: {} as ManagedBootstrapAdapter, + route: "none", + persistStartupCommand: false, + sandboxName: "alpha", + sandboxGpuConfig: { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + requiredLimits: [], + timeoutSecs: 30, + onPatchFailure, + network: { + inferenceProvider: "openai", + dockerDriverGateway: false, + gatewayPort: 0, + }, + dependencies: {}, + }); + + await expect( + lifecycle.runCreate(async () => ({ value: "launched", receipt: seed.handle.createReceipt })), + ).resolves.toBe("launched"); + const failure = (await Promise.resolve(lifecycle.patch.commitAfterReady()).catch( + (error: unknown) => error, + )) as Error & { managedBootstrapRollbackError?: Error }; + + expect(failure).toBeInstanceOf(Error); + expect(failure.message).toBe("commit acknowledgement lost"); + expect(failure.managedBootstrapRollbackError?.message).toBe( + "Managed bootstrap rollback is no longer legal after commit finalization began.", + ); + expect(adapterMocks.finalize).toHaveBeenCalledOnce(); + expect(adapterMocks.finalize).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ outcome: "commit", transaction: activated }), + ); + expect(onPatchFailure).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index dba07739d18..a05a982ec90 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -31,6 +31,7 @@ import type { ManagedBootstrapRuntimeCreateLifecycleInput, ManagedBootstrapRuntimeOnboardRoutingInput, } from "./runtime-create"; +import { createManagedBootstrapTerminalFinalizer } from "./runtime-create"; type SupportedBootstrapSurface = Extract< RuntimeProviderBootstrapSurface, @@ -191,7 +192,12 @@ function createDockerLifecycle( }); throw new Error("Managed bootstrap did not return its OpenShell create receipt."); } - let finalized = false; + const finalizer = createManagedBootstrapTerminalFinalizer((outcome) => + finalizeManagedBootstrapSequence(adapter, { + outcome, + transaction: activated, + }).then(() => undefined), + ); patch.attachManagedBootstrapCutover({ selectedMode: mode, failureContext: { @@ -201,22 +207,8 @@ function createDockerLifecycle( backupContainerName: null, selectedMode: mode, }, - async rollback() { - if (finalized) return; - await finalizeManagedBootstrapSequence(adapter, { - outcome: "rollback", - transaction: activated, - }); - finalized = true; - }, - async commit() { - if (finalized) return; - await finalizeManagedBootstrapSequence(adapter, { - outcome: "commit", - transaction: activated, - }); - finalized = true; - }, + rollback: finalizer.rollback, + commit: finalizer.commit, }); return launched.value; }, diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.test.ts b/src/lib/onboard/managed-bootstrap/runtime-create.test.ts new file mode 100644 index 00000000000..ad591f19277 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/runtime-create.test.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createManagedBootstrapTerminalFinalizer } from "./runtime-create"; + +describe("managed bootstrap terminal finalizer", () => { + it("shares one in-flight outcome and rejects an opposite concurrent outcome", async () => { + let release = (): void => {}; + const finalize = vi.fn( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const finalizer = createManagedBootstrapTerminalFinalizer(finalize); + + const firstCommit = finalizer.commit(); + const duplicateCommit = finalizer.commit(); + + expect(duplicateCommit).toBe(firstCommit); + await expect(finalizer.rollback()).rejects.toThrow( + "rollback is no longer legal after commit finalization began", + ); + release(); + await expect(Promise.all([firstCommit, duplicateCommit])).resolves.toEqual([ + undefined, + undefined, + ]); + expect(finalize).toHaveBeenCalledExactlyOnceWith("commit"); + }); + + it("retains the claimed outcome after a lost finalization acknowledgement", async () => { + const finalize = vi.fn(async () => { + throw new Error("commit acknowledgement lost"); + }); + const finalizer = createManagedBootstrapTerminalFinalizer(finalize); + + await expect(finalizer.commit()).rejects.toThrow("commit acknowledgement lost"); + await expect(finalizer.rollback()).rejects.toThrow( + "rollback is no longer legal after commit finalization began", + ); + expect(finalize).toHaveBeenCalledExactlyOnceWith("commit"); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 6ffcc028966..73efeac0cbb 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -90,6 +90,42 @@ export interface ManagedBootstrapRuntimeCreateLaunchResult { readonly receipt: ManagedBootstrapCreateReceipt; } +export type ManagedBootstrapTerminalOutcome = "commit" | "rollback"; + +export interface ManagedBootstrapTerminalFinalizer { + commit(): Promise; + rollback(): Promise; +} + +/** + * Claim one terminal outcome before driver finalization starts. Duplicate calls + * for that outcome share the in-flight promise; the opposite outcome fails + * closed even when finalization loses acknowledgement. + */ +export function createManagedBootstrapTerminalFinalizer( + finalize: (outcome: ManagedBootstrapTerminalOutcome) => Promise, +): ManagedBootstrapTerminalFinalizer { + let claimedOutcome: ManagedBootstrapTerminalOutcome | null = null; + let pending: Promise | null = null; + const run = (outcome: ManagedBootstrapTerminalOutcome): Promise => { + if (claimedOutcome === outcome && pending !== null) return pending; + if (claimedOutcome !== null) { + return Promise.reject( + new Error( + `Managed bootstrap ${outcome} is no longer legal after ${claimedOutcome} finalization began.`, + ), + ); + } + claimedOutcome = outcome; + pending = Promise.resolve().then(() => finalize(outcome)); + return pending; + }; + return Object.freeze({ + commit: () => run("commit"), + rollback: () => run("rollback"), + }); +} + export interface ManagedBootstrapRuntimeCreateLifecycle { readonly launchArgv: readonly string[]; readonly patch: ManagedBootstrapRuntimePatch; diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index f6f7b91f7e3..fe5079b9b94 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -63,7 +63,13 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; - /** Dormant until a complete runtime bundle and durable authority store are selected. */ + /** + * Intentional partial migration: remains unset until production selects a + * complete runtime bundle with supported bootstrap after epic #7744's durable + * lifecycle, recovery, and rollback gates plus exact-head/base protected + * all-agent amd64/arm64, GPU/local-inference, and regression matrix pass. + * https://github.com/NVIDIA/NemoClaw/issues/7744 + */ managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } From 897408f74dcc055e7e9d263985931cf664a1c439 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 05:50:32 -0700 Subject: [PATCH 2/4] fix(onboard): preserve shared-state commit authority Reconstruct the net #8078 shared-state authority slice on current main. Signed-off-by: Aaron Erickson (cherry picked from commit 31236f767aa79c9e110be55bc1bf56b5396b227a) --- src/lib/onboard/managed-bootstrap/README.md | 7 + ...er-shared-state-rollback-authority.test.ts | 279 ++++++++++++++++++ .../managed-bootstrap/docker-shared-state.ts | 27 +- ...d-startup-shared-state-transaction.test.ts | 89 +++++- .../shared-state-transaction.ts | 49 ++- 5 files changed, 423 insertions(+), 28 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index b737ab6cb37..437164f324c 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -105,6 +105,13 @@ commit atomically moves its pending manifest and backups into a durable receipt namespace, compacts that state to an exact commit receipt, and rejects rollback after a restart. The provider may retire that receipt only after it proves the external rollback backup is gone, leaving the next bootstrap attempt unblocked. +The parser accepts the exact canonical schema-v1 manifest written before +`bootstrapIdentity` was added only for the legacy null-identity path. It rejects +additional fields, missing historical fields, and legacy state presented as +identity-bound authority. Before rollback, the Docker adapter stops the +replacement and copies its writable-layer commit receipt to a protected host +path for verification. The immutable helper cannot obtain that receipt through +`--volumes-from`, which exposes volumes but not the replacement writable layer. Direct identity lookup reconstructs one known transaction record, while managed create-lifecycle startup uses unfinished-record enumeration to ask the selected provider to reconcile every identity-addressed record before a new sandbox diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts new file mode 100644 index 00000000000..34eca762bcb --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts @@ -0,0 +1,279 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "../managed-startup/shared-state-transaction"; +import { + type DockerManagedBootstrapSharedStateTransaction, + finalizeDockerManagedStartupSharedState, +} from "./docker-shared-state"; + +const CONTAINER_ID = "c".repeat(64); +const TRANSACTION: DockerManagedBootstrapSharedStateTransaction = { + agent: "openclaw", + bootstrapIdentity: "b".repeat(64), + containerId: CONTAINER_ID, + image: `sha256:${"a".repeat(64)}`, + profileFingerprint: "d".repeat(64), +}; + +interface SharedStateFixture { + readonly commands: readonly (readonly string[])[]; + readonly deps: DockerGpuPatchDeps; + readonly events: readonly string[]; + readonly state: () => "committed" | "none" | "pending"; +} + +interface SharedStateFixtureOptions { + readonly stateAfterCommitFailure?: "committed" | "none" | "pending"; + readonly stateAfterStop?: "committed" | "none" | "pending"; +} + +const copiedReceiptPaths: string[] = []; + +afterEach(() => { + vi.restoreAllMocks(); + for (const receiptPath of copiedReceiptPaths.splice(0)) { + fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true }); + } +}); + +function fixture( + initialState: "committed" | "none" | "pending", + options: SharedStateFixtureOptions = {}, +): SharedStateFixture { + let state = initialState; + const commands: string[][] = []; + const events: string[] = []; + const copyPresentReceipt = (destination: string) => { + fs.mkdirSync(destination, { recursive: true }); + copiedReceiptPaths.push(destination); + return { status: 0 }; + }; + const copyMissingReceipt = (sourcePath: string) => ({ + status: 1, + stderr: `Error response from daemon: Could not find the file ${sourcePath} in container ${CONTAINER_ID}`, + }); + const dockerRun = vi.fn((args: readonly string[]) => { + commands.push([...args]); + switch (args[0]) { + case "cp": { + const source = String(args[2] ?? ""); + const destination = String(args[3] ?? ""); + const sourcePath = source.slice(`${CONTAINER_ID}:`.length); + const present = + (sourcePath === MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY && + state === "committed") || + (sourcePath === MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY && state === "pending"); + events.push(`copy:${path.basename(sourcePath)}:${present ? "present" : "absent"}`); + return present ? copyPresentReceipt(destination) : copyMissingReceipt(sourcePath); + } + case "run": { + const action = args.includes("--shared-state-transaction-status") + ? "status" + : args.includes("--rollback-shared-state-transaction") + ? "rollback" + : "unexpected"; + switch (action) { + case "status": + events.push(`status:${state}`); + return { status: 0, stdout: `${state}\n` }; + case "rollback": + events.push("rollback"); + state = "none"; + return { status: 0 }; + default: + throw new Error(`Unexpected Docker command: ${args.join(" ")}`); + } + } + case "exec": + switch (args.includes("--commit-shared-state-transaction")) { + case true: + events.push("commit:failed"); + state = options.stateAfterCommitFailure ?? state; + return { status: 1, stderr: "commit helper failed" }; + default: + throw new Error("Unexpected Docker commit command"); + } + default: + throw new Error(`Unexpected Docker command: ${args.join(" ")}`); + } + }); + return { + commands, + deps: { + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRun, + dockerStop: vi.fn(() => { + events.push("stop"); + state = options.stateAfterStop ?? state; + return { status: 0 }; + }), + }, + events, + state: () => state, + }; +} + +describe("Docker managed-bootstrap shared-state rollback authority", () => { + it("copies and verifies writable-layer commit authority before rollback", () => { + const fake = fixture("committed"); + + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: false }, + fake.deps, + ), + ).toThrow(/durably committed and cannot be rolled back/u); + + expect(fake.events).toEqual([ + "stop", + `copy:${path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)}:present`, + "status:committed", + ]); + const statusCommand = fake.commands.find((args) => + args.includes("--shared-state-transaction-status"), + ); + expect(statusCommand).toContainEqual( + expect.stringMatching( + new RegExp( + `^type=bind,src=.+,dst=${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY},readonly$`, + "u", + ), + ), + ); + expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe( + false, + ); + }); + + it("proves pending authority after quiescence before starting the rollback helper", () => { + const fake = fixture("pending"); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: false }, + fake.deps, + ), + ).toEqual({ supervisorReady: false, failure: null }); + + expect(fake.events[0]).toBe("stop"); + expect(fake.events.indexOf("status:pending")).toBeLessThan(fake.events.indexOf("rollback")); + expect(fake.state()).toBe("none"); + const rollbackCommand = fake.commands.find((args) => + args.includes("--rollback-shared-state-transaction"), + ); + expect(rollbackCommand).toContainEqual( + expect.stringMatching( + new RegExp( + `^type=bind,src=.+,dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly$`, + "u", + ), + ), + ); + }); + + it("removes the exact failed container without rollback when both receipts are absent", () => { + const fake = fixture("none"); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: false }, + fake.deps, + ), + ).toEqual({ supervisorReady: false, failure: null }); + + expect(fake.events).toEqual([ + "stop", + `copy:${path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)}:absent`, + `copy:${path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY)}:absent`, + ]); + expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe( + false, + ); + expect(fake.deps.dockerRm).toHaveBeenCalledTimes(1); + expect(fake.deps.dockerRm).toHaveBeenCalledWith(CONTAINER_ID, expect.any(Object)); + expect(fake.state()).toBe("none"); + }); + + it("reuses one preserved pending receipt when commit validation fails", () => { + const fake = fixture("pending", { stateAfterCommitFailure: "none" }); + + const outcome = finalizeDockerManagedStartupSharedState( + { + retainContainerAfterRollback: true, + transaction: TRANSACTION, + supervisorReady: true, + }, + fake.deps, + ); + + expect(outcome.supervisorReady).toBe(false); + expect(outcome.failure).toEqual( + expect.objectContaining({ + message: expect.stringContaining("commit helper failed"), + }), + ); + const pendingSource = CONTAINER_ID + ":" + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY; + const pendingCopies = fake.commands.filter( + (args) => args[0] === "cp" && args[2] === pendingSource, + ); + expect( + fake.events.filter( + (event) => + event === + "copy:" + path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY) + ":present", + ), + ).toHaveLength(1); + const preservedReceiptPath = String(pendingCopies[0]?.[3] ?? ""); + const rollbackCommand = fake.commands.find((args) => + args.includes("--rollback-shared-state-transaction"), + ); + expect(rollbackCommand).toContain( + "type=bind,src=" + + preservedReceiptPath + + ",dst=" + + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY + + ",readonly", + ); + expect(fake.deps.dockerRm).not.toHaveBeenCalled(); + }); + + it("rejects rollback when a failed commit becomes durable during quiescence", () => { + const fake = fixture("pending", { + stateAfterCommitFailure: "none", + stateAfterStop: "committed", + }); + + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: true }, + fake.deps, + ), + ).toThrow(/durably committed and cannot be rolled back/u); + + expect( + fake.events.filter( + (event) => + event === + "copy:" + path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY) + ":present", + ), + ).toHaveLength(1); + expect(fake.events).toContain("commit:failed"); + expect(fake.events).toContain("status:committed"); + expect(fake.events).not.toContain("rollback"); + expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe( + false, + ); + expect(fake.deps.dockerRm).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index a92d2e335ed..2fcd8595802 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -461,10 +461,21 @@ function copyManagedStartupReceipt( function rollbackManagedStartupSharedState( transaction: DockerManagedBootstrapSharedStateTransaction, - receiptPath: string, deps: DockerGpuPatchDeps, -): void { + preservedReceiptPath?: string, +): boolean { const dockerRun = deps.dockerRun ?? defaultDockerRun; + const status = probeDockerManagedStartupSharedState( + { transaction, profileFingerprint: transaction.profileFingerprint }, + deps, + ); + if (status === "committed") { + throw new Error("Managed-startup shared state is durably committed and cannot be rolled back."); + } + const receiptPath = + preservedReceiptPath ?? + (status === "pending" ? copyManagedStartupReceipt(transaction, deps) : null); + if (!receiptPath) return false; let restored = false; try { // The immutable image owns the canonical receipt parser and exact @@ -519,6 +530,7 @@ function rollbackManagedStartupSharedState( cleanupReceiptBestEffort(receiptPath); } } + return true; } function removeFailedUnbackedContainer( @@ -616,7 +628,7 @@ export function finalizeDockerManagedStartupSharedState( { cause: stopError }, ); } - rollbackManagedStartupSharedState(transaction, receiptPath, deps); + rollbackManagedStartupSharedState(transaction, deps, receiptPath); if (!input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } @@ -624,14 +636,7 @@ export function finalizeDockerManagedStartupSharedState( } quiesceManagedStartupContainer(transaction, deps); - const receiptPath = copyManagedStartupReceipt(transaction, deps, true); - if (!receiptPath) { - if (!input.patchResult && !input.retainContainerAfterRollback) { - removeFailedUnbackedContainer(transaction, deps); - } - return { supervisorReady: false, failure: null }; - } - rollbackManagedStartupSharedState(transaction, receiptPath, deps); + rollbackManagedStartupSharedState(transaction, deps); if (!input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 6ec1563e73e..90f283e1a09 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -78,6 +78,20 @@ describe("managed startup shared-state transaction", () => { ); } + function rewriteManifest( + rewrite: (manifest: Record) => Record = (manifest) => + manifest, + ): void { + const manifestFile = path.join(transactionDirectory, "manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestFile, "utf8")) as Record; + expect(manifest.bootstrapIdentity).toBeNull(); + delete manifest.bootstrapIdentity; + const rewritten = rewrite(manifest); + fs.chmodSync(manifestFile, 0o600); + fs.writeFileSync(manifestFile, `${JSON.stringify(rewritten, null, 2)}\n`); + fs.chmodSync(manifestFile, 0o400); + } + it.each([ "openclaw", "hermes", @@ -257,6 +271,70 @@ describe("managed startup shared-state transaction", () => { expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); }); + it.each([ + ["commits", "commit"], + ["rolls back", "rollback"], + ] as const)("%s an exact historical schema-v1 manifest without bootstrap identity", (_description, action) => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, "before\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + rewriteManifest(); + fs.writeFileSync(config, "after\n"); + + const result = + action === "commit" + ? commitManagedStartupSharedStateTransaction("openclaw", options) + : rollbackManagedStartupSharedStateTransaction("openclaw", options); + + expect(result).toBe(true); + expect(fs.readFileSync(config, "utf8")).toBe(action === "commit" ? "after\n" : "before\n"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(fs.existsSync(commitReceiptDirectory())).toBe(false); + }); + + it.each([ + ["an extra field", (manifest: Record) => ({ ...manifest, extra: true })], + [ + "a missing historical field", + (manifest: Record) => { + delete manifest.directories; + return manifest; + }, + ], + ] as const)("rejects a schema-v1 legacy manifest with %s", (_case, rewrite) => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + rewriteManifest(rewrite); + + expect(() => commitManagedStartupSharedStateTransaction("openclaw", options)).toThrow( + /unexpected fields/u, + ); + expect(fs.existsSync(transactionDirectory)).toBe(true); + }); + + it("does not treat a legacy manifest as authority for an identity-bound bootstrap", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + const boundOptions = { ...options, bootstrapIdentity: "b".repeat(64) }; + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), boundOptions); + const manifestFile = path.join(transactionDirectory, "manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestFile, "utf8")) as Record; + delete manifest.bootstrapIdentity; + fs.chmodSync(manifestFile, 0o600); + fs.writeFileSync(manifestFile, `${JSON.stringify(manifest, null, 2)}\n`); + fs.chmodSync(manifestFile, 0o400); + + expect(() => rollbackManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /different bootstrap attempt/u, + ); + expect(fs.existsSync(transactionDirectory)).toBe(true); + }); + it("fsyncs every transaction namespace before exposing a pending receipt", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); @@ -403,10 +481,13 @@ describe("managed startup shared-state transaction", () => { throw new Error("injected post-rename cleanup interruption"); })() : originalRmSync(target, removeOptions)) as typeof fs.rmSync); - expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( - /injected post-rename cleanup interruption/u, - ); - rm.mockRestore(); + try { + expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /injected post-rename cleanup interruption/u, + ); + } finally { + rm.mockRestore(); + } expect(fs.existsSync(transactionDirectory)).toBe(false); const committedDirectory = commitReceiptDirectory(); diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts index 62894abeb5c..fd83d10c0c8 100644 --- a/src/lib/onboard/managed-startup/shared-state-transaction.ts +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -538,6 +538,20 @@ function canonicalManifest(manifest: TransactionManifest): string { return `${JSON.stringify(manifest, null, 2)}\n`; } +function canonicalLegacyManifest(manifest: TransactionManifest): string { + return `${JSON.stringify( + { + schemaVersion: manifest.schemaVersion, + agent: manifest.agent, + profileFingerprint: manifest.profileFingerprint, + files: manifest.files, + directories: manifest.directories, + }, + null, + 2, + )}\n`; +} + function canonicalCommitReceipt(receipt: CommitReceipt): string { return `${JSON.stringify(receipt, null, 2)}\n`; } @@ -597,23 +611,29 @@ function parseManifest(text: string): TransactionManifest { fail("transaction manifest must be an object"); } const record = parsed as Record; - requireExactKeys(record, [ - "agent", - "bootstrapIdentity", - "directories", - "files", - "profileFingerprint", - "schemaVersion", - ]); + const hasBootstrapIdentity = Object.hasOwn(record, "bootstrapIdentity"); + requireExactKeys( + record, + hasBootstrapIdentity + ? [ + "agent", + "bootstrapIdentity", + "directories", + "files", + "profileFingerprint", + "schemaVersion", + ] + : ["agent", "directories", "files", "profileFingerprint", "schemaVersion"], + ); + const bootstrapIdentity = hasBootstrapIdentity ? record.bootstrapIdentity : null; if ( record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || typeof record.profileFingerprint !== "string" || !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || !( - record.bootstrapIdentity === null || - (typeof record.bootstrapIdentity === "string" && - /^[a-f0-9]{64}$/u.test(record.bootstrapIdentity)) + bootstrapIdentity === null || + (typeof bootstrapIdentity === "string" && /^[a-f0-9]{64}$/u.test(bootstrapIdentity)) ) || !Array.isArray(record.files) || !Array.isArray(record.directories) || @@ -709,11 +729,14 @@ function parseManifest(text: string): TransactionManifest { schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: record.agent as ManagedStartupAgent, profileFingerprint: record.profileFingerprint, - bootstrapIdentity: record.bootstrapIdentity as string | null, + bootstrapIdentity, files, directories, }; - if (canonicalManifest(manifest) !== text) { + const canonical = hasBootstrapIdentity + ? canonicalManifest(manifest) + : canonicalLegacyManifest(manifest); + if (canonical !== text) { fail("transaction manifest is not canonical"); } return manifest; From 08c0d3bbc58157a165e0492352bef44586b3db08 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 06:00:45 -0700 Subject: [PATCH 3/4] fix(onboard): preserve durable journal compatibility Reconstruct the net #8080 journal-compatibility slice on current main. Signed-off-by: Aaron Erickson (cherry picked from commit c52370db1119ec1b8f3365a0ce4c22beebdd28e4) --- .../onboard/managed-bootstrap/adapter.test.ts | 43 ++++++++++++++++ src/lib/onboard/managed-bootstrap/adapter.ts | 16 ++++++ .../managed-bootstrap/docker-journal.test.ts | 51 +++++++++++++------ .../managed-bootstrap/docker-journal.ts | 26 ---------- .../managed-bootstrap/docker-runtime.test.ts | 2 +- .../docker-shared-state.test.ts | 7 +-- .../managed-bootstrap/docker-test-fixture.ts | 4 +- .../onboard/managed-bootstrap/docker.test.ts | 25 +++++++++ src/lib/onboard/managed-bootstrap/docker.ts | 24 +++++---- src/lib/onboard/managed-bootstrap/index.ts | 2 + .../managed-bootstrap-test-fixture.ts | 6 +++ test/runtime-provider-source-shape.test.ts | 1 + 12 files changed, 150 insertions(+), 57 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index c076af031cc..fa21399c97a 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -20,6 +20,7 @@ import { type ManagedBootstrapAuthorityStore, type ManagedBootstrapCompletionReceipt, type ManagedBootstrapCreateReceipt, + type ManagedBootstrapDurablePreparationReceipt, type ManagedBootstrapFinalizationReceipt, type ManagedBootstrapHeldWorkloadHandle, type ManagedBootstrapObservedSnapshot, @@ -29,7 +30,10 @@ import { prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, renderManagedBootstrapHeldCommand, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; +import { reverseKeys } from "./managed-bootstrap-test-fixture"; const IDENTITY = "1".repeat(64); const CONFIG_ID = `sha256:${"2".repeat(64)}`; @@ -356,6 +360,45 @@ async function captureFailure(promise: Promise) { } describe("managed bootstrap adapter contract", () => { + it("compares provider-neutral durable receipts by canonical value", () => { + const handle = handleFor(requestFor("hermes")); + const preparation: ManagedBootstrapDurablePreparationReceipt = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: "a".repeat(64), + recordId: "mxc-durable-authority", + recordedAt: "2026-07-29T12:00:30.000Z", + }; + const reorderedPreparation = reverseKeys({ + ...preparation, + sandbox: reverseKeys({ ...preparation.sandbox }), + }); + expect(sameManagedBootstrapDurablePreparationReceipt(preparation, reorderedPreparation)).toBe( + true, + ); + expect( + sameManagedBootstrapDurablePreparationReceipt(preparation, { + ...reorderedPreparation, + recordId: "changed-authority", + }), + ).toBe(false); + + const completion = completionFor(requestFor("hermes"), handle); + const reorderedCompletion = reverseKeys({ + ...completion, + image: reverseKeys({ ...completion.image }), + sandbox: reverseKeys({ ...completion.sandbox }), + }); + expect(sameManagedBootstrapCompletionReceipt(completion, reorderedCompletion)).toBe(true); + expect( + sameManagedBootstrapCompletionReceipt(completion, { + ...reorderedCompletion, + transactionPending: false, + }), + ).toBe(false); + }); + it.each( MANAGED_STARTUP_AGENTS, )("prepares, durably records, and only then activates %s through a provider-neutral adapter", async (agent) => { diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 19d0651881f..637a2f993e8 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -905,6 +905,22 @@ function canonicalJson(value: unknown): string { .join(",")}}`; } +/** Compare durable provider receipts by canonical value, independent of object key order. */ +export function sameManagedBootstrapDurablePreparationReceipt( + left: ManagedBootstrapDurablePreparationReceipt, + right: ManagedBootstrapDurablePreparationReceipt, +): boolean { + return canonicalJson(left) === canonicalJson(right); +} + +/** Compare completion receipts by canonical value, independent of object key order. */ +export function sameManagedBootstrapCompletionReceipt( + left: ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapCompletionReceipt, +): boolean { + return canonicalJson(left) === canonicalJson(right); +} + export function assertManagedBootstrapIdentity(value: string): void { if (!SHA256_RE.test(value)) { protocolFail("identity must be 32 random bytes encoded as lowercase hex"); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 457e9b4e154..087e188cb58 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -7,6 +7,10 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, +} from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, @@ -20,10 +24,10 @@ import { normalizeDockerManagedBootstrapJournal, parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, - sameDockerManagedBootstrapReceipt, serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; +import { reverseKeys } from "./managed-bootstrap-test-fixture"; const roots: string[] = []; const IDENTITY = "1".repeat(64); @@ -366,9 +370,9 @@ describe("Docker managed bootstrap journal", () => { }, schemaVersion: preparation.schemaVersion, } satisfies typeof preparation; - expect( - sameDockerManagedBootstrapReceipt("preparation", preparation, reorderedPreparation), - ).toBe(true); + expect(sameManagedBootstrapDurablePreparationReceipt(preparation, reorderedPreparation)).toBe( + true, + ); const completion = finalization.commitReceipt; const reorderedCompletion = { @@ -391,9 +395,7 @@ describe("Docker managed bootstrap journal", () => { }, schemaVersion: completion.schemaVersion, } satisfies typeof completion; - expect(sameDockerManagedBootstrapReceipt("completion", completion, reorderedCompletion)).toBe( - true, - ); + expect(sameManagedBootstrapCompletionReceipt(completion, reorderedCompletion)).toBe(true); }); it("reloads exact terminal receipts from a new journal store", () => { @@ -440,15 +442,11 @@ describe("Docker managed bootstrap journal", () => { const store = createFileDockerManagedBootstrapJournalStore(root); store.create(journal); const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); - fs.writeFileSync( - path.join(directory, `.${IDENTITY}.json${suffix}.1234.deadbeef.tmp`), - "partial", - { - mode: 0o600, - }, - ); + const target = path.join(directory, `.${IDENTITY}.json${suffix}.1234.deadbeef.tmp`); + fs.writeFileSync(target, "partial", { mode: 0o600 }); expect(loadUnfinished(store)).toEqual([journal]); + expect(fs.existsSync(target)).toBe(true); }); it("rejects an unsupported journal-directory entry during enumeration", () => { @@ -466,6 +464,24 @@ describe("Docker managed bootstrap journal", () => { ); }); + it.each([ + `.${IDENTITY}.json.commit.123.a0.tmp`, + `.${IDENTITY}.json.decision.pid.a0.tmp`, + `.${IDENTITY}.json.finalized.123.A0.tmp`, + `${IDENTITY}.json.decision.123.a0.tmp`, + `.${IDENTITY}.json.decision.123.a0.tmp.extra`, + ])("rejects and retains near-miss atomic entry %s", (name) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(loadUnfinished(store)).toEqual([]); + const target = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, name); + fs.writeFileSync(target, "near miss\n", { mode: 0o600 }); + + expect(() => store.listUnfinishedIdentities()).toThrow("unsupported entry"); + expect(fs.existsSync(target)).toBe(true); + }); + it("reloads the exact completion receipt from a new journal store", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); @@ -476,7 +492,12 @@ describe("Docker managed bootstrap journal", () => { expect(completed.commitReceipt).toEqual(finalization.commitReceipt); const restarted = createFileDockerManagedBootstrapJournalStore(root); - expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + const reorderedReceipt = reverseKeys({ + ...finalization.commitReceipt, + image: reverseKeys({ ...finalization.commitReceipt.image }), + sandbox: reverseKeys({ ...finalization.commitReceipt.sandbox }), + }); + expect(restarted.recordCompletion(IDENTITY, reorderedReceipt)).toEqual(completed); expect(loadUnfinished(restarted)).toEqual([completed]); expect(() => restarted.recordCompletion(IDENTITY, { diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 47f1022e884..d0e1f4eaee0 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -740,32 +740,6 @@ function exactCompletionReceipt(value: unknown): ManagedBootstrapCompletionRecei }); } -export function sameDockerManagedBootstrapReceipt( - kind: "preparation", - left: ManagedBootstrapDurablePreparationReceipt, - right: ManagedBootstrapDurablePreparationReceipt, -): boolean; -export function sameDockerManagedBootstrapReceipt( - kind: "completion", - left: ManagedBootstrapCompletionReceipt, - right: ManagedBootstrapCompletionReceipt, -): boolean; -export function sameDockerManagedBootstrapReceipt( - kind: "preparation" | "completion", - left: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, - right: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, -): boolean { - if (kind === "preparation") { - return ( - JSON.stringify(exactPreparationReceipt(left)) === - JSON.stringify(exactPreparationReceipt(right)) - ); - } - return ( - JSON.stringify(exactCompletionReceipt(left)) === JSON.stringify(exactCompletionReceipt(right)) - ); -} - function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceipt { if (typeof value !== "object" || value === null || Array.isArray(value)) { fail("cleanup receipt must be an object"); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts index e9b40f6340b..954de3a49c0 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -78,7 +78,7 @@ describe("Docker managed-bootstrap lifecycle composition", () => { onPatchFailure, network: { inferenceProvider: "openai", - dockerDriverGateway: false, + gatewayUsesContainerBridge: false, gatewayPort: 0, }, dependencies: {}, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts index ae965865eb7..02db650a396 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts @@ -112,9 +112,10 @@ describe("Docker managed-bootstrap shared-state helper environment", () => { expect(outcome).toEqual({ supervisorReady: false, failure: null }); const helpers = nodeHelperCalls(fake.deps); - expect(helpers).toHaveLength(1); - expect(helpers[0]).toContain("--rollback-shared-state-transaction"); - expectCleanRunNodeHelper(helpers[0]!); + expect(helpers).toHaveLength(2); + expect(helpers.some((args) => args.includes("--shared-state-transaction-status"))).toBe(true); + expect(helpers.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe(true); + helpers.forEach(expectCleanRunNodeHelper); }); it("clears arbitrary container environment before the durable receipt-clear helper", () => { diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 7c0f52b99c7..6e52d0c6a4d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -18,6 +18,7 @@ import { type ManagedBootstrapObservedSnapshot, type ManagedBootstrapPreparedReplacementHandle, type ManagedBootstrapReplacementHandle, + sameManagedBootstrapCompletionReceipt, } from "./adapter"; import type { DockerManagedBootstrapDeps } from "./docker"; import { @@ -26,7 +27,6 @@ import { DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalPhase, type DockerManagedBootstrapJournalStore, - sameDockerManagedBootstrapReceipt, serializeDockerManagedBootstrapFinalizationRecord, } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; @@ -277,7 +277,7 @@ export function fixture(options: DockerFixtureOptions = {}) { } if ( journal.commitReceipt !== null && - !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, receipt) + !sameManagedBootstrapCompletionReceipt(journal.commitReceipt, receipt) ) { throw new Error("completion changed"); } diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index cf1e1e63fdd..a72236b3b41 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -377,6 +377,31 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.replacement).toBeNull(); }); + it("rejects a divergent snapshot image before creating durable recovery state", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot: { + ...snapshot, + image: { + ...snapshot.image, + repository: "registry.example/nemoclaw/divergent", + }, + }, + request: rootRequest, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow("replacement snapshot image does not match its plan"); + expect(fake.replacement).toBeNull(); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.events).not.toContain("journal:staged"); + }); + it.each( SUPPORTED_AGENTS, )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 8dc146f1df7..817263d729c 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -65,6 +65,8 @@ import { type ManagedBootstrapReplacementOptions, type ManagedBootstrapSandboxIdentity, renderManagedBootstrapHeldCommand, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, @@ -77,7 +79,6 @@ import { type DockerManagedBootstrapJournalStore, DockerManagedBootstrapLegacyRecordRequiresAgentError, parseDockerManagedBootstrapJournal, - sameDockerManagedBootstrapReceipt, serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; @@ -1629,8 +1630,7 @@ function assertDockerBootstrapTransactionAuthority( (durablePreparation !== undefined && durablePreparation !== null && (transaction.preparationReceipt === null || - !sameDockerManagedBootstrapReceipt( - "preparation", + !sameManagedBootstrapDurablePreparationReceipt( transaction.preparationReceipt, durablePreparation, ))) || @@ -2042,11 +2042,7 @@ export function createDockerManagedBootstrapAdapter( journal.phase === "shared-state-committed" && finalization.commitReceipt !== null && journal.commitReceipt !== null && - sameDockerManagedBootstrapReceipt( - "completion", - finalization.commitReceipt, - journal.commitReceipt, - )) || + sameManagedBootstrapCompletionReceipt(finalization.commitReceipt, journal.commitReceipt)) || (finalization.phase === "rolled-back" && (journal.phase === "staged" || journal.phase === "rollback-authorized" || @@ -2800,7 +2796,7 @@ export function createDockerManagedBootstrapAdapter( if ( finalized.phase !== "committed" || !finalized.commitReceipt || - !sameDockerManagedBootstrapReceipt("completion", finalized.commitReceipt, completion) + !sameManagedBootstrapCompletionReceipt(finalized.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, @@ -2880,7 +2876,7 @@ export function createDockerManagedBootstrapAdapter( ); if ( journal.commitReceipt === null || - !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, completion) + !sameManagedBootstrapCompletionReceipt(journal.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: journal.bootstrapIdentity, @@ -3155,6 +3151,14 @@ export function createDockerManagedBootstrapAdapter( ) { throw new Error("Managed bootstrap Docker replacement identities do not match."); } + if ( + snapshot.image.repository !== handle.plan.image.repository || + snapshot.image.manifestDigest !== handle.plan.image.manifestDigest + ) { + throw new Error( + "Managed bootstrap Docker replacement snapshot image does not match its plan.", + ); + } const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); if (normalizedOriginal.hash !== snapshot.specHash) { diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 1c6378642e4..1e1f2c67fd9 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -17,6 +17,8 @@ export { type ManagedBootstrapRecoveryReport, prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; export { MANAGED_BOOTSTRAP_COMPLETION_FILE, diff --git a/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts b/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts new file mode 100644 index 00000000000..c4eb9cfb957 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function reverseKeys(value: T): T { + return Object.fromEntries(Object.entries(value).reverse()) as T; +} diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 7bcfd06ec4d..abd9e01f2da 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -143,6 +143,7 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/image-runtime.ts", "src/lib/onboard/managed-bootstrap/index.ts", + "src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts", "src/lib/onboard/managed-bootstrap/runtime-create.ts", ]); }); From b3973cebb50d1841dda57b2b883dcfd1795beff6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 07:05:15 -0700 Subject: [PATCH 4/4] fix(onboard): retain failed terminal authority Signed-off-by: Aaron Erickson --- .../onboard/docker-gpu-sandbox-create-lifecycle.test.ts | 4 ++++ src/lib/onboard/docker-gpu-sandbox-create.ts | 5 +++++ .../docker-shared-state-rollback-authority.test.ts | 2 +- src/lib/onboard/managed-bootstrap/docker-shared-state.ts | 8 ++++---- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index cf614528c94..0206ce9a80d 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -159,6 +159,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(onPatchFailureExit).not.toHaveBeenCalled(); + await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup"); await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup"); expect(onPatchFailureExit).toHaveBeenCalledOnce(); @@ -197,6 +198,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); + await expect(patch.commitAfterReady()).rejects.toThrow( + "cannot commit before the recreated OpenShell supervisor reconnects", + ); await expect(patch.commitAfterReady()).rejects.toThrow( "cannot commit before the recreated OpenShell supervisor reconnects", ); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index cd7e9aa8af6..b1e566785fc 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -147,6 +147,7 @@ export function createDockerGpuSandboxCreatePatch( let cutoverFinalized = false; let cutoverFinalization: Promise | null = null; let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; + let cutoverFinalizationFailure: Error | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -374,6 +375,7 @@ export function createDockerGpuSandboxCreatePatch( }, async commitAfterReady() { + if (cutoverFinalizationFailure) throw cutoverFinalizationFailure; if (cutoverFinalized || (!managedBootstrapCutover && !result)) return; if (needsSupervisorWait) { const error = new Error( @@ -383,6 +385,7 @@ export function createDockerGpuSandboxCreatePatch( const failure = rollbackError ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) : error; + cutoverFinalizationFailure = failure; onPatchFailureExit(options.sandboxName, failure, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -417,6 +420,7 @@ export function createDockerGpuSandboxCreatePatch( failure as Error & { managedBootstrapRollbackError?: unknown } ).managedBootstrapRollbackError = rollbackError; } + cutoverFinalizationFailure = failure; onPatchFailureExit(options.sandboxName, failure, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -437,6 +441,7 @@ export function createDockerGpuSandboxCreatePatch( const failure = new Error( "Managed startup passed Ready, but its rollback backup could not be removed.", ); + cutoverFinalizationFailure = failure; onPatchFailureExit(options.sandboxName, failure, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts index 34eca762bcb..2e88d215f1b 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts @@ -180,6 +180,7 @@ describe("Docker managed-bootstrap shared-state rollback authority", () => { ), ), ); + expect(fake.deps.dockerRm).not.toHaveBeenCalled(); }); it("removes the exact failed container without rollback when both receipts are absent", () => { @@ -210,7 +211,6 @@ describe("Docker managed-bootstrap shared-state rollback authority", () => { const outcome = finalizeDockerManagedStartupSharedState( { - retainContainerAfterRollback: true, transaction: TRANSACTION, supervisorReady: true, }, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index 2fcd8595802..6d616a9dc07 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -628,16 +628,16 @@ export function finalizeDockerManagedStartupSharedState( { cause: stopError }, ); } - rollbackManagedStartupSharedState(transaction, deps, receiptPath); - if (!input.patchResult && !input.retainContainerAfterRollback) { + const restoredSharedState = rollbackManagedStartupSharedState(transaction, deps, receiptPath); + if (!restoredSharedState && !input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } return { supervisorReady: false, failure }; } quiesceManagedStartupContainer(transaction, deps); - rollbackManagedStartupSharedState(transaction, deps); - if (!input.patchResult && !input.retainContainerAfterRollback) { + const restoredSharedState = rollbackManagedStartupSharedState(transaction, deps); + if (!restoredSharedState && !input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } return { supervisorReady: false, failure: null };