From d96f0f103da2a572e937e2cb73fa77e5fa592a45 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 09:05:48 -0700 Subject: [PATCH 1/5] fix(onboard): preserve durable record compatibility Recognize exact atomic-write leftovers and fail closed on agentless legacy journals. Upgrade legacy finalizations only with durable context. Compare provider-neutral durable receipts by canonical value. Signed-off-by: Aaron Erickson --- .../onboard/managed-bootstrap/adapter.test.ts | 46 ++ src/lib/onboard/managed-bootstrap/adapter.ts | 16 + .../managed-bootstrap/docker-journal.test.ts | 239 +++++++- .../managed-bootstrap/docker-journal.ts | 526 ++++++++++++++++-- .../managed-bootstrap/docker-test-fixture.ts | 16 +- .../onboard/managed-bootstrap/docker.test.ts | 18 +- src/lib/onboard/managed-bootstrap/docker.ts | 61 +- src/lib/onboard/managed-bootstrap/index.ts | 2 + 8 files changed, 846 insertions(+), 78 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 6e3a5c1feb9..666dfdcc317 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -19,6 +19,7 @@ import { type ManagedBootstrapAuthorityStore, type ManagedBootstrapCompletionReceipt, type ManagedBootstrapCreateReceipt, + type ManagedBootstrapDurablePreparationReceipt, type ManagedBootstrapFinalizationReceipt, type ManagedBootstrapHeldWorkloadHandle, type ManagedBootstrapObservedSnapshot, @@ -27,6 +28,8 @@ import { prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, renderManagedBootstrapHeldCommand, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; const IDENTITY = "1".repeat(64); @@ -41,6 +44,10 @@ const ACTIVATED_HASH = createHash("sha256").update(ACTIVATED_SPEC_JSON, "utf8"). const RUNTIME_ID = "7".repeat(64); const PREPARED_ID = "8".repeat(64); +function reverseKeys(value: T): T { + return Object.fromEntries(Object.entries(value).reverse()) as T; +} + function requestFor(agent: ManagedStartupAgent) { return createManagedStartupRootApplyRequest({ agent, @@ -354,6 +361,45 @@ async function captureFailure(promise: Promise) { } describe("managed bootstrap adapter contract", () => { + it("compares provider-neutral durable receipts by canonical value", () => { + const handle = handleFor(requestFor("hermes")); + const preparation: ManagedBootstrapDurablePreparationReceipt = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: "a".repeat(64), + recordId: "mxc-durable-authority", + recordedAt: "2026-07-29T12:00:30.000Z", + }; + const reorderedPreparation = reverseKeys({ + ...preparation, + sandbox: reverseKeys({ ...preparation.sandbox }), + }); + expect(sameManagedBootstrapDurablePreparationReceipt(preparation, reorderedPreparation)).toBe( + true, + ); + expect( + sameManagedBootstrapDurablePreparationReceipt(preparation, { + ...reorderedPreparation, + recordId: "changed-authority", + }), + ).toBe(false); + + const completion = completionFor(requestFor("hermes"), handle); + const reorderedCompletion = reverseKeys({ + ...completion, + image: reverseKeys({ ...completion.image }), + sandbox: reverseKeys({ ...completion.sandbox }), + }); + expect(sameManagedBootstrapCompletionReceipt(completion, reorderedCompletion)).toBe(true); + expect( + sameManagedBootstrapCompletionReceipt(completion, { + ...reorderedCompletion, + transactionPending: false, + }), + ).toBe(false); + }); + it.each( MANAGED_STARTUP_AGENTS, )("prepares, durably records, and only then activates %s through a provider-neutral adapter", async (agent) => { diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index bff6c6cb0ff..a1ca17f983f 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -768,6 +768,22 @@ function canonicalJson(value: unknown): string { .join(",")}}`; } +/** Compare durable provider receipts by canonical value, independent of object key order. */ +export function sameManagedBootstrapDurablePreparationReceipt( + left: ManagedBootstrapDurablePreparationReceipt, + right: ManagedBootstrapDurablePreparationReceipt, +): boolean { + return canonicalJson(left) === canonicalJson(right); +} + +/** Compare completion receipts by canonical value, independent of object key order. */ +export function sameManagedBootstrapCompletionReceipt( + left: ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapCompletionReceipt, +): boolean { + return canonicalJson(left) === canonicalJson(right); +} + export function assertManagedBootstrapIdentity(value: string): void { if (!SHA256_RE.test(value)) { protocolFail("identity must be 32 random bytes encoded as lowercase hex"); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 5e1838e280f..96f20db72bf 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -12,8 +12,10 @@ import { DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationContext, type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, + DockerManagedBootstrapLegacyRecordRequiresAgentError, parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, serializeDockerManagedBootstrapFinalizationRecord, @@ -22,6 +24,12 @@ import { const roots: string[] = []; const IDENTITY = "1".repeat(64); +const OTHER_IDENTITY = "0".repeat(64); + +function reverseKeys(value: T): T { + return Object.fromEntries(Object.entries(value).reverse()) as T; +} + const journal = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", @@ -99,6 +107,75 @@ const finalization = Object.freeze({ }, } satisfies DockerManagedBootstrapFinalizationRecord); +const finalizationContext = Object.freeze({ + bootstrapIdentity: finalization.bootstrapIdentity, + providerId: finalization.providerId, + agent: finalization.agent, + sandbox: finalization.sandbox, + planFingerprint: finalization.planFingerprint, + profileFingerprint: finalization.profileFingerprint, + imageReference: finalization.imageReference, +} satisfies DockerManagedBootstrapFinalizationContext); + +function legacyJournalV1() { + return Object.freeze({ + schemaVersion: 1 as const, + phase: journal.phase, + bootstrapIdentity: journal.bootstrapIdentity, + sandbox: journal.sandbox, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + runtimeImageContentId: journal.runtimeImageContentId, + originalRuntimeId: journal.originalRuntimeId, + replacementRuntimeId: journal.replacementRuntimeId, + originalName: journal.originalName, + replacementStagingName: journal.replacementStagingName, + backupName: journal.backupName, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + }); +} + +function legacyJournalV2() { + return Object.freeze({ + schemaVersion: 2 as const, + phase: journal.phase, + bootstrapIdentity: journal.bootstrapIdentity, + providerId: journal.providerId, + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + runtimeImageContentId: journal.runtimeImageContentId, + originalRuntimeId: journal.originalRuntimeId, + replacementRuntimeId: journal.replacementRuntimeId, + originalName: journal.originalName, + replacementStagingName: journal.replacementStagingName, + backupName: journal.backupName, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + rollbackTargetRuntimeId: journal.rollbackTargetRuntimeId, + rollbackTargetSpecHash: journal.rollbackTargetSpecHash, + preparationReceipt: journal.preparationReceipt, + commitReceipt: journal.commitReceipt, + }); +} + +function legacyFinalizationV1() { + return Object.freeze({ + schemaVersion: 1 as const, + phase: finalization.phase, + bootstrapIdentity: finalization.bootstrapIdentity, + providerId: finalization.providerId, + sandbox: finalization.sandbox, + planFingerprint: finalization.planFingerprint, + profileFingerprint: finalization.profileFingerprint, + imageReference: finalization.imageReference, + commitReceipt: finalization.commitReceipt, + cleanupReceipt: finalization.cleanupReceipt, + }); +} + function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); try { @@ -273,7 +350,12 @@ describe("Docker managed bootstrap journal", () => { const restarted = createFileDockerManagedBootstrapJournalStore(root); expect(restarted.listUnfinished()).toEqual([completed]); - expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + const reorderedReceipt = reverseKeys({ + ...finalization.commitReceipt, + image: reverseKeys({ ...finalization.commitReceipt.image }), + sandbox: reverseKeys({ ...finalization.commitReceipt.sandbox }), + }); + expect(restarted.recordCompletion(IDENTITY, reorderedReceipt)).toEqual(completed); expect(() => restarted.recordCompletion(IDENTITY, { ...finalization.commitReceipt, @@ -282,6 +364,161 @@ describe("Docker managed bootstrap journal", () => { ).toThrow("completion receipt changed"); }); + it("retains exact journal, decision, and finalization atomic leftovers during enumeration", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + store.create(journal); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const leftovers = [ + `.${IDENTITY}.json.123.a0.tmp`, + `.${IDENTITY}.json.decision.123.a0.tmp`, + `.${IDENTITY}.json.finalized.123.a0.tmp`, + ]; + for (const name of leftovers) + fs.writeFileSync(path.join(directory, name), "orphan\n", { mode: 0o600 }); + + expect(store.listUnfinished()).toEqual([journal]); + for (const name of leftovers) expect(fs.existsSync(path.join(directory, name))).toBe(true); + }); + + it.each([ + `.${IDENTITY}.json.commit.123.a0.tmp`, + `.${IDENTITY}.json.decision.pid.a0.tmp`, + `.${IDENTITY}.json.finalized.123.A0.tmp`, + `${IDENTITY}.json.decision.123.a0.tmp`, + `.${IDENTITY}.json.decision.123.a0.tmp.extra`, + ])("rejects and retains near-miss atomic entry %s", (name) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(store.listUnfinished()).toEqual([]); + const target = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, name); + fs.writeFileSync(target, "near miss\n", { mode: 0o600 }); + + expect(() => store.listUnfinished()).toThrow("unsupported entry"); + expect(fs.existsSync(target)).toBe(true); + }); + + it.each([ + [1, legacyJournalV1], + [2, legacyJournalV2], + ] as const)("fails typed and closed for exact legacy journal schema %i", (schemaVersion, legacy) => { + const serialized = `${JSON.stringify(legacy())}\n`; + let failure: unknown; + try { + parseDockerManagedBootstrapJournal(serialized); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(DockerManagedBootstrapLegacyRecordRequiresAgentError); + expect(failure).toMatchObject({ + bootstrapIdentity: IDENTITY, + recordKind: "journal", + schemaVersion, + }); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(store.listUnfinished()).toEqual([]); + const target = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, `${IDENTITY}.json`); + fs.writeFileSync(target, serialized, { mode: 0o600 }); + expect(() => store.listUnfinished()).toThrowError( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(readPinnedPrivateFile(target).text).toBe(serialized); + }); + + it("does not classify a malformed legacy journal as upgradeable authority", () => { + const malformed = { ...legacyJournalV2(), agent: "hermes" }; + expect(() => parseDockerManagedBootstrapJournal(`${JSON.stringify(malformed)}\n`)).toThrow( + "legacy journal schema is invalid", + ); + try { + parseDockerManagedBootstrapJournal(`${JSON.stringify(malformed)}\n`); + } catch (error) { + expect(error).not.toBeInstanceOf(DockerManagedBootstrapLegacyRecordRequiresAgentError); + } + }); + + it("upgrades legacy finalization only with exact immutable transaction context", () => { + const serialized = `${JSON.stringify(legacyFinalizationV1())}\n`; + expect(() => parseDockerManagedBootstrapFinalizationRecord(serialized)).toThrowError( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(() => + parseDockerManagedBootstrapFinalizationRecord(serialized, { + ...finalizationContext, + planFingerprint: "0".repeat(64), + }), + ).toThrowError(DockerManagedBootstrapLegacyRecordRequiresAgentError); + expect(parseDockerManagedBootstrapFinalizationRecord(serialized, finalizationContext)).toEqual( + finalization, + ); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(store.listUnfinished()).toEqual([]); + const target = path.join( + root, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + `${IDENTITY}.json.finalized`, + ); + fs.writeFileSync(target, serialized, { mode: 0o600 }); + expect(() => store.loadFinalization(IDENTITY)).toThrowError( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(() => store.recordFinalization(finalization)).toThrowError( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(readPinnedPrivateFile(target).text).toBe(serialized); + + store.recordFinalization(finalization, finalizationContext); + expect(store.loadFinalization(IDENTITY)).toEqual(finalization); + expect(readPinnedPrivateFile(target).text).toBe( + serializeDockerManagedBootstrapFinalizationRecord(finalization), + ); + }); + + it("rejects a current finalization that contradicts supplied durable context", () => { + const wrongAgent = Object.freeze({ ...finalization, agent: "openclaw" as const }); + expect(() => + parseDockerManagedBootstrapFinalizationRecord( + serializeDockerManagedBootstrapFinalizationRecord(wrongAgent), + finalizationContext, + ), + ).toThrow("does not match supplied durable context"); + }); + + it("rejects current journal and finalization records stored under another identity", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(store.listUnfinished()).toEqual([]); + const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); + const misplacedJournal = path.join(directory, `${OTHER_IDENTITY}.json`); + const misplacedFinalization = `${misplacedJournal}.finalized`; + fs.writeFileSync(misplacedJournal, serializeDockerManagedBootstrapJournal(journal), { + mode: 0o600, + }); + fs.writeFileSync( + misplacedFinalization, + serializeDockerManagedBootstrapFinalizationRecord(finalization), + { mode: 0o600 }, + ); + + expect(() => store.load(OTHER_IDENTITY)).toThrow( + "journal bootstrap identity does not match its file name", + ); + expect(() => store.loadFinalization(OTHER_IDENTITY)).toThrow( + "finalization bootstrap identity does not match its file name", + ); + expect(fs.existsSync(misplacedJournal)).toBe(true); + expect(fs.existsSync(misplacedFinalization)).toBe(true); + }); + it("fails closed when enumeration encounters an unsupported state entry", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 58d7514ffd4..9b4670196d6 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -69,6 +69,17 @@ export interface DockerManagedBootstrapFinalizationRecord { readonly cleanupReceipt: ManagedBootstrapFinalizationReceipt; } +export type DockerManagedBootstrapFinalizationContext = Pick< + DockerManagedBootstrapFinalizationRecord, + | "agent" + | "bootstrapIdentity" + | "imageReference" + | "planFingerprint" + | "profileFingerprint" + | "providerId" + | "sandbox" +>; + export interface DockerManagedBootstrapJournalStore { create(journal: DockerManagedBootstrapJournal): void; load(bootstrapIdentity: string): DockerManagedBootstrapJournal | null; @@ -83,8 +94,14 @@ export interface DockerManagedBootstrapJournalStore { receipt: ManagedBootstrapCompletionReceipt, ): DockerManagedBootstrapJournal; remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; - recordFinalization(record: DockerManagedBootstrapFinalizationRecord): void; - loadFinalization(bootstrapIdentity: string): DockerManagedBootstrapFinalizationRecord | null; + recordFinalization( + record: DockerManagedBootstrapFinalizationRecord, + context?: DockerManagedBootstrapFinalizationContext, + ): void; + loadFinalization( + bootstrapIdentity: string, + context?: DockerManagedBootstrapFinalizationContext, + ): DockerManagedBootstrapFinalizationRecord | null; } /** @@ -99,6 +116,27 @@ export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error } } +export class DockerManagedBootstrapLegacyRecordRequiresAgentError extends Error { + readonly bootstrapIdentity: string; + readonly recordKind: "finalization" | "journal"; + readonly schemaVersion: number; + + constructor(input: { + readonly bootstrapIdentity: string; + readonly recordKind: "finalization" | "journal"; + readonly schemaVersion: number; + }) { + super( + `Managed bootstrap Docker ${input.recordKind} schema ${input.schemaVersion} for ` + + `${input.bootstrapIdentity} lacks durable agent identity`, + ); + this.name = "DockerManagedBootstrapLegacyRecordRequiresAgentError"; + this.bootstrapIdentity = input.bootstrapIdentity; + this.recordKind = input.recordKind; + this.schemaVersion = input.schemaVersion; + } +} + class DockerManagedBootstrapJournalExistsError extends Error { constructor() { super( @@ -118,6 +156,10 @@ function fail(message: string): never { throw new Error(`Managed bootstrap Docker journal is invalid: ${message}`); } +function hasExactKeys(record: Readonly>, expected: readonly string[]) { + return Object.keys(record).sort().join(",") === [...expected].sort().join(","); +} + function exactString(value: unknown, label: string, maxBytes = 4096): string { if ( typeof value !== "string" || @@ -169,6 +211,171 @@ function exactSandbox(value: unknown): ManagedBootstrapSandboxIdentity { }); } +function sameSandboxIdentity( + left: ManagedBootstrapSandboxIdentity, + right: ManagedBootstrapSandboxIdentity, +): boolean { + return ( + left.sandboxName === right.sandboxName && + left.sandboxId === right.sandboxId && + left.driverId === right.driverId + ); +} + +function normalizeLegacyDockerManagedBootstrapJournal( + journal: Readonly>, + schemaVersion: 1 | 2, +): { readonly bootstrapIdentity: string; readonly canonical: string } { + if (schemaVersion === 1) { + const expectedKeys = [ + "backupName", + "bootstrapIdentity", + "imageReference", + "originalName", + "originalRuntimeId", + "originalSpecHash", + "phase", + "profileFingerprint", + "replacementRuntimeId", + "replacementSpecHash", + "replacementStagingName", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + ]; + if (!hasExactKeys(journal, expectedKeys)) fail("legacy journal schema is invalid"); + const normalized = Object.freeze({ + schemaVersion: 1 as const, + phase: exactPhase(journal.phase), + bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + sandbox: exactSandbox(journal.sandbox), + profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), + imageReference: exactString(journal.imageReference, "image reference"), + runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), + originalRuntimeId: exactSha256(journal.originalRuntimeId, "original runtime ID"), + replacementRuntimeId: exactSha256(journal.replacementRuntimeId, "replacement runtime ID"), + originalName: exactString(journal.originalName, "original name", 253), + replacementStagingName: exactString( + journal.replacementStagingName, + "replacement staging name", + 253, + ), + backupName: exactString(journal.backupName, "backup name", 253), + originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), + replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + }); + if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { + fail("original and replacement runtime IDs must differ"); + } + if ( + new Set([normalized.originalName, normalized.replacementStagingName, normalized.backupName]) + .size !== 3 + ) { + fail("original, staging, and backup names must be distinct"); + } + return { + bootstrapIdentity: normalized.bootstrapIdentity, + canonical: `${JSON.stringify(normalized)}\n`, + }; + } + + const expectedKeys = [ + "backupName", + "bootstrapIdentity", + "commitReceipt", + "imageReference", + "originalName", + "originalRuntimeId", + "originalSpecHash", + "phase", + "planFingerprint", + "preparationReceipt", + "profileFingerprint", + "providerId", + "replacementRuntimeId", + "replacementSpecHash", + "replacementStagingName", + "rollbackTargetRuntimeId", + "rollbackTargetSpecHash", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + ]; + if (!hasExactKeys(journal, expectedKeys)) fail("legacy journal schema is invalid"); + const normalized = Object.freeze({ + schemaVersion: 2 as const, + phase: exactPhase(journal.phase), + bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + providerId: exactString(journal.providerId, "provider ID"), + sandbox: exactSandbox(journal.sandbox), + planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), + profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), + imageReference: exactString(journal.imageReference, "image reference"), + runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), + originalRuntimeId: exactSha256(journal.originalRuntimeId, "original runtime ID"), + replacementRuntimeId: exactSha256(journal.replacementRuntimeId, "replacement runtime ID"), + originalName: exactString(journal.originalName, "original name", 253), + replacementStagingName: exactString( + journal.replacementStagingName, + "replacement staging name", + 253, + ), + backupName: exactString(journal.backupName, "backup name", 253), + originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), + replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + rollbackTargetRuntimeId: exactSha256( + journal.rollbackTargetRuntimeId, + "rollback target runtime ID", + ), + rollbackTargetSpecHash: exactSha256( + journal.rollbackTargetSpecHash, + "rollback target spec hash", + ), + preparationReceipt: + journal.preparationReceipt === null + ? null + : exactPreparationReceipt(journal.preparationReceipt), + commitReceipt: + journal.commitReceipt === null ? null : exactCompletionReceipt(journal.commitReceipt), + }); + if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { + fail("original and replacement runtime IDs must differ"); + } + if ( + new Set([normalized.originalName, normalized.replacementStagingName, normalized.backupName]) + .size !== 3 + ) { + fail("original, staging, and backup names must be distinct"); + } + if ( + normalized.providerId !== normalized.sandbox.driverId || + normalized.rollbackTargetRuntimeId !== normalized.originalRuntimeId || + normalized.rollbackTargetSpecHash !== normalized.originalSpecHash + ) { + fail("provider or rollback authority does not match the transaction identity"); + } + if ( + (normalized.preparationReceipt !== null && + (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + !sameSandboxIdentity(normalized.preparationReceipt.sandbox, normalized.sandbox))) || + (normalized.commitReceipt !== null && + (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + !sameSandboxIdentity(normalized.commitReceipt.sandbox, normalized.sandbox) || + normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || + normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || + normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || + normalized.commitReceipt.replacementSpecHash !== normalized.replacementSpecHash || + `${normalized.commitReceipt.image.repository}@${normalized.commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("durable preparation or commit receipt does not match the transaction identity"); + } + return { + bootstrapIdentity: normalized.bootstrapIdentity, + canonical: `${JSON.stringify(normalized)}\n`, + }; +} + export function normalizeDockerManagedBootstrapJournal( value: unknown, ): DockerManagedBootstrapJournal { @@ -176,6 +383,14 @@ export function normalizeDockerManagedBootstrapJournal( fail("journal must be an object"); } const journal = value as Record; + if (journal.schemaVersion === 1 || journal.schemaVersion === 2) { + const legacy = normalizeLegacyDockerManagedBootstrapJournal(journal, journal.schemaVersion); + throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "journal", + schemaVersion: journal.schemaVersion, + }); + } const expectedKeys = [ "agent", "backupName", @@ -200,7 +415,7 @@ export function normalizeDockerManagedBootstrapJournal( "schemaVersion", ]; if ( - Object.keys(journal).sort().join(",") !== expectedKeys.sort().join(",") || + !hasExactKeys(journal, expectedKeys) || journal.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION ) { fail("journal schema is invalid"); @@ -261,14 +476,10 @@ export function normalizeDockerManagedBootstrapJournal( if ( (normalized.preparationReceipt !== null && (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || - normalized.preparationReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || - normalized.preparationReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || - normalized.preparationReceipt.sandbox.driverId !== normalized.sandbox.driverId)) || + !sameSandboxIdentity(normalized.preparationReceipt.sandbox, normalized.sandbox))) || (normalized.commitReceipt !== null && (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || - normalized.commitReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || - normalized.commitReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || - normalized.commitReceipt.sandbox.driverId !== normalized.sandbox.driverId || + !sameSandboxIdentity(normalized.commitReceipt.sandbox, normalized.sandbox) || normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || @@ -306,6 +517,22 @@ export function parseDockerManagedBootstrapJournal(text: string): DockerManagedB } catch { fail("serialized journal is not valid JSON"); } + if ( + typeof parsed === "object" && + parsed !== null && + !Array.isArray(parsed) && + ((parsed as Record).schemaVersion === 1 || + (parsed as Record).schemaVersion === 2) + ) { + const record = parsed as Record & { readonly schemaVersion: 1 | 2 }; + const legacy = normalizeLegacyDockerManagedBootstrapJournal(record, record.schemaVersion); + if (legacy.canonical !== text) fail("serialized legacy journal is not canonical"); + throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "journal", + schemaVersion: record.schemaVersion, + }); + } const journal = normalizeDockerManagedBootstrapJournal(parsed); if (serializeDockerManagedBootstrapJournal(journal) !== text) { fail("serialized journal is not canonical"); @@ -462,32 +689,16 @@ function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceip }); } -export function normalizeDockerManagedBootstrapFinalizationRecord( - value: unknown, -): DockerManagedBootstrapFinalizationRecord { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - fail("finalization record must be an object"); - } - const record = value as Record; - const expectedKeys = [ - "agent", - "bootstrapIdentity", - "cleanupReceipt", - "commitReceipt", - "imageReference", - "phase", - "planFingerprint", - "profileFingerprint", - "providerId", - "sandbox", - "schemaVersion", - ]; - if ( - Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || - record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION || - !["committed", "rolled-back"].includes(String(record.phase)) - ) { - fail("finalization record schema is invalid"); +type DockerManagedBootstrapFinalizationWithoutAgent = Omit< + DockerManagedBootstrapFinalizationRecord, + "agent" | "schemaVersion" +>; + +function normalizeFinalizationWithoutAgent( + record: Readonly>, +): DockerManagedBootstrapFinalizationWithoutAgent { + if (!["committed", "rolled-back"].includes(String(record.phase))) { + fail("finalization phase is invalid"); } const phase = record.phase as "committed" | "rolled-back"; const sandbox = exactSandbox(record.sandbox); @@ -495,28 +706,26 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( record.commitReceipt === null ? null : exactCompletionReceipt(record.commitReceipt); const cleanupReceipt = exactCleanupReceipt(record.cleanupReceipt); const normalized = Object.freeze({ - schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, phase, bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), providerId: exactString(record.providerId, "finalization provider ID"), - agent: exactAgent(record.agent), sandbox, planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), imageReference: exactString(record.imageReference, "finalization image reference"), commitReceipt, cleanupReceipt, - } satisfies DockerManagedBootstrapFinalizationRecord); + } satisfies DockerManagedBootstrapFinalizationWithoutAgent); if ( normalized.providerId !== sandbox.driverId || normalized.bootstrapIdentity !== cleanupReceipt.bootstrapIdentity || normalized.phase !== cleanupReceipt.outcome || - JSON.stringify(normalized.sandbox) !== JSON.stringify(cleanupReceipt.sandbox) || + !sameSandboxIdentity(normalized.sandbox, cleanupReceipt.sandbox) || (phase === "committed") !== (commitReceipt !== null) || (commitReceipt !== null && (commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || commitReceipt.profileFingerprint !== normalized.profileFingerprint || - JSON.stringify(commitReceipt.sandbox) !== JSON.stringify(normalized.sandbox) || + !sameSandboxIdentity(commitReceipt.sandbox, normalized.sandbox) || `${commitReceipt.image.repository}@${commitReceipt.image.manifestDigest}` !== normalized.imageReference)) ) { @@ -525,6 +734,160 @@ export function normalizeDockerManagedBootstrapFinalizationRecord( return normalized; } +function normalizeLegacyFinalizationShape(value: unknown): { + readonly canonical: string; + readonly record: DockerManagedBootstrapFinalizationWithoutAgent; +} { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if (!hasExactKeys(record, expectedKeys) || record.schemaVersion !== 1) { + fail("legacy finalization record schema is invalid"); + } + const normalized = normalizeFinalizationWithoutAgent(record); + const legacy = Object.freeze({ + schemaVersion: 1 as const, + phase: normalized.phase, + bootstrapIdentity: normalized.bootstrapIdentity, + providerId: normalized.providerId, + sandbox: normalized.sandbox, + planFingerprint: normalized.planFingerprint, + profileFingerprint: normalized.profileFingerprint, + imageReference: normalized.imageReference, + commitReceipt: normalized.commitReceipt, + cleanupReceipt: normalized.cleanupReceipt, + }); + return { canonical: `${JSON.stringify(legacy)}\n`, record: normalized }; +} + +function normalizeFinalizationContext( + context: DockerManagedBootstrapFinalizationContext, +): DockerManagedBootstrapFinalizationContext { + return Object.freeze({ + bootstrapIdentity: exactSha256( + context.bootstrapIdentity, + "finalization context bootstrap identity", + ), + providerId: exactString(context.providerId, "finalization context provider ID"), + agent: exactAgent(context.agent), + sandbox: exactSandbox(context.sandbox), + planFingerprint: exactSha256(context.planFingerprint, "finalization context plan fingerprint"), + profileFingerprint: exactSha256( + context.profileFingerprint, + "finalization context profile fingerprint", + ), + imageReference: exactString(context.imageReference, "finalization context image reference"), + }); +} + +function matchesFinalizationContext( + record: DockerManagedBootstrapFinalizationWithoutAgent, + context: DockerManagedBootstrapFinalizationContext, +): boolean { + return ( + context.bootstrapIdentity === record.bootstrapIdentity && + context.providerId === record.providerId && + sameSandboxIdentity(context.sandbox, record.sandbox) && + context.planFingerprint === record.planFingerprint && + context.profileFingerprint === record.profileFingerprint && + context.imageReference === record.imageReference + ); +} + +function upgradeLegacyFinalization( + legacy: DockerManagedBootstrapFinalizationWithoutAgent, + context: DockerManagedBootstrapFinalizationContext | undefined, +): DockerManagedBootstrapFinalizationRecord { + const missingAgent = () => + new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "finalization", + schemaVersion: 1, + }); + // Runtime names and image repositories are mutable descriptions, never + // agent authority. Only an exact live handle or current journal may supply + // the field omitted by schema v1. + if (!context) throw missingAgent(); + const normalizedContext = normalizeFinalizationContext(context); + if (!matchesFinalizationContext(legacy, normalizedContext)) throw missingAgent(); + return Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: legacy.phase, + bootstrapIdentity: legacy.bootstrapIdentity, + providerId: legacy.providerId, + agent: normalizedContext.agent, + sandbox: legacy.sandbox, + planFingerprint: legacy.planFingerprint, + profileFingerprint: legacy.profileFingerprint, + imageReference: legacy.imageReference, + commitReceipt: legacy.commitReceipt, + cleanupReceipt: legacy.cleanupReceipt, + }); +} + +export function normalizeDockerManagedBootstrapFinalizationRecord( + value: unknown, +): DockerManagedBootstrapFinalizationRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + if (record.schemaVersion === 1) { + const legacy = normalizeLegacyFinalizationShape(record).record; + throw new DockerManagedBootstrapLegacyRecordRequiresAgentError({ + bootstrapIdentity: legacy.bootstrapIdentity, + recordKind: "finalization", + schemaVersion: 1, + }); + } + const expectedKeys = [ + "agent", + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if ( + !hasExactKeys(record, expectedKeys) || + record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION + ) { + fail("finalization record schema is invalid"); + } + const normalized = normalizeFinalizationWithoutAgent(record); + return Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: normalized.phase, + bootstrapIdentity: normalized.bootstrapIdentity, + providerId: normalized.providerId, + agent: exactAgent(record.agent), + sandbox: normalized.sandbox, + planFingerprint: normalized.planFingerprint, + profileFingerprint: normalized.profileFingerprint, + imageReference: normalized.imageReference, + commitReceipt: normalized.commitReceipt, + cleanupReceipt: normalized.cleanupReceipt, + }); +} + export function serializeDockerManagedBootstrapFinalizationRecord( record: DockerManagedBootstrapFinalizationRecord, ): string { @@ -535,9 +898,10 @@ export function serializeDockerManagedBootstrapFinalizationRecord( return serialized; } -export function parseDockerManagedBootstrapFinalizationRecord( +function parseDockerManagedBootstrapFinalizationRecordWithContext( text: string, -): DockerManagedBootstrapFinalizationRecord { + context?: DockerManagedBootstrapFinalizationContext, +): { readonly record: DockerManagedBootstrapFinalizationRecord; readonly upgradedLegacy: boolean } { if ( text.length === 0 || text.includes("\0") || @@ -551,11 +915,37 @@ export function parseDockerManagedBootstrapFinalizationRecord( } catch { fail("serialized finalization record is not valid JSON"); } + if ( + typeof parsed === "object" && + parsed !== null && + !Array.isArray(parsed) && + (parsed as Record).schemaVersion === 1 + ) { + const legacy = normalizeLegacyFinalizationShape(parsed); + if (legacy.canonical !== text) fail("serialized legacy finalization record is not canonical"); + return { record: upgradeLegacyFinalization(legacy.record, context), upgradedLegacy: true }; + } const record = normalizeDockerManagedBootstrapFinalizationRecord(parsed); if (serializeDockerManagedBootstrapFinalizationRecord(record) !== text) { fail("serialized finalization record is not canonical"); } - return record; + if (context) { + const normalizedContext = normalizeFinalizationContext(context); + if ( + record.agent !== normalizedContext.agent || + !matchesFinalizationContext(record, normalizedContext) + ) { + fail("finalization record does not match supplied durable context"); + } + } + return { record, upgradedLegacy: false }; +} + +export function parseDockerManagedBootstrapFinalizationRecord( + text: string, + context?: DockerManagedBootstrapFinalizationContext, +): DockerManagedBootstrapFinalizationRecord { + return parseDockerManagedBootstrapFinalizationRecordWithContext(text, context).record; } function assertDirectory(directory: string): void { @@ -722,6 +1112,9 @@ export function createFileDockerManagedBootstrapJournalStore( const contents = readPrivateFile(target, "journal"); if (contents === null) return null; const journal = parseDockerManagedBootstrapJournal(contents); + if (journal.bootstrapIdentity !== bootstrapIdentity) { + fail("journal bootstrap identity does not match its file name"); + } const decision = readPrivateFile(decisionPath(target), "decision"); if (decision === null) return journal; const phase = decision.endsWith("\n") ? decision.slice(0, -1) : ""; @@ -739,13 +1132,24 @@ export function createFileDockerManagedBootstrapJournalStore( }; const loadFinalization = ( bootstrapIdentity: string, + context?: DockerManagedBootstrapFinalizationContext, ): DockerManagedBootstrapFinalizationRecord | null => { assertDirectory(directory); - const contents = readPrivateFile( - finalizationPath(journalPath(directory, bootstrapIdentity)), - "finalization", - ); - return contents === null ? null : parseDockerManagedBootstrapFinalizationRecord(contents); + const target = finalizationPath(journalPath(directory, bootstrapIdentity)); + const contents = readPrivateFile(target, "finalization"); + if (contents === null) return null; + const parsed = parseDockerManagedBootstrapFinalizationRecordWithContext(contents, context); + if (parsed.record.bootstrapIdentity !== bootstrapIdentity) { + fail("finalization bootstrap identity does not match its file name"); + } + if (parsed.upgradedLegacy) { + const serialized = serializeDockerManagedBootstrapFinalizationRecord(parsed.record); + atomicWrite(directory, target, serialized, false); + if (readPrivateFile(target, "finalization") !== serialized) { + fail("upgraded finalization record was not durably re-readable"); + } + } + return parsed.record; }; return Object.freeze({ create(journal: DockerManagedBootstrapJournal) { @@ -775,7 +1179,7 @@ export function createFileDockerManagedBootstrapJournalStore( continue; } if ( - /^\.[a-f0-9]{64}\.json\.[0-9]+\.[a-f0-9]+\.tmp$/u.test(name) || + /^\.[a-f0-9]{64}\.json(?:\.decision|\.finalized)?\.[0-9]+\.[a-f0-9]+\.tmp$/u.test(name) || /^[a-f0-9]{64}\.json\.(?:decision|finalized)$/u.test(name) ) { continue; @@ -864,23 +1268,37 @@ export function createFileDockerManagedBootstrapJournalStore( fs.unlinkSync(target); fsyncDirectory(directory); }, - recordFinalization(record: DockerManagedBootstrapFinalizationRecord) { + recordFinalization( + record: DockerManagedBootstrapFinalizationRecord, + context?: DockerManagedBootstrapFinalizationContext, + ) { const normalized = normalizeDockerManagedBootstrapFinalizationRecord(record); assertDirectory(directory); const target = finalizationPath(journalPath(directory, normalized.bootstrapIdentity)); const serialized = serializeDockerManagedBootstrapFinalizationRecord(normalized); - const existing = readPrivateFile(target, "finalization"); + const existing = loadFinalization(normalized.bootstrapIdentity, context); if (existing !== null) { - if (existing !== serialized) + if (serializeDockerManagedBootstrapFinalizationRecord(existing) !== serialized) { fail("finalization record changed for this bootstrap identity"); + } return; } try { atomicWrite(directory, target, serialized, true); } catch (error) { - if (readPrivateFile(target, "finalization") !== serialized) throw error; + const recovered = loadFinalization(normalized.bootstrapIdentity, context); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } } - if (readPrivateFile(target, "finalization") !== serialized) { + const persisted = loadFinalization(normalized.bootstrapIdentity, context); + if ( + !persisted || + serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized + ) { fail("finalization record was not durably re-readable"); } }, diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index c272c4183b6..b3f53a1e7b2 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -18,6 +18,7 @@ import { type ManagedBootstrapObservedSnapshot, type ManagedBootstrapPreparedReplacementHandle, type ManagedBootstrapReplacementHandle, + sameManagedBootstrapCompletionReceipt, } from "./adapter"; import type { DockerManagedBootstrapDeps } from "./docker"; import { @@ -26,6 +27,7 @@ import { DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalPhase, type DockerManagedBootstrapJournalStore, + serializeDockerManagedBootstrapFinalizationRecord, } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; import { @@ -58,6 +60,7 @@ export type DockerFixtureAcknowledgement = | "container:stop" | "journal:create" | "journal:cutover" + | "journal:completion" | "journal:remove" | "journal:rollback-authorized" | "journal:staged" @@ -265,12 +268,17 @@ export function fixture(options: DockerFixtureOptions = {}) { } if ( journal.commitReceipt !== null && - JSON.stringify(journal.commitReceipt) !== JSON.stringify(receipt) + !sameManagedBootstrapCompletionReceipt(journal.commitReceipt, receipt) ) { throw new Error("completion changed"); } journal = { ...journal, commitReceipt: structuredClone(receipt) }; events.push("journal:completion"); + if (losesAcknowledgement("journal:completion")) { + throw new DockerManagedBootstrapJournalAcknowledgementLostError( + "lost journal completion acknowledgement", + ); + } return structuredClone(journal); }, remove(_identity, expected) { @@ -294,7 +302,11 @@ export function fixture(options: DockerFixtureOptions = {}) { } }, recordFinalization(value) { - if (finalization && JSON.stringify(finalization) !== JSON.stringify(value)) { + if ( + finalization && + serializeDockerManagedBootstrapFinalizationRecord(finalization) !== + serializeDockerManagedBootstrapFinalizationRecord(value) + ) { throw new Error("finalization changed"); } finalization = structuredClone(value); diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 21407385918..ea23d4bbb95 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -17,6 +17,10 @@ import { SUPPORTED_AGENTS, } from "./docker-test-fixture"; +function reverseKeys(value: T): T { + return Object.fromEntries(Object.entries(value).reverse()) as T; +} + describe("Docker managed bootstrap adapter", () => { it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { const fake = fixture({ @@ -28,6 +32,7 @@ describe("Docker managed bootstrap adapter", () => { "container:stop", "journal:create", "journal:cutover", + "journal:completion", "journal:remove", "journal:shared-state-committed", ], @@ -69,14 +74,23 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events.indexOf("journal:completion")).toBeGreaterThan( fake.events.indexOf(`start:${NEW_ID}`), ); + const reorderedDurable = reverseKeys({ + ...durable, + sandbox: reverseKeys({ ...durable.sandbox }), + }); + const reorderedCommitReceipt = reverseKeys({ + ...commitReceipt, + image: reverseKeys({ ...commitReceipt.image }), + sandbox: reverseKeys({ ...commitReceipt.sandbox }), + }); const finalized = await adapter.finalizeBootstrap({ outcome: "commit", handle, snapshot, prepared, - durablePreparation: durable, + durablePreparation: reorderedDurable, replacement, - completion: commitReceipt, + completion: reorderedCommitReceipt, }); expect(finalized).toMatchObject({ outcome: "committed" }); expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 3202b6907e1..fe1fdf0787a 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -64,6 +64,8 @@ import { type ManagedBootstrapReplacementOptions, type ManagedBootstrapSandboxIdentity, renderManagedBootstrapHeldCommand, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, @@ -1446,13 +1448,6 @@ function sameDockerBootstrapPreparedAuthority( ); } -function sameDurablePreparationReceipt( - left: ManagedBootstrapDurablePreparationReceipt, - right: ManagedBootstrapDurablePreparationReceipt, -): boolean { - return JSON.stringify(left) === JSON.stringify(right); -} - function createDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1573,7 +1568,10 @@ function assertDockerBootstrapTransactionAuthority( (durablePreparation !== undefined && durablePreparation !== null && (transaction.preparationReceipt === null || - !sameDurablePreparationReceipt(transaction.preparationReceipt, durablePreparation))) || + !sameManagedBootstrapDurablePreparationReceipt( + transaction.preparationReceipt, + durablePreparation, + ))) || (prepared !== undefined && prepared !== null && (transaction.originalRuntimeId !== prepared.originalRuntimeId || @@ -1764,13 +1762,31 @@ export function createDockerManagedBootstrapAdapter( dependencies: DockerManagedBootstrapDeps = {}, ): DockerManagedBootstrapAdapter { const deps = resolveDeps(dependencies); + const finalizationContext = (handle: ManagedBootstrapHeldWorkloadHandle) => + Object.freeze({ + bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, + agent: handle.plan.profile.agent, + sandbox: handle.sandbox, + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + }); const finalizationRecord = ( handle: ManagedBootstrapHeldWorkloadHandle, ): DockerManagedBootstrapFinalizationRecord | null => { - const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + const record = deps.journalStore.loadFinalization( + handle.bootstrapIdentity, + finalizationContext(handle), + ); if (!record) return null; if ( + record.bootstrapIdentity !== handle.bootstrapIdentity || record.providerId !== handle.sandbox.driverId || + record.agent !== handle.plan.profile.agent || record.sandbox.sandboxName !== handle.sandbox.sandboxName || record.sandbox.sandboxId !== handle.sandbox.sandboxId || record.sandbox.driverId !== handle.sandbox.driverId || @@ -1807,9 +1823,12 @@ export function createDockerManagedBootstrapAdapter( } satisfies DockerManagedBootstrapFinalizationRecord); const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); try { - deps.journalStore.recordFinalization(record); + deps.journalStore.recordFinalization(record, finalizationContext(handle)); } catch (error) { - const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + const recovered = deps.journalStore.loadFinalization( + handle.bootstrapIdentity, + finalizationContext(handle), + ); if ( !recovered || serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized @@ -1817,7 +1836,10 @@ export function createDockerManagedBootstrapAdapter( throw error; } } - const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + const persisted = deps.journalStore.loadFinalization( + handle.bootstrapIdentity, + finalizationContext(handle), + ); if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); } @@ -1910,9 +1932,9 @@ export function createDockerManagedBootstrapAdapter( } satisfies DockerManagedBootstrapFinalizationRecord); const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); try { - deps.journalStore.recordFinalization(record); + deps.journalStore.recordFinalization(record, journal); } catch (error) { - const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + const recovered = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); if ( !recovered || serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized @@ -1920,7 +1942,7 @@ export function createDockerManagedBootstrapAdapter( throw error; } } - const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + const persisted = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { throw new Error("Managed bootstrap recovered finalization was not durably re-readable."); } @@ -1944,13 +1966,14 @@ export function createDockerManagedBootstrapAdapter( journal: DockerBootstrapTransaction, sourcePhase: DockerBootstrapTransaction["phase"], ): ManagedBootstrapRecoveryReceipt | null => { - const finalization = deps.journalStore.loadFinalization(journal.bootstrapIdentity); + const finalization = deps.journalStore.loadFinalization(journal.bootstrapIdentity, journal); if (!finalization) return null; const phaseMatches = (finalization.phase === "committed" && journal.phase === "shared-state-committed" && finalization.commitReceipt !== null && - JSON.stringify(finalization.commitReceipt) === JSON.stringify(journal.commitReceipt)) || + journal.commitReceipt !== null && + sameManagedBootstrapCompletionReceipt(finalization.commitReceipt, journal.commitReceipt)) || (finalization.phase === "rolled-back" && (journal.phase === "staged" || journal.phase === "rollback-authorized") && finalization.commitReceipt === null); @@ -2684,7 +2707,7 @@ export function createDockerManagedBootstrapAdapter( if ( finalized.phase !== "committed" || !finalized.commitReceipt || - JSON.stringify(finalized.commitReceipt) !== JSON.stringify(completion) + !sameManagedBootstrapCompletionReceipt(finalized.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, @@ -2764,7 +2787,7 @@ export function createDockerManagedBootstrapAdapter( ); if ( journal.commitReceipt === null || - JSON.stringify(journal.commitReceipt) !== JSON.stringify(completion) + !sameManagedBootstrapCompletionReceipt(journal.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: journal.bootstrapIdentity, diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index abeddd0a06d..f5f6f545869 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -13,6 +13,8 @@ export { type ManagedBootstrapRecoveryReceipt, prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; export { MANAGED_BOOTSTRAP_COMPLETION_FILE, From 563213b62f7336091d18492b845f99eb13ae1f61 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 09:09:12 -0700 Subject: [PATCH 2/5] test(onboard): require durable event evidence Assert both events before comparing lifecycle order and require exact rollback finalization. Signed-off-by: Aaron Erickson --- .../onboard/managed-bootstrap/docker.test.ts | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index ea23d4bbb95..32d6a81e75d 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -21,6 +21,12 @@ function reverseKeys(value: T): T { return Object.fromEntries(Object.entries(value).reverse()) as T; } +function expectEventBefore(events: readonly string[], before: string, after: string): void { + expect(events).toContain(before); + expect(events).toContain(after); + expect(events.indexOf(before)).toBeLessThan(events.indexOf(after)); +} + describe("Docker managed bootstrap adapter", () => { it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { const fake = fixture({ @@ -57,8 +63,8 @@ describe("Docker managed bootstrap adapter", () => { durablePreparation: durable, }); const order = fake.events; - expect(order.indexOf("journal:staged")).toBeGreaterThan(order.indexOf("authority:recorded")); - expect(order.indexOf("journal:cutover")).toBeLessThan(order.indexOf(`stop:${OLD_ID}`)); + expectEventBefore(order, "authority:recorded", "journal:staged"); + expectEventBefore(order, "journal:cutover", `stop:${OLD_ID}`); expect(fake.journal).toMatchObject({ phase: "cutover", originalRuntimeId: OLD_ID, @@ -71,9 +77,7 @@ describe("Docker managed bootstrap adapter", () => { replacement, timeoutSecs: 1, }); - expect(fake.events.indexOf("journal:completion")).toBeGreaterThan( - fake.events.indexOf(`start:${NEW_ID}`), - ); + expectEventBefore(fake.events, `start:${NEW_ID}`, "journal:completion"); const reorderedDurable = reverseKeys({ ...durable, sandbox: reverseKeys({ ...durable.sandbox }), @@ -93,12 +97,8 @@ describe("Docker managed bootstrap adapter", () => { completion: reorderedCommitReceipt, }); expect(finalized).toMatchObject({ outcome: "committed" }); - expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( - fake.events.indexOf(`rm:${OLD_ID}`), - ); - expect(fake.events.indexOf("finalization:committed")).toBeLessThan( - fake.events.indexOf("journal:removed"), - ); + expectEventBefore(fake.events, "journal:shared-state-committed", `rm:${OLD_ID}`); + expectEventBefore(fake.events, "finalization:committed", "journal:removed"); expect(fake.journal).toBeNull(); expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); expect(fake.sharedState).toBe("none"); @@ -156,12 +156,9 @@ describe("Docker managed bootstrap adapter", () => { completion: null, }), ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); - expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( - fake.events.indexOf(`rm:${NEW_ID}`), - ); - expect(fake.events.indexOf("finalization:rolled-back")).toBeLessThan( - fake.events.indexOf("journal:removed"), - ); + expectEventBefore(fake.events, "journal:rollback-authorized", `rm:${NEW_ID}`); + expectEventBefore(fake.events, "finalization:rolled-back", "journal:removed"); + expect(fake.finalization).toMatchObject({ phase: "rolled-back" }); expect(fake.journal).toBeNull(); expect(fake.replacement).toBeNull(); expect(fake.original.Name).toBe("/openshell-alpha"); @@ -203,9 +200,8 @@ describe("Docker managed bootstrap adapter", () => { completion: null, }), ).rejects.toBeInstanceOf(ManagedBootstrapOwnerCleanupRequiredError); - expect(fake.events.indexOf("journal:rollback-authorized")).toBeLessThan( - fake.events.indexOf(`rm:${NEW_ID}`), - ); + expectEventBefore(fake.events, "journal:rollback-authorized", `rm:${NEW_ID}`); + expect(fake.finalization).toMatchObject({ phase: "rolled-back" }); expect(fake.journal).toBeNull(); }); From 24a49d0329fab17677eb522ffe24934a3c0988bf Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 09:50:18 -0700 Subject: [PATCH 3/5] fix(onboard): validate durable bootstrap authority Signed-off-by: Aaron Erickson --- src/lib/onboard/managed-bootstrap/README.md | 4 +-- .../managed-bootstrap/docker-journal.test.ts | 28 +++++++++++++++++++ .../managed-bootstrap/docker-journal.ts | 24 ++++++++++------ .../onboard/managed-bootstrap/docker.test.ts | 25 +++++++++++++++++ src/lib/onboard/managed-bootstrap/docker.ts | 8 ++++++ 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 40dac4dfa3c..77bece4661e 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -74,8 +74,8 @@ work completes exact restore and cleanup; and shared-state-committed work completes exact backup cleanup and commit. Recovery persists an identity-bound finalization receipt before removing the active journal, is idempotent across another interruption, and returns normalized, provider-owned receipts in stable -identity order. Mutable OpenShell names are read only to detect ownership reuse, -and unsafe name-only deletion returns a typed retention error. Multi-process +identity order. The code reads mutable OpenShell names only to detect ownership +reuse, and unsafe name-only deletion returns a typed retention error. Multi-process lease/arbitration remains an explicit production-activation gate. Activation must also inject the selected gateway's canonical state root. diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 96f20db72bf..d4672208518 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -475,6 +475,14 @@ describe("Docker managed bootstrap journal", () => { ); expect(readPinnedPrivateFile(target).text).toBe(serialized); + expect(() => + store.recordFinalization(finalization, { + ...finalizationContext, + agent: "openclaw", + }), + ).toThrow("does not match supplied durable context"); + expect(readPinnedPrivateFile(target).text).toBe(serialized); + store.recordFinalization(finalization, finalizationContext); expect(store.loadFinalization(IDENTITY)).toEqual(finalization); expect(readPinnedPrivateFile(target).text).toBe( @@ -492,6 +500,26 @@ describe("Docker managed bootstrap journal", () => { ).toThrow("does not match supplied durable context"); }); + it("does not create a finalization before validating durable context", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(store.listUnfinished()).toEqual([]); + const target = path.join( + root, + DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, + `${IDENTITY}.json.finalized`, + ); + + expect(() => + store.recordFinalization(finalization, { + ...finalizationContext, + agent: "openclaw", + }), + ).toThrow("does not match supplied durable context"); + expect(fs.existsSync(target)).toBe(false); + }); + it("rejects current journal and finalization records stored under another identity", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 9b4670196d6..70942ceebb4 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -807,6 +807,19 @@ function matchesFinalizationContext( ); } +function assertFinalizationMatchesContext( + record: DockerManagedBootstrapFinalizationRecord, + context: DockerManagedBootstrapFinalizationContext, +): void { + const normalizedContext = normalizeFinalizationContext(context); + if ( + record.agent !== normalizedContext.agent || + !matchesFinalizationContext(record, normalizedContext) + ) { + fail("finalization record does not match supplied durable context"); + } +} + function upgradeLegacyFinalization( legacy: DockerManagedBootstrapFinalizationWithoutAgent, context: DockerManagedBootstrapFinalizationContext | undefined, @@ -929,15 +942,7 @@ function parseDockerManagedBootstrapFinalizationRecordWithContext( if (serializeDockerManagedBootstrapFinalizationRecord(record) !== text) { fail("serialized finalization record is not canonical"); } - if (context) { - const normalizedContext = normalizeFinalizationContext(context); - if ( - record.agent !== normalizedContext.agent || - !matchesFinalizationContext(record, normalizedContext) - ) { - fail("finalization record does not match supplied durable context"); - } - } + if (context) assertFinalizationMatchesContext(record, context); return { record, upgradedLegacy: false }; } @@ -1273,6 +1278,7 @@ export function createFileDockerManagedBootstrapJournalStore( context?: DockerManagedBootstrapFinalizationContext, ) { const normalized = normalizeDockerManagedBootstrapFinalizationRecord(record); + if (context) assertFinalizationMatchesContext(normalized, context); assertDirectory(directory); const target = finalizationPath(journalPath(directory, normalized.bootstrapIdentity)); const serialized = serializeDockerManagedBootstrapFinalizationRecord(normalized); diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 32d6a81e75d..91f27c915fb 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -276,6 +276,31 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.replacement).toBeNull(); }); + it("rejects a divergent snapshot image before creating durable recovery state", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot: { + ...snapshot, + image: { + ...snapshot.image, + repository: "registry.example/nemoclaw/divergent", + }, + }, + request: rootRequest, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow("replacement snapshot image does not match its plan"); + expect(fake.replacement).toBeNull(); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.events).not.toContain("journal:staged"); + }); + it.each( SUPPORTED_AGENTS, )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index fe1fdf0787a..4d071be70b0 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -3040,6 +3040,14 @@ export function createDockerManagedBootstrapAdapter( ) { throw new Error("Managed bootstrap Docker replacement identities do not match."); } + if ( + snapshot.image.repository !== handle.plan.image.repository || + snapshot.image.manifestDigest !== handle.plan.image.manifestDigest + ) { + throw new Error( + "Managed bootstrap Docker replacement snapshot image does not match its plan.", + ); + } const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); if (normalizedOriginal.hash !== snapshot.specHash) { From 5e255a76365f0764d5a637d2e24708b431813f6a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 11:15:51 -0700 Subject: [PATCH 4/5] fix(onboard): close durable journal review feedback Signed-off-by: Aaron Erickson --- .../onboard/managed-bootstrap/adapter.test.ts | 5 +- .../managed-bootstrap/docker-journal.test.ts | 43 ++++++++++++--- .../managed-bootstrap/docker-journal.ts | 26 +++++++-- .../onboard/managed-bootstrap/docker.test.ts | 5 +- src/lib/onboard/managed-bootstrap/docker.ts | 55 ++++++++----------- .../managed-bootstrap-test-fixture.ts | 6 ++ 6 files changed, 87 insertions(+), 53 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index 666dfdcc317..06dec76255f 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -31,6 +31,7 @@ import { sameManagedBootstrapCompletionReceipt, sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; +import { reverseKeys } from "./managed-bootstrap-test-fixture"; const IDENTITY = "1".repeat(64); const CONFIG_ID = `sha256:${"2".repeat(64)}`; @@ -44,10 +45,6 @@ const ACTIVATED_HASH = createHash("sha256").update(ACTIVATED_SPEC_JSON, "utf8"). const RUNTIME_ID = "7".repeat(64); const PREPARED_ID = "8".repeat(64); -function reverseKeys(value: T): T { - return Object.fromEntries(Object.entries(value).reverse()) as T; -} - function requestFor(agent: ManagedStartupAgent) { return createManagedStartupRootApplyRequest({ agent, diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index d4672208518..f59ffe5188a 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -16,20 +16,18 @@ import { type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapLegacyRecordRequiresAgentError, + normalizeDockerManagedBootstrapJournal, parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; +import { reverseKeys } from "./managed-bootstrap-test-fixture"; const roots: string[] = []; const IDENTITY = "1".repeat(64); const OTHER_IDENTITY = "0".repeat(64); -function reverseKeys(value: T): T { - return Object.fromEntries(Object.entries(value).reverse()) as T; -} - const journal = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", @@ -198,6 +196,17 @@ afterEach(() => { }); describe("Docker managed bootstrap journal", () => { + it("rejects comma-joined keys as a different schema", () => { + const { agent: _agent, backupName: _backupName, ...withoutSeparateKeys } = journal; + + expect(() => + normalizeDockerManagedBootstrapJournal({ + ...withoutSeparateKeys, + "agent,backupName": "hermes", + }), + ).toThrow("journal schema is invalid"); + }); + it("publishes private canonical state through only monotonic phases", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); @@ -444,15 +453,33 @@ describe("Docker managed bootstrap journal", () => { it("upgrades legacy finalization only with exact immutable transaction context", () => { const serialized = `${JSON.stringify(legacyFinalizationV1())}\n`; - expect(() => parseDockerManagedBootstrapFinalizationRecord(serialized)).toThrowError( + let missingContextFailure: unknown; + try { + parseDockerManagedBootstrapFinalizationRecord(serialized); + } catch (error) { + missingContextFailure = error; + } + expect(missingContextFailure).toBeInstanceOf( DockerManagedBootstrapLegacyRecordRequiresAgentError, ); - expect(() => + expect(missingContextFailure).toMatchObject({ reason: "missing-context" }); + + let contextMismatchFailure: unknown; + try { parseDockerManagedBootstrapFinalizationRecord(serialized, { ...finalizationContext, planFingerprint: "0".repeat(64), - }), - ).toThrowError(DockerManagedBootstrapLegacyRecordRequiresAgentError); + }); + } catch (error) { + contextMismatchFailure = error; + } + expect(contextMismatchFailure).toBeInstanceOf( + DockerManagedBootstrapLegacyRecordRequiresAgentError, + ); + expect(contextMismatchFailure).toMatchObject({ + message: expect.stringContaining("supplied durable context does not match this record"), + reason: "context-mismatch", + }); expect(parseDockerManagedBootstrapFinalizationRecord(serialized, finalizationContext)).toEqual( finalization, ); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 70942ceebb4..acef0d35d70 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -119,20 +119,27 @@ export class DockerManagedBootstrapJournalAcknowledgementLostError extends Error export class DockerManagedBootstrapLegacyRecordRequiresAgentError extends Error { readonly bootstrapIdentity: string; readonly recordKind: "finalization" | "journal"; + readonly reason: "context-mismatch" | "missing-context" | undefined; readonly schemaVersion: number; constructor(input: { readonly bootstrapIdentity: string; readonly recordKind: "finalization" | "journal"; + readonly reason?: "context-mismatch" | "missing-context"; readonly schemaVersion: number; }) { + const reason = input.reason; super( `Managed bootstrap Docker ${input.recordKind} schema ${input.schemaVersion} for ` + - `${input.bootstrapIdentity} lacks durable agent identity`, + `${input.bootstrapIdentity} lacks durable agent identity` + + (reason === "context-mismatch" + ? "; supplied durable context does not match this record" + : ""), ); this.name = "DockerManagedBootstrapLegacyRecordRequiresAgentError"; this.bootstrapIdentity = input.bootstrapIdentity; this.recordKind = input.recordKind; + this.reason = reason; this.schemaVersion = input.schemaVersion; } } @@ -157,7 +164,12 @@ function fail(message: string): never { } function hasExactKeys(record: Readonly>, expected: readonly string[]) { - return Object.keys(record).sort().join(",") === [...expected].sort().join(","); + const actualKeys = Object.keys(record).sort(); + const expectedKeys = [...expected].sort(); + return ( + actualKeys.length === expectedKeys.length && + actualKeys.every((key, index) => key === expectedKeys[index]) + ); } function exactString(value: unknown, label: string, maxBytes = 4096): string { @@ -222,6 +234,9 @@ function sameSandboxIdentity( ); } +// Frozen historical schemas: these branches must reproduce the exact canonical +// bytes written by schema 1 and schema 2. Do not share their implementation with +// the current normalizer or update them when the current schema changes. function normalizeLegacyDockerManagedBootstrapJournal( journal: Readonly>, schemaVersion: 1 | 2, @@ -824,10 +839,11 @@ function upgradeLegacyFinalization( legacy: DockerManagedBootstrapFinalizationWithoutAgent, context: DockerManagedBootstrapFinalizationContext | undefined, ): DockerManagedBootstrapFinalizationRecord { - const missingAgent = () => + const missingAgent = (reason: "context-mismatch" | "missing-context" = "missing-context") => new DockerManagedBootstrapLegacyRecordRequiresAgentError({ bootstrapIdentity: legacy.bootstrapIdentity, recordKind: "finalization", + reason, schemaVersion: 1, }); // Runtime names and image repositories are mutable descriptions, never @@ -835,7 +851,9 @@ function upgradeLegacyFinalization( // the field omitted by schema v1. if (!context) throw missingAgent(); const normalizedContext = normalizeFinalizationContext(context); - if (!matchesFinalizationContext(legacy, normalizedContext)) throw missingAgent(); + if (!matchesFinalizationContext(legacy, normalizedContext)) { + throw missingAgent("context-mismatch"); + } return Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, phase: legacy.phase, diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 91f27c915fb..8c64d46cf39 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -16,10 +16,7 @@ import { OLD_ID, SUPPORTED_AGENTS, } from "./docker-test-fixture"; - -function reverseKeys(value: T): T { - return Object.fromEntries(Object.entries(value).reverse()) as T; -} +import { reverseKeys } from "./managed-bootstrap-test-fixture"; function expectEventBefore(events: readonly string[], before: string, after: string): void { expect(events).toContain(before); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 4d071be70b0..24071816c79 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -1778,22 +1778,19 @@ export function createDockerManagedBootstrapAdapter( const finalizationRecord = ( handle: ManagedBootstrapHeldWorkloadHandle, ): DockerManagedBootstrapFinalizationRecord | null => { - const record = deps.journalStore.loadFinalization( - handle.bootstrapIdentity, - finalizationContext(handle), - ); + const context = finalizationContext(handle); + const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); if (!record) return null; if ( - record.bootstrapIdentity !== handle.bootstrapIdentity || - record.providerId !== handle.sandbox.driverId || - record.agent !== handle.plan.profile.agent || - record.sandbox.sandboxName !== handle.sandbox.sandboxName || - record.sandbox.sandboxId !== handle.sandbox.sandboxId || - record.sandbox.driverId !== handle.sandbox.driverId || - record.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || - record.profileFingerprint !== handle.plan.profile.fingerprint || - record.imageReference !== - expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + record.bootstrapIdentity !== context.bootstrapIdentity || + record.providerId !== context.providerId || + record.agent !== context.agent || + record.sandbox.sandboxName !== context.sandbox.sandboxName || + record.sandbox.sandboxId !== context.sandbox.sandboxId || + record.sandbox.driverId !== context.sandbox.driverId || + record.planFingerprint !== context.planFingerprint || + record.profileFingerprint !== context.profileFingerprint || + record.imageReference !== context.imageReference ) { throw new Error("Managed bootstrap finalization record does not match its durable identity."); } @@ -1805,30 +1802,25 @@ export function createDockerManagedBootstrapAdapter( commitReceipt: ManagedBootstrapCompletionReceipt | null, cleanupReceipt: ManagedBootstrapFinalizationReceipt, ): ManagedBootstrapFinalizationReceipt => { + const context = finalizationContext(handle); const record = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, phase, - bootstrapIdentity: handle.bootstrapIdentity, - providerId: handle.sandbox.driverId, - agent: handle.plan.profile.agent, - sandbox: handle.sandbox, - planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), - profileFingerprint: handle.plan.profile.fingerprint, - imageReference: expectedImageReference( - handle.plan.image.repository, - handle.plan.image.manifestDigest, - ), + bootstrapIdentity: context.bootstrapIdentity, + providerId: context.providerId, + agent: context.agent, + sandbox: context.sandbox, + planFingerprint: context.planFingerprint, + profileFingerprint: context.profileFingerprint, + imageReference: context.imageReference, commitReceipt, cleanupReceipt, } satisfies DockerManagedBootstrapFinalizationRecord); const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); try { - deps.journalStore.recordFinalization(record, finalizationContext(handle)); + deps.journalStore.recordFinalization(record, context); } catch (error) { - const recovered = deps.journalStore.loadFinalization( - handle.bootstrapIdentity, - finalizationContext(handle), - ); + const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); if ( !recovered || serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized @@ -1836,10 +1828,7 @@ export function createDockerManagedBootstrapAdapter( throw error; } } - const persisted = deps.journalStore.loadFinalization( - handle.bootstrapIdentity, - finalizationContext(handle), - ); + const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity, context); if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); } diff --git a/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts b/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts new file mode 100644 index 00000000000..c4eb9cfb957 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function reverseKeys(value: T): T { + return Object.fromEntries(Object.entries(value).reverse()) as T; +} From 976a316b99ac871beffa53220644f708e1eda244 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 12:22:12 -0700 Subject: [PATCH 5/5] test(onboard): inventory provider-neutral bootstrap fixture Signed-off-by: Aaron Erickson --- test/runtime-provider-source-shape.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 8d718b95d13..20212473ae1 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -143,6 +143,7 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/managed-bootstrap/docker.ts", "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/index.ts", + "src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts", "src/lib/onboard/managed-bootstrap/runtime-create.ts", ]); expect(providerPaths).toEqual([