diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 159ab401e86..b5f8434f79f 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -85,9 +85,24 @@ and sandbox ID and then enter the destructive cutover. Post-cutover rollback publishes `rollback-authorized` before exact replacement deletion; pre-cutover staged cleanup removes only the exact prepared replacement without that journal transition. Commit publishes `shared-state-committed` before exact backup -deletion. Cleanup is bound to full runtime IDs. Mutable OpenShell names are read -only to detect ownership reuse, and unsafe name-only deletion returns a typed -retention error. The dormant adapter assumes the protocol's single coordinator; +deletion. Cleanup is bound to full runtime IDs. Its private state root retains +versioned, identity-addressed transaction records containing the provider and +sandbox identities, plan and profile +fingerprints, exact original and replacement IDs, rollback target, and phase. +Exact commit and cleanup receipts are durable terminal records, so adapter +recreation does not depend on process-local transaction sets or tombstone maps. +The image-owned shared-state transaction uses the same identity-bound model: a +commit atomically moves its pending manifest and backups into a durable receipt +namespace, compacts that state to an exact commit receipt, and rejects rollback +when a later image-runtime invocation reads that receipt. The provider may +retire that receipt only after it proves the external rollback backup is gone, +so that this receipt does not block the next bootstrap attempt. +Direct identity lookup reconstructs one known transaction record. The bounded +[3.12b recovery slice](https://github.com/NVIDIA/NemoClaw/issues/7744) introduces +unfinished-record enumeration together with phase reconciliation and +cross-surface resume or rollback. The adapter reads mutable OpenShell names only +to detect ownership reuse. Unsafe name-only deletion returns a typed retention +error. The dormant adapter assumes the protocol's single coordinator; multi-process lease/arbitration remains an explicit production-activation gate. Activation must also inject the selected gateway's canonical state root. diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 415fced86cd..ee1a834985f 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -933,13 +933,15 @@ function normalizePreparedReplacement( }); } +export function createManagedBootstrapPlanFingerprint(plan: ManagedBootstrapExpectedPlan): string { + return createHash("sha256").update(canonicalJson(plan), "utf8").digest("hex"); +} + export function createManagedBootstrapPreparedAuthority( transaction: ManagedBootstrapPreparedTransaction, ): ManagedBootstrapPreparedAuthority { const { handle, snapshot, prepared } = transaction; - const planFingerprint = createHash("sha256") - .update(canonicalJson(handle.plan), "utf8") - .digest("hex"); + const planFingerprint = createManagedBootstrapPlanFingerprint(handle.plan); const bound = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, phase: "prepared" as const, diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 7ed0d8bba6a..25cd9ec0e50 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -9,10 +9,15 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, + parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, + sameDockerManagedBootstrapReceipt, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; @@ -22,11 +27,13 @@ const journal = Object.freeze({ schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: IDENTITY, + providerId: "docker", sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "docker", }, + planFingerprint: "9".repeat(64), profileFingerprint: "2".repeat(64), imageReference: `registry.example/image@sha256:${"3".repeat(64)}`, runtimeImageContentId: `sha256:${"4".repeat(64)}`, @@ -37,7 +44,59 @@ const journal = Object.freeze({ backupName: "openshell-alpha-backup", originalSpecHash: "7".repeat(64), replacementSpecHash: "8".repeat(64), + rollbackTargetRuntimeId: "5".repeat(64), + rollbackTargetSpecHash: "7".repeat(64), + preparationReceipt: { + schemaVersion: 1, + sandbox: { + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + driverId: "docker", + }, + bootstrapIdentity: IDENTITY, + authorityFingerprint: "a".repeat(64), + recordId: "prepared-alpha", + recordedAt: "2026-07-31T19:59:59.000Z", + }, + commitReceipt: null, } satisfies DockerManagedBootstrapJournal); +const finalization = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase: "committed", + bootstrapIdentity: IDENTITY, + providerId: "docker", + sandbox: journal.sandbox, + planFingerprint: journal.planFingerprint, + profileFingerprint: journal.profileFingerprint, + imageReference: journal.imageReference, + commitReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + runtimeId: journal.replacementRuntimeId, + image: { + repository: "registry.example/image", + manifestDigest: `sha256:${"3".repeat(64)}` as const, + }, + runtimeImageContentId: journal.runtimeImageContentId, + originalSpecHash: journal.originalSpecHash, + replacementSpecHash: journal.replacementSpecHash, + profileFingerprint: journal.profileFingerprint, + bootstrapIdentity: IDENTITY, + transactionPending: false, + completedAt: "2026-07-31T20:00:00.000Z", + }, + cleanupReceipt: { + schemaVersion: 1, + sandbox: journal.sandbox, + bootstrapIdentity: IDENTITY, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: "2026-07-31T20:00:01.000Z", + }, +} satisfies DockerManagedBootstrapFinalizationRecord); function readPinnedPrivateFile(target: string): { readonly mode: number; readonly text: string } { const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); @@ -78,6 +137,9 @@ describe("Docker managed bootstrap journal", () => { ); expect(store.transition(IDENTITY, "staged", "cutover").phase).toBe("cutover"); + expect(store.recordCompletion(IDENTITY, finalization.commitReceipt).commitReceipt).toEqual( + finalization.commitReceipt, + ); expect(store.transition(IDENTITY, "cutover", "shared-state-committed").phase).toBe( "shared-state-committed", ); @@ -172,4 +234,97 @@ describe("Docker managed bootstrap journal", () => { serializeDockerManagedBootstrapJournal(Object.freeze({ ...journal, phase: "staged" })), ).toBe(`${JSON.stringify(journal)}\n`); }); + + it("compares equivalent receipts independent of property insertion order", () => { + const preparation = journal.preparationReceipt; + const reorderedPreparation = { + recordedAt: preparation.recordedAt, + recordId: preparation.recordId, + authorityFingerprint: preparation.authorityFingerprint, + bootstrapIdentity: preparation.bootstrapIdentity, + sandbox: { + driverId: preparation.sandbox.driverId, + sandboxId: preparation.sandbox.sandboxId, + sandboxName: preparation.sandbox.sandboxName, + }, + schemaVersion: preparation.schemaVersion, + } satisfies typeof preparation; + expect( + sameDockerManagedBootstrapReceipt("preparation", preparation, reorderedPreparation), + ).toBe(true); + + const completion = finalization.commitReceipt; + const reorderedCompletion = { + completedAt: completion.completedAt, + transactionPending: completion.transactionPending, + bootstrapIdentity: completion.bootstrapIdentity, + profileFingerprint: completion.profileFingerprint, + replacementSpecHash: completion.replacementSpecHash, + originalSpecHash: completion.originalSpecHash, + runtimeImageContentId: completion.runtimeImageContentId, + image: { + manifestDigest: completion.image.manifestDigest, + repository: completion.image.repository, + }, + runtimeId: completion.runtimeId, + sandbox: { + driverId: completion.sandbox.driverId, + sandboxId: completion.sandbox.sandboxId, + sandboxName: completion.sandbox.sandboxName, + }, + schemaVersion: completion.schemaVersion, + } satisfies typeof completion; + expect(sameDockerManagedBootstrapReceipt("completion", completion, reorderedCompletion)).toBe( + true, + ); + }); + + it("reloads exact terminal receipts from a new journal store", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + + first.recordFinalization(finalization); + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.loadFinalization(IDENTITY)).toEqual(finalization); + expect( + parseDockerManagedBootstrapFinalizationRecord( + serializeDockerManagedBootstrapFinalizationRecord(finalization), + ), + ).toEqual(finalization); + expect(() => + restarted.recordFinalization({ + ...finalization, + cleanupReceipt: { ...finalization.cleanupReceipt, finalizedAt: "2026-07-31T20:00:02.000Z" }, + }), + ).toThrow("finalization record changed"); + expect(() => + restarted.recordFinalization({ + ...finalization, + phase: "rolled-back", + commitReceipt: null, + cleanupReceipt: { ...finalization.cleanupReceipt, outcome: "rolled-back" }, + }), + ).toThrow("finalization record changed"); + }); + + it("reloads the exact completion receipt from a new journal store", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const first = createFileDockerManagedBootstrapJournalStore(root); + first.create(journal); + first.transition(IDENTITY, "staged", "cutover"); + const completed = first.recordCompletion(IDENTITY, finalization.commitReceipt); + expect(completed.commitReceipt).toEqual(finalization.commitReceipt); + + const restarted = createFileDockerManagedBootstrapJournalStore(root); + expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + expect(() => + restarted.recordCompletion(IDENTITY, { + ...finalization.commitReceipt, + completedAt: "2026-07-31T20:00:02.000Z", + }), + ).toThrow("completion receipt changed"); + }); }); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index c6043409d7d..015bda1d12d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -4,12 +4,19 @@ import fs from "node:fs"; import path from "node:path"; -import type { ManagedBootstrapSandboxIdentity } from "./adapter"; +import type { + ManagedBootstrapCompletionReceipt, + ManagedBootstrapDurablePreparationReceipt, + ManagedBootstrapFinalizationReceipt, + ManagedBootstrapSandboxIdentity, +} from "./adapter"; -export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 1 as const; +export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION = 2 as const; export const DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY = "managed-bootstrap"; +export const DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION = 1 as const; const SHA256_RE = /^[a-f0-9]{64}$/u; +const MANIFEST_DIGEST_RE = /^sha256:[a-f0-9]{64}$/u; const MAX_JOURNAL_BYTES = 32 * 1024; const JOURNAL_DIRECTORY_MODE = 0o700; const JOURNAL_FILE_MODE = 0o600; @@ -28,7 +35,9 @@ export interface DockerManagedBootstrapJournal { readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; readonly phase: DockerManagedBootstrapJournalPhase; readonly bootstrapIdentity: string; + readonly providerId: string; readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; readonly profileFingerprint: string; readonly imageReference: string; readonly runtimeImageContentId: string; @@ -39,6 +48,23 @@ export interface DockerManagedBootstrapJournal { readonly backupName: string; readonly originalSpecHash: string; readonly replacementSpecHash: string; + readonly rollbackTargetRuntimeId: string; + readonly rollbackTargetSpecHash: string; + readonly preparationReceipt: ManagedBootstrapDurablePreparationReceipt | null; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; +} + +export interface DockerManagedBootstrapFinalizationRecord { + readonly schemaVersion: typeof DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION; + readonly phase: "committed" | "rolled-back"; + readonly bootstrapIdentity: string; + readonly providerId: string; + readonly sandbox: ManagedBootstrapSandboxIdentity; + readonly planFingerprint: string; + readonly profileFingerprint: string; + readonly imageReference: string; + readonly commitReceipt: ManagedBootstrapCompletionReceipt | null; + readonly cleanupReceipt: ManagedBootstrapFinalizationReceipt; } export interface DockerManagedBootstrapJournalStore { @@ -49,7 +75,13 @@ export interface DockerManagedBootstrapJournalStore { expected: DockerManagedBootstrapJournalPhase, next: DockerManagedBootstrapJournalPhase, ): DockerManagedBootstrapJournal; + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal; remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]): void; + recordFinalization(record: DockerManagedBootstrapFinalizationRecord): void; + loadFinalization(bootstrapIdentity: string): DockerManagedBootstrapFinalizationRecord | null; } /** @@ -137,15 +169,21 @@ export function normalizeDockerManagedBootstrapJournal( const expectedKeys = [ "backupName", "bootstrapIdentity", + "commitReceipt", "imageReference", "originalName", "originalRuntimeId", "originalSpecHash", "phase", + "planFingerprint", + "preparationReceipt", "profileFingerprint", + "providerId", "replacementRuntimeId", "replacementSpecHash", "replacementStagingName", + "rollbackTargetRuntimeId", + "rollbackTargetSpecHash", "runtimeImageContentId", "sandbox", "schemaVersion", @@ -160,7 +198,9 @@ export function normalizeDockerManagedBootstrapJournal( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: exactPhase(journal.phase), bootstrapIdentity: exactSha256(journal.bootstrapIdentity, "bootstrap identity"), + providerId: exactString(journal.providerId, "provider ID"), sandbox: exactSandbox(journal.sandbox), + planFingerprint: exactSha256(journal.planFingerprint, "plan fingerprint"), profileFingerprint: exactSha256(journal.profileFingerprint, "profile fingerprint"), imageReference: exactString(journal.imageReference, "image reference"), runtimeImageContentId: exactString(journal.runtimeImageContentId, "runtime image content ID"), @@ -175,6 +215,20 @@ export function normalizeDockerManagedBootstrapJournal( backupName: exactString(journal.backupName, "backup name", 253), originalSpecHash: exactSha256(journal.originalSpecHash, "original spec hash"), replacementSpecHash: exactSha256(journal.replacementSpecHash, "replacement spec hash"), + rollbackTargetRuntimeId: exactSha256( + journal.rollbackTargetRuntimeId, + "rollback target runtime ID", + ), + rollbackTargetSpecHash: exactSha256( + journal.rollbackTargetSpecHash, + "rollback target spec hash", + ), + preparationReceipt: + journal.preparationReceipt === null + ? null + : exactPreparationReceipt(journal.preparationReceipt), + commitReceipt: + journal.commitReceipt === null ? null : exactCompletionReceipt(journal.commitReceipt), } satisfies DockerManagedBootstrapJournal); if (normalized.originalRuntimeId === normalized.replacementRuntimeId) { fail("original and replacement runtime IDs must differ"); @@ -185,6 +239,33 @@ export function normalizeDockerManagedBootstrapJournal( ) { fail("original, staging, and backup names must be distinct"); } + if ( + normalized.providerId !== normalized.sandbox.driverId || + normalized.rollbackTargetRuntimeId !== normalized.originalRuntimeId || + normalized.rollbackTargetSpecHash !== normalized.originalSpecHash + ) { + fail("provider or rollback authority does not match the transaction identity"); + } + if ( + (normalized.preparationReceipt !== null && + (normalized.preparationReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.preparationReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.preparationReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.preparationReceipt.sandbox.driverId !== normalized.sandbox.driverId)) || + (normalized.commitReceipt !== null && + (normalized.commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + normalized.commitReceipt.sandbox.sandboxName !== normalized.sandbox.sandboxName || + normalized.commitReceipt.sandbox.sandboxId !== normalized.sandbox.sandboxId || + normalized.commitReceipt.sandbox.driverId !== normalized.sandbox.driverId || + normalized.commitReceipt.runtimeId !== normalized.replacementRuntimeId || + normalized.commitReceipt.profileFingerprint !== normalized.profileFingerprint || + normalized.commitReceipt.originalSpecHash !== normalized.originalSpecHash || + normalized.commitReceipt.replacementSpecHash !== normalized.replacementSpecHash || + `${normalized.commitReceipt.image.repository}@${normalized.commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("durable preparation or commit receipt does not match the transaction identity"); + } return normalized; } @@ -220,6 +301,275 @@ export function parseDockerManagedBootstrapJournal(text: string): DockerManagedB return journal; } +function exactTimestamp(value: unknown, label: string): string { + const timestamp = exactString(value, label, 128); + const parsed = new Date(timestamp); + if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== timestamp) { + fail(`${label} must be one canonical timestamp`); + } + return timestamp; +} + +function exactBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") fail(`${label} must be boolean`); + return value; +} + +function exactNullableSha256(value: unknown, label: string): string | null { + return value === null ? null : exactSha256(value, label); +} + +function exactImage(value: unknown): ManagedBootstrapCompletionReceipt["image"] { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("completion image identity must be an object"); + } + const image = value as Record; + if (Object.keys(image).sort().join(",") !== "manifestDigest,repository") { + fail("completion image identity schema is invalid"); + } + const manifestDigest = exactString(image.manifestDigest, "completion manifest digest", 128); + if (!MANIFEST_DIGEST_RE.test(manifestDigest)) { + fail("completion manifest digest must be canonical sha256"); + } + return Object.freeze({ + repository: exactString(image.repository, "completion image repository"), + manifestDigest: manifestDigest as `sha256:${string}`, + }); +} + +function exactPreparationReceipt(value: unknown): ManagedBootstrapDurablePreparationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("durable preparation receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "authorityFingerprint", + "bootstrapIdentity", + "recordId", + "recordedAt", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("durable preparation receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256( + receipt.bootstrapIdentity, + "durable preparation bootstrap identity", + ), + authorityFingerprint: exactSha256( + receipt.authorityFingerprint, + "durable preparation authority fingerprint", + ), + recordId: exactString(receipt.recordId, "durable preparation record ID", 1024), + recordedAt: exactTimestamp(receipt.recordedAt, "durable preparation timestamp"), + }); +} + +function exactCompletionReceipt(value: unknown): ManagedBootstrapCompletionReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("commit receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "completedAt", + "image", + "originalSpecHash", + "profileFingerprint", + "replacementSpecHash", + "runtimeId", + "runtimeImageContentId", + "sandbox", + "schemaVersion", + "transactionPending", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 + ) { + fail("commit receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + runtimeId: exactSha256(receipt.runtimeId, "commit runtime ID"), + image: exactImage(receipt.image), + runtimeImageContentId: exactString( + receipt.runtimeImageContentId, + "commit runtime image content ID", + ), + originalSpecHash: exactSha256(receipt.originalSpecHash, "commit original spec hash"), + replacementSpecHash: exactSha256(receipt.replacementSpecHash, "commit replacement spec hash"), + profileFingerprint: exactSha256(receipt.profileFingerprint, "commit profile fingerprint"), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "commit bootstrap identity"), + transactionPending: exactBoolean(receipt.transactionPending, "commit transaction pending"), + completedAt: exactTimestamp(receipt.completedAt, "commit completion timestamp"), + }); +} + +export function sameDockerManagedBootstrapReceipt( + kind: "preparation", + left: ManagedBootstrapDurablePreparationReceipt, + right: ManagedBootstrapDurablePreparationReceipt, +): boolean; +export function sameDockerManagedBootstrapReceipt( + kind: "completion", + left: ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapCompletionReceipt, +): boolean; +export function sameDockerManagedBootstrapReceipt( + kind: "preparation" | "completion", + left: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, +): boolean { + if (kind === "preparation") { + return ( + JSON.stringify(exactPreparationReceipt(left)) === + JSON.stringify(exactPreparationReceipt(right)) + ); + } + return ( + JSON.stringify(exactCompletionReceipt(left)) === JSON.stringify(exactCompletionReceipt(right)) + ); +} + +function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("cleanup receipt must be an object"); + } + const receipt = value as Record; + const expectedKeys = [ + "alreadyRolledBack", + "bootstrapIdentity", + "finalizedAt", + "heldWorkloadRemoved", + "outcome", + "restoredRuntimeId", + "restoredSpecHash", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(receipt).sort().join(",") !== expectedKeys.sort().join(",") || + receipt.schemaVersion !== 1 || + !["committed", "rolled-back"].includes(String(receipt.outcome)) + ) { + fail("cleanup receipt schema is invalid"); + } + return Object.freeze({ + schemaVersion: 1, + sandbox: exactSandbox(receipt.sandbox), + bootstrapIdentity: exactSha256(receipt.bootstrapIdentity, "cleanup bootstrap identity"), + outcome: receipt.outcome as "committed" | "rolled-back", + restoredRuntimeId: exactNullableSha256(receipt.restoredRuntimeId, "restored runtime ID"), + restoredSpecHash: exactNullableSha256(receipt.restoredSpecHash, "restored spec hash"), + heldWorkloadRemoved: exactBoolean(receipt.heldWorkloadRemoved, "held workload removed"), + alreadyRolledBack: exactBoolean(receipt.alreadyRolledBack, "already rolled back"), + finalizedAt: exactTimestamp(receipt.finalizedAt, "cleanup finalization timestamp"), + }); +} + +export function normalizeDockerManagedBootstrapFinalizationRecord( + value: unknown, +): DockerManagedBootstrapFinalizationRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail("finalization record must be an object"); + } + const record = value as Record; + const expectedKeys = [ + "bootstrapIdentity", + "cleanupReceipt", + "commitReceipt", + "imageReference", + "phase", + "planFingerprint", + "profileFingerprint", + "providerId", + "sandbox", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION || + !["committed", "rolled-back"].includes(String(record.phase)) + ) { + fail("finalization record schema is invalid"); + } + const phase = record.phase as "committed" | "rolled-back"; + const sandbox = exactSandbox(record.sandbox); + const commitReceipt = + record.commitReceipt === null ? null : exactCompletionReceipt(record.commitReceipt); + const cleanupReceipt = exactCleanupReceipt(record.cleanupReceipt); + const normalized = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: exactSha256(record.bootstrapIdentity, "finalization bootstrap identity"), + providerId: exactString(record.providerId, "finalization provider ID"), + sandbox, + planFingerprint: exactSha256(record.planFingerprint, "finalization plan fingerprint"), + profileFingerprint: exactSha256(record.profileFingerprint, "finalization profile fingerprint"), + imageReference: exactString(record.imageReference, "finalization image reference"), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + if ( + normalized.providerId !== sandbox.driverId || + normalized.bootstrapIdentity !== cleanupReceipt.bootstrapIdentity || + normalized.phase !== cleanupReceipt.outcome || + JSON.stringify(normalized.sandbox) !== JSON.stringify(cleanupReceipt.sandbox) || + (phase === "committed") !== (commitReceipt !== null) || + (commitReceipt !== null && + (commitReceipt.bootstrapIdentity !== normalized.bootstrapIdentity || + commitReceipt.profileFingerprint !== normalized.profileFingerprint || + JSON.stringify(commitReceipt.sandbox) !== JSON.stringify(normalized.sandbox) || + `${commitReceipt.image.repository}@${commitReceipt.image.manifestDigest}` !== + normalized.imageReference)) + ) { + fail("finalization receipts do not match their durable transaction identity"); + } + return normalized; +} + +export function serializeDockerManagedBootstrapFinalizationRecord( + record: DockerManagedBootstrapFinalizationRecord, +): string { + const serialized = `${JSON.stringify(normalizeDockerManagedBootstrapFinalizationRecord(record))}\n`; + if (Buffer.byteLength(serialized, "utf8") > MAX_JOURNAL_BYTES) { + fail("serialized finalization record exceeds its bounded transport"); + } + return serialized; +} + +export function parseDockerManagedBootstrapFinalizationRecord( + text: string, +): DockerManagedBootstrapFinalizationRecord { + if ( + text.length === 0 || + text.includes("\0") || + Buffer.byteLength(text, "utf8") > MAX_JOURNAL_BYTES + ) { + fail("serialized finalization record is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized finalization record is not valid JSON"); + } + const record = normalizeDockerManagedBootstrapFinalizationRecord(parsed); + if (serializeDockerManagedBootstrapFinalizationRecord(record) !== text) { + fail("serialized finalization record is not canonical"); + } + return record; +} + function assertDirectory(directory: string): void { fs.mkdirSync(directory, { recursive: true, mode: JOURNAL_DIRECTORY_MODE }); const stat = fs.lstatSync(directory); @@ -237,6 +587,10 @@ function decisionPath(target: string): string { return `${target}.decision`; } +function finalizationPath(target: string): string { + return `${target}.finalized`; +} + function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { return ( left.dev === right.dev && @@ -361,6 +715,15 @@ function atomicWrite( if (cleanupFailure !== null) throw cleanupFailure.error; } +function sameSerializedJournal( + left: DockerManagedBootstrapJournal, + right: DockerManagedBootstrapJournal, +): boolean { + return ( + serializeDockerManagedBootstrapJournal(left) === serializeDockerManagedBootstrapJournal(right) + ); +} + export function createFileDockerManagedBootstrapJournalStore( stateRoot: string, ): DockerManagedBootstrapJournalStore { @@ -386,9 +749,26 @@ export function createFileDockerManagedBootstrapJournalStore( } return decided; }; + const loadFinalization = ( + bootstrapIdentity: string, + ): DockerManagedBootstrapFinalizationRecord | null => { + assertDirectory(directory); + const contents = readPrivateFile( + finalizationPath(journalPath(directory, bootstrapIdentity)), + "finalization", + ); + return contents === null ? null : parseDockerManagedBootstrapFinalizationRecord(contents); + }; return Object.freeze({ create(journal: DockerManagedBootstrapJournal) { const normalized = normalizeDockerManagedBootstrapJournal(journal); + if ( + normalized.phase !== "staged" || + normalized.preparationReceipt === null || + normalized.commitReceipt !== null + ) { + fail("a new journal requires staged durable preparation authority"); + } assertDirectory(directory); const target = journalPath(directory, normalized.bootstrapIdentity); if (readPrivateFile(decisionPath(target), "decision") !== null) { @@ -429,6 +809,33 @@ export function createFileDockerManagedBootstrapJournalStore( atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); return updated; }, + recordCompletion( + bootstrapIdentity: string, + receipt: ManagedBootstrapCompletionReceipt, + ): DockerManagedBootstrapJournal { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || current.phase !== "cutover") { + fail(`completion recording requires phase cutover, found ${current?.phase ?? "absent"}`); + } + const updated = normalizeDockerManagedBootstrapJournal({ + ...current, + commitReceipt: receipt, + }); + if (current.commitReceipt !== null) { + if (!sameSerializedJournal(current, updated)) { + fail("completion receipt changed for this bootstrap identity"); + } + return current; + } + atomicWrite(directory, target, serializeDockerManagedBootstrapJournal(updated), false); + const persisted = load(bootstrapIdentity); + if (!persisted || !sameSerializedJournal(persisted, updated)) { + fail("completion receipt was not durably re-readable"); + } + return persisted; + }, remove(bootstrapIdentity: string, expected: readonly DockerManagedBootstrapJournalPhase[]) { assertDirectory(directory); const target = journalPath(directory, bootstrapIdentity); @@ -444,5 +851,26 @@ export function createFileDockerManagedBootstrapJournalStore( fs.unlinkSync(target); fsyncDirectory(directory); }, + recordFinalization(record: DockerManagedBootstrapFinalizationRecord) { + const normalized = normalizeDockerManagedBootstrapFinalizationRecord(record); + assertDirectory(directory); + const target = finalizationPath(journalPath(directory, normalized.bootstrapIdentity)); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(normalized); + const existing = readPrivateFile(target, "finalization"); + if (existing !== null) { + if (existing !== serialized) + fail("finalization record changed for this bootstrap identity"); + return; + } + try { + atomicWrite(directory, target, serialized, true); + } catch (error) { + if (readPrivateFile(target, "finalization") !== serialized) throw error; + } + if (readPrivateFile(target, "finalization") !== serialized) { + fail("finalization record was not durably re-readable"); + } + }, + loadFinalization, }); } diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index 415fe56fb82..a92d2e335ed 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -17,6 +17,7 @@ import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-pat import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "../managed-startup/image-runtime"; import { MANAGED_STARTUP_AGENTS, type ManagedStartupAgent } from "../managed-startup/profile"; import { + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, } from "../managed-startup/shared-state-transaction"; @@ -24,8 +25,6 @@ import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers import { cleanupTempDir, secureTempFile } from "../temp-files"; const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; -const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = - "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; const DURABLE_IDENTITY_RE = /^[a-f0-9]{64}$/u; const DOCKER_MUTATION_OPTIONS = { diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index cb069cb5bca..088a3ed7a05 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -21,13 +21,18 @@ import { } from "./adapter"; import type { DockerManagedBootstrapDeps } from "./docker"; import { + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalPhase, type DockerManagedBootstrapJournalStore, } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; -import { parseManagedBootstrapEnvelope } from "./envelope"; +import { + MANAGED_BOOTSTRAP_COMPLETION_FILE, + parseManagedBootstrapEnvelope, + serializeManagedBootstrapImageCompletion, +} from "./envelope"; export const IDENTITY = "1".repeat(64); export const OLD_ID = "2".repeat(64); @@ -206,6 +211,7 @@ export function fixture(options: DockerFixtureOptions = {}) { let original: DockerContainerInspect | null = originalInspect(agentInputs(options.agent)); let replacement: DockerContainerInspect | null = null; let journal: DockerManagedBootstrapJournal | null = null; + let finalization: DockerManagedBootstrapFinalizationRecord | null = null; let sharedState: "committed" | "none" | "pending" = options.sharedState ?? "none"; const events: string[] = []; const lostAcknowledgements = new Set(options.lostAcknowledgements ?? []); @@ -240,6 +246,20 @@ export function fixture(options: DockerFixtureOptions = {}) { } return structuredClone(journal); }, + recordCompletion(_identity, receipt) { + if (!journal || journal.phase !== "cutover") { + throw new Error("completion requires cutover journal"); + } + if ( + journal.commitReceipt !== null && + JSON.stringify(journal.commitReceipt) !== JSON.stringify(receipt) + ) { + throw new Error("completion changed"); + } + journal = { ...journal, commitReceipt: structuredClone(receipt) }; + events.push("journal:completion"); + return structuredClone(journal); + }, remove(_identity, expected) { const current = journal; void (current !== null && expected.includes(current.phase) @@ -253,6 +273,14 @@ export function fixture(options: DockerFixtureOptions = {}) { ); } }, + recordFinalization(value) { + if (finalization && JSON.stringify(finalization) !== JSON.stringify(value)) { + throw new Error("finalization changed"); + } + finalization = structuredClone(value); + events.push(`finalization:${value.phase}`); + }, + loadFinalization: () => (finalization ? structuredClone(finalization) : null), }; const inspect = (reference: string): DockerContainerInspect => { const candidates = [original, replacement].filter( @@ -323,6 +351,20 @@ export function fixture(options: DockerFixtureOptions = {}) { return ok(); }; const copyFromContainer = () => { + if (source === `${NEW_ID}:${MANAGED_BOOTSTRAP_COMPLETION_FILE}`) { + fs.writeFileSync( + destination, + serializeManagedBootstrapImageCompletion({ + bootstrapIdentity: IDENTITY, + agent: options.agent ?? "hermes", + profileFingerprint: agentInputs(options.agent).request.profileFingerprint, + transactionPending: sharedState === "pending", + }), + { mode: 0o444 }, + ); + fs.chmodSync(destination, 0o444); + return ok(); + } const receipt = source.split(":")[1]; const expected = receipt?.includes("shared-state-commit") ? "committed" : "pending"; return sharedState === expected @@ -433,6 +475,9 @@ export function fixture(options: DockerFixtureOptions = {}) { get journal() { return journal; }, + get finalization() { + return finalization; + }, get original() { return original; }, diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 7979330b180..8e94b573589 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -3,7 +3,10 @@ import { assert, describe, expect, it, vi } from "vitest"; -import { ManagedBootstrapOwnerCleanupRequiredError } from "./adapter"; +import { + ManagedBootstrapDurableCommitCleanupPendingError, + ManagedBootstrapOwnerCleanupRequiredError, +} from "./adapter"; import { createDockerManagedBootstrapAdapter } from "./docker"; import { normalizeDockerManagedBootstrapLaunchSpec, @@ -49,11 +52,23 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.events).not.toContain(`stop:${OLD_ID}`); fake.events.push("authority:recorded"); const durable = durablePreparation(handle, snapshot, prepared); + const reorderedDurable = { + recordedAt: durable.recordedAt, + recordId: durable.recordId, + authorityFingerprint: durable.authorityFingerprint, + bootstrapIdentity: durable.bootstrapIdentity, + sandbox: { + driverId: durable.sandbox.driverId, + sandboxId: durable.sandbox.sandboxId, + sandboxName: durable.sandbox.sandboxName, + }, + schemaVersion: durable.schemaVersion, + } satisfies typeof durable; const replacement = await adapter.activateBootstrapReplacement({ handle, snapshot, prepared, - durablePreparation: durable, + durablePreparation: reorderedDurable, }); const order = fake.events; expect(order).toContain("authority:recorded"); @@ -68,17 +83,70 @@ describe("Docker managed bootstrap adapter", () => { replacementRuntimeId: NEW_ID, }); + const commitReceipt = await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); + const reorderedCommitReceipt = { + completedAt: commitReceipt.completedAt, + transactionPending: commitReceipt.transactionPending, + bootstrapIdentity: commitReceipt.bootstrapIdentity, + profileFingerprint: commitReceipt.profileFingerprint, + replacementSpecHash: commitReceipt.replacementSpecHash, + originalSpecHash: commitReceipt.originalSpecHash, + runtimeImageContentId: commitReceipt.runtimeImageContentId, + image: { + manifestDigest: commitReceipt.image.manifestDigest, + repository: commitReceipt.image.repository, + }, + runtimeId: commitReceipt.runtimeId, + sandbox: { + driverId: commitReceipt.sandbox.driverId, + sandboxId: commitReceipt.sandbox.sandboxId, + sandboxName: commitReceipt.sandbox.sandboxName, + }, + schemaVersion: commitReceipt.schemaVersion, + } satisfies typeof commitReceipt; + expect(fake.events).toContain("journal:completion"); + expect(fake.events).toContain(`start:${NEW_ID}`); + expect(fake.events.indexOf("journal:completion")).toBeGreaterThan( + fake.events.indexOf(`start:${NEW_ID}`), + ); + const finalized = await adapter.finalizeBootstrap({ + outcome: "commit", + handle, + snapshot, + prepared, + durablePreparation: reorderedDurable, + replacement, + completion: reorderedCommitReceipt, + }); + expect(finalized).toMatchObject({ outcome: "committed" }); + expect(fake.events).toContain("journal:shared-state-committed"); + expect(fake.events).toContain(`rm:${OLD_ID}`); + expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( + fake.events.indexOf(`rm:${OLD_ID}`), + ); + expect(fake.journal).toBeNull(); + expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); + expect(fake.sharedState).toBe("none"); + expect(fake.replacement?.Id).toBe(NEW_ID); + + const eventCount = fake.events.length; await expect( - adapter.finalizeBootstrap({ + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ outcome: "commit", handle, snapshot, prepared, - durablePreparation: durable, + durablePreparation: reorderedDurable, replacement, - completion: completion(replacement), + completion: reorderedCommitReceipt, }), - ).resolves.toMatchObject({ outcome: "committed" }); + ).resolves.toEqual(finalized); + expect(fake.events).toHaveLength(eventCount); expect(fake.events).toContain("journal:shared-state-committed"); expect(fake.events).toContain(`rm:${OLD_ID}`); expect(fake.events.indexOf("journal:shared-state-committed")).toBeLessThan( @@ -87,6 +155,20 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.journal).toBeNull(); expect(fake.sharedState).toBe("none"); expect(fake.replacement?.Id).toBe(NEW_ID); + + await expect( + createDockerManagedBootstrapAdapter(fake.deps).finalizeBootstrap({ + outcome: "rollback", + handle, + snapshot, + prepared, + durablePreparation: reorderedDurable, + replacement, + completion: null, + }), + ).rejects.toBeInstanceOf(ManagedBootstrapDurableCommitCleanupPendingError); + expect(fake.events).toHaveLength(eventCount); + expect(fake.finalization).toMatchObject({ phase: "committed", commitReceipt }); }); it("preserves commit validation failure details when the replacement cannot be quiesced", async () => { @@ -109,6 +191,12 @@ describe("Docker managed bootstrap adapter", () => { prepared, durablePreparation: durable, }); + const commitReceipt = await adapter.awaitBootstrap({ + handle, + snapshot, + replacement, + timeoutSecs: 1, + }); vi.mocked(fake.deps.dockerStop!).mockReturnValue({ status: 1, stderr: "injected quiesce failure", @@ -122,7 +210,7 @@ describe("Docker managed bootstrap adapter", () => { prepared, durablePreparation: durable, replacement, - completion: completion(replacement), + completion: commitReceipt, }), ).rejects.toThrow( /logical commit validation failed: Managed-startup shared-state commit helper failed.*injected commit failure.*new workload could not be quiesced.*injected quiesce failure/u, diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 03afb2ba280..e633ee9f2ed 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -42,6 +42,7 @@ import { assertManagedBootstrapSafeProcessEnvironmentKey, attachManagedBootstrapRollbackError, createManagedBootstrapIdentity, + createManagedBootstrapPlanFingerprint, createManagedBootstrapPreparedAuthority, MANAGED_BOOTSTRAP_SCHEMA_VERSION, type ManagedBootstrapAdapter, @@ -64,11 +65,15 @@ import { } from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, + DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type DockerManagedBootstrapFinalizationRecord, type DockerManagedBootstrapJournal, DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalStore, parseDockerManagedBootstrapJournal, + sameDockerManagedBootstrapReceipt, + serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; import { @@ -144,12 +149,6 @@ type ResolvedDeps = Required< type DockerBootstrapTransaction = DockerManagedBootstrapJournal; -interface DockerBootstrapRollbackTombstone { - readonly profileFingerprint: string; - readonly imageReference: string; - readonly receipt: ManagedBootstrapFinalizationReceipt; -} - export interface DockerManagedBootstrapAdapter extends ManagedBootstrapAdapter {} function resolveDeps(deps: DockerManagedBootstrapDeps): ResolvedDeps { @@ -1426,6 +1425,26 @@ function sameDockerBootstrapJournal( ); } +function sameDockerBootstrapPreparedAuthority( + left: DockerBootstrapTransaction, + right: DockerBootstrapTransaction, +): boolean { + return sameDockerBootstrapJournal( + Object.freeze({ + ...left, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + Object.freeze({ + ...right, + phase: "staged" as const, + preparationReceipt: null, + commitReceipt: null, + }), + ); +} + function createDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1467,6 +1486,27 @@ function transitionDockerBootstrapJournalDurably( return persisted; } +function recordDockerBootstrapCompletionDurably( + journal: DockerBootstrapTransaction, + receipt: ManagedBootstrapCompletionReceipt, + deps: ResolvedDeps, +): DockerBootstrapTransaction { + const expected = Object.freeze({ ...journal, commitReceipt: receipt }); + try { + deps.journalStore.recordCompletion(journal.bootstrapIdentity, receipt); + } catch (error) { + if (!(error instanceof DockerManagedBootstrapJournalAcknowledgementLostError)) throw error; + const recovered = deps.journalStore.load(journal.bootstrapIdentity); + if (!recovered || !sameDockerBootstrapJournal(recovered, expected)) throw error; + return recovered; + } + const persisted = deps.journalStore.load(journal.bootstrapIdentity); + if (!persisted || !sameDockerBootstrapJournal(persisted, expected)) { + throw new Error("Managed bootstrap Docker completion receipt was not durably re-readable."); + } + return persisted; +} + function removeDockerBootstrapJournalDurably( journal: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1490,6 +1530,7 @@ function assertDockerBootstrapTransactionAuthority( snapshot: ManagedBootstrapObservedSnapshot, prepared?: ManagedBootstrapPreparedReplacementHandle | null, replacement?: ManagedBootstrapReplacementHandle | null, + durablePreparation?: ManagedBootstrapDurablePreparationReceipt | null, ): void { const originalName = dockerContainerName( parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson).inspect, @@ -1498,9 +1539,11 @@ function assertDockerBootstrapTransactionAuthority( if ( transaction.schemaVersion !== DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION || transaction.bootstrapIdentity !== handle.bootstrapIdentity || + transaction.providerId !== expectedSandbox.driverId || transaction.sandbox.sandboxName !== expectedSandbox.sandboxName || transaction.sandbox.sandboxId !== expectedSandbox.sandboxId || transaction.sandbox.driverId !== expectedSandbox.driverId || + transaction.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || transaction.profileFingerprint !== handle.plan.profile.fingerprint || transaction.imageReference !== expectedImageReference(snapshot.image.repository, snapshot.image.manifestDigest) || @@ -1511,6 +1554,22 @@ function assertDockerBootstrapTransactionAuthority( replacementStagingName(originalName, handle.bootstrapIdentity) || transaction.backupName !== backupName(originalName, handle.bootstrapIdentity) || transaction.originalSpecHash !== snapshot.specHash || + transaction.rollbackTargetRuntimeId !== snapshot.runtimeId || + transaction.rollbackTargetSpecHash !== snapshot.specHash || + (transaction.preparationReceipt !== null && + prepared !== undefined && + prepared !== null && + transaction.preparationReceipt.authorityFingerprint !== + createManagedBootstrapPreparedAuthority({ handle, snapshot, prepared }) + .authorityFingerprint) || + (durablePreparation !== undefined && + durablePreparation !== null && + (transaction.preparationReceipt === null || + !sameDockerManagedBootstrapReceipt( + "preparation", + transaction.preparationReceipt, + durablePreparation, + ))) || (prepared !== undefined && prepared !== null && (transaction.originalRuntimeId !== prepared.originalRuntimeId || @@ -1677,8 +1736,64 @@ export function createDockerManagedBootstrapAdapter( dependencies: DockerManagedBootstrapDeps = {}, ): DockerManagedBootstrapAdapter { const deps = resolveDeps(dependencies); - const committedTransactions = new Set(); - const rollbackTombstones = new Map(); + const finalizationRecord = ( + handle: ManagedBootstrapHeldWorkloadHandle, + ): DockerManagedBootstrapFinalizationRecord | null => { + const record = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!record) return null; + if ( + record.providerId !== handle.sandbox.driverId || + record.sandbox.sandboxName !== handle.sandbox.sandboxName || + record.sandbox.sandboxId !== handle.sandbox.sandboxId || + record.sandbox.driverId !== handle.sandbox.driverId || + record.planFingerprint !== createManagedBootstrapPlanFingerprint(handle.plan) || + record.profileFingerprint !== handle.plan.profile.fingerprint || + record.imageReference !== + expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) + ) { + throw new Error("Managed bootstrap finalization record does not match its durable identity."); + } + return record; + }; + const persistFinalization = ( + handle: ManagedBootstrapHeldWorkloadHandle, + phase: "committed" | "rolled-back", + commitReceipt: ManagedBootstrapCompletionReceipt | null, + cleanupReceipt: ManagedBootstrapFinalizationReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const record = Object.freeze({ + schemaVersion: DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, + phase, + bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, + sandbox: handle.sandbox, + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), + profileFingerprint: handle.plan.profile.fingerprint, + imageReference: expectedImageReference( + handle.plan.image.repository, + handle.plan.image.manifestDigest, + ), + commitReceipt, + cleanupReceipt, + } satisfies DockerManagedBootstrapFinalizationRecord); + const serialized = serializeDockerManagedBootstrapFinalizationRecord(record); + try { + deps.journalStore.recordFinalization(record); + } catch (error) { + const recovered = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if ( + !recovered || + serializeDockerManagedBootstrapFinalizationRecord(recovered) !== serialized + ) { + throw error; + } + } + const persisted = deps.journalStore.loadFinalization(handle.bootstrapIdentity); + if (!persisted || serializeDockerManagedBootstrapFinalizationRecord(persisted) !== serialized) { + throw new Error("Managed bootstrap finalization receipt was not durably re-readable."); + } + return persisted.cleanupReceipt; + }; const completedRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, alreadyRolledBack: boolean, @@ -1694,34 +1809,39 @@ export function createDockerManagedBootstrapAdapter( alreadyRolledBack, finalizedAt: deps.now().toISOString(), } satisfies ManagedBootstrapFinalizationReceipt); - rollbackTombstones.set(handle.bootstrapIdentity, { - profileFingerprint: handle.plan.profile.fingerprint, - imageReference: expectedImageReference( - handle.plan.image.repository, - handle.plan.image.manifestDigest, - ), - receipt, - }); - return receipt; + return persistFinalization(handle, "rolled-back", null, receipt); + }; + const completedCommit = ( + handle: ManagedBootstrapHeldWorkloadHandle, + commitReceipt: ManagedBootstrapCompletionReceipt, + ): ManagedBootstrapFinalizationReceipt => { + const cleanupReceipt = Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: deps.now().toISOString(), + } satisfies ManagedBootstrapFinalizationReceipt); + return persistFinalization(handle, "committed", commitReceipt, cleanupReceipt); }; const priorRollback = ( handle: ManagedBootstrapHeldWorkloadHandle, ): ManagedBootstrapFinalizationReceipt | null => { - const tombstone = rollbackTombstones.get(handle.bootstrapIdentity); - if (!tombstone) return null; - const receipt = tombstone.receipt; - if ( - receipt.sandbox.sandboxName !== handle.sandbox.sandboxName || - receipt.sandbox.sandboxId !== handle.sandbox.sandboxId || - receipt.sandbox.driverId !== handle.sandbox.driverId || - tombstone.profileFingerprint !== handle.plan.profile.fingerprint || - tombstone.imageReference !== - expectedImageReference(handle.plan.image.repository, handle.plan.image.manifestDigest) - ) { - throw new Error("Managed bootstrap rollback tombstone does not match its durable identity."); + const finalized = finalizationRecord(handle); + if (!finalized) return null; + if (finalized.phase === "committed") { + throw new ManagedBootstrapDurableCommitCleanupPendingError({ + bootstrapIdentity: handle.bootstrapIdentity, + cleanupRuntimeId: finalized.commitReceipt?.runtimeId ?? "unknown", + detail: "rollback is no longer legal after the durable finalization receipt", + }); } return Object.freeze({ - ...receipt, + ...finalized.cleanupReceipt, alreadyRolledBack: true, }); }; @@ -1743,10 +1863,7 @@ export function createDockerManagedBootstrapAdapter( const finalized = priorRollback(handle); if (finalized) return finalized; const journal = deps.journalStore.load(handle.bootstrapIdentity); - if ( - committedTransactions.has(handle.bootstrapIdentity) || - journal?.phase === "shared-state-committed" - ) { + if (journal?.phase === "shared-state-committed") { throw new ManagedBootstrapDurableCommitCleanupPendingError({ bootstrapIdentity: handle.bootstrapIdentity, cleanupRuntimeId: journal?.originalRuntimeId ?? snapshot?.runtimeId ?? "unknown", @@ -1854,15 +1971,21 @@ export function createDockerManagedBootstrapAdapter( detail: "durable Docker cutover lacks its coordinator-recorded prepared authority", }); } - const stagedJournal = Object.freeze({ ...journal, phase: "staged" as const }); - if (!sameDockerBootstrapJournal(stagedJournal, preparedAuthority)) { + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, detail: "durable Docker cutover changed its prepared rollback authority", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); const original = inspectTransactionRuntime(journal, journal.originalRuntimeId, deps); if (!original) { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2082,14 +2205,14 @@ export function createDockerManagedBootstrapAdapter( return completedRollback(handle, false); }; const commitBootstrapNow = ( + handle: ManagedBootstrapHeldWorkloadHandle, receipt: ManagedBootstrapCompletionReceipt, transaction: DockerBootstrapTransaction, input: { readonly sharedStateStatus: "committed" | "none"; readonly sharedStateTransaction: ReturnType; }, - ): void => { - if (committedTransactions.has(receipt.bootstrapIdentity)) return; + ): ManagedBootstrapFinalizationReceipt => { if ( transaction.phase !== "shared-state-committed" || transaction.replacementRuntimeId !== receipt.runtimeId || @@ -2181,7 +2304,7 @@ export function createDockerManagedBootstrapAdapter( } } removeDockerBootstrapJournalDurably(transaction, deps); - committedTransactions.add(receipt.bootstrapIdentity); + return completedCommit(handle, receipt); }; const finalizeBootstrap = async ( input: Parameters[0], @@ -2193,6 +2316,21 @@ export function createDockerManagedBootstrapAdapter( if (!completion || !snapshot || !prepared || !durablePreparation || !replacement) { throw new Error("Managed bootstrap commit requires one complete cutover receipt."); } + const finalized = finalizationRecord(handle); + if (finalized) { + if ( + finalized.phase !== "committed" || + !finalized.commitReceipt || + !sameDockerManagedBootstrapReceipt("completion", finalized.commitReceipt, completion) + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: handle.bootstrapIdentity, + runtimeId: replacement.replacementRuntimeId, + detail: "durable finalization cannot change outcome or commit receipt", + }); + } + return finalized.cleanupReceipt; + } const preparedAuthority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); const sharedTransaction = managedSharedStateTransaction( @@ -2210,19 +2348,6 @@ export function createDockerManagedBootstrapAdapter( let journal = deps.journalStore.load(handle.bootstrapIdentity); if (!journal) { - if (committedTransactions.has(completion.bootstrapIdentity)) { - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); - } const originalPresence = probeExactDockerContainerAbsence(snapshot.runtimeId, deps); if (originalPresence === "unknown") { throw new ManagedBootstrapCommitStateIndeterminateError({ @@ -2256,33 +2381,34 @@ export function createDockerManagedBootstrapAdapter( detail: "the retired-journal replacement does not match the exact completion receipt", }); } - committedTransactions.add(completion.bootstrapIdentity); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, + return completedCommit(handle, completion); + } + + if (!sameDockerBootstrapPreparedAuthority(journal, preparedAuthority)) { + throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), + runtimeId: journal.replacementRuntimeId, + detail: "durable Docker commit changed its prepared rollback authority", }); } - + assertDockerBootstrapTransactionAuthority( + journal, + handle, + snapshot, + prepared, + replacement, + durablePreparation, + ); if ( - !sameDockerBootstrapJournal( - Object.freeze({ ...journal, phase: "staged" as const }), - preparedAuthority, - ) + journal.commitReceipt === null || + !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ - bootstrapIdentity: handle.bootstrapIdentity, + bootstrapIdentity: journal.bootstrapIdentity, runtimeId: journal.replacementRuntimeId, - detail: "durable Docker commit changed its prepared rollback authority", + detail: "commit requires the exact durable completion receipt", }); } - assertDockerBootstrapTransactionAuthority(journal, handle, snapshot, prepared, replacement); if (journal.phase === "staged" || journal.phase === "rollback-authorized") { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: journal.bootstrapIdentity, @@ -2361,21 +2487,10 @@ export function createDockerManagedBootstrapAdapter( }); } - commitBootstrapNow(completion, journal, { + return commitBootstrapNow(handle, completion, journal, { sharedStateStatus: sharedStatus === "committed" ? "committed" : "none", sharedStateTransaction: sharedTransaction, }); - return Object.freeze({ - schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, - sandbox: handle.sandbox, - bootstrapIdentity: handle.bootstrapIdentity, - outcome: "committed", - restoredRuntimeId: null, - restoredSpecHash: null, - heldWorkloadRemoved: false, - alreadyRolledBack: false, - finalizedAt: deps.now().toISOString(), - }); }; return { async createHeldWorkload(input) { @@ -2631,7 +2746,9 @@ export function createDockerManagedBootstrapAdapter( schemaVersion: DOCKER_MANAGED_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, phase: "staged", bootstrapIdentity: handle.bootstrapIdentity, + providerId: handle.sandbox.driverId, sandbox: Object.freeze({ ...handle.sandbox }), + planFingerprint: createManagedBootstrapPlanFingerprint(handle.plan), profileFingerprint: handle.plan.profile.fingerprint, imageReference: expectedImageReference( snapshot.image.repository, @@ -2645,6 +2762,10 @@ export function createDockerManagedBootstrapAdapter( backupName: backupContainerName, originalSpecHash: snapshot.specHash, replacementSpecHash: expectedActivatedSpecHash, + rollbackTargetRuntimeId: snapshot.runtimeId, + rollbackTargetSpecHash: snapshot.specHash, + preparationReceipt: null, + commitReceipt: null, }); requestFile = writeProtectedEnvelope(handle.bootstrapIdentity, request); @@ -2724,9 +2845,20 @@ export function createDockerManagedBootstrapAdapter( async activateBootstrapReplacement({ handle, snapshot, prepared, durablePreparation }) { const authority = transactionFromPreparedAuthority(handle, snapshot, prepared); assertDurablePreparationAuthority(handle, snapshot, prepared, durablePreparation); + const durableAuthority = Object.freeze({ + ...authority, + preparationReceipt: durablePreparation, + }); const existingJournal = deps.journalStore.load(handle.bootstrapIdentity); if (existingJournal) { - assertDockerBootstrapTransactionAuthority(existingJournal, handle, snapshot, prepared); + assertDockerBootstrapTransactionAuthority( + existingJournal, + handle, + snapshot, + prepared, + null, + durablePreparation, + ); throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: existingJournal.bootstrapIdentity, runtimeId: existingJournal.replacementRuntimeId, @@ -2756,7 +2888,7 @@ export function createDockerManagedBootstrapAdapter( ); } - let journal = createDockerBootstrapJournalDurably(authority, deps); + let journal = createDockerBootstrapJournalDurably(durableAuthority, deps); const originalAtFence = inspectExact(snapshot.runtimeId, deps); const replacementAtFence = inspectExact(prepared.preparedRuntimeId, deps); assertTransactionOriginal(journal, originalAtFence); @@ -2929,7 +3061,19 @@ export function createDockerManagedBootstrapAdapter( "Managed bootstrap Docker image completion identities do not match the transaction.", ); } - return Object.freeze({ + if (afterWaitJournal.commitReceipt !== null) { + if ( + afterWaitJournal.commitReceipt.transactionPending !== imageCompletion.transactionPending + ) { + throw new ManagedBootstrapCommitStateIndeterminateError({ + bootstrapIdentity: afterWaitJournal.bootstrapIdentity, + runtimeId: afterWaitJournal.replacementRuntimeId, + detail: "durable completion disagrees with the image-owned transaction receipt", + }); + } + return afterWaitJournal.commitReceipt; + } + const completion = Object.freeze({ schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, sandbox: handle.sandbox, runtimeId: replacement.replacementRuntimeId, @@ -2942,6 +3086,15 @@ export function createDockerManagedBootstrapAdapter( transactionPending: imageCompletion.transactionPending, completedAt: deps.now().toISOString(), }); + const completedJournal = recordDockerBootstrapCompletionDurably( + afterWaitJournal, + completion, + deps, + ); + if (completedJournal.commitReceipt === null) { + throw new Error("Managed bootstrap Docker completion receipt disappeared after recording."); + } + return completedJournal.commitReceipt; }, finalizeBootstrap, diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 3022fc71850..6ec1563e73e 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -10,9 +10,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; import type { SandboxMessagingPlan } from "../messaging/manifest"; import type { ManagedStartupAgent, ManagedStartupProfile } from "./managed-startup/profile"; +import { fingerprintManagedStartupProfile } from "./managed-startup/profile"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, type ManagedStartupSharedTransactionOptions, rollbackManagedStartupSharedStateTransaction, } from "./managed-startup/shared-state-transaction"; @@ -67,6 +71,13 @@ describe("managed startup shared-state transaction", () => { ); } + function commitReceiptDirectory(): string { + return path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); + } + it.each([ "openclaw", "hermes", @@ -246,6 +257,192 @@ describe("managed startup shared-state transaction", () => { expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); }); + it("fsyncs every transaction namespace before exposing a pending receipt", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + const open = vi.spyOn(fs, "openSync"); + const fsync = vi.spyOn(fs, "fsyncSync"); + + expect( + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toBe(true); + + const transactionParent = path.dirname(transactionDirectory); + const backupDirectory = path.join(transactionDirectory, "backups"); + expect(open).toHaveBeenCalledWith(transactionParent, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(transactionDirectory, fs.constants.O_RDONLY); + expect(open).toHaveBeenCalledWith(backupDirectory, fs.constants.O_RDONLY); + // File contents plus parent, backup, and manifest directory entries all + // reach stable storage before the transaction is returned as pending. + expect(fsync.mock.calls.length).toBeGreaterThanOrEqual(6); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("persists one exact compact %s bootstrap commit across fresh calls, forbids rollback, and retires it for the next attempt", (agent) => { + const profile = managedStartupE2eProfile(agent); + const bootstrapIdentity = "b".repeat(64); + const nextBootstrapIdentity = "d".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot(agent); + fs.mkdirSync(root); + const config = path.join( + root, + agent === "openclaw" ? "openclaw.json" : agent === "hermes" ? "config.yaml" : "config.toml", + ); + fs.writeFileSync(config, "before\n"); + + expect(beginManagedStartupSharedStateTransaction(profile, boundOptions)).toBe(true); + fs.writeFileSync(config, "committed\n"); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("pending"); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: "e".repeat(64), + bootstrapIdentity, + }, + options, + ), + ).toThrow(/expected agent, profile fingerprint, or bootstrap identity/u); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + + const receiptDirectory = commitReceiptDirectory(); + const receiptFile = path.join(receiptDirectory, "receipt.json"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(fs.readdirSync(receiptDirectory)).toEqual(["receipt.json"]); + expect(mode(receiptDirectory)).toBe(0o700); + expect(mode(receiptFile)).toBe(0o400); + expect(JSON.parse(fs.readFileSync(receiptFile, "utf8"))).toEqual({ + schemaVersion: 1, + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }); + + // These calls reconstruct state solely from the image-owned receipt. + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction(agent, boundOptions)).toBe(true); + expect(() => rollbackManagedStartupSharedStateTransaction(agent, boundOptions)).toThrow( + /durably committed and cannot be rolled back/u, + ); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: "e".repeat(64), + bootstrapIdentity, + }, + options, + ), + ).toThrow(/different bootstrap attempt/u); + expect(fs.readFileSync(config, "utf8")).toBe("committed\n"); + + expect(clearManagedStartupSharedStateCommitReceipt(agent, boundOptions)).toBe(true); + expect(fs.existsSync(receiptDirectory)).toBe(false); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent, + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("none"); + + const nextOptions = { ...options, bootstrapIdentity: nextBootstrapIdentity }; + expect(beginManagedStartupSharedStateTransaction(profile, nextOptions)).toBe(true); + expect(rollbackManagedStartupSharedStateTransaction(agent, nextOptions)).toBe(true); + }); + + it.each([ + "during-compact-receipt-write", + "before-backup-removal", + "during-backup-removal", + "after-backup-removal", + "after-manifest-removal", + ] as const)("recovers an atomically established commit interrupted %s", (interruption) => { + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + beginManagedStartupSharedStateTransaction(profile, boundOptions); + fs.writeFileSync(path.join(root, "openclaw.json"), "committed\n"); + + const originalRmSync = fs.rmSync.bind(fs); + const rm = vi.spyOn(fs, "rmSync").mockImplementation((( + target: fs.PathLike, + removeOptions?: fs.RmDirOptions, + ) => + String(target).endsWith(`${path.sep}backups`) + ? (() => { + throw new Error("injected post-rename cleanup interruption"); + })() + : originalRmSync(target, removeOptions)) as typeof fs.rmSync); + expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /injected post-rename cleanup interruption/u, + ); + rm.mockRestore(); + + expect(fs.existsSync(transactionDirectory)).toBe(false); + const committedDirectory = commitReceiptDirectory(); + const backups = path.join(committedDirectory, "backups"); + const manifest = path.join(committedDirectory, "manifest.json"); + const applyInterruption: Record void> = { + "during-compact-receipt-write": () => + fs.renameSync( + path.join(committedDirectory, "receipt.json"), + path.join(committedDirectory, ".receipt.json.1234567890abcdef12345678"), + ), + "before-backup-removal": () => undefined, + "during-backup-removal": () => { + const [firstBackup] = fs.readdirSync(backups); + expect(firstBackup).toBeTruthy(); + fs.unlinkSync(path.join(backups, firstBackup!)); + }, + "after-backup-removal": () => originalRmSync(backups, { force: false, recursive: true }), + "after-manifest-removal": () => fs.unlinkSync(manifest), + }; + applyInterruption[interruption](); + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + options, + ), + ).toBe("committed"); + expect(commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toBe(true); + expect(fs.readdirSync(committedDirectory)).toEqual(["receipt.json"]); + expect(clearManagedStartupSharedStateCommitReceipt("openclaw", boundOptions)).toBe(true); + }); + it("resumes the same pending profile idempotently and rejects profile drift", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); @@ -258,10 +455,62 @@ describe("managed startup shared-state transaction", () => { managedStartupE2eProfile("openclaw", true), options, ), - ).toThrow(/belongs to a different profile/u); + ).toThrow(/belongs to a different agent, profile fingerprint, or bootstrap attempt/u); expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); }); + it("validates a directly mounted copied receipt under an unchanged 0755 image parent", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const profile = managedStartupE2eProfile("openclaw"); + const bootstrapIdentity = "b".repeat(64); + const boundOptions = { ...options, bootstrapIdentity }; + beginManagedStartupSharedStateTransaction(profile, boundOptions); + + const imageParent = path.join(temporaryRoot, "image-var-lib-nemoclaw"); + const copiedReceipt = path.join(imageParent, "managed-startup-shared-state-transaction-v1"); + fs.mkdirSync(imageParent, { mode: 0o755 }); + fs.chmodSync(imageParent, 0o755); + fs.cpSync(transactionDirectory, copiedReceipt, { + recursive: true, + preserveTimestamps: true, + }); + // Node 22.23 normalizes copied directory modes to 0755. Recreate the + // protected modes that the container-copy fixture is intended to model. + fs.chmodSync(copiedReceipt, 0o700); + fs.chmodSync(path.join(copiedReceipt, "backups"), 0o700); + + expect( + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toBe("pending"); + + fs.chmodSync(imageParent, 0o700); + expect(() => + getManagedStartupSharedStateTransactionStatus( + { + agent: "openclaw", + profileFingerprint: fingerprintManagedStartupProfile(profile), + bootstrapIdentity, + }, + { + ...boundOptions, + transactionDirectory: copiedReceipt, + }, + ), + ).toThrow(/must be .* mode 755/u); + }); + it("rejects planted target and ancestor symlinks before creating a receipt", () => { const outside = path.join(temporaryRoot, "outside"); fs.mkdirSync(outside); diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts index f8e5c7f4edd..898783c7f30 100644 --- a/src/lib/onboard/managed-startup/image-runtime.ts +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -35,7 +35,9 @@ import { } from "./root-apply"; import { beginManagedStartupSharedStateTransaction, + clearManagedStartupSharedStateCommitReceipt, commitManagedStartupSharedStateTransaction, + getManagedStartupSharedStateTransactionStatus, MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, rollbackManagedStartupSharedStateTransaction, } from "./shared-state-transaction"; @@ -1540,7 +1542,7 @@ function readCliAgent(argv: readonly string[], expectedLength = 2): string { const index = argv.indexOf("--agent"); if (index < 0 || index + 1 >= argv.length || argv.length !== expectedLength) { fail( - "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction] --agent ", + "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction|--clear-shared-state-commit-receipt|--shared-state-transaction-status] --agent ", ); } return argv[index + 1] as string; @@ -1554,6 +1556,14 @@ function readCliFingerprint(argv: readonly string[]): string { return argv[index + 1] as string; } +function readCliBootstrapIdentity(argv: readonly string[]): string { + const index = argv.indexOf("--bootstrap-identity"); + if (index < 0 || index + 1 >= argv.length || !SHA256_RE.test(String(argv[index + 1] ?? ""))) { + fail("managed bootstrap identity argument is missing or invalid"); + } + return argv[index + 1] as string; +} + export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { if (argv.length === 1 && argv[0] === "--internal-write-openclaw-hash") { internalWriteOpenClawHash(); @@ -1600,29 +1610,58 @@ export async function main(argv: readonly string[] = process.argv.slice(2)): Pro return; } if ( - argv.length === 4 && + (argv.length === 4 || argv.length === 6) && argv[0] === "--rollback-shared-state-transaction" && - argv[3] === "--read-only-receipt" + argv[argv.length - 1] === "--read-only-receipt" ) { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 4)); + const agent = exactAgent(readCliAgent(argv, argv.length)); const rolledBack = rollbackManagedStartupSharedStateTransaction(agent, { transactionDirectory: MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, readOnlyReceipt: true, + bootstrapIdentity: argv.length === 6 ? readCliBootstrapIdentity(argv) : null, }); if (!rolledBack) fail("read-only shared-state rollback receipt is missing"); console.log(`[managed-startup] verified and restored ${agent} shared state`); return; } - if (argv.length === 3 && argv[0] === "--commit-shared-state-transaction") { + if ((argv.length === 3 || argv.length === 5) && argv[0] === "--commit-shared-state-transaction") { requireRoot(); - const agent = exactAgent(readCliAgent(argv, 3)); - if (!commitManagedStartupSharedStateTransaction(agent)) { + const agent = exactAgent(readCliAgent(argv, argv.length)); + if ( + !commitManagedStartupSharedStateTransaction(agent, { + bootstrapIdentity: argv.length === 5 ? readCliBootstrapIdentity(argv) : null, + }) + ) { fail("managed startup transaction is missing at commit"); } console.log(`[managed-startup] committed ${agent} shared state`); return; } + if (argv.length === 5 && argv[0] === "--clear-shared-state-commit-receipt") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 5)); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + if (!clearManagedStartupSharedStateCommitReceipt(agent, { bootstrapIdentity })) { + fail("managed startup durable commit receipt is missing at cleanup"); + } + console.log(`[managed-startup] cleared ${agent} durable shared-state commit receipt`); + return; + } + if (argv.length === 7 && argv[0] === "--shared-state-transaction-status") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 7)); + const profileFingerprint = readCliFingerprint(argv); + const bootstrapIdentity = readCliBootstrapIdentity(argv); + process.stdout.write( + `${getManagedStartupSharedStateTransactionStatus({ + agent, + profileFingerprint, + bootstrapIdentity, + })}\n`, + ); + return; + } const result = await applyManagedStartupImageProfile(readCliAgent(argv)); console.log( result.adapterApplied diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts index d921ec36d89..62894abeb5c 100644 --- a/src/lib/onboard/managed-startup/shared-state-transaction.ts +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -20,14 +20,19 @@ const MAX_TRANSACTION_FILES = 128; const MAX_TRANSACTION_FILE_BYTES = 8 * 1024 * 1024; const MAX_TRANSACTION_TOTAL_BYTES = 32 * 1024 * 1024; const MAX_MANIFEST_BYTES = 256 * 1024; +const MAX_COMMIT_RECEIPT_BYTES = 4096; const TRANSACTION_PARENT_DIRECTORY_MODE = 0o755; const TRANSACTION_DIRECTORY_MODE = 0o700; const TRANSACTION_FILE_MODE = 0o400; +const ATOMIC_TEMPORARY_FILE_MODE = 0o600; export const MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY = "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; export const MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY = "/run/nemoclaw/managed-startup-shared-rollback-receipt-v1"; +export const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-commit-v1"; +const MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE = "receipt.json"; interface FilePresentReceipt { readonly path: string; @@ -66,13 +71,23 @@ interface TransactionManifest { readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; readonly agent: ManagedStartupAgent; readonly profileFingerprint: string; + readonly bootstrapIdentity: string | null; readonly files: readonly FileReceipt[]; readonly directories: readonly DirectoryReceipt[]; } +interface CommitReceipt { + readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; +} + export interface ManagedStartupSharedTransactionOptions { readonly sandboxRoot?: string; readonly transactionDirectory?: string; + /** Test/helper seam. Production derives the fixed image-owned commit receipt path. */ + readonly commitReceiptDirectory?: string; /** Test seam. Production always retains the root:root defaults. */ readonly trustedUid?: number; /** Test seam. Production always retains the root:root defaults. */ @@ -82,6 +97,8 @@ export interface ManagedStartupSharedTransactionOptions { * so ownership may reflect the Docker CLI user instead of container root. */ readonly readOnlyReceipt?: boolean; + /** One-attempt identity for managed bootstrap; null for legacy root application. */ + readonly bootstrapIdentity?: string | null; } interface ResolvedOptions { @@ -90,9 +107,12 @@ interface ResolvedOptions { readonly transactionDirectory: string; readonly backupDirectory: string; readonly manifestFile: string; + readonly commitReceiptDirectory: string; + readonly commitReceiptFile: string; readonly trustedUid: number; readonly trustedGid: number; readonly readOnlyReceipt: boolean; + readonly bootstrapIdentity: string | null; } interface StableFile { @@ -109,11 +129,28 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R const transactionDirectory = path.resolve( options.transactionDirectory ?? MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, ); + const commitReceiptDirectory = path.resolve( + options.commitReceiptDirectory ?? + (options.transactionDirectory + ? path.join( + path.dirname(transactionDirectory), + path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ) + : MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY), + ); if ( transactionDirectory === sandboxRoot || - transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) + transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + commitReceiptDirectory === sandboxRoot || + commitReceiptDirectory.startsWith(`${sandboxRoot}${path.sep}`) || + path.dirname(commitReceiptDirectory) !== path.dirname(transactionDirectory) || + commitReceiptDirectory === transactionDirectory ) { - fail("transaction receipts must not be stored in sandbox-shared state"); + fail("transaction and commit receipts require distinct paths outside sandbox-shared state"); + } + const bootstrapIdentity = options.bootstrapIdentity ?? null; + if (bootstrapIdentity !== null && !/^[a-f0-9]{64}$/u.test(bootstrapIdentity)) { + fail("bootstrap identity must encode 32 lowercase-hex bytes"); } return { sandboxRoot, @@ -121,9 +158,15 @@ function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): R transactionDirectory, backupDirectory: path.join(transactionDirectory, "backups"), manifestFile: path.join(transactionDirectory, "manifest.json"), + commitReceiptDirectory, + commitReceiptFile: path.join( + commitReceiptDirectory, + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + ), trustedUid: options.trustedUid ?? 0, trustedGid: options.trustedGid ?? 0, readOnlyReceipt: options.readOnlyReceipt ?? false, + bootstrapIdentity, }; } @@ -482,16 +525,63 @@ function atomicWriteTrustedFile( } } +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + function canonicalManifest(manifest: TransactionManifest): string { return `${JSON.stringify(manifest, null, 2)}\n`; } +function canonicalCommitReceipt(receipt: CommitReceipt): string { + return `${JSON.stringify(receipt, null, 2)}\n`; +} + function requireExactKeys(record: Record, keys: readonly string[]): void { if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { fail("transaction manifest contains unexpected fields"); } } +function parseCommitReceipt(text: string): CommitReceipt { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("commit receipt is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("commit receipt must be an object"); + } + const record = parsed as Record; + requireExactKeys(record, ["agent", "bootstrapIdentity", "profileFingerprint", "schemaVersion"]); + if ( + record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || + typeof record.profileFingerprint !== "string" || + !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + typeof record.bootstrapIdentity !== "string" || + !/^[a-f0-9]{64}$/u.test(record.bootstrapIdentity) + ) { + fail("commit receipt has an invalid envelope"); + } + const receipt: CommitReceipt = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity, + }; + if (canonicalCommitReceipt(receipt) !== text) { + fail("commit receipt is not canonical"); + } + return receipt; +} + function safeMetadata(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 0; } @@ -509,6 +599,7 @@ function parseManifest(text: string): TransactionManifest { const record = parsed as Record; requireExactKeys(record, [ "agent", + "bootstrapIdentity", "directories", "files", "profileFingerprint", @@ -519,6 +610,11 @@ function parseManifest(text: string): TransactionManifest { !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || typeof record.profileFingerprint !== "string" || !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + !( + record.bootstrapIdentity === null || + (typeof record.bootstrapIdentity === "string" && + /^[a-f0-9]{64}$/u.test(record.bootstrapIdentity)) + ) || !Array.isArray(record.files) || !Array.isArray(record.directories) || record.files.length > MAX_TRANSACTION_FILES || @@ -613,6 +709,7 @@ function parseManifest(text: string): TransactionManifest { schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: record.agent as ManagedStartupAgent, profileFingerprint: record.profileFingerprint, + bootstrapIdentity: record.bootstrapIdentity as string | null, files, directories, }; @@ -639,9 +736,9 @@ function requireTrustedTransactionPath( } } -function requireReadOnlyReceiptMount(options: ResolvedOptions): void { +function requireReadOnlyReceiptMount(target: string, options: ResolvedOptions): void { if (!options.readOnlyReceipt) return; - const probe = path.join(options.transactionDirectory, ".nemoclaw-write-probe"); + const probe = path.join(target, ".nemoclaw-write-probe"); let descriptor: number | undefined; try { descriptor = fs.openSync( @@ -664,7 +761,7 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { requireTransactionBoundaries(options); if (!pathExistsNoFollow(options.transactionDirectory)) return null; requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); - requireReadOnlyReceiptMount(options); + requireReadOnlyReceiptMount(options.transactionDirectory, options); requireTrustedTransactionPath(options.backupDirectory, TRANSACTION_DIRECTORY_MODE, options); requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); @@ -679,6 +776,59 @@ function loadManifest(options: ResolvedOptions): TransactionManifest | null { return parseManifest(stable.bytes.toString("utf8")); } +function transactionOptionsAt( + options: ResolvedOptions, + transactionDirectory: string, +): ResolvedOptions { + return { + ...options, + transactionDirectory, + backupDirectory: path.join(transactionDirectory, "backups"), + manifestFile: path.join(transactionDirectory, "manifest.json"), + }; +} + +function loadCommitReceipt( + options: ResolvedOptions, +): { readonly receipt: CommitReceipt; readonly compact: boolean } | null { + requireTransactionBoundaries(options); + if (!pathExistsNoFollow(options.commitReceiptDirectory)) return null; + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + if (pathExistsNoFollow(options.commitReceiptFile)) { + requireReadOnlyReceiptMount(options.commitReceiptDirectory, options); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.commitReceiptFile, MAX_COMMIT_RECEIPT_BYTES); + if ( + (!options.readOnlyReceipt && + (Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid)) || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("commit receipt ownership changed while it was read"); + } + return { receipt: parseCommitReceipt(stable.bytes.toString("utf8")), compact: true }; + } + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const staged = loadManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt is incomplete"); + } + verifyAllBackups(staged.files, stagedOptions); + return { + receipt: { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }, + compact: false, + }; +} + function verifyBackup(receipt: FilePresentReceipt, options: ResolvedOptions): Buffer { const backupPath = path.join(options.backupDirectory, receipt.backup); requireTrustedTransactionPath(backupPath, TRANSACTION_FILE_MODE, options); @@ -742,11 +892,162 @@ function directoryMatchesReceipt(target: string, receipt: DirectoryPresentReceip function removeTransactionDirectory(options: ResolvedOptions): void { requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); fs.rmSync(options.transactionDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); if (pathExistsNoFollow(options.transactionDirectory)) { fail("transaction directory remained after cleanup"); } } +function assertCommitReceiptMatches( + receipt: CommitReceipt, + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint?: string; + readonly bootstrapIdentity: string; + }, +): void { + if ( + receipt.agent !== expected.agent || + (expected.profileFingerprint !== undefined && + receipt.profileFingerprint !== expected.profileFingerprint) || + receipt.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail("durable commit receipt belongs to a different bootstrap attempt"); + } +} + +function loadCommitStagingManifest(options: ResolvedOptions): TransactionManifest | null { + if (!pathExistsNoFollow(options.manifestFile)) return null; + requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("durable commit staging manifest ownership changed while it was read"); + } + return parseManifest(stable.bytes.toString("utf8")); +} + +function retireInterruptedCommitReceiptWrites( + receipt: CommitReceipt, + options: ResolvedOptions, +): void { + const temporaryPattern = new RegExp( + `^\\.${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE.replace(".", "\\.")}\\.[a-f0-9]{24}$`, + "u", + ); + for (const entry of fs.readdirSync(options.commitReceiptDirectory)) { + if (!temporaryPattern.test(entry)) continue; + const target = path.join(options.commitReceiptDirectory, entry); + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== options.trustedUid || + stat.gid !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(modeOf(stat)) + ) { + fail("interrupted durable commit receipt write has unsafe metadata"); + } + const stable = readStableFile(target, MAX_COMMIT_RECEIPT_BYTES); + const mode = Number(stable.stat.mode & 0o7777n); + if ( + Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid || + ![ATOMIC_TEMPORARY_FILE_MODE, TRANSACTION_FILE_MODE].includes(mode) + ) { + fail("interrupted durable commit receipt write changed during verification"); + } + if (stable.bytes.length > 0) { + let interruptedReceipt: CommitReceipt | null = null; + try { + interruptedReceipt = parseCommitReceipt(stable.bytes.toString("utf8")); + } catch { + // The atomic writer may have crashed after any partial write. The + // trusted 0700 directory and exact random temp-name shape bind this + // artifact to that interrupted write; the established receipt is now + // authoritative. + } + if (interruptedReceipt) assertCommitReceiptMatches(interruptedReceipt, receipt); + } + fs.unlinkSync(target); + fsyncDirectory(options.commitReceiptDirectory); + } +} + +function compactDurableCommitReceipt( + state: { readonly receipt: CommitReceipt; readonly compact: boolean }, + options: ResolvedOptions, +): void { + if (!state.compact) { + atomicWriteTrustedFile( + options.commitReceiptFile, + canonicalCommitReceipt(state.receipt), + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + fsyncDirectory(options.commitReceiptDirectory); + } + retireInterruptedCommitReceiptWrites(state.receipt, options); + const stagedOptions = transactionOptionsAt(options, options.commitReceiptDirectory); + const manifestExists = pathExistsNoFollow(stagedOptions.manifestFile); + const backupsExist = pathExistsNoFollow(stagedOptions.backupDirectory); + const unexpectedBeforeCleanup = fs + .readdirSync(options.commitReceiptDirectory) + .filter( + (entry) => + ![ + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE, + path.basename(stagedOptions.backupDirectory), + path.basename(stagedOptions.manifestFile), + ].includes(entry), + ); + if (unexpectedBeforeCleanup.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + if (manifestExists) { + // The fsynced compact receipt is authoritative after commit. Validate the + // remaining manifest identity without requiring a complete backup tree: + // recursive backup deletion may have been interrupted at any point. + const staged = loadCommitStagingManifest(stagedOptions); + if (!staged || staged.bootstrapIdentity === null) { + fail("durable commit staging receipt disappeared during cleanup"); + } + assertCommitReceiptMatches(state.receipt, { + agent: staged.agent, + profileFingerprint: staged.profileFingerprint, + bootstrapIdentity: staged.bootstrapIdentity, + }); + } + if (backupsExist) { + requireTrustedTransactionPath( + stagedOptions.backupDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + fs.rmSync(stagedOptions.backupDirectory, { force: false, recursive: true }); + fsyncDirectory(options.commitReceiptDirectory); + } + if (manifestExists) { + requireTrustedTransactionPath(stagedOptions.manifestFile, TRANSACTION_FILE_MODE, options); + fs.unlinkSync(stagedOptions.manifestFile); + fsyncDirectory(options.commitReceiptDirectory); + } + const unexpected = fs + .readdirSync(options.commitReceiptDirectory) + .filter((entry) => entry !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE); + if (unexpected.length !== 0) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + const verified = loadCommitReceipt(options); + if (!verified?.compact) fail("durable commit receipt did not compact successfully"); + assertCommitReceiptMatches(verified.receipt, state.receipt); +} + export function beginManagedStartupSharedStateTransaction( profile: ManagedStartupProfile, inputOptions: ManagedStartupSharedTransactionOptions = {}, @@ -758,10 +1059,28 @@ export function beginManagedStartupSharedStateTransaction( } requireTransactionBoundaries(options); const profileFingerprint = fingerprintManagedStartupProfile(profile); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("a durable managed bootstrap commit receipt already exists"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: profile.agent, + profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("this managed bootstrap attempt is already durably committed"); + } const pending = loadManifest(options); if (pending) { - if (pending.agent !== profile.agent || pending.profileFingerprint !== profileFingerprint) { - fail("a pending managed startup transaction belongs to a different profile"); + if ( + pending.agent !== profile.agent || + pending.profileFingerprint !== profileFingerprint || + pending.bootstrapIdentity !== options.bootstrapIdentity + ) { + fail( + "a pending managed startup transaction belongs to a different agent, profile fingerprint, or bootstrap attempt", + ); } verifyAllBackups(pending.files, options); return false; @@ -780,6 +1099,7 @@ export function beginManagedStartupSharedStateTransaction( schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: profile.agent, profileFingerprint, + bootstrapIdentity: options.bootstrapIdentity, files: snapshots.map(({ receipt }) => receipt), directories, }; @@ -801,9 +1121,11 @@ export function beginManagedStartupSharedStateTransaction( }; fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionParentDirectory); fs.mkdirSync(options.backupDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); fs.chownSync(options.backupDirectory, options.trustedUid, options.trustedGid); fs.chmodSync(options.backupDirectory, TRANSACTION_DIRECTORY_MODE); + fsyncDirectory(options.transactionDirectory); for (const snapshot of snapshots) { if (snapshot.receipt.state !== "file" || snapshot.bytes === null) continue; atomicWriteTrustedFile( @@ -814,6 +1136,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedGid, ); } + fsyncDirectory(options.backupDirectory); atomicWriteTrustedFile( options.manifestFile, canonicalManifest(manifest), @@ -821,6 +1144,7 @@ export function beginManagedStartupSharedStateTransaction( options.trustedUid, options.trustedGid, ); + fsyncDirectory(options.transactionDirectory); loadManifest(options); } catch (error) { try { @@ -990,11 +1314,25 @@ export function rollbackManagedStartupSharedStateTransaction( ): boolean { const options = resolveOptions(inputOptions); requireTransactionIdentity(options); + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("shared state is already durably committed"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + fail("shared state is already durably committed and cannot be rolled back"); + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } const backups = verifyAllBackups(manifest.files, options); ensureOriginalDirectories(manifest.directories, options); restoreFiles(manifest.files, backups, options); @@ -1015,11 +1353,123 @@ export function commitManagedStartupSharedStateTransaction( if (options.readOnlyReceipt) { fail("cannot commit a read-only rollback receipt"); } + const committed = loadCommitReceipt(options); + if (committed) { + if (options.bootstrapIdentity === null) { + fail("durable commit receipt is missing its expected bootstrap identity"); + } + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + return true; + } const manifest = loadManifest(options); if (!manifest) return false; if (manifest.agent !== expectedAgent) { fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); } - removeTransactionDirectory(options); + if (manifest.bootstrapIdentity !== options.bootstrapIdentity) { + fail("pending transaction belongs to a different bootstrap attempt"); + } + if (manifest.bootstrapIdentity === null) { + removeTransactionDirectory(options); + return true; + } + verifyAllBackups(manifest.files, options); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt path appeared before transaction commit"); + } + try { + fs.renameSync(options.transactionDirectory, options.commitReceiptDirectory); + fsyncDirectory(options.transactionParentDirectory); + } catch (error) { + fail(`could not atomically establish durable commit state: ${(error as Error).message}`); + } + const renamed = loadCommitReceipt(options); + if (!renamed) fail("durable commit state disappeared after atomic rename"); + assertCommitReceiptMatches(renamed.receipt, { + agent: expectedAgent, + profileFingerprint: manifest.profileFingerprint, + bootstrapIdentity: manifest.bootstrapIdentity, + }); + compactDurableCommitReceipt(renamed, options); return true; } + +/** + * Retire one exact durable bootstrap commit only after the runtime owner has + * proven its external rollback backup is gone. This prevents a completed + * attempt's image-owned receipt from blocking a later bootstrap with a + * different identity in the same persisted workload. + */ +export function clearManagedStartupSharedStateCommitReceipt( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot clear a durable commit from a read-only receipt"); + } + if (options.bootstrapIdentity === null) { + fail("durable commit cleanup requires its bootstrap identity"); + } + const committed = loadCommitReceipt(options); + if (!committed) return false; + assertCommitReceiptMatches(committed.receipt, { + agent: expectedAgent, + bootstrapIdentity: options.bootstrapIdentity, + }); + compactDurableCommitReceipt(committed, options); + requireTrustedTransactionPath( + options.commitReceiptDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + requireTrustedTransactionPath(options.commitReceiptFile, TRANSACTION_FILE_MODE, options); + const entries = fs.readdirSync(options.commitReceiptDirectory); + if (entries.length !== 1 || entries[0] !== MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE) { + fail("durable commit receipt directory contains unexpected artifacts"); + } + fs.rmSync(options.commitReceiptDirectory, { force: false, recursive: true }); + fsyncDirectory(options.transactionParentDirectory); + if (pathExistsNoFollow(options.commitReceiptDirectory)) { + fail("durable commit receipt remained after cleanup"); + } + return true; +} + +export function getManagedStartupSharedStateTransactionStatus( + expected: { + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly bootstrapIdentity: string; + }, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): "committed" | "none" | "pending" { + const options = resolveOptions({ + ...inputOptions, + bootstrapIdentity: expected.bootstrapIdentity, + }); + requireTransactionIdentity(options); + const manifest = loadManifest(options); + if (manifest) { + if ( + manifest.agent !== expected.agent || + manifest.profileFingerprint !== expected.profileFingerprint || + manifest.bootstrapIdentity !== expected.bootstrapIdentity + ) { + fail( + "pending transaction does not match the expected agent, profile fingerprint, or bootstrap identity", + ); + } + verifyAllBackups(manifest.files, options); + return "pending"; + } + const committed = loadCommitReceipt(options); + if (!committed) return "none"; + assertCommitReceiptMatches(committed.receipt, expected); + return "committed"; +}