From f7f9e4a04691e999ff892a26ecb4f6ade8222134 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 09:01:13 -0400 Subject: [PATCH 01/14] refactor(rebuild): add durable transaction store Signed-off-by: Julie Yaunches --- src/lib/state/rebuild-transaction.test.ts | 428 +++++++++++++ src/lib/state/rebuild-transaction.ts | 749 ++++++++++++++++++++++ 2 files changed, 1177 insertions(+) create mode 100644 src/lib/state/rebuild-transaction.test.ts create mode 100644 src/lib/state/rebuild-transaction.ts diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts new file mode 100644 index 00000000000..8e12cbd68a0 --- /dev/null +++ b/src/lib/state/rebuild-transaction.test.ts @@ -0,0 +1,428 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + getRebuildTransactionPath, + REBUILD_TRANSACTION_DIRNAME, + RebuildTransactionError, + type RebuildTransactionErrorCode, + type RebuildTransactionIntentV1, + type RebuildTransactionReceiptsV1, + type RebuildTransactionRecordV1, + RebuildTransactionStore, +} from "./rebuild-transaction"; + +const SANDBOX = "transaction-test"; +const TRANSACTION_ID = "11111111-1111-4111-8111-111111111111"; +const FP_A = `sha256:${"a".repeat(64)}`; +const FP_B = `sha256:${"b".repeat(64)}`; +const FP_C = `sha256:${"c".repeat(64)}`; +const FP_D = `sha256:${"d".repeat(64)}`; +const START = Date.parse("2026-07-08T00:00:00.000Z"); + +const tempDirs: string[] = []; + +function tempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-transaction-")); + tempDirs.push(dir); + return dir; +} + +function intent(overrides: Partial = {}): RebuildTransactionIntentV1 { + return { + sandboxName: SANDBOX, + source: { agent: "openclaw", registryFingerprint: FP_A }, + target: { + agent: "openclaw", + provider: "nvidia", + model: "nvidia/test-model", + credentialEnv: "NVIDIA_API_KEY", + endpointFingerprint: FP_B, + imageFingerprint: FP_C, + configurationFingerprint: FP_D, + gatewayName: "nemoclaw", + gatewayPort: 18000, + toolDisclosure: "progressive", + observabilityEnabled: false, + }, + ...overrides, + }; +} + +function preparedReceipts(): RebuildTransactionReceiptsV1 { + return { + backup: { + manifestTimestamp: "2026-07-08T00:00:00.000Z", + manifestFingerprint: FP_A, + }, + }; +} + +function deletedReceipts(): RebuildTransactionReceiptsV1 { + return { + ...preparedReceipts(), + registryRemoval: { + entryFingerprint: FP_B, + wasDefault: true, + fallbackDefault: "another-sandbox", + postRemovalDefaultSelectionRevision: 4, + }, + oldSandboxDeletion: { observedAt: "2026-07-08T00:01:00.000Z" }, + }; +} + +function replacementReceipts(): RebuildTransactionReceiptsV1 { + return { + ...deletedReceipts(), + replacement: { + identityFingerprint: FP_C, + observedAt: "2026-07-08T00:02:00.000Z", + }, + }; +} + +function makeStore(root = tempDir()): { stateDir: string; store: RebuildTransactionStore } { + let tick = 0; + const stateDir = path.join(root, ".nemoclaw", "state"); + return { + stateDir, + store: new RebuildTransactionStore({ + stateDir, + now: () => new Date(START + tick++ * 1_000), + transactionId: () => TRANSACTION_ID, + }), + }; +} + +function expectCode(action: () => unknown, code: RebuildTransactionErrorCode): void { + try { + action(); + throw new Error(`Expected ${code}`); + } catch (error) { + expect(error).toBeInstanceOf(RebuildTransactionError); + expect((error as RebuildTransactionError).code).toBe(code); + } +} + +function writeRawRecord(filePath: string, update: (record: Record) => void): void { + const record = JSON.parse(fs.readFileSync(filePath, "utf8")) as Record; + update(record); + fs.writeFileSync(filePath, JSON.stringify(record), { mode: 0o600 }); +} + +function advanceToReplacement(store: RebuildTransactionStore): RebuildTransactionRecordV1 { + const prepared = store.create(intent(), preparedReceipts()); + const deleted = store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()); + return store.transition(SANDBOX, deleted.revision, "replacement_created", replacementReceipts()); +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe("RebuildTransactionStore", () => { + it("round-trips the versioned prepared record with secure paths and permissions", () => { + const { stateDir, store } = makeStore(); + + const created = store.create(intent(), preparedReceipts()); + const filePath = getRebuildTransactionPath(SANDBOX, stateDir); + + expect(store.load(SANDBOX)).toEqual(created); + expect(created).toMatchObject({ + version: 1, + transactionId: TRANSACTION_ID, + revision: 1, + status: "active", + phase: "prepared", + failure: null, + completedAt: null, + }); + expect(path.basename(filePath)).not.toContain(SANDBOX); + expect(fs.statSync(path.dirname(filePath)).mode & 0o777).toBe(0o700); + expect(fs.statSync(filePath).mode & 0o777).toBe(0o600); + expect(fs.readdirSync(path.dirname(filePath)).filter((name) => name.endsWith(".tmp"))).toEqual( + [], + ); + }); + + it("returns null only when no transaction exists", () => { + const { store } = makeStore(); + expect(store.load(SANDBOX)).toBeNull(); + }); + + it("advances every V1 phase with monotonic revisions and clears prior failures", () => { + const { store } = makeStore(); + const prepared = store.create(intent(), preparedReceipts()); + const failed = store.recordFailure(SANDBOX, prepared.revision, { + code: "DELETE_RETRY_REQUIRED", + recordedAt: "2026-07-08T00:00:30.000Z", + retryable: true, + }); + const deleted = store.transition(SANDBOX, failed.revision, "old_deleted", deletedReceipts()); + const replacement = store.transition( + SANDBOX, + deleted.revision, + "replacement_created", + replacementReceipts(), + ); + const completed = store.complete(SANDBOX, replacement.revision); + + expect([prepared.phase, deleted.phase, replacement.phase, completed.phase]).toEqual([ + "prepared", + "old_deleted", + "replacement_created", + "completed", + ]); + expect([ + prepared.revision, + failed.revision, + deleted.revision, + replacement.revision, + completed.revision, + ]).toEqual([1, 2, 3, 4, 5]); + expect(deleted.failure).toBeNull(); + expect(completed).toMatchObject({ status: "completed", failure: null }); + expect(completed.completedAt).not.toBeNull(); + expect(store.load(SANDBOX)).toEqual(completed); + }); + + it("rejects stale revisions without overwriting the newer generation", () => { + const { store } = makeStore(); + const prepared = store.create(intent(), preparedReceipts()); + const failed = store.recordFailure(SANDBOX, prepared.revision, { + code: "FIRST_WRITER", + recordedAt: "2026-07-08T00:00:30.000Z", + retryable: true, + }); + + expectCode( + () => + store.recordFailure(SANDBOX, prepared.revision, { + code: "STALE_WRITER", + recordedAt: "2026-07-08T00:00:31.000Z", + retryable: true, + }), + "REVISION_CONFLICT", + ); + expect(store.load(SANDBOX)).toEqual(failed); + }); + + it("allows only one active record per sandbox", () => { + const { store } = makeStore(); + const first = store.create(intent(), preparedReceipts()); + + expectCode(() => store.create(intent(), preparedReceipts()), "ALREADY_EXISTS"); + expect(store.load(SANDBOX)).toEqual(first); + }); + + it("makes completion idempotent without allowing a terminal record to become active", () => { + const { store } = makeStore(); + const replacement = advanceToReplacement(store); + const completed = store.complete(SANDBOX, replacement.revision); + + expect(store.complete(SANDBOX, replacement.revision)).toEqual(completed); + expectCode( + () => + store.transition(SANDBOX, completed.revision, "replacement_created", replacementReceipts()), + "INVALID_TRANSITION", + ); + expect(store.load(SANDBOX)).toEqual(completed); + }); + + it("rejects skipped, reversed, and prematurely completed transitions", () => { + const { store } = makeStore(); + const prepared = store.create(intent(), preparedReceipts()); + + expectCode( + () => + store.transition(SANDBOX, prepared.revision, "replacement_created", replacementReceipts()), + "INVALID_TRANSITION", + ); + expectCode(() => store.complete(SANDBOX, prepared.revision), "INVALID_TRANSITION"); + expectCode(() => makeStore().store.create(intent(), replacementReceipts()), "INVALID_INPUT"); + }); + + it("does not allow a later phase to replace an existing receipt", () => { + const { store } = makeStore(); + const prepared = store.create(intent(), preparedReceipts()); + const changedBackup: RebuildTransactionReceiptsV1 = { + ...deletedReceipts(), + backup: { ...preparedReceipts().backup, manifestFingerprint: FP_D }, + }; + + expectCode( + () => store.transition(SANDBOX, prepared.revision, "old_deleted", changedBackup), + "INVALID_TRANSITION", + ); + expect(store.load(SANDBOX)).toEqual(prepared); + }); + + it("leaves the prior valid record when atomic replacement fails", () => { + const { stateDir, store } = makeStore(); + const prepared = store.create(intent(), preparedReceipts()); + vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { + throw Object.assign(new Error("simulated rename failure"), { code: "EIO" }); + }); + + expect(() => + store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()), + ).toThrow("simulated rename failure"); + + expect(store.load(SANDBOX)).toEqual(prepared); + const transactionDir = path.join(stateDir, REBUILD_TRANSACTION_DIRNAME); + expect(fs.readdirSync(transactionDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); + }); + + it("fails closed for malformed JSON and unknown future versions", () => { + const { stateDir, store } = makeStore(); + store.create(intent(), preparedReceipts()); + const filePath = getRebuildTransactionPath(SANDBOX, stateDir); + + writeRawRecord(filePath, (record) => { + record.version = 2; + }); + expectCode(() => store.load(SANDBOX), "UNSUPPORTED_VERSION"); + + fs.writeFileSync(filePath, "{not-json", { mode: 0o600 }); + expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + + it.each([ + [ + "transaction ID", + (record: Record) => { + record.transactionId = "not-a-uuid"; + }, + ], + [ + "revision", + (record: Record) => { + record.revision = -1; + }, + ], + [ + "phase/status", + (record: Record) => { + record.status = "completed"; + }, + ], + [ + "timestamp", + (record: Record) => { + record.updatedAt = "yesterday"; + }, + ], + ])("rejects a malformed %s", (_label, mutate) => { + const { stateDir, store } = makeStore(); + store.create(intent(), preparedReceipts()); + writeRawRecord(getRebuildTransactionPath(SANDBOX, stateDir), mutate); + expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + + it("rejects an incomplete registry-removal receipt", () => { + const { stateDir, store } = makeStore(); + const prepared = store.create(intent(), preparedReceipts()); + store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()); + writeRawRecord(getRebuildTransactionPath(SANDBOX, stateDir), (record) => { + const receipts = record.receipts as Record; + const removal = receipts.registryRemoval as Record; + delete removal.wasDefault; + }); + + expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + + it("rejects invalid and traversal-shaped sandbox names before path construction", () => { + const { stateDir, store } = makeStore(); + for (const name of ["../escape", "has/slash", "UPPER", "", "a".repeat(64)]) { + expectCode(() => store.load(name), "INVALID_INPUT"); + expectCode(() => getRebuildTransactionPath(name, stateDir), "INVALID_INPUT"); + } + }); + + it("does not adopt a valid record stored under another sandbox key", () => { + const { stateDir, store } = makeStore(); + store.create(intent(), preparedReceipts()); + const otherName = "other-sandbox"; + const otherPath = getRebuildTransactionPath(otherName, stateDir); + fs.copyFileSync(getRebuildTransactionPath(SANDBOX, stateDir), otherPath); + + expectCode(() => store.load(otherName), "CORRUPT"); + }); + + it("rejects symlinked transaction directories and record files", () => { + const root = tempDir(); + vi.stubEnv("HOME", root); + const stateDir = path.join(root, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + const attackerDir = path.join(root, "attacker"); + fs.mkdirSync(attackerDir); + fs.symlinkSync(attackerDir, path.join(stateDir, REBUILD_TRANSACTION_DIRNAME)); + const store = makeStore(root).store; + + expect(() => store.create(intent(), preparedReceipts())).toThrow(/symbolic link/); + + fs.unlinkSync(path.join(stateDir, REBUILD_TRANSACTION_DIRNAME)); + const created = store.create(intent(), preparedReceipts()); + const filePath = getRebuildTransactionPath(SANDBOX, stateDir); + fs.unlinkSync(filePath); + const attackerFile = path.join(attackerDir, "record.json"); + fs.writeFileSync(attackerFile, JSON.stringify(created), { mode: 0o600 }); + fs.symlinkSync(attackerFile, filePath); + + expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + + it("normalizes allow-listed data and emits a redacted diagnostic projection", () => { + const { stateDir, store } = makeStore(); + const secret = "secret-sentinel-do-not-persist"; + const untrustedIntent = { + ...intent(), + token: secret, + target: { + ...intent().target, + endpointUrl: `https://user:${secret}@example.test/v1`, + environment: { NVIDIA_API_KEY: secret }, + }, + } as unknown as RebuildTransactionIntentV1; + + const record = store.create(untrustedIntent, preparedReceipts()); + const serialized = fs.readFileSync(getRebuildTransactionPath(SANDBOX, stateDir), "utf8"); + const diagnostic = JSON.stringify(store.diagnostic(record)); + + expect(serialized).not.toContain(secret); + expect(serialized).not.toContain("endpointUrl"); + expect(serialized).not.toContain("environment"); + expect(diagnostic).not.toContain("NVIDIA_API_KEY"); + expect(diagnostic).not.toContain("provider"); + expect(store.diagnostic(record)).toMatchObject({ + sandboxName: SANDBOX, + phase: "prepared", + receipts: { + backup: true, + registryRemoval: false, + oldSandboxDeletion: false, + replacement: false, + }, + }); + }); + + it("repairs loose state-directory and record permissions while loading", () => { + const { stateDir, store } = makeStore(); + store.create(intent(), preparedReceipts()); + const filePath = getRebuildTransactionPath(SANDBOX, stateDir); + fs.chmodSync(path.dirname(filePath), 0o755); + fs.chmodSync(filePath, 0o644); + + expect(store.load(SANDBOX)).not.toBeNull(); + expect(fs.statSync(path.dirname(filePath)).mode & 0o777).toBe(0o700); + expect(fs.statSync(filePath).mode & 0o777).toBe(0o600); + }); +}); diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts new file mode 100644 index 00000000000..3859a7c0565 --- /dev/null +++ b/src/lib/state/rebuild-transaction.ts @@ -0,0 +1,749 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; + +import { isErrnoException } from "../core/errno"; +import { isRecord, type UnknownRecord } from "../core/json-types"; +import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../name-validation"; +import type { ToolDisclosure } from "../tool-disclosure"; +import { ensureConfigDir } from "./config-io"; +import { resolveNemoclawStateDir } from "./paths"; + +export const REBUILD_TRANSACTION_VERSION = 1 as const; +export const REBUILD_TRANSACTION_DIRNAME = "rebuild-transactions"; + +const MAX_TRANSACTION_BYTES = 256 * 1024; +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const FINGERPRINT_PATTERN = /^sha256:[0-9a-f]{64}$/; +const FAILURE_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; + +export type RebuildTransactionPhaseV1 = + | "prepared" + | "old_deleted" + | "replacement_created" + | "completed"; +export type RebuildTransactionStatusV1 = "active" | "completed"; + +export interface RebuildTransactionIntentV1 { + readonly sandboxName: string; + readonly source: { + readonly agent: string | null; + readonly registryFingerprint: string; + }; + readonly target: { + readonly agent: string | null; + readonly provider: string; + readonly model: string; + readonly credentialEnv: string | null; + readonly endpointFingerprint: string | null; + readonly imageFingerprint: string; + readonly configurationFingerprint: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly toolDisclosure: ToolDisclosure; + readonly observabilityEnabled: boolean; + }; +} + +export interface RebuildTransactionReceiptsV1 { + readonly backup: { + readonly manifestTimestamp: string; + readonly manifestFingerprint: string; + }; + readonly registryRemoval?: { + readonly entryFingerprint: string; + readonly wasDefault: boolean; + readonly fallbackDefault: string | null; + readonly postRemovalDefaultSelectionRevision: number; + }; + readonly oldSandboxDeletion?: { + readonly observedAt: string; + }; + readonly replacement?: { + readonly identityFingerprint: string; + readonly observedAt: string; + }; +} + +export interface RebuildTransactionFailureV1 { + readonly code: string; + readonly recordedAt: string; + readonly retryable: boolean; +} + +export interface RebuildTransactionRecordV1 { + readonly version: typeof REBUILD_TRANSACTION_VERSION; + readonly transactionId: string; + readonly revision: number; + readonly status: RebuildTransactionStatusV1; + readonly phase: RebuildTransactionPhaseV1; + readonly intent: RebuildTransactionIntentV1; + readonly receipts: RebuildTransactionReceiptsV1; + readonly failure: RebuildTransactionFailureV1 | null; + readonly createdAt: string; + readonly updatedAt: string; + readonly completedAt: string | null; +} + +export type RebuildTransactionErrorCode = + | "ALREADY_EXISTS" + | "CORRUPT" + | "INVALID_INPUT" + | "INVALID_TRANSITION" + | "NOT_FOUND" + | "REVISION_CONFLICT" + | "UNSUPPORTED_VERSION"; + +export class RebuildTransactionError extends Error { + constructor( + readonly code: RebuildTransactionErrorCode, + readonly sandboxName: string, + message: string, + options?: ErrorOptions, + ) { + super(message, options); + this.name = "RebuildTransactionError"; + } +} + +export interface RebuildTransactionDiagnosticV1 { + version: typeof REBUILD_TRANSACTION_VERSION; + transactionId: string; + sandboxName: string; + revision: number; + status: RebuildTransactionStatusV1; + phase: RebuildTransactionPhaseV1; + failureCode: string | null; + createdAt: string; + updatedAt: string; + completedAt: string | null; + receipts: { + backup: boolean; + registryRemoval: boolean; + oldSandboxDeletion: boolean; + replacement: boolean; + }; +} + +export interface RebuildTransactionStoreOptions { + stateDir?: string; + now?: () => Date; + transactionId?: () => string; +} + +function transactionFileStem(sandboxName: string): string { + return crypto.createHash("sha256").update(sandboxName).digest("hex"); +} + +export function getRebuildTransactionPath( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): string { + assertSandboxName(sandboxName); + return path.join( + stateDir, + REBUILD_TRANSACTION_DIRNAME, + `${transactionFileStem(sandboxName)}.json`, + ); +} + +function transactionError( + code: RebuildTransactionErrorCode, + sandboxName: string, + detail: string, + cause?: unknown, +): RebuildTransactionError { + return new RebuildTransactionError(code, sandboxName, `Rebuild transaction ${detail}`, { + cause: cause instanceof Error ? cause : undefined, + }); +} + +function assertSandboxName(sandboxName: string): void { + if ( + typeof sandboxName !== "string" || + sandboxName.length === 0 || + sandboxName.length > NAME_MAX_LENGTH || + !NAME_VALID_PATTERN.test(sandboxName) + ) { + throw transactionError("INVALID_INPUT", String(sandboxName), "has an invalid sandbox name"); + } +} + +function requiredRecord(value: unknown, label: string, sandboxName: string): UnknownRecord { + if (!isRecord(value)) throw transactionError("CORRUPT", sandboxName, `${label} is invalid`); + return value; +} + +function requiredString(value: unknown, label: string, sandboxName: string): string { + if (typeof value !== "string" || value.length === 0 || value.length > 1_024) { + throw transactionError("CORRUPT", sandboxName, `${label} is invalid`); + } + return value; +} + +function nullableString(value: unknown, label: string, sandboxName: string): string | null { + if (value === null) return null; + return requiredString(value, label, sandboxName); +} + +function timestamp(value: unknown, label: string, sandboxName: string): string { + const candidate = requiredString(value, label, sandboxName); + if (!Number.isFinite(Date.parse(candidate))) { + throw transactionError("CORRUPT", sandboxName, `${label} is not a timestamp`); + } + return candidate; +} + +function fingerprint(value: unknown, label: string, sandboxName: string): string { + const candidate = requiredString(value, label, sandboxName); + if (!FINGERPRINT_PATTERN.test(candidate)) { + throw transactionError("CORRUPT", sandboxName, `${label} is not a SHA-256 fingerprint`); + } + return candidate; +} + +function safeRevision(value: unknown, label: string, sandboxName: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 0) { + throw transactionError("CORRUPT", sandboxName, `${label} is invalid`); + } + return Number(value); +} + +function normalizeIntent(value: unknown, sandboxName: string): RebuildTransactionIntentV1 { + const intent = requiredRecord(value, "intent", sandboxName); + const source = requiredRecord(intent.source, "intent.source", sandboxName); + const target = requiredRecord(intent.target, "intent.target", sandboxName); + const normalizedSandboxName = requiredString( + intent.sandboxName, + "intent.sandboxName", + sandboxName, + ); + if (normalizedSandboxName !== sandboxName) { + throw transactionError("CORRUPT", sandboxName, "intent belongs to another sandbox"); + } + const gatewayPort = safeRevision(target.gatewayPort, "intent.target.gatewayPort", sandboxName); + if (gatewayPort < 1 || gatewayPort > 65_535) { + throw transactionError("CORRUPT", sandboxName, "intent.target.gatewayPort is invalid"); + } + if (target.toolDisclosure !== "progressive" && target.toolDisclosure !== "direct") { + throw transactionError("CORRUPT", sandboxName, "intent.target.toolDisclosure is invalid"); + } + if (typeof target.observabilityEnabled !== "boolean") { + throw transactionError("CORRUPT", sandboxName, "intent.target.observabilityEnabled is invalid"); + } + return { + sandboxName, + source: { + agent: nullableString(source.agent, "intent.source.agent", sandboxName), + registryFingerprint: fingerprint( + source.registryFingerprint, + "intent.source.registryFingerprint", + sandboxName, + ), + }, + target: { + agent: nullableString(target.agent, "intent.target.agent", sandboxName), + provider: requiredString(target.provider, "intent.target.provider", sandboxName), + model: requiredString(target.model, "intent.target.model", sandboxName), + credentialEnv: nullableString( + target.credentialEnv, + "intent.target.credentialEnv", + sandboxName, + ), + endpointFingerprint: + target.endpointFingerprint === null + ? null + : fingerprint( + target.endpointFingerprint, + "intent.target.endpointFingerprint", + sandboxName, + ), + imageFingerprint: fingerprint( + target.imageFingerprint, + "intent.target.imageFingerprint", + sandboxName, + ), + configurationFingerprint: fingerprint( + target.configurationFingerprint, + "intent.target.configurationFingerprint", + sandboxName, + ), + gatewayName: requiredString(target.gatewayName, "intent.target.gatewayName", sandboxName), + gatewayPort, + toolDisclosure: target.toolDisclosure, + observabilityEnabled: target.observabilityEnabled, + }, + }; +} + +function normalizeReceipts(value: unknown, sandboxName: string): RebuildTransactionReceiptsV1 { + const receipts = requiredRecord(value, "receipts", sandboxName); + const backup = requiredRecord(receipts.backup, "receipts.backup", sandboxName); + const registryRemovalWasDefault = isRecord(receipts.registryRemoval) + ? receipts.registryRemoval.wasDefault + : undefined; + if (isRecord(receipts.registryRemoval) && typeof registryRemovalWasDefault !== "boolean") { + throw transactionError( + "CORRUPT", + sandboxName, + "receipts.registryRemoval.wasDefault is invalid", + ); + } + const registryRemoval = isRecord(receipts.registryRemoval) + ? { + entryFingerprint: fingerprint( + receipts.registryRemoval.entryFingerprint, + "receipts.registryRemoval.entryFingerprint", + sandboxName, + ), + wasDefault: registryRemovalWasDefault as boolean, + fallbackDefault: + receipts.registryRemoval.fallbackDefault === null + ? null + : requiredString( + receipts.registryRemoval.fallbackDefault, + "receipts.registryRemoval.fallbackDefault", + sandboxName, + ), + postRemovalDefaultSelectionRevision: safeRevision( + receipts.registryRemoval.postRemovalDefaultSelectionRevision, + "receipts.registryRemoval.postRemovalDefaultSelectionRevision", + sandboxName, + ), + } + : undefined; + const oldSandboxDeletion = isRecord(receipts.oldSandboxDeletion) + ? { + observedAt: timestamp( + receipts.oldSandboxDeletion.observedAt, + "receipts.oldSandboxDeletion.observedAt", + sandboxName, + ), + } + : undefined; + const replacement = isRecord(receipts.replacement) + ? { + identityFingerprint: fingerprint( + receipts.replacement.identityFingerprint, + "receipts.replacement.identityFingerprint", + sandboxName, + ), + observedAt: timestamp( + receipts.replacement.observedAt, + "receipts.replacement.observedAt", + sandboxName, + ), + } + : undefined; + return { + backup: { + manifestTimestamp: timestamp( + backup.manifestTimestamp, + "receipts.backup.manifestTimestamp", + sandboxName, + ), + manifestFingerprint: fingerprint( + backup.manifestFingerprint, + "receipts.backup.manifestFingerprint", + sandboxName, + ), + }, + ...(registryRemoval ? { registryRemoval } : {}), + ...(oldSandboxDeletion ? { oldSandboxDeletion } : {}), + ...(replacement ? { replacement } : {}), + }; +} + +function normalizeFailure(value: unknown, sandboxName: string): RebuildTransactionFailureV1 | null { + if (value === null) return null; + const failure = requiredRecord(value, "failure", sandboxName); + const code = requiredString(failure.code, "failure.code", sandboxName); + if (!FAILURE_CODE_PATTERN.test(code) || typeof failure.retryable !== "boolean") { + throw transactionError("CORRUPT", sandboxName, "failure is invalid"); + } + return { + code, + recordedAt: timestamp(failure.recordedAt, "failure.recordedAt", sandboxName), + retryable: failure.retryable, + }; +} + +function normalizeRecord(value: unknown, sandboxName: string): RebuildTransactionRecordV1 { + const record = requiredRecord(value, "record", sandboxName); + if (record.version !== REBUILD_TRANSACTION_VERSION) { + if (typeof record.version === "number" && record.version > REBUILD_TRANSACTION_VERSION) { + throw transactionError( + "UNSUPPORTED_VERSION", + sandboxName, + `uses unsupported schema version ${String(record.version)}`, + ); + } + throw transactionError("CORRUPT", sandboxName, "has an invalid schema version"); + } + const transactionId = requiredString(record.transactionId, "transactionId", sandboxName); + if (!UUID_PATTERN.test(transactionId)) { + throw transactionError("CORRUPT", sandboxName, "has an invalid transaction ID"); + } + const revision = safeRevision(record.revision, "revision", sandboxName); + if (revision < 1) throw transactionError("CORRUPT", sandboxName, "has an invalid revision"); + const phase = record.phase; + const status = record.status; + if ( + phase !== "prepared" && + phase !== "old_deleted" && + phase !== "replacement_created" && + phase !== "completed" + ) { + throw transactionError("CORRUPT", sandboxName, "has an invalid phase"); + } + if (status !== "active" && status !== "completed") { + throw transactionError("CORRUPT", sandboxName, "has an invalid status"); + } + const receipts = normalizeReceipts(record.receipts, sandboxName); + const failure = normalizeFailure(record.failure, sandboxName); + const completedAt = + record.completedAt === null ? null : timestamp(record.completedAt, "completedAt", sandboxName); + if ( + (status === "completed" && (phase !== "completed" || completedAt === null || failure)) || + (status === "active" && (phase === "completed" || completedAt !== null)) + ) { + throw transactionError("CORRUPT", sandboxName, "has an invalid phase/status combination"); + } + if (phase !== "prepared" && !receipts.oldSandboxDeletion) { + throw transactionError("CORRUPT", sandboxName, "is missing the old-sandbox deletion receipt"); + } + if ((phase === "replacement_created" || phase === "completed") && !receipts.replacement) { + throw transactionError("CORRUPT", sandboxName, "is missing the replacement receipt"); + } + if ( + (phase === "prepared" && + (receipts.registryRemoval || receipts.oldSandboxDeletion || receipts.replacement)) || + (phase === "old_deleted" && receipts.replacement) + ) { + throw transactionError("CORRUPT", sandboxName, "contains receipts from a future phase"); + } + return { + version: REBUILD_TRANSACTION_VERSION, + transactionId, + revision, + status, + phase, + intent: normalizeIntent(record.intent, sandboxName), + receipts, + failure, + createdAt: timestamp(record.createdAt, "createdAt", sandboxName), + updatedAt: timestamp(record.updatedAt, "updatedAt", sandboxName), + completedAt, + }; +} + +function syncDirectory(dirPath: string): void { + const fd = fs.openSync(dirPath, fs.constants.O_RDONLY); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function durablePublish( + filePath: string, + record: RebuildTransactionRecordV1, + createOnly: boolean, +): void { + const dirPath = path.dirname(filePath); + ensureConfigDir(dirPath); + const candidatePath = path.join( + dirPath, + `.${path.basename(filePath)}.${String(process.pid)}.${crypto.randomUUID()}.tmp`, + ); + let fd: number | null = null; + try { + fd = fs.openSync( + candidatePath, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, + 0o600, + ); + fs.writeFileSync(fd, `${JSON.stringify(record, null, 2)}\n`, "utf8"); + fs.fsyncSync(fd); + fs.closeSync(fd); + fd = null; + if (createOnly) { + fs.linkSync(candidatePath, filePath); + fs.unlinkSync(candidatePath); + } else { + fs.renameSync(candidatePath, filePath); + } + syncDirectory(dirPath); + } finally { + if (fd !== null) fs.closeSync(fd); + try { + fs.unlinkSync(candidatePath); + } catch { + // Best-effort cleanup; the canonical record is published only by link/rename. + } + } +} + +function readStrictRecord( + filePath: string, + sandboxName: string, +): RebuildTransactionRecordV1 | null { + ensureConfigDir(path.dirname(filePath)); + let fd: number; + try { + fd = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return null; + throw transactionError("CORRUPT", sandboxName, "cannot be opened safely", error); + } + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile() || stat.size > MAX_TRANSACTION_BYTES) { + throw transactionError("CORRUPT", sandboxName, "is not a valid state file"); + } + if ((stat.mode & 0o077) !== 0) fs.fchmodSync(fd, 0o600); + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(fd, "utf8")); + } catch (error) { + throw transactionError("CORRUPT", sandboxName, "contains invalid JSON", error); + } + return normalizeRecord(parsed, sandboxName); + } finally { + fs.closeSync(fd); + } +} + +const NEXT_PHASE: Readonly< + Record, RebuildTransactionPhaseV1> +> = { + prepared: "old_deleted", + old_deleted: "replacement_created", + replacement_created: "completed", +}; + +function assertReceiptHistoryUnchanged( + current: RebuildTransactionReceiptsV1, + next: RebuildTransactionReceiptsV1, + sandboxName: string, +): void { + for (const key of ["backup", "registryRemoval", "oldSandboxDeletion", "replacement"] as const) { + if (current[key] !== undefined && !isDeepStrictEqual(current[key], next[key])) { + throw transactionError( + "INVALID_TRANSITION", + sandboxName, + `cannot replace the existing ${key} receipt`, + ); + } + } +} + +/** + * Durable state for the rebuild coordinator. Mutation calls are synchronous so + * their revision check and atomic publication cannot interleave in one process. + * Cross-process callers must hold the existing per-sandbox MCP lifecycle lock; + * revisions then reject state loaded before the current lock generation. + */ +export class RebuildTransactionStore { + private readonly stateDir: string; + private readonly now: () => Date; + private readonly transactionId: () => string; + + constructor(options: RebuildTransactionStoreOptions = {}) { + this.stateDir = options.stateDir ?? resolveNemoclawStateDir(); + this.now = options.now ?? (() => new Date()); + this.transactionId = options.transactionId ?? (() => crypto.randomUUID()); + } + + create( + intent: RebuildTransactionIntentV1, + receipts: RebuildTransactionReceiptsV1, + ): RebuildTransactionRecordV1 { + assertSandboxName(intent.sandboxName); + const now = this.now().toISOString(); + let record: RebuildTransactionRecordV1; + try { + record = normalizeRecord( + { + version: REBUILD_TRANSACTION_VERSION, + transactionId: this.transactionId(), + revision: 1, + status: "active", + phase: "prepared", + intent, + receipts, + failure: null, + createdAt: now, + updatedAt: now, + completedAt: null, + }, + intent.sandboxName, + ); + } catch (error) { + if (error instanceof RebuildTransactionError && error.code === "CORRUPT") { + throw transactionError( + "INVALID_INPUT", + intent.sandboxName, + "creation input is invalid", + error, + ); + } + throw error; + } + try { + durablePublish(this.path(intent.sandboxName), record, true); + } catch (error) { + if (isErrnoException(error) && error.code === "EEXIST") { + throw transactionError( + "ALREADY_EXISTS", + intent.sandboxName, + "already exists; load and reconcile it before starting another rebuild", + error, + ); + } + throw error; + } + return record; + } + + load(sandboxName: string): RebuildTransactionRecordV1 | null { + assertSandboxName(sandboxName); + return readStrictRecord(this.path(sandboxName), sandboxName); + } + + transition( + sandboxName: string, + expectedRevision: number, + phase: Exclude, + receipts: RebuildTransactionReceiptsV1, + ): RebuildTransactionRecordV1 { + const current = this.requireActive(sandboxName, expectedRevision); + if (NEXT_PHASE[current.phase as Exclude] !== phase) { + throw transactionError( + "INVALID_TRANSITION", + sandboxName, + `cannot advance from ${current.phase} to ${phase}`, + ); + } + assertReceiptHistoryUnchanged(current.receipts, receipts, sandboxName); + const updated = normalizeRecord( + { + ...current, + phase, + receipts, + failure: null, + revision: current.revision + 1, + updatedAt: this.now().toISOString(), + }, + sandboxName, + ); + durablePublish(this.path(sandboxName), updated, false); + return updated; + } + + recordFailure( + sandboxName: string, + expectedRevision: number, + failure: RebuildTransactionFailureV1, + ): RebuildTransactionRecordV1 { + const current = this.requireActive(sandboxName, expectedRevision); + const updated = normalizeRecord( + { + ...current, + failure, + revision: current.revision + 1, + updatedAt: this.now().toISOString(), + }, + sandboxName, + ); + durablePublish(this.path(sandboxName), updated, false); + return updated; + } + + complete(sandboxName: string, expectedRevision: number): RebuildTransactionRecordV1 { + const current = this.require(sandboxName); + if (current.status === "completed") return current; + this.assertRevision(current, expectedRevision); + if (current.phase !== "replacement_created") { + throw transactionError( + "INVALID_TRANSITION", + sandboxName, + `cannot complete from ${current.phase}`, + ); + } + const completedAt = this.now().toISOString(); + const completed = normalizeRecord( + { + ...current, + phase: "completed", + status: "completed", + failure: null, + revision: current.revision + 1, + updatedAt: completedAt, + completedAt, + }, + sandboxName, + ); + durablePublish(this.path(sandboxName), completed, false); + return completed; + } + + diagnostic(record: RebuildTransactionRecordV1): RebuildTransactionDiagnosticV1 { + return { + version: record.version, + transactionId: record.transactionId, + sandboxName: record.intent.sandboxName, + revision: record.revision, + status: record.status, + phase: record.phase, + failureCode: record.failure?.code ?? null, + createdAt: record.createdAt, + updatedAt: record.updatedAt, + completedAt: record.completedAt, + receipts: { + backup: true, + registryRemoval: record.receipts.registryRemoval !== undefined, + oldSandboxDeletion: record.receipts.oldSandboxDeletion !== undefined, + replacement: record.receipts.replacement !== undefined, + }, + }; + } + + private path(sandboxName: string): string { + return getRebuildTransactionPath(sandboxName, this.stateDir); + } + + private require(sandboxName: string): RebuildTransactionRecordV1 { + const current = this.load(sandboxName); + if (!current) throw transactionError("NOT_FOUND", sandboxName, "does not exist"); + return current; + } + + private requireActive(sandboxName: string, expectedRevision: number): RebuildTransactionRecordV1 { + const current = this.require(sandboxName); + if (current.status !== "active") { + throw transactionError( + "INVALID_TRANSITION", + sandboxName, + "is completed and cannot become active again", + ); + } + this.assertRevision(current, expectedRevision); + return current; + } + + private assertRevision(record: RebuildTransactionRecordV1, expectedRevision: number): void { + if (record.revision !== expectedRevision) { + throw transactionError( + "REVISION_CONFLICT", + record.intent.sandboxName, + `revision conflict: expected ${String(expectedRevision)}, found ${String(record.revision)}`, + ); + } + } +} From bf6b29b2517d83c554fd2505565febbd2aeb992f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 09:10:13 -0400 Subject: [PATCH 02/14] fix(rebuild): enforce transaction revision checks Signed-off-by: Julie Yaunches --- src/lib/state/rebuild-transaction.test.ts | 32 ++++++++++++++++++++--- src/lib/state/rebuild-transaction.ts | 4 +-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 8e12cbd68a0..acc136c6f40 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -86,7 +86,10 @@ function replacementReceipts(): RebuildTransactionReceiptsV1 { }; } -function makeStore(root = tempDir()): { stateDir: string; store: RebuildTransactionStore } { +function makeStore(root = tempDir()): { + stateDir: string; + store: RebuildTransactionStore; +} { let tick = 0; const stateDir = path.join(root, ".nemoclaw", "state"); return { @@ -227,7 +230,8 @@ describe("RebuildTransactionStore", () => { const replacement = advanceToReplacement(store); const completed = store.complete(SANDBOX, replacement.revision); - expect(store.complete(SANDBOX, replacement.revision)).toEqual(completed); + expect(store.complete(SANDBOX, completed.revision)).toEqual(completed); + expectCode(() => store.complete(SANDBOX, replacement.revision), "REVISION_CONFLICT"); expectCode( () => store.transition(SANDBOX, completed.revision, "replacement_created", replacementReceipts()), @@ -236,6 +240,26 @@ describe("RebuildTransactionStore", () => { expect(store.load(SANDBOX)).toEqual(completed); }); + it("persists a failure after deletion and clears it when replacement is observed", () => { + const { store } = makeStore(); + const prepared = store.create(intent(), preparedReceipts()); + const deleted = store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()); + const failed = store.recordFailure(SANDBOX, deleted.revision, { + code: "REPLACEMENT_RETRY_REQUIRED", + recordedAt: "2026-07-08T00:01:30.000Z", + retryable: true, + }); + + expect(store.load(SANDBOX)).toEqual(failed); + const replacement = store.transition( + SANDBOX, + failed.revision, + "replacement_created", + replacementReceipts(), + ); + expect(replacement.failure).toBeNull(); + }); + it("rejects skipped, reversed, and prematurely completed transitions", () => { const { store } = makeStore(); const prepared = store.create(intent(), preparedReceipts()); @@ -268,7 +292,9 @@ describe("RebuildTransactionStore", () => { const { stateDir, store } = makeStore(); const prepared = store.create(intent(), preparedReceipts()); vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { - throw Object.assign(new Error("simulated rename failure"), { code: "EIO" }); + throw Object.assign(new Error("simulated rename failure"), { + code: "EIO", + }); }); expect(() => diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index 3859a7c0565..f876f689daa 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -623,7 +623,7 @@ export class RebuildTransactionStore { receipts: RebuildTransactionReceiptsV1, ): RebuildTransactionRecordV1 { const current = this.requireActive(sandboxName, expectedRevision); - if (NEXT_PHASE[current.phase as Exclude] !== phase) { + if (current.phase === "completed" || NEXT_PHASE[current.phase] !== phase) { throw transactionError( "INVALID_TRANSITION", sandboxName, @@ -667,8 +667,8 @@ export class RebuildTransactionStore { complete(sandboxName: string, expectedRevision: number): RebuildTransactionRecordV1 { const current = this.require(sandboxName); - if (current.status === "completed") return current; this.assertRevision(current, expectedRevision); + if (current.status === "completed") return current; if (current.phase !== "replacement_created") { throw transactionError( "INVALID_TRANSITION", From 50e03b49f0556df8169e068128a6d1702e4110b5 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 09:16:05 -0400 Subject: [PATCH 03/14] fix(rebuild): accept persisted backup identifiers Signed-off-by: Julie Yaunches --- src/lib/state/rebuild-transaction.test.ts | 2 +- src/lib/state/rebuild-transaction.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index acc136c6f40..36e8f3a9089 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -57,7 +57,7 @@ function intent(overrides: Partial = {}): RebuildTra function preparedReceipts(): RebuildTransactionReceiptsV1 { return { backup: { - manifestTimestamp: "2026-07-08T00:00:00.000Z", + manifestTimestamp: "2026-07-08T00-00-00-000Z", manifestFingerprint: FP_A, }, }; diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index f876f689daa..8921493f9bf 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -341,7 +341,7 @@ function normalizeReceipts(value: unknown, sandboxName: string): RebuildTransact : undefined; return { backup: { - manifestTimestamp: timestamp( + manifestTimestamp: requiredString( backup.manifestTimestamp, "receipts.backup.manifestTimestamp", sandboxName, From 18e00342bdcf1eeae39ad5246caa30cc25eff6a2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 09:24:28 -0400 Subject: [PATCH 04/14] refactor(rebuild): keep registry outside transaction receipts Signed-off-by: Julie Yaunches --- src/lib/state/rebuild-transaction.test.ts | 20 ---------- src/lib/state/rebuild-transaction.ts | 47 +---------------------- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 36e8f3a9089..8b69ae45e2a 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -66,12 +66,6 @@ function preparedReceipts(): RebuildTransactionReceiptsV1 { function deletedReceipts(): RebuildTransactionReceiptsV1 { return { ...preparedReceipts(), - registryRemoval: { - entryFingerprint: FP_B, - wasDefault: true, - fallbackDefault: "another-sandbox", - postRemovalDefaultSelectionRevision: 4, - }, oldSandboxDeletion: { observedAt: "2026-07-08T00:01:00.000Z" }, }; } @@ -352,19 +346,6 @@ describe("RebuildTransactionStore", () => { expectCode(() => store.load(SANDBOX), "CORRUPT"); }); - it("rejects an incomplete registry-removal receipt", () => { - const { stateDir, store } = makeStore(); - const prepared = store.create(intent(), preparedReceipts()); - store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()); - writeRawRecord(getRebuildTransactionPath(SANDBOX, stateDir), (record) => { - const receipts = record.receipts as Record; - const removal = receipts.registryRemoval as Record; - delete removal.wasDefault; - }); - - expectCode(() => store.load(SANDBOX), "CORRUPT"); - }); - it("rejects invalid and traversal-shaped sandbox names before path construction", () => { const { stateDir, store } = makeStore(); for (const name of ["../escape", "has/slash", "UPPER", "", "a".repeat(64)]) { @@ -433,7 +414,6 @@ describe("RebuildTransactionStore", () => { phase: "prepared", receipts: { backup: true, - registryRemoval: false, oldSandboxDeletion: false, replacement: false, }, diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index 8921493f9bf..10dbcb11430 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -54,12 +54,6 @@ export interface RebuildTransactionReceiptsV1 { readonly manifestTimestamp: string; readonly manifestFingerprint: string; }; - readonly registryRemoval?: { - readonly entryFingerprint: string; - readonly wasDefault: boolean; - readonly fallbackDefault: string | null; - readonly postRemovalDefaultSelectionRevision: number; - }; readonly oldSandboxDeletion?: { readonly observedAt: string; }; @@ -123,7 +117,6 @@ export interface RebuildTransactionDiagnosticV1 { completedAt: string | null; receipts: { backup: boolean; - registryRemoval: boolean; oldSandboxDeletion: boolean; replacement: boolean; }; @@ -283,39 +276,6 @@ function normalizeIntent(value: unknown, sandboxName: string): RebuildTransactio function normalizeReceipts(value: unknown, sandboxName: string): RebuildTransactionReceiptsV1 { const receipts = requiredRecord(value, "receipts", sandboxName); const backup = requiredRecord(receipts.backup, "receipts.backup", sandboxName); - const registryRemovalWasDefault = isRecord(receipts.registryRemoval) - ? receipts.registryRemoval.wasDefault - : undefined; - if (isRecord(receipts.registryRemoval) && typeof registryRemovalWasDefault !== "boolean") { - throw transactionError( - "CORRUPT", - sandboxName, - "receipts.registryRemoval.wasDefault is invalid", - ); - } - const registryRemoval = isRecord(receipts.registryRemoval) - ? { - entryFingerprint: fingerprint( - receipts.registryRemoval.entryFingerprint, - "receipts.registryRemoval.entryFingerprint", - sandboxName, - ), - wasDefault: registryRemovalWasDefault as boolean, - fallbackDefault: - receipts.registryRemoval.fallbackDefault === null - ? null - : requiredString( - receipts.registryRemoval.fallbackDefault, - "receipts.registryRemoval.fallbackDefault", - sandboxName, - ), - postRemovalDefaultSelectionRevision: safeRevision( - receipts.registryRemoval.postRemovalDefaultSelectionRevision, - "receipts.registryRemoval.postRemovalDefaultSelectionRevision", - sandboxName, - ), - } - : undefined; const oldSandboxDeletion = isRecord(receipts.oldSandboxDeletion) ? { observedAt: timestamp( @@ -352,7 +312,6 @@ function normalizeReceipts(value: unknown, sandboxName: string): RebuildTransact sandboxName, ), }, - ...(registryRemoval ? { registryRemoval } : {}), ...(oldSandboxDeletion ? { oldSandboxDeletion } : {}), ...(replacement ? { replacement } : {}), }; @@ -420,8 +379,7 @@ function normalizeRecord(value: unknown, sandboxName: string): RebuildTransactio throw transactionError("CORRUPT", sandboxName, "is missing the replacement receipt"); } if ( - (phase === "prepared" && - (receipts.registryRemoval || receipts.oldSandboxDeletion || receipts.replacement)) || + (phase === "prepared" && (receipts.oldSandboxDeletion || receipts.replacement)) || (phase === "old_deleted" && receipts.replacement) ) { throw transactionError("CORRUPT", sandboxName, "contains receipts from a future phase"); @@ -532,7 +490,7 @@ function assertReceiptHistoryUnchanged( next: RebuildTransactionReceiptsV1, sandboxName: string, ): void { - for (const key of ["backup", "registryRemoval", "oldSandboxDeletion", "replacement"] as const) { + for (const key of ["backup", "oldSandboxDeletion", "replacement"] as const) { if (current[key] !== undefined && !isDeepStrictEqual(current[key], next[key])) { throw transactionError( "INVALID_TRANSITION", @@ -707,7 +665,6 @@ export class RebuildTransactionStore { completedAt: record.completedAt, receipts: { backup: true, - registryRemoval: record.receipts.registryRemoval !== undefined, oldSandboxDeletion: record.receipts.oldSandboxDeletion !== undefined, replacement: record.receipts.replacement !== undefined, }, From 59e2b99025d8878339f50a0aa659fd9618106a5d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 11:50:40 -0400 Subject: [PATCH 05/14] fix(rebuild): serialize transaction mutations --- .../rebuild-transaction-concurrency.test.ts | 73 +++++ src/lib/state/rebuild-transaction.test.ts | 282 +++++++----------- src/lib/state/rebuild-transaction.ts | 233 ++++++++------- test/helpers/rebuild-transaction-store.ts | 138 +++++++++ 4 files changed, 448 insertions(+), 278 deletions(-) create mode 100644 src/lib/state/rebuild-transaction-concurrency.test.ts create mode 100644 test/helpers/rebuild-transaction-store.ts diff --git a/src/lib/state/rebuild-transaction-concurrency.test.ts b/src/lib/state/rebuild-transaction-concurrency.test.ts new file mode 100644 index 00000000000..bbfd1267d28 --- /dev/null +++ b/src/lib/state/rebuild-transaction-concurrency.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it } from "vitest"; + +import { + advanceToReplacement, + cleanupRebuildTransactionTests, + deletedReceipts, + intent, + makeStore, + preparedReceipts, + SANDBOX, +} from "../../../test/helpers/rebuild-transaction-store"; +import type { RebuildTransactionRecordV1, RebuildTransactionStore } from "./rebuild-transaction"; + +async function expectOneRevisionWinner( + store: RebuildTransactionStore, + operations: [Promise, Promise], +): Promise { + const results = await Promise.allSettled(operations); + const fulfilled = results.filter( + (result): result is PromiseFulfilledResult => + result.status === "fulfilled", + ); + const rejected = results.filter( + (result): result is PromiseRejectedResult => result.status === "rejected", + ); + + expect(fulfilled).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0]?.reason).toMatchObject({ code: "REVISION_CONFLICT" }); + expect(store.load(SANDBOX)).toEqual(fulfilled[0]?.value); +} + +afterEach(cleanupRebuildTransactionTests); + +describe("RebuildTransactionStore concurrency", () => { + it("serializes competing transitions before revision validation", async () => { + const { store } = makeStore(); + const prepared = await store.create(intent(), preparedReceipts()); + + await expectOneRevisionWinner(store, [ + store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()), + store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()), + ]); + }); + + it("serializes competing failure records before revision validation", async () => { + const { store } = makeStore(); + const prepared = await store.create(intent(), preparedReceipts()); + const failure = (code: string) => ({ + code, + recordedAt: "2026-07-08T00:00:30.000Z", + retryable: true, + }); + + await expectOneRevisionWinner(store, [ + store.recordFailure(SANDBOX, prepared.revision, failure("FIRST_WRITER")), + store.recordFailure(SANDBOX, prepared.revision, failure("SECOND_WRITER")), + ]); + }); + + it("serializes competing completions before revision validation", async () => { + const { store } = makeStore(); + const replacement = await advanceToReplacement(store); + + await expectOneRevisionWinner(store, [ + store.complete(SANDBOX, replacement.revision), + store.complete(SANDBOX, replacement.revision), + ]); + }); +}); diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 8b69ae45e2a..f5327a6e5b8 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -2,133 +2,40 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + advanceToReplacement, + cleanupRebuildTransactionTests, + deletedReceipts, + expectCode, + FP_A, + FP_D, + intent, + makeStore, + preparedReceipts, + replacementReceipts, + SANDBOX, + tempDir, + TRANSACTION_ID, + writeRawRecord, +} from "../../../test/helpers/rebuild-transaction-store"; + import { getRebuildTransactionPath, REBUILD_TRANSACTION_DIRNAME, - RebuildTransactionError, - type RebuildTransactionErrorCode, type RebuildTransactionIntentV1, type RebuildTransactionReceiptsV1, - type RebuildTransactionRecordV1, - RebuildTransactionStore, } from "./rebuild-transaction"; -const SANDBOX = "transaction-test"; -const TRANSACTION_ID = "11111111-1111-4111-8111-111111111111"; -const FP_A = `sha256:${"a".repeat(64)}`; -const FP_B = `sha256:${"b".repeat(64)}`; -const FP_C = `sha256:${"c".repeat(64)}`; -const FP_D = `sha256:${"d".repeat(64)}`; -const START = Date.parse("2026-07-08T00:00:00.000Z"); - -const tempDirs: string[] = []; - -function tempDir(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-transaction-")); - tempDirs.push(dir); - return dir; -} - -function intent(overrides: Partial = {}): RebuildTransactionIntentV1 { - return { - sandboxName: SANDBOX, - source: { agent: "openclaw", registryFingerprint: FP_A }, - target: { - agent: "openclaw", - provider: "nvidia", - model: "nvidia/test-model", - credentialEnv: "NVIDIA_API_KEY", - endpointFingerprint: FP_B, - imageFingerprint: FP_C, - configurationFingerprint: FP_D, - gatewayName: "nemoclaw", - gatewayPort: 18000, - toolDisclosure: "progressive", - observabilityEnabled: false, - }, - ...overrides, - }; -} - -function preparedReceipts(): RebuildTransactionReceiptsV1 { - return { - backup: { - manifestTimestamp: "2026-07-08T00-00-00-000Z", - manifestFingerprint: FP_A, - }, - }; -} - -function deletedReceipts(): RebuildTransactionReceiptsV1 { - return { - ...preparedReceipts(), - oldSandboxDeletion: { observedAt: "2026-07-08T00:01:00.000Z" }, - }; -} - -function replacementReceipts(): RebuildTransactionReceiptsV1 { - return { - ...deletedReceipts(), - replacement: { - identityFingerprint: FP_C, - observedAt: "2026-07-08T00:02:00.000Z", - }, - }; -} - -function makeStore(root = tempDir()): { - stateDir: string; - store: RebuildTransactionStore; -} { - let tick = 0; - const stateDir = path.join(root, ".nemoclaw", "state"); - return { - stateDir, - store: new RebuildTransactionStore({ - stateDir, - now: () => new Date(START + tick++ * 1_000), - transactionId: () => TRANSACTION_ID, - }), - }; -} - -function expectCode(action: () => unknown, code: RebuildTransactionErrorCode): void { - try { - action(); - throw new Error(`Expected ${code}`); - } catch (error) { - expect(error).toBeInstanceOf(RebuildTransactionError); - expect((error as RebuildTransactionError).code).toBe(code); - } -} - -function writeRawRecord(filePath: string, update: (record: Record) => void): void { - const record = JSON.parse(fs.readFileSync(filePath, "utf8")) as Record; - update(record); - fs.writeFileSync(filePath, JSON.stringify(record), { mode: 0o600 }); -} - -function advanceToReplacement(store: RebuildTransactionStore): RebuildTransactionRecordV1 { - const prepared = store.create(intent(), preparedReceipts()); - const deleted = store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()); - return store.transition(SANDBOX, deleted.revision, "replacement_created", replacementReceipts()); -} - -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllEnvs(); - for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); -}); +afterEach(cleanupRebuildTransactionTests); describe("RebuildTransactionStore", () => { - it("round-trips the versioned prepared record with secure paths and permissions", () => { + it("round-trips the versioned prepared record with secure paths and permissions", async () => { const { stateDir, store } = makeStore(); - const created = store.create(intent(), preparedReceipts()); + const created = await store.create(intent(), preparedReceipts()); const filePath = getRebuildTransactionPath(SANDBOX, stateDir); expect(store.load(SANDBOX)).toEqual(created); @@ -149,27 +56,51 @@ describe("RebuildTransactionStore", () => { ); }); - it("returns null only when no transaction exists", () => { + it("returns null only when no transaction exists", async () => { const { store } = makeStore(); expect(store.load(SANDBOX)).toBeNull(); }); - it("advances every V1 phase with monotonic revisions and clears prior failures", () => { + it("reports NOT_FOUND for every mutation without a transaction", async () => { const { store } = makeStore(); - const prepared = store.create(intent(), preparedReceipts()); - const failed = store.recordFailure(SANDBOX, prepared.revision, { + + await expectCode( + () => store.transition(SANDBOX, 1, "old_deleted", deletedReceipts()), + "NOT_FOUND", + ); + await expectCode( + () => + store.recordFailure(SANDBOX, 1, { + code: "MISSING", + recordedAt: "2026-07-08T00:00:30.000Z", + retryable: true, + }), + "NOT_FOUND", + ); + await expectCode(() => store.complete(SANDBOX, 1), "NOT_FOUND"); + }); + + it("advances every V1 phase with monotonic revisions and clears prior failures", async () => { + const { store } = makeStore(); + const prepared = await store.create(intent(), preparedReceipts()); + const failed = await store.recordFailure(SANDBOX, prepared.revision, { code: "DELETE_RETRY_REQUIRED", recordedAt: "2026-07-08T00:00:30.000Z", retryable: true, }); - const deleted = store.transition(SANDBOX, failed.revision, "old_deleted", deletedReceipts()); - const replacement = store.transition( + const deleted = await store.transition( + SANDBOX, + failed.revision, + "old_deleted", + deletedReceipts(), + ); + const replacement = await store.transition( SANDBOX, deleted.revision, "replacement_created", replacementReceipts(), ); - const completed = store.complete(SANDBOX, replacement.revision); + const completed = await store.complete(SANDBOX, replacement.revision); expect([prepared.phase, deleted.phase, replacement.phase, completed.phase]).toEqual([ "prepared", @@ -190,16 +121,16 @@ describe("RebuildTransactionStore", () => { expect(store.load(SANDBOX)).toEqual(completed); }); - it("rejects stale revisions without overwriting the newer generation", () => { + it("rejects stale revisions without overwriting the newer generation", async () => { const { store } = makeStore(); - const prepared = store.create(intent(), preparedReceipts()); - const failed = store.recordFailure(SANDBOX, prepared.revision, { + const prepared = await store.create(intent(), preparedReceipts()); + const failed = await store.recordFailure(SANDBOX, prepared.revision, { code: "FIRST_WRITER", recordedAt: "2026-07-08T00:00:30.000Z", retryable: true, }); - expectCode( + await expectCode( () => store.recordFailure(SANDBOX, prepared.revision, { code: "STALE_WRITER", @@ -211,22 +142,22 @@ describe("RebuildTransactionStore", () => { expect(store.load(SANDBOX)).toEqual(failed); }); - it("allows only one active record per sandbox", () => { + it("allows only one active record per sandbox", async () => { const { store } = makeStore(); - const first = store.create(intent(), preparedReceipts()); + const first = await store.create(intent(), preparedReceipts()); - expectCode(() => store.create(intent(), preparedReceipts()), "ALREADY_EXISTS"); + await expectCode(() => store.create(intent(), preparedReceipts()), "ALREADY_EXISTS"); expect(store.load(SANDBOX)).toEqual(first); }); - it("makes completion idempotent without allowing a terminal record to become active", () => { + it("makes completion idempotent without allowing a terminal record to become active", async () => { const { store } = makeStore(); - const replacement = advanceToReplacement(store); - const completed = store.complete(SANDBOX, replacement.revision); + const replacement = await advanceToReplacement(store); + const completed = await store.complete(SANDBOX, replacement.revision); - expect(store.complete(SANDBOX, completed.revision)).toEqual(completed); - expectCode(() => store.complete(SANDBOX, replacement.revision), "REVISION_CONFLICT"); - expectCode( + expect(await store.complete(SANDBOX, completed.revision)).toEqual(completed); + await expectCode(() => store.complete(SANDBOX, replacement.revision), "REVISION_CONFLICT"); + await expectCode( () => store.transition(SANDBOX, completed.revision, "replacement_created", replacementReceipts()), "INVALID_TRANSITION", @@ -234,18 +165,23 @@ describe("RebuildTransactionStore", () => { expect(store.load(SANDBOX)).toEqual(completed); }); - it("persists a failure after deletion and clears it when replacement is observed", () => { + it("persists a failure after deletion and clears it when replacement is observed", async () => { const { store } = makeStore(); - const prepared = store.create(intent(), preparedReceipts()); - const deleted = store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()); - const failed = store.recordFailure(SANDBOX, deleted.revision, { + const prepared = await store.create(intent(), preparedReceipts()); + const deleted = await store.transition( + SANDBOX, + prepared.revision, + "old_deleted", + deletedReceipts(), + ); + const failed = await store.recordFailure(SANDBOX, deleted.revision, { code: "REPLACEMENT_RETRY_REQUIRED", recordedAt: "2026-07-08T00:01:30.000Z", retryable: true, }); expect(store.load(SANDBOX)).toEqual(failed); - const replacement = store.transition( + const replacement = await store.transition( SANDBOX, failed.revision, "replacement_created", @@ -254,64 +190,67 @@ describe("RebuildTransactionStore", () => { expect(replacement.failure).toBeNull(); }); - it("rejects skipped, reversed, and prematurely completed transitions", () => { + it("rejects skipped, reversed, and prematurely completed transitions", async () => { const { store } = makeStore(); - const prepared = store.create(intent(), preparedReceipts()); + const prepared = await store.create(intent(), preparedReceipts()); - expectCode( + await expectCode( () => store.transition(SANDBOX, prepared.revision, "replacement_created", replacementReceipts()), "INVALID_TRANSITION", ); - expectCode(() => store.complete(SANDBOX, prepared.revision), "INVALID_TRANSITION"); - expectCode(() => makeStore().store.create(intent(), replacementReceipts()), "INVALID_INPUT"); + await expectCode(() => store.complete(SANDBOX, prepared.revision), "INVALID_TRANSITION"); + await expectCode( + () => makeStore().store.create(intent(), replacementReceipts()), + "INVALID_INPUT", + ); }); - it("does not allow a later phase to replace an existing receipt", () => { + it("does not allow a later phase to replace an existing receipt", async () => { const { store } = makeStore(); - const prepared = store.create(intent(), preparedReceipts()); + const prepared = await store.create(intent(), preparedReceipts()); const changedBackup: RebuildTransactionReceiptsV1 = { ...deletedReceipts(), backup: { ...preparedReceipts().backup, manifestFingerprint: FP_D }, }; - expectCode( + await expectCode( () => store.transition(SANDBOX, prepared.revision, "old_deleted", changedBackup), "INVALID_TRANSITION", ); expect(store.load(SANDBOX)).toEqual(prepared); }); - it("leaves the prior valid record when atomic replacement fails", () => { + it("leaves the prior valid record when atomic replacement fails", async () => { const { stateDir, store } = makeStore(); - const prepared = store.create(intent(), preparedReceipts()); + const prepared = await store.create(intent(), preparedReceipts()); vi.spyOn(fs, "renameSync").mockImplementationOnce(() => { throw Object.assign(new Error("simulated rename failure"), { code: "EIO", }); }); - expect(() => + await expect( store.transition(SANDBOX, prepared.revision, "old_deleted", deletedReceipts()), - ).toThrow("simulated rename failure"); + ).rejects.toThrow("simulated rename failure"); expect(store.load(SANDBOX)).toEqual(prepared); const transactionDir = path.join(stateDir, REBUILD_TRANSACTION_DIRNAME); expect(fs.readdirSync(transactionDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); }); - it("fails closed for malformed JSON and unknown future versions", () => { + it("fails closed for malformed JSON and unknown future versions", async () => { const { stateDir, store } = makeStore(); - store.create(intent(), preparedReceipts()); + await store.create(intent(), preparedReceipts()); const filePath = getRebuildTransactionPath(SANDBOX, stateDir); writeRawRecord(filePath, (record) => { record.version = 2; }); - expectCode(() => store.load(SANDBOX), "UNSUPPORTED_VERSION"); + await expectCode(() => store.load(SANDBOX), "UNSUPPORTED_VERSION"); fs.writeFileSync(filePath, "{not-json", { mode: 0o600 }); - expectCode(() => store.load(SANDBOX), "CORRUPT"); + await expectCode(() => store.load(SANDBOX), "CORRUPT"); }); it.each([ @@ -339,33 +278,34 @@ describe("RebuildTransactionStore", () => { record.updatedAt = "yesterday"; }, ], - ])("rejects a malformed %s", (_label, mutate) => { + ])("rejects a malformed %s", async (_label, mutate) => { const { stateDir, store } = makeStore(); - store.create(intent(), preparedReceipts()); + await store.create(intent(), preparedReceipts()); writeRawRecord(getRebuildTransactionPath(SANDBOX, stateDir), mutate); - expectCode(() => store.load(SANDBOX), "CORRUPT"); + await expectCode(() => store.load(SANDBOX), "CORRUPT"); }); - it("rejects invalid and traversal-shaped sandbox names before path construction", () => { + it("rejects invalid and traversal-shaped sandbox names before path construction", async () => { const { stateDir, store } = makeStore(); for (const name of ["../escape", "has/slash", "UPPER", "", "a".repeat(64)]) { - expectCode(() => store.load(name), "INVALID_INPUT"); - expectCode(() => getRebuildTransactionPath(name, stateDir), "INVALID_INPUT"); + await expectCode(() => store.load(name), "INVALID_INPUT"); + await expectCode(() => getRebuildTransactionPath(name, stateDir), "INVALID_INPUT"); } }); - it("does not adopt a valid record stored under another sandbox key", () => { + it("does not adopt a valid record stored under another sandbox key", async () => { const { stateDir, store } = makeStore(); - store.create(intent(), preparedReceipts()); + await store.create(intent(), preparedReceipts()); const otherName = "other-sandbox"; const otherPath = getRebuildTransactionPath(otherName, stateDir); fs.copyFileSync(getRebuildTransactionPath(SANDBOX, stateDir), otherPath); - expectCode(() => store.load(otherName), "CORRUPT"); + await expectCode(() => store.load(otherName), "CORRUPT"); }); - it("rejects symlinked transaction directories and record files", () => { + it("rejects symlinked transaction directories and record files", async () => { const root = tempDir(); + // rejectSymlinksOnPath scopes user-controlled components beneath HOME. vi.stubEnv("HOME", root); const stateDir = path.join(root, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); @@ -374,20 +314,20 @@ describe("RebuildTransactionStore", () => { fs.symlinkSync(attackerDir, path.join(stateDir, REBUILD_TRANSACTION_DIRNAME)); const store = makeStore(root).store; - expect(() => store.create(intent(), preparedReceipts())).toThrow(/symbolic link/); + await expect(store.create(intent(), preparedReceipts())).rejects.toThrow(/symbolic link/); fs.unlinkSync(path.join(stateDir, REBUILD_TRANSACTION_DIRNAME)); - const created = store.create(intent(), preparedReceipts()); + const created = await store.create(intent(), preparedReceipts()); const filePath = getRebuildTransactionPath(SANDBOX, stateDir); fs.unlinkSync(filePath); const attackerFile = path.join(attackerDir, "record.json"); fs.writeFileSync(attackerFile, JSON.stringify(created), { mode: 0o600 }); fs.symlinkSync(attackerFile, filePath); - expectCode(() => store.load(SANDBOX), "CORRUPT"); + await expectCode(() => store.load(SANDBOX), "CORRUPT"); }); - it("normalizes allow-listed data and emits a redacted diagnostic projection", () => { + it("normalizes allow-listed data and emits a redacted diagnostic projection", async () => { const { stateDir, store } = makeStore(); const secret = "secret-sentinel-do-not-persist"; const untrustedIntent = { @@ -400,7 +340,7 @@ describe("RebuildTransactionStore", () => { }, } as unknown as RebuildTransactionIntentV1; - const record = store.create(untrustedIntent, preparedReceipts()); + const record = await store.create(untrustedIntent, preparedReceipts()); const serialized = fs.readFileSync(getRebuildTransactionPath(SANDBOX, stateDir), "utf8"); const diagnostic = JSON.stringify(store.diagnostic(record)); @@ -420,9 +360,9 @@ describe("RebuildTransactionStore", () => { }); }); - it("repairs loose state-directory and record permissions while loading", () => { + it("repairs loose state-directory and record permissions while loading", async () => { const { stateDir, store } = makeStore(); - store.create(intent(), preparedReceipts()); + await store.create(intent(), preparedReceipts()); const filePath = getRebuildTransactionPath(SANDBOX, stateDir); fs.chmodSync(path.dirname(filePath), 0o755); fs.chmodSync(filePath, 0o644); diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index 10dbcb11430..bf2568bb8e2 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -11,6 +11,7 @@ import { isRecord, type UnknownRecord } from "../core/json-types"; import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../name-validation"; import type { ToolDisclosure } from "../tool-disclosure"; import { ensureConfigDir } from "./config-io"; +import { withMcpLifecycleLock } from "./mcp-lifecycle-lock"; import { resolveNemoclawStateDir } from "./paths"; export const REBUILD_TRANSACTION_VERSION = 1 as const; @@ -400,6 +401,8 @@ function normalizeRecord(value: unknown, sandboxName: string): RebuildTransactio } function syncDirectory(dirPath: string): void { + // NemoClaw's supported Linux filesystems persist the directory entry after + // fsync. Other platforms may provide weaker directory-fsync guarantees. const fd = fs.openSync(dirPath, fs.constants.O_RDONLY); try { fs.fsyncSync(fd); @@ -431,6 +434,8 @@ function durablePublish( fs.closeSync(fd); fd = null; if (createOnly) { + // Both paths are deliberately in dirPath, so this hard link cannot cross + // a filesystem boundary and fail with EXDEV. fs.linkSync(candidatePath, filePath); fs.unlinkSync(candidatePath); } else { @@ -501,11 +506,9 @@ function assertReceiptHistoryUnchanged( } } -/** - * Durable state for the rebuild coordinator. Mutation calls are synchronous so - * their revision check and atomic publication cannot interleave in one process. - * Cross-process callers must hold the existing per-sandbox MCP lifecycle lock; - * revisions then reject state loaded before the current lock generation. +/** Durable state for the rebuild coordinator. Each mutation acquires the + * existing per-sandbox lifecycle lock before revision validation and durable + * publication. Nested calls from an already-locked rebuild are reentrant. */ export class RebuildTransactionStore { private readonly stateDir: string; @@ -518,55 +521,57 @@ export class RebuildTransactionStore { this.transactionId = options.transactionId ?? (() => crypto.randomUUID()); } - create( + async create( intent: RebuildTransactionIntentV1, receipts: RebuildTransactionReceiptsV1, - ): RebuildTransactionRecordV1 { + ): Promise { assertSandboxName(intent.sandboxName); - const now = this.now().toISOString(); - let record: RebuildTransactionRecordV1; - try { - record = normalizeRecord( - { - version: REBUILD_TRANSACTION_VERSION, - transactionId: this.transactionId(), - revision: 1, - status: "active", - phase: "prepared", - intent, - receipts, - failure: null, - createdAt: now, - updatedAt: now, - completedAt: null, - }, - intent.sandboxName, - ); - } catch (error) { - if (error instanceof RebuildTransactionError && error.code === "CORRUPT") { - throw transactionError( - "INVALID_INPUT", + return this.withMutationLock(intent.sandboxName, () => { + const now = this.now().toISOString(); + let record: RebuildTransactionRecordV1; + try { + record = normalizeRecord( + { + version: REBUILD_TRANSACTION_VERSION, + transactionId: this.transactionId(), + revision: 1, + status: "active", + phase: "prepared", + intent, + receipts, + failure: null, + createdAt: now, + updatedAt: now, + completedAt: null, + }, intent.sandboxName, - "creation input is invalid", - error, ); + } catch (error) { + if (error instanceof RebuildTransactionError && error.code === "CORRUPT") { + throw transactionError( + "INVALID_INPUT", + intent.sandboxName, + "creation input is invalid", + error, + ); + } + throw error; } - throw error; - } - try { - durablePublish(this.path(intent.sandboxName), record, true); - } catch (error) { - if (isErrnoException(error) && error.code === "EEXIST") { - throw transactionError( - "ALREADY_EXISTS", - intent.sandboxName, - "already exists; load and reconcile it before starting another rebuild", - error, - ); + try { + durablePublish(this.path(intent.sandboxName), record, true); + } catch (error) { + if (isErrnoException(error) && error.code === "EEXIST") { + throw transactionError( + "ALREADY_EXISTS", + intent.sandboxName, + "already exists; load and reconcile it before starting another rebuild", + error, + ); + } + throw error; } - throw error; - } - return record; + return record; + }); } load(sandboxName: string): RebuildTransactionRecordV1 | null { @@ -574,81 +579,90 @@ export class RebuildTransactionStore { return readStrictRecord(this.path(sandboxName), sandboxName); } - transition( + async transition( sandboxName: string, expectedRevision: number, phase: Exclude, receipts: RebuildTransactionReceiptsV1, - ): RebuildTransactionRecordV1 { - const current = this.requireActive(sandboxName, expectedRevision); - if (current.phase === "completed" || NEXT_PHASE[current.phase] !== phase) { - throw transactionError( - "INVALID_TRANSITION", + ): Promise { + return this.withMutationLock(sandboxName, () => { + const current = this.requireActive(sandboxName, expectedRevision); + if (current.phase === "completed" || NEXT_PHASE[current.phase] !== phase) { + throw transactionError( + "INVALID_TRANSITION", + sandboxName, + `cannot advance from ${current.phase} to ${phase}`, + ); + } + assertReceiptHistoryUnchanged(current.receipts, receipts, sandboxName); + const updated = normalizeRecord( + { + ...current, + phase, + receipts, + failure: null, + revision: current.revision + 1, + updatedAt: this.now().toISOString(), + }, sandboxName, - `cannot advance from ${current.phase} to ${phase}`, ); - } - assertReceiptHistoryUnchanged(current.receipts, receipts, sandboxName); - const updated = normalizeRecord( - { - ...current, - phase, - receipts, - failure: null, - revision: current.revision + 1, - updatedAt: this.now().toISOString(), - }, - sandboxName, - ); - durablePublish(this.path(sandboxName), updated, false); - return updated; + durablePublish(this.path(sandboxName), updated, false); + return updated; + }); } - recordFailure( + async recordFailure( sandboxName: string, expectedRevision: number, failure: RebuildTransactionFailureV1, - ): RebuildTransactionRecordV1 { - const current = this.requireActive(sandboxName, expectedRevision); - const updated = normalizeRecord( - { - ...current, - failure, - revision: current.revision + 1, - updatedAt: this.now().toISOString(), - }, - sandboxName, - ); - durablePublish(this.path(sandboxName), updated, false); - return updated; + ): Promise { + return this.withMutationLock(sandboxName, () => { + const current = this.requireActive(sandboxName, expectedRevision); + const updated = normalizeRecord( + { + ...current, + failure, + revision: current.revision + 1, + updatedAt: this.now().toISOString(), + }, + sandboxName, + ); + durablePublish(this.path(sandboxName), updated, false); + return updated; + }); } - complete(sandboxName: string, expectedRevision: number): RebuildTransactionRecordV1 { - const current = this.require(sandboxName); - this.assertRevision(current, expectedRevision); - if (current.status === "completed") return current; - if (current.phase !== "replacement_created") { - throw transactionError( - "INVALID_TRANSITION", + async complete( + sandboxName: string, + expectedRevision: number, + ): Promise { + return this.withMutationLock(sandboxName, () => { + const current = this.require(sandboxName); + this.assertRevision(current, expectedRevision); + if (current.status === "completed") return current; + if (current.phase !== "replacement_created") { + throw transactionError( + "INVALID_TRANSITION", + sandboxName, + `cannot complete from ${current.phase}`, + ); + } + const completedAt = this.now().toISOString(); + const completed = normalizeRecord( + { + ...current, + phase: "completed", + status: "completed", + failure: null, + revision: current.revision + 1, + updatedAt: completedAt, + completedAt, + }, sandboxName, - `cannot complete from ${current.phase}`, ); - } - const completedAt = this.now().toISOString(); - const completed = normalizeRecord( - { - ...current, - phase: "completed", - status: "completed", - failure: null, - revision: current.revision + 1, - updatedAt: completedAt, - completedAt, - }, - sandboxName, - ); - durablePublish(this.path(sandboxName), completed, false); - return completed; + durablePublish(this.path(sandboxName), completed, false); + return completed; + }); } diagnostic(record: RebuildTransactionRecordV1): RebuildTransactionDiagnosticV1 { @@ -675,6 +689,11 @@ export class RebuildTransactionStore { return getRebuildTransactionPath(sandboxName, this.stateDir); } + private withMutationLock(sandboxName: string, operation: () => T): Promise { + assertSandboxName(sandboxName); + return withMcpLifecycleLock(sandboxName, operation, { stateDir: this.stateDir }); + } + private require(sandboxName: string): RebuildTransactionRecordV1 { const current = this.load(sandboxName); if (!current) throw transactionError("NOT_FOUND", sandboxName, "does not exist"); diff --git a/test/helpers/rebuild-transaction-store.ts b/test/helpers/rebuild-transaction-store.ts new file mode 100644 index 00000000000..1870e3e1650 --- /dev/null +++ b/test/helpers/rebuild-transaction-store.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect, vi } from "vitest"; + +import { + RebuildTransactionError, + type RebuildTransactionErrorCode, + type RebuildTransactionIntentV1, + type RebuildTransactionReceiptsV1, + type RebuildTransactionRecordV1, + RebuildTransactionStore, +} from "../../src/lib/state/rebuild-transaction"; + +export const SANDBOX = "transaction-test"; +export const TRANSACTION_ID = "11111111-1111-4111-8111-111111111111"; +export const FP_A = `sha256:${"a".repeat(64)}`; +export const FP_B = `sha256:${"b".repeat(64)}`; +export const FP_C = `sha256:${"c".repeat(64)}`; +export const FP_D = `sha256:${"d".repeat(64)}`; +const START = Date.parse("2026-07-08T00:00:00.000Z"); + +const tempDirs: string[] = []; + +export function tempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-transaction-")); + tempDirs.push(dir); + return dir; +} + +export function intent( + overrides: Partial = {}, +): RebuildTransactionIntentV1 { + return { + sandboxName: SANDBOX, + source: { agent: "openclaw", registryFingerprint: FP_A }, + target: { + agent: "openclaw", + provider: "nvidia", + model: "nvidia/test-model", + credentialEnv: "NVIDIA_API_KEY", + endpointFingerprint: FP_B, + imageFingerprint: FP_C, + configurationFingerprint: FP_D, + gatewayName: "nemoclaw", + gatewayPort: 18000, + toolDisclosure: "progressive", + observabilityEnabled: false, + }, + ...overrides, + }; +} + +export function preparedReceipts(): RebuildTransactionReceiptsV1 { + return { + backup: { + manifestTimestamp: "2026-07-08T00-00-00-000Z", + manifestFingerprint: FP_A, + }, + }; +} + +export function deletedReceipts(): RebuildTransactionReceiptsV1 { + return { + ...preparedReceipts(), + oldSandboxDeletion: { observedAt: "2026-07-08T00:01:00.000Z" }, + }; +} + +export function replacementReceipts(): RebuildTransactionReceiptsV1 { + return { + ...deletedReceipts(), + replacement: { + identityFingerprint: FP_C, + observedAt: "2026-07-08T00:02:00.000Z", + }, + }; +} + +export function makeStore(root = tempDir()): { + stateDir: string; + store: RebuildTransactionStore; +} { + let tick = 0; + const stateDir = path.join(root, ".nemoclaw", "state"); + return { + stateDir, + store: new RebuildTransactionStore({ + stateDir, + now: () => new Date(START + tick++ * 1_000), + transactionId: () => TRANSACTION_ID, + }), + }; +} + +export async function expectCode( + action: () => unknown | Promise, + code: RebuildTransactionErrorCode, +): Promise { + try { + await action(); + throw new Error(`Expected ${code}`); + } catch (error) { + expect(error).toBeInstanceOf(RebuildTransactionError); + expect((error as RebuildTransactionError).code).toBe(code); + } +} + +export function writeRawRecord( + filePath: string, + update: (record: Record) => void, +): void { + const record = JSON.parse(fs.readFileSync(filePath, "utf8")) as Record; + update(record); + fs.writeFileSync(filePath, JSON.stringify(record), { mode: 0o600 }); +} + +export async function advanceToReplacement( + store: RebuildTransactionStore, +): Promise { + const prepared = await store.create(intent(), preparedReceipts()); + const deleted = await store.transition( + SANDBOX, + prepared.revision, + "old_deleted", + deletedReceipts(), + ); + return store.transition(SANDBOX, deleted.revision, "replacement_created", replacementReceipts()); +} + +export function cleanupRebuildTransactionTests(): void { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +} From 42a356af6fbdd362082c3f90fe01e4bb68191114 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 12:17:03 -0400 Subject: [PATCH 06/14] fix(rebuild): validate transaction receipts --- src/lib/state/rebuild-transaction.test.ts | 61 ++++++++++++ src/lib/state/rebuild-transaction.ts | 88 +++++++++++++---- .../rebuild-transaction-store-process.test.ts | 99 +++++++++++++++++++ 3 files changed, 229 insertions(+), 19 deletions(-) create mode 100644 test/rebuild-transaction-store-process.test.ts diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index f5327a6e5b8..f834c53930e 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -239,6 +239,36 @@ describe("RebuildTransactionStore", () => { expect(fs.readdirSync(transactionDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); }); + it("fails safely when atomic create publication reports a cross-device invariant", async () => { + const { stateDir, store } = makeStore(); + vi.spyOn(fs, "linkSync").mockImplementationOnce(() => { + throw Object.assign(new Error("simulated cross-device link"), { code: "EXDEV" }); + }); + + await expect(store.create(intent(), preparedReceipts())).rejects.toThrow( + "state directory changed filesystem", + ); + expect(store.load(SANDBOX)).toBeNull(); + const transactionDir = path.join(stateDir, REBUILD_TRANSACTION_DIRNAME); + expect(fs.readdirSync(transactionDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); + }); + + it("commits create when only temporary-link cleanup initially fails", async () => { + const { stateDir, store } = makeStore(); + const unlinkSync = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync") + .mockImplementationOnce(() => { + throw Object.assign(new Error("simulated cleanup failure"), { code: "EIO" }); + }) + .mockImplementation(unlinkSync); + + const created = await store.create(intent(), preparedReceipts()); + + expect(store.load(SANDBOX)).toEqual(created); + const transactionDir = path.join(stateDir, REBUILD_TRANSACTION_DIRNAME); + expect(fs.readdirSync(transactionDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); + }); + it("fails closed for malformed JSON and unknown future versions", async () => { const { stateDir, store } = makeStore(); await store.create(intent(), preparedReceipts()); @@ -278,6 +308,25 @@ describe("RebuildTransactionStore", () => { record.updatedAt = "yesterday"; }, ], + [ + "backup timestamp", + (record: Record) => { + const receipts = record.receipts as Record; + (receipts.backup as Record).manifestTimestamp = "not-a-date"; + }, + ], + [ + "old-sandbox deletion receipt", + (record: Record) => { + (record.receipts as Record).oldSandboxDeletion = "bad"; + }, + ], + [ + "replacement receipt", + (record: Record) => { + (record.receipts as Record).replacement = "bad"; + }, + ], ])("rejects a malformed %s", async (_label, mutate) => { const { stateDir, store } = makeStore(); await store.create(intent(), preparedReceipts()); @@ -285,6 +334,18 @@ describe("RebuildTransactionStore", () => { await expectCode(() => store.load(SANDBOX), "CORRUPT"); }); + it("rejects replacement evidence timestamped before deletion", async () => { + const { stateDir, store } = makeStore(); + await advanceToReplacement(store); + const filePath = getRebuildTransactionPath(SANDBOX, stateDir); + writeRawRecord(filePath, (record) => { + const receipts = record.receipts as Record>; + receipts.replacement!.observedAt = "2026-07-08T00:00:30.000Z"; + }); + + await expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + it("rejects invalid and traversal-shaped sandbox names before path construction", async () => { const { stateDir, store } = makeStore(); for (const name of ["../escape", "has/slash", "UPPER", "", "a".repeat(64)]) { diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index bf2568bb8e2..716eabd9939 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -21,6 +21,7 @@ const MAX_TRANSACTION_BYTES = 256 * 1024; const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const FINGERPRINT_PATTERN = /^sha256:[0-9a-f]{64}$/; const FAILURE_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; +const BACKUP_TIMESTAMP_PATTERN = /^(\d{4}-\d{2}-\d{2}T)(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/; export type RebuildTransactionPhaseV1 = | "prepared" @@ -192,6 +193,22 @@ function timestamp(value: unknown, label: string, sandboxName: string): string { return candidate; } +function backupManifestTimestamp(value: unknown, sandboxName: string): string { + const candidate = requiredString(value, "receipts.backup.manifestTimestamp", sandboxName); + const match = BACKUP_TIMESTAMP_PATTERN.exec(candidate); + const parseable = match + ? `${match[1]}${match[2]}:${match[3]}:${match[4]}.${match[5]}Z` + : candidate; + if (!Number.isFinite(Date.parse(parseable))) { + throw transactionError( + "CORRUPT", + sandboxName, + "receipts.backup.manifestTimestamp is not a timestamp", + ); + } + return candidate; +} + function fingerprint(value: unknown, label: string, sandboxName: string): string { const candidate = requiredString(value, label, sandboxName); if (!FINGERPRINT_PATTERN.test(candidate)) { @@ -277,24 +294,34 @@ function normalizeIntent(value: unknown, sandboxName: string): RebuildTransactio function normalizeReceipts(value: unknown, sandboxName: string): RebuildTransactionReceiptsV1 { const receipts = requiredRecord(value, "receipts", sandboxName); const backup = requiredRecord(receipts.backup, "receipts.backup", sandboxName); - const oldSandboxDeletion = isRecord(receipts.oldSandboxDeletion) + const oldSandboxDeletionValue = receipts.oldSandboxDeletion; + const oldSandboxDeletion = + oldSandboxDeletionValue === undefined + ? undefined + : requiredRecord(oldSandboxDeletionValue, "receipts.oldSandboxDeletion", sandboxName); + const normalizedOldSandboxDeletion = oldSandboxDeletion ? { observedAt: timestamp( - receipts.oldSandboxDeletion.observedAt, + oldSandboxDeletion.observedAt, "receipts.oldSandboxDeletion.observedAt", sandboxName, ), } : undefined; - const replacement = isRecord(receipts.replacement) + const replacementValue = receipts.replacement; + const replacement = + replacementValue === undefined + ? undefined + : requiredRecord(replacementValue, "receipts.replacement", sandboxName); + const normalizedReplacement = replacement ? { identityFingerprint: fingerprint( - receipts.replacement.identityFingerprint, + replacement.identityFingerprint, "receipts.replacement.identityFingerprint", sandboxName, ), observedAt: timestamp( - receipts.replacement.observedAt, + replacement.observedAt, "receipts.replacement.observedAt", sandboxName, ), @@ -302,19 +329,15 @@ function normalizeReceipts(value: unknown, sandboxName: string): RebuildTransact : undefined; return { backup: { - manifestTimestamp: requiredString( - backup.manifestTimestamp, - "receipts.backup.manifestTimestamp", - sandboxName, - ), + manifestTimestamp: backupManifestTimestamp(backup.manifestTimestamp, sandboxName), manifestFingerprint: fingerprint( backup.manifestFingerprint, "receipts.backup.manifestFingerprint", sandboxName, ), }, - ...(oldSandboxDeletion ? { oldSandboxDeletion } : {}), - ...(replacement ? { replacement } : {}), + ...(normalizedOldSandboxDeletion ? { oldSandboxDeletion: normalizedOldSandboxDeletion } : {}), + ...(normalizedReplacement ? { replacement: normalizedReplacement } : {}), }; } @@ -385,6 +408,13 @@ function normalizeRecord(value: unknown, sandboxName: string): RebuildTransactio ) { throw transactionError("CORRUPT", sandboxName, "contains receipts from a future phase"); } + if ( + receipts.oldSandboxDeletion && + receipts.replacement && + Date.parse(receipts.replacement.observedAt) < Date.parse(receipts.oldSandboxDeletion.observedAt) + ) { + throw transactionError("CORRUPT", sandboxName, "has out-of-order phase receipts"); + } return { version: REBUILD_TRANSACTION_VERSION, transactionId, @@ -401,8 +431,9 @@ function normalizeRecord(value: unknown, sandboxName: string): RebuildTransactio } function syncDirectory(dirPath: string): void { - // NemoClaw's supported Linux filesystems persist the directory entry after - // fsync. Other platforms may provide weaker directory-fsync guarantees. + // Linux persists the directory entry after fsync. Node does not expose + // macOS F_FULLFSYNC, so macOS provides best-effort rather than power-loss + // durability; schema validation still fails closed after any torn record. const fd = fs.openSync(dirPath, fs.constants.O_RDONLY); try { fs.fsyncSync(fd); @@ -434,10 +465,28 @@ function durablePublish( fs.closeSync(fd); fd = null; if (createOnly) { - // Both paths are deliberately in dirPath, so this hard link cannot cross - // a filesystem boundary and fail with EXDEV. - fs.linkSync(candidatePath, filePath); - fs.unlinkSync(candidatePath); + // Both directory entries are in dirPath. Verify the invariant before the + // atomic no-replace link; a copy/rename fallback would reintroduce the + // concurrent-creator overwrite this path exists to prevent. + if (fs.statSync(candidatePath).dev !== fs.statSync(dirPath).dev) { + throw new Error("Rebuild transaction candidate crossed a filesystem boundary"); + } + try { + fs.linkSync(candidatePath, filePath); + } catch (error) { + if (isErrnoException(error) && error.code === "EXDEV") { + throw new Error( + "Rebuild transaction state directory changed filesystem during atomic publication", + { cause: error }, + ); + } + throw error; + } + try { + fs.unlinkSync(candidatePath); + } catch { + // The canonical hard link is already durable; finally retries cleanup. + } } else { fs.renameSync(candidatePath, filePath); } @@ -508,7 +557,8 @@ function assertReceiptHistoryUnchanged( /** Durable state for the rebuild coordinator. Each mutation acquires the * existing per-sandbox lifecycle lock before revision validation and durable - * publication. Nested calls from an already-locked rebuild are reentrant. + * publication. AsyncLocalStorage makes nested calls reentrant within one + * process; separate processes always contend on the filesystem lock. */ export class RebuildTransactionStore { private readonly stateDir: string; diff --git a/test/rebuild-transaction-store-process.test.ts b/test/rebuild-transaction-store-process.test.ts new file mode 100644 index 00000000000..ca51fc912d9 --- /dev/null +++ b/test/rebuild-transaction-store-process.test.ts @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { RebuildTransactionStore } from "../src/lib/state/rebuild-transaction"; +import { intent, preparedReceipts, SANDBOX } from "./helpers/rebuild-transaction-store"; + +const roots: string[] = []; +const requireSource = createRequire(import.meta.url); +const storeModule = requireSource.resolve("../src/lib/state/rebuild-transaction.js"); + +const writerScript = String.raw` +const fs = require("node:fs"); +const { RebuildTransactionStore } = require(process.argv[1]); +const stateDir = process.argv[2]; +const gate = process.argv[3]; +const code = process.argv[4]; +fs.writeFileSync(gate + "." + code + ".ready", ""); +(async () => { + while (!fs.existsSync(gate)) await new Promise((resolve) => setTimeout(resolve, 5)); + try { + const record = await new RebuildTransactionStore({ stateDir }).recordFailure("transaction-test", 1, { + code, + recordedAt: "2026-07-08T00:00:30.000Z", + retryable: true, + }); + process.stdout.write(JSON.stringify({ ok: true, record })); + } catch (error) { + process.stdout.write(JSON.stringify({ ok: false, code: error.code, message: error.message })); + } +})().catch((error) => { + process.stderr.write(String(error)); + process.exitCode = 1; +}); +`; + +function startWriter( + stateDir: string, + gate: string, + code: string, +): Promise<{ ok: boolean; code?: string; record?: unknown }> { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["-e", writerScript, storeModule, stateDir, gate, code], { + cwd: process.cwd(), + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + child.on("error", reject); + child.on("close", (exitCode) => { + if (exitCode !== 0) reject(new Error(stderr || `writer exited ${String(exitCode)}`)); + else resolve(JSON.parse(stdout)); + }); + }); +} + +async function waitForReady(gate: string, codes: string[]): Promise { + for (let attempt = 0; attempt < 200; attempt++) { + if (codes.every((code) => fs.existsSync(`${gate}.${code}.ready`))) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error("Timed out waiting for transaction writers"); +} + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe("RebuildTransactionStore cross-process serialization", () => { + it("allows one writer to advance and rejects the stale process", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-transaction-process-")); + roots.push(root); + const stateDir = path.join(root, "state"); + const gate = path.join(root, "start"); + const store = new RebuildTransactionStore({ stateDir }); + await store.create(intent(), preparedReceipts()); + + const writers = [ + startWriter(stateDir, gate, "PROCESS_A"), + startWriter(stateDir, gate, "PROCESS_B"), + ]; + await waitForReady(gate, ["PROCESS_A", "PROCESS_B"]); + fs.writeFileSync(gate, "go"); + const results = await Promise.all(writers); + + expect(results.filter((result) => result.ok)).toHaveLength(1); + expect(results.filter((result) => result.code === "REVISION_CONFLICT")).toHaveLength(1); + expect(store.load(SANDBOX)).toEqual(results.find((result) => result.ok)?.record); + }, 15_000); +}); From b98da8450264025cd07f6f307b1687fee2a382ed Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 12:19:30 -0400 Subject: [PATCH 07/14] test(rebuild): keep process race linear --- test/rebuild-transaction-store-process.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/rebuild-transaction-store-process.test.ts b/test/rebuild-transaction-store-process.test.ts index ca51fc912d9..b7991adbf5b 100644 --- a/test/rebuild-transaction-store-process.test.ts +++ b/test/rebuild-transaction-store-process.test.ts @@ -57,18 +57,22 @@ function startWriter( child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); child.on("error", reject); child.on("close", (exitCode) => { - if (exitCode !== 0) reject(new Error(stderr || `writer exited ${String(exitCode)}`)); - else resolve(JSON.parse(stdout)); + exitCode === 0 + ? resolve(JSON.parse(stdout)) + : reject(new Error(stderr || `writer exited ${String(exitCode)}`)); }); }); } async function waitForReady(gate: string, codes: string[]): Promise { - for (let attempt = 0; attempt < 200; attempt++) { - if (codes.every((code) => fs.existsSync(`${gate}.${code}.ready`))) return; + let attemptsRemaining = 200; + while ( + !codes.every((code) => fs.existsSync(`${gate}.${code}.ready`)) && + attemptsRemaining-- > 0 + ) { await new Promise((resolve) => setTimeout(resolve, 10)); } - throw new Error("Timed out waiting for transaction writers"); + expect(codes.every((code) => fs.existsSync(`${gate}.${code}.ready`))).toBe(true); } afterEach(() => { From f7eccf4b11711c895d9f4927c610e077daec12ea Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 12:21:58 -0400 Subject: [PATCH 08/14] test(rebuild): remove stale fixture import --- src/lib/state/rebuild-transaction.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index f834c53930e..76e0d12ffc9 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -10,7 +10,6 @@ import { cleanupRebuildTransactionTests, deletedReceipts, expectCode, - FP_A, FP_D, intent, makeStore, From 417beffe85e0a4430bb63f90d98a8f51dc1a14ef Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 12:30:06 -0400 Subject: [PATCH 09/14] docs(rebuild): define durability boundaries --- src/lib/state/README.md | 22 +++++++++++++ src/lib/state/rebuild-transaction.test.ts | 6 ++-- src/lib/state/rebuild-transaction.ts | 39 +++++++++++++++-------- 3 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 src/lib/state/README.md diff --git a/src/lib/state/README.md b/src/lib/state/README.md new file mode 100644 index 00000000000..0b4bb233fc0 --- /dev/null +++ b/src/lib/state/README.md @@ -0,0 +1,22 @@ + + + +# Local state durability + +State modules own persisted NemoClaw files and must fail closed when durable state is missing, +malformed, or from an unsupported schema version. + +The rebuild transaction store publishes a fully synced candidate from the destination directory. +Initial publication uses a same-directory hard link for atomic create-only semantics; update +publication uses rename. A copy fallback is intentionally excluded because it cannot retain the +same no-replace guarantee for competing creators. An unexpected cross-device error is therefore an +invariant failure, not a recoverable publication mode. + +After publication, the store calls `fsync` on the containing directory. Common Linux filesystems +use this to persist the directory entry, but exact guarantees remain filesystem and device +specific. On macOS, Node.js does not expose `F_FULLFSYNC`; strict persistence through sudden power +loss is unsupported and best-effort until Node exposes that primitive or NemoClaw adopts a native +adapter. Atomic visibility and fail-closed validation still apply on macOS. + +Mutation reentrancy is limited to nested calls in one process. Independent processes always +contend on the per-sandbox filesystem lifecycle lock before revision validation and publication. diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 76e0d12ffc9..8de52b6ee3e 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -245,7 +245,7 @@ describe("RebuildTransactionStore", () => { }); await expect(store.create(intent(), preparedReceipts())).rejects.toThrow( - "state directory changed filesystem", + "atomic-publication invariant failed", ); expect(store.load(SANDBOX)).toBeNull(); const transactionDir = path.join(stateDir, REBUILD_TRANSACTION_DIRNAME); @@ -424,8 +424,8 @@ describe("RebuildTransactionStore", () => { const { stateDir, store } = makeStore(); await store.create(intent(), preparedReceipts()); const filePath = getRebuildTransactionPath(SANDBOX, stateDir); - fs.chmodSync(path.dirname(filePath), 0o755); - fs.chmodSync(filePath, 0o644); + fs.chmodSync(path.dirname(filePath), 0o777); + fs.chmodSync(filePath, 0o666); expect(store.load(SANDBOX)).not.toBeNull(); expect(fs.statSync(path.dirname(filePath)).mode & 0o777).toBe(0o700); diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index 716eabd9939..eadc5393c6f 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -431,9 +431,16 @@ function normalizeRecord(value: unknown, sandboxName: string): RebuildTransactio } function syncDirectory(dirPath: string): void { - // Linux persists the directory entry after fsync. Node does not expose - // macOS F_FULLFSYNC, so macOS provides best-effort rather than power-loss - // durability; schema validation still fails closed after any torn record. + // Durability boundary review: + // - Invalid state: a published directory entry can be lost after power loss. + // - Source boundary: filesystem/device-specific fsync semantics, outside NemoClaw. + // - Constraint: Node exposes fsync but not macOS F_FULLFSYNC/fcntl. + // - Regression evidence: atomicity and torn-record validation are testable; + // physical power-loss persistence requires platform/storage fault injection. + // - Removal condition: use F_FULLFSYNC when Node exposes it (or a native + // adapter is adopted). Until then macOS power-loss durability is best-effort. + // Linux ext4/xfs directory fsync commonly persists entries; other filesystems + // and storage devices may provide weaker guarantees. const fd = fs.openSync(dirPath, fs.constants.O_RDONLY); try { fs.fsyncSync(fd); @@ -465,18 +472,19 @@ function durablePublish( fs.closeSync(fd); fd = null; if (createOnly) { - // Both directory entries are in dirPath. Verify the invariant before the - // atomic no-replace link; a copy/rename fallback would reintroduce the - // concurrent-creator overwrite this path exists to prevent. - if (fs.statSync(candidatePath).dev !== fs.statSync(dirPath).dev) { - throw new Error("Rebuild transaction candidate crossed a filesystem boundary"); - } + // Publication boundary review: + // - Invalid state: a second creator overwrites the first transaction. + // - Source boundary: both names are created directly inside dirPath, so a + // hard link is same-directory and same-filesystem by construction. + // - Constraint: copy/rename cannot preserve link(2)'s atomic no-replace. + // - Regression evidence: EEXIST and injected EXDEV both fail closed. + // - Removal condition: replace this when Node exposes renameat2(RENAME_NOREPLACE). try { fs.linkSync(candidatePath, filePath); } catch (error) { if (isErrnoException(error) && error.code === "EXDEV") { throw new Error( - "Rebuild transaction state directory changed filesystem during atomic publication", + "Rebuild transaction atomic-publication invariant failed: candidate and record must share a filesystem", { cause: error }, ); } @@ -496,7 +504,9 @@ function durablePublish( try { fs.unlinkSync(candidatePath); } catch { - // Best-effort cleanup; the canonical record is published only by link/rename. + // A persistent cleanup failure can leave a 0600 dotfile hard link inside + // the 0700 state directory. It is inert: only the canonical hashed path is + // loaded, and publication already fsynced the same inode. } } } @@ -557,8 +567,11 @@ function assertReceiptHistoryUnchanged( /** Durable state for the rebuild coordinator. Each mutation acquires the * existing per-sandbox lifecycle lock before revision validation and durable - * publication. AsyncLocalStorage makes nested calls reentrant within one - * process; separate processes always contend on the filesystem lock. + * publication. Reentrancy is intra-process only via AsyncLocalStorage; + * cross-process callers always contend on the filesystem lock. + * + * On macOS, Node's lack of F_FULLFSYNC means strict power-loss durability is + * unsupported; atomic publication and fail-closed record validation still hold. */ export class RebuildTransactionStore { private readonly stateDir: string; From 6c5c5185c402cd50c73641ef7d8096660c896af8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 12:34:25 -0400 Subject: [PATCH 10/14] fix(rebuild): require canonical receipt timestamps --- src/lib/state/rebuild-transaction.test.ts | 4 ++-- src/lib/state/rebuild-transaction.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 8de52b6ee3e..77aa11cd056 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -304,14 +304,14 @@ describe("RebuildTransactionStore", () => { [ "timestamp", (record: Record) => { - record.updatedAt = "yesterday"; + record.updatedAt = "2026-07-08"; }, ], [ "backup timestamp", (record: Record) => { const receipts = record.receipts as Record; - (receipts.backup as Record).manifestTimestamp = "not-a-date"; + (receipts.backup as Record).manifestTimestamp = "2026-07-08T00:00:00Z"; }, ], [ diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index eadc5393c6f..1d2d17d72c9 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -21,6 +21,7 @@ const MAX_TRANSACTION_BYTES = 256 * 1024; const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const FINGERPRINT_PATTERN = /^sha256:[0-9a-f]{64}$/; const FAILURE_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; +const ISO_TIMESTAMP_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; const BACKUP_TIMESTAMP_PATTERN = /^(\d{4}-\d{2}-\d{2}T)(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/; export type RebuildTransactionPhaseV1 = @@ -185,9 +186,15 @@ function nullableString(value: unknown, label: string, sandboxName: string): str return requiredString(value, label, sandboxName); } +function isExactIsoTimestamp(candidate: string): boolean { + if (!ISO_TIMESTAMP_PATTERN.test(candidate)) return false; + const milliseconds = Date.parse(candidate); + return Number.isFinite(milliseconds) && new Date(milliseconds).toISOString() === candidate; +} + function timestamp(value: unknown, label: string, sandboxName: string): string { const candidate = requiredString(value, label, sandboxName); - if (!Number.isFinite(Date.parse(candidate))) { + if (!isExactIsoTimestamp(candidate)) { throw transactionError("CORRUPT", sandboxName, `${label} is not a timestamp`); } return candidate; @@ -199,7 +206,7 @@ function backupManifestTimestamp(value: unknown, sandboxName: string): string { const parseable = match ? `${match[1]}${match[2]}:${match[3]}:${match[4]}.${match[5]}Z` : candidate; - if (!Number.isFinite(Date.parse(parseable))) { + if (!isExactIsoTimestamp(parseable)) { throw transactionError( "CORRUPT", sandboxName, From 7ec1122e55f33a61e15755e46139f25a1b161cb9 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 12:43:48 -0400 Subject: [PATCH 11/14] feat(rebuild): persist legacy recovery authority --- src/lib/state/README.md | 6 ++++++ src/lib/state/rebuild-transaction.test.ts | 14 ++++++++++++++ src/lib/state/rebuild-transaction.ts | 20 +++++++++++++++++++- test/helpers/rebuild-transaction-store.ts | 6 +++++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/lib/state/README.md b/src/lib/state/README.md index 0b4bb233fc0..244442d7388 100644 --- a/src/lib/state/README.md +++ b/src/lib/state/README.md @@ -12,6 +12,12 @@ publication uses rename. A copy fallback is intentionally excluded because it ca same no-replace guarantee for competing creators. An unexpected cross-device error is therefore an invariant failure, not a recoverable publication mode. +Transaction filenames are deterministic SHA-256 hashes of validated sandbox names. The hash is a +stable traversal-safe key, not a confidentiality boundary; transaction contents and names remain +protected by the `0700` state directory and `0600` record mode. Backup receipt timestamps preserve +the product's filename-safe dashed format and are validated by a bijective conversion to canonical +UTC ISO time. + After publication, the store calls `fsync` on the containing directory. Common Linux filesystems use this to persist the directory entry, but exact guarantees remain filesystem and device specific. On macOS, Node.js does not expose `F_FULLFSYNC`; strict persistence through sudden power diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 77aa11cd056..055933ad95c 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -301,6 +301,20 @@ describe("RebuildTransactionStore", () => { record.status = "completed"; }, ], + [ + "legacy recovery authorization", + (record: Record) => { + const intent = record.intent as Record>; + intent.source!.legacyManagedImageRecoveryAuthorized = "yes"; + }, + ], + [ + "credential environment variable", + (record: Record) => { + const intent = record.intent as Record>; + intent.target!.credentialEnv = "not-an-env-name"; + }, + ], [ "timestamp", (record: Record) => { diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index 1d2d17d72c9..f82c374f3de 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -21,6 +21,7 @@ const MAX_TRANSACTION_BYTES = 256 * 1024; const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const FINGERPRINT_PATTERN = /^sha256:[0-9a-f]{64}$/; const FAILURE_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; +const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; const ISO_TIMESTAMP_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; const BACKUP_TIMESTAMP_PATTERN = /^(\d{4}-\d{2}-\d{2}T)(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/; @@ -36,6 +37,7 @@ export interface RebuildTransactionIntentV1 { readonly source: { readonly agent: string | null; readonly registryFingerprint: string; + readonly legacyManagedImageRecoveryAuthorized: boolean; }; readonly target: { readonly agent: string | null; @@ -186,6 +188,14 @@ function nullableString(value: unknown, label: string, sandboxName: string): str return requiredString(value, label, sandboxName); } +function nullableEnvName(value: unknown, label: string, sandboxName: string): string | null { + const candidate = nullableString(value, label, sandboxName); + if (candidate !== null && !ENV_NAME_PATTERN.test(candidate)) { + throw transactionError("CORRUPT", sandboxName, `${label} is not an environment variable name`); + } + return candidate; +} + function isExactIsoTimestamp(candidate: string): boolean { if (!ISO_TIMESTAMP_PATTERN.test(candidate)) return false; const milliseconds = Date.parse(candidate); @@ -253,6 +263,13 @@ function normalizeIntent(value: unknown, sandboxName: string): RebuildTransactio if (typeof target.observabilityEnabled !== "boolean") { throw transactionError("CORRUPT", sandboxName, "intent.target.observabilityEnabled is invalid"); } + if (typeof source.legacyManagedImageRecoveryAuthorized !== "boolean") { + throw transactionError( + "CORRUPT", + sandboxName, + "intent.source.legacyManagedImageRecoveryAuthorized is invalid", + ); + } return { sandboxName, source: { @@ -262,12 +279,13 @@ function normalizeIntent(value: unknown, sandboxName: string): RebuildTransactio "intent.source.registryFingerprint", sandboxName, ), + legacyManagedImageRecoveryAuthorized: source.legacyManagedImageRecoveryAuthorized, }, target: { agent: nullableString(target.agent, "intent.target.agent", sandboxName), provider: requiredString(target.provider, "intent.target.provider", sandboxName), model: requiredString(target.model, "intent.target.model", sandboxName), - credentialEnv: nullableString( + credentialEnv: nullableEnvName( target.credentialEnv, "intent.target.credentialEnv", sandboxName, diff --git a/test/helpers/rebuild-transaction-store.ts b/test/helpers/rebuild-transaction-store.ts index 1870e3e1650..fb632e31b39 100644 --- a/test/helpers/rebuild-transaction-store.ts +++ b/test/helpers/rebuild-transaction-store.ts @@ -36,7 +36,11 @@ export function intent( ): RebuildTransactionIntentV1 { return { sandboxName: SANDBOX, - source: { agent: "openclaw", registryFingerprint: FP_A }, + source: { + agent: "openclaw", + registryFingerprint: FP_A, + legacyManagedImageRecoveryAuthorized: false, + }, target: { agent: "openclaw", provider: "nvidia", From ad173d08f7b364e92cab09a2fd5caa6fefae0386 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 12:55:39 -0400 Subject: [PATCH 12/14] refactor(rebuild): isolate record validation tests --- src/lib/state/README.md | 7 +- .../rebuild-transaction-validation.test.ts | 131 ++++++++++++++++++ src/lib/state/rebuild-transaction.test.ts | 92 ------------ src/lib/state/rebuild-transaction.ts | 71 +++++----- 4 files changed, 175 insertions(+), 126 deletions(-) create mode 100644 src/lib/state/rebuild-transaction-validation.test.ts diff --git a/src/lib/state/README.md b/src/lib/state/README.md index 244442d7388..94690ad96cf 100644 --- a/src/lib/state/README.md +++ b/src/lib/state/README.md @@ -14,9 +14,10 @@ invariant failure, not a recoverable publication mode. Transaction filenames are deterministic SHA-256 hashes of validated sandbox names. The hash is a stable traversal-safe key, not a confidentiality boundary; transaction contents and names remain -protected by the `0700` state directory and `0600` record mode. Backup receipt timestamps preserve -the product's filename-safe dashed format and are validated by a bijective conversion to canonical -UTC ISO time. +protected by the `0700` state directory and `0600` record mode. Backup receipts accept canonical +millisecond UTC ISO timestamps and the filename-safe dashed form emitted by backup generation. The +dashed form is validated by a bijective conversion to canonical UTC ISO time and can be removed if +backup manifests later standardize on a filename-safe encoding separate from their timestamp. After publication, the store calls `fsync` on the containing directory. Common Linux filesystems use this to persist the directory entry, but exact guarantees remain filesystem and device diff --git a/src/lib/state/rebuild-transaction-validation.test.ts b/src/lib/state/rebuild-transaction-validation.test.ts new file mode 100644 index 00000000000..88eb5b2231e --- /dev/null +++ b/src/lib/state/rebuild-transaction-validation.test.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { afterEach, describe, it } from "vitest"; + +import { + advanceToReplacement, + cleanupRebuildTransactionTests, + deletedReceipts, + expectCode, + intent, + makeStore, + preparedReceipts, + SANDBOX, + writeRawRecord, +} from "../../../test/helpers/rebuild-transaction-store"; +import { getRebuildTransactionPath } from "./rebuild-transaction"; + +afterEach(cleanupRebuildTransactionTests); + +describe("RebuildTransactionStore validation", () => { + it.each([ + "2026-07-08T00:00:00.000Z", + "2026-07-08T00-00-00-000Z", + ])("accepts the documented backup timestamp shape %s", async (manifestTimestamp) => { + const { store } = makeStore(); + await store.create(intent(), { + backup: { ...preparedReceipts().backup, manifestTimestamp }, + }); + }); + + it("fails closed for malformed JSON and unknown future versions", async () => { + const { stateDir, store } = makeStore(); + await store.create(intent(), preparedReceipts()); + const filePath = getRebuildTransactionPath(SANDBOX, stateDir); + + writeRawRecord(filePath, (record) => { + record.version = 2; + }); + await expectCode(() => store.load(SANDBOX), "UNSUPPORTED_VERSION"); + + fs.writeFileSync(filePath, "{not-json", { mode: 0o600 }); + await expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + + it.each([ + ["transaction ID", (record: Record) => (record.transactionId = "bad")], + ["revision", (record: Record) => (record.revision = -1)], + ["phase/status", (record: Record) => (record.status = "completed")], + [ + "legacy recovery authorization", + (record: Record) => { + const value = record.intent as Record>; + value.source!.legacyManagedImageRecoveryAuthorized = "yes"; + }, + ], + [ + "credential environment variable", + (record: Record) => { + const value = record.intent as Record>; + value.target!.credentialEnv = "not-an-env-name"; + }, + ], + ["timestamp", (record: Record) => (record.updatedAt = "2026-07-08")], + [ + "backup timestamp", + (record: Record) => { + const receipts = record.receipts as Record; + (receipts.backup as Record).manifestTimestamp = "2026-07-08T00:00:00Z"; + }, + ], + [ + "old-sandbox deletion receipt", + (record: Record) => { + (record.receipts as Record).oldSandboxDeletion = "bad"; + }, + ], + [ + "replacement receipt", + (record: Record) => { + (record.receipts as Record).replacement = "bad"; + }, + ], + ])("rejects a malformed %s", async (_label, mutate) => { + const { stateDir, store } = makeStore(); + await store.create(intent(), preparedReceipts()); + writeRawRecord(getRebuildTransactionPath(SANDBOX, stateDir), mutate); + await expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + + it("rejects replacement evidence timestamped before deletion", async () => { + const { stateDir, store } = makeStore(); + await advanceToReplacement(store); + writeRawRecord(getRebuildTransactionPath(SANDBOX, stateDir), (record) => { + const receipts = record.receipts as Record>; + receipts.replacement!.observedAt = "2026-07-08T00:00:30.000Z"; + }); + await expectCode(() => store.load(SANDBOX), "CORRUPT"); + }); + + it("classifies malformed mutation arguments separately from corrupt state", async () => { + const { store } = makeStore(); + await expectCode( + () => + store.create( + intent({ target: { ...intent().target, credentialEnv: "bad-name" } }), + preparedReceipts(), + ), + "INVALID_INPUT", + ); + const prepared = await store.create(intent(), preparedReceipts()); + await expectCode( + () => + store.transition(SANDBOX, prepared.revision, "old_deleted", { + ...deletedReceipts(), + oldSandboxDeletion: { observedAt: "2026-07-08" }, + }), + "INVALID_TRANSITION", + ); + await expectCode( + () => + store.recordFailure(SANDBOX, prepared.revision, { + code: "bad-code", + recordedAt: "2026-07-08T00:00:30.000Z", + retryable: true, + }), + "INVALID_INPUT", + ); + }); +}); diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 055933ad95c..18cf45afa3b 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -18,7 +18,6 @@ import { SANDBOX, tempDir, TRANSACTION_ID, - writeRawRecord, } from "../../../test/helpers/rebuild-transaction-store"; import { @@ -268,97 +267,6 @@ describe("RebuildTransactionStore", () => { expect(fs.readdirSync(transactionDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); }); - it("fails closed for malformed JSON and unknown future versions", async () => { - const { stateDir, store } = makeStore(); - await store.create(intent(), preparedReceipts()); - const filePath = getRebuildTransactionPath(SANDBOX, stateDir); - - writeRawRecord(filePath, (record) => { - record.version = 2; - }); - await expectCode(() => store.load(SANDBOX), "UNSUPPORTED_VERSION"); - - fs.writeFileSync(filePath, "{not-json", { mode: 0o600 }); - await expectCode(() => store.load(SANDBOX), "CORRUPT"); - }); - - it.each([ - [ - "transaction ID", - (record: Record) => { - record.transactionId = "not-a-uuid"; - }, - ], - [ - "revision", - (record: Record) => { - record.revision = -1; - }, - ], - [ - "phase/status", - (record: Record) => { - record.status = "completed"; - }, - ], - [ - "legacy recovery authorization", - (record: Record) => { - const intent = record.intent as Record>; - intent.source!.legacyManagedImageRecoveryAuthorized = "yes"; - }, - ], - [ - "credential environment variable", - (record: Record) => { - const intent = record.intent as Record>; - intent.target!.credentialEnv = "not-an-env-name"; - }, - ], - [ - "timestamp", - (record: Record) => { - record.updatedAt = "2026-07-08"; - }, - ], - [ - "backup timestamp", - (record: Record) => { - const receipts = record.receipts as Record; - (receipts.backup as Record).manifestTimestamp = "2026-07-08T00:00:00Z"; - }, - ], - [ - "old-sandbox deletion receipt", - (record: Record) => { - (record.receipts as Record).oldSandboxDeletion = "bad"; - }, - ], - [ - "replacement receipt", - (record: Record) => { - (record.receipts as Record).replacement = "bad"; - }, - ], - ])("rejects a malformed %s", async (_label, mutate) => { - const { stateDir, store } = makeStore(); - await store.create(intent(), preparedReceipts()); - writeRawRecord(getRebuildTransactionPath(SANDBOX, stateDir), mutate); - await expectCode(() => store.load(SANDBOX), "CORRUPT"); - }); - - it("rejects replacement evidence timestamped before deletion", async () => { - const { stateDir, store } = makeStore(); - await advanceToReplacement(store); - const filePath = getRebuildTransactionPath(SANDBOX, stateDir); - writeRawRecord(filePath, (record) => { - const receipts = record.receipts as Record>; - receipts.replacement!.observedAt = "2026-07-08T00:00:30.000Z"; - }); - - await expectCode(() => store.load(SANDBOX), "CORRUPT"); - }); - it("rejects invalid and traversal-shaped sandbox names before path construction", async () => { const { stateDir, store } = makeStore(); for (const name of ["../escape", "has/slash", "UPPER", "", "a".repeat(64)]) { diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index f82c374f3de..0547bd3e9fd 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -455,6 +455,22 @@ function normalizeRecord(value: unknown, sandboxName: string): RebuildTransactio }; } +function normalizeMutationRecord( + value: unknown, + sandboxName: string, + code: "INVALID_INPUT" | "INVALID_TRANSITION", + detail: string, +): RebuildTransactionRecordV1 { + try { + return normalizeRecord(value, sandboxName); + } catch (error) { + if (error instanceof RebuildTransactionError && error.code === "CORRUPT") { + throw transactionError(code, sandboxName, detail, error); + } + throw error; + } +} + function syncDirectory(dirPath: string): void { // Durability boundary review: // - Invalid state: a published directory entry can be lost after power loss. @@ -616,35 +632,24 @@ export class RebuildTransactionStore { assertSandboxName(intent.sandboxName); return this.withMutationLock(intent.sandboxName, () => { const now = this.now().toISOString(); - let record: RebuildTransactionRecordV1; - try { - record = normalizeRecord( - { - version: REBUILD_TRANSACTION_VERSION, - transactionId: this.transactionId(), - revision: 1, - status: "active", - phase: "prepared", - intent, - receipts, - failure: null, - createdAt: now, - updatedAt: now, - completedAt: null, - }, - intent.sandboxName, - ); - } catch (error) { - if (error instanceof RebuildTransactionError && error.code === "CORRUPT") { - throw transactionError( - "INVALID_INPUT", - intent.sandboxName, - "creation input is invalid", - error, - ); - } - throw error; - } + const record = normalizeMutationRecord( + { + version: REBUILD_TRANSACTION_VERSION, + transactionId: this.transactionId(), + revision: 1, + status: "active", + phase: "prepared", + intent, + receipts, + failure: null, + createdAt: now, + updatedAt: now, + completedAt: null, + }, + intent.sandboxName, + "INVALID_INPUT", + "creation input is invalid", + ); try { durablePublish(this.path(intent.sandboxName), record, true); } catch (error) { @@ -683,7 +688,7 @@ export class RebuildTransactionStore { ); } assertReceiptHistoryUnchanged(current.receipts, receipts, sandboxName); - const updated = normalizeRecord( + const updated = normalizeMutationRecord( { ...current, phase, @@ -693,6 +698,8 @@ export class RebuildTransactionStore { updatedAt: this.now().toISOString(), }, sandboxName, + "INVALID_TRANSITION", + "transition input is invalid", ); durablePublish(this.path(sandboxName), updated, false); return updated; @@ -706,7 +713,7 @@ export class RebuildTransactionStore { ): Promise { return this.withMutationLock(sandboxName, () => { const current = this.requireActive(sandboxName, expectedRevision); - const updated = normalizeRecord( + const updated = normalizeMutationRecord( { ...current, failure, @@ -714,6 +721,8 @@ export class RebuildTransactionStore { updatedAt: this.now().toISOString(), }, sandboxName, + "INVALID_INPUT", + "failure input is invalid", ); durablePublish(this.path(sandboxName), updated, false); return updated; From b6c333b83c5d7ad21337f8ec01a4e67091cd2034 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 13:03:25 -0400 Subject: [PATCH 13/14] fix(rebuild): secure custom state roots --- src/lib/state/rebuild-transaction.test.ts | 16 ++++++++++++++++ src/lib/state/rebuild-transaction.ts | 13 +++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/lib/state/rebuild-transaction.test.ts b/src/lib/state/rebuild-transaction.test.ts index 18cf45afa3b..0e94487e358 100644 --- a/src/lib/state/rebuild-transaction.test.ts +++ b/src/lib/state/rebuild-transaction.test.ts @@ -25,11 +25,27 @@ import { REBUILD_TRANSACTION_DIRNAME, type RebuildTransactionIntentV1, type RebuildTransactionReceiptsV1, + RebuildTransactionStore, } from "./rebuild-transaction"; afterEach(cleanupRebuildTransactionTests); describe("RebuildTransactionStore", () => { + it("secures a custom state root and rejects a symlinked root", () => { + const root = tempDir(); + const stateDir = path.join(root, "state"); + fs.mkdirSync(stateDir, { mode: 0o777 }); + + new RebuildTransactionStore({ stateDir }); + expect(fs.statSync(stateDir).mode & 0o777).toBe(0o700); + + fs.rmSync(stateDir, { recursive: true }); + const attackerDir = path.join(root, "attacker"); + fs.mkdirSync(attackerDir); + fs.symlinkSync(attackerDir, stateDir); + expect(() => new RebuildTransactionStore({ stateDir })).toThrow("untrusted"); + }); + it("round-trips the versioned prepared record with secure paths and permissions", async () => { const { stateDir, store } = makeStore(); diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index 0547bd3e9fd..a2ddc74ade3 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -137,6 +137,18 @@ function transactionFileStem(sandboxName: string): string { return crypto.createHash("sha256").update(sandboxName).digest("hex"); } +function ensureTransactionStateRoot(stateDir: string): void { + try { + const stat = fs.lstatSync(stateDir); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error(`Refusing untrusted rebuild transaction state root: ${stateDir}`); + } + } catch (error) { + if (!(isErrnoException(error) && error.code === "ENOENT")) throw error; + } + ensureConfigDir(stateDir); +} + export function getRebuildTransactionPath( sandboxName: string, stateDir = resolveNemoclawStateDir(), @@ -621,6 +633,7 @@ export class RebuildTransactionStore { constructor(options: RebuildTransactionStoreOptions = {}) { this.stateDir = options.stateDir ?? resolveNemoclawStateDir(); + ensureTransactionStateRoot(this.stateDir); this.now = options.now ?? (() => new Date()); this.transactionId = options.transactionId ?? (() => crypto.randomUUID()); } From 21467fd1bdf600d77d83d9accd08763b6855e820 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 8 Jul 2026 13:13:23 -0400 Subject: [PATCH 14/14] feat(rebuild): record original shields posture --- src/lib/state/rebuild-transaction-validation.test.ts | 7 +++++++ src/lib/state/rebuild-transaction.ts | 5 +++++ test/helpers/rebuild-transaction-store.ts | 1 + 3 files changed, 13 insertions(+) diff --git a/src/lib/state/rebuild-transaction-validation.test.ts b/src/lib/state/rebuild-transaction-validation.test.ts index 88eb5b2231e..c7e52e6fb23 100644 --- a/src/lib/state/rebuild-transaction-validation.test.ts +++ b/src/lib/state/rebuild-transaction-validation.test.ts @@ -55,6 +55,13 @@ describe("RebuildTransactionStore validation", () => { value.source!.legacyManagedImageRecoveryAuthorized = "yes"; }, ], + [ + "source shields posture", + (record: Record) => { + const value = record.intent as Record>; + value.source!.shieldsLocked = "yes"; + }, + ], [ "credential environment variable", (record: Record) => { diff --git a/src/lib/state/rebuild-transaction.ts b/src/lib/state/rebuild-transaction.ts index a2ddc74ade3..72bbb121fc6 100644 --- a/src/lib/state/rebuild-transaction.ts +++ b/src/lib/state/rebuild-transaction.ts @@ -38,6 +38,7 @@ export interface RebuildTransactionIntentV1 { readonly agent: string | null; readonly registryFingerprint: string; readonly legacyManagedImageRecoveryAuthorized: boolean; + readonly shieldsLocked: boolean; }; readonly target: { readonly agent: string | null; @@ -282,6 +283,9 @@ function normalizeIntent(value: unknown, sandboxName: string): RebuildTransactio "intent.source.legacyManagedImageRecoveryAuthorized is invalid", ); } + if (typeof source.shieldsLocked !== "boolean") { + throw transactionError("CORRUPT", sandboxName, "intent.source.shieldsLocked is invalid"); + } return { sandboxName, source: { @@ -292,6 +296,7 @@ function normalizeIntent(value: unknown, sandboxName: string): RebuildTransactio sandboxName, ), legacyManagedImageRecoveryAuthorized: source.legacyManagedImageRecoveryAuthorized, + shieldsLocked: source.shieldsLocked, }, target: { agent: nullableString(target.agent, "intent.target.agent", sandboxName), diff --git a/test/helpers/rebuild-transaction-store.ts b/test/helpers/rebuild-transaction-store.ts index fb632e31b39..119319a1632 100644 --- a/test/helpers/rebuild-transaction-store.ts +++ b/test/helpers/rebuild-transaction-store.ts @@ -40,6 +40,7 @@ export function intent( agent: "openclaw", registryFingerprint: FP_A, legacyManagedImageRecoveryAuthorized: false, + shieldsLocked: false, }, target: { agent: "openclaw",