From 82ac3b0e580c20393c89befca790e6239001c14a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 27 Jul 2026 22:45:47 -0700 Subject: [PATCH 1/5] fix(onboard): journal resumed sandbox recreation Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 68 +++- src/lib/onboard/checkpoint-replay.test.ts | 1 + .../onboard/checkpoint-resume-guard.test.ts | 1 + src/lib/onboard/lifecycle-contracts.md | 18 + .../sandbox-checkpoint-crash-recovery.test.ts | 1 + .../handlers/sandbox-messaging.test.ts | 1 + .../handlers/sandbox-recreate-journal.test.ts | 91 +++++ .../machine/handlers/sandbox-test-fixtures.ts | 2 + .../onboard/machine/handlers/sandbox.test.ts | 5 + src/lib/onboard/machine/handlers/sandbox.ts | 196 ++++++++++- src/lib/onboard/not-ready-recreate.ts | 12 +- src/lib/onboard/sandbox-lifecycle.test.ts | 23 +- src/lib/onboard/sandbox-lifecycle.ts | 5 + .../sandbox-recreate-transaction.test.ts | 316 ++++++++++++++++++ .../onboard/sandbox-recreate-transaction.ts | 275 +++++++++++++++ src/lib/onboard/sandbox-registration.test.ts | 2 + src/lib/onboard/sandbox-registration.ts | 2 + src/lib/onboard/sandbox-reuse.test.ts | 44 ++- src/lib/onboard/sandbox-reuse.ts | 27 +- src/lib/onboard/session-bootstrap.test.ts | 2 + src/lib/onboard/types.ts | 6 + .../state/onboard-checkpoint-migrate.test.ts | 1 + src/lib/state/onboard-checkpoint-migrate.ts | 1 + src/lib/state/onboard-checkpoint-types.ts | 33 +- src/lib/state/onboard-checkpoint.test.ts | 113 ++++++- src/lib/state/onboard-checkpoint.ts | 106 +++++- src/lib/state/registry.ts | 2 + 27 files changed, 1312 insertions(+), 42 deletions(-) create mode 100644 src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts create mode 100644 src/lib/onboard/sandbox-recreate-transaction.test.ts create mode 100644 src/lib/onboard/sandbox-recreate-transaction.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 7b00aa1ad82..dcc28a31124 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -440,6 +440,8 @@ const sandboxAgent: typeof import("./onboard/sandbox-agent") = require("./onboar const sandboxLifecycle: typeof import("./onboard/sandbox-lifecycle") = require("./onboard/sandbox-lifecycle"); const sandboxRegistryMetadata: typeof import("./onboard/sandbox-registry-metadata") = require("./onboard/sandbox-registry-metadata"); const sandboxReuse: typeof import("./onboard/sandbox-reuse") = require("./onboard/sandbox-reuse"); +const sandboxRecreateTransaction: typeof import("./onboard/sandbox-recreate-transaction") = + require("./onboard/sandbox-recreate-transaction"); const sandboxRegistration: typeof import("./onboard/sandbox-registration") = require("./onboard/sandbox-registration"); const { @@ -786,12 +788,13 @@ const { getGatewayReuseSnapshot, selectNamedGatewayForReuseIfNeeded } = cliDisplayName, }); -const { getSandboxReuseState, repairRecordedSandbox } = sandboxReuse.createSandboxReuseHelpers({ - runCaptureOpenshell, - runOpenshell, - getSandboxStateFromOutputs, - note, -}); +const { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox } = + sandboxReuse.createSandboxReuseHelpers({ + runCaptureOpenshell, + runOpenshell, + getSandboxStateFromOutputs, + note, + }); const { executeSandboxCommandForVerification, @@ -2289,6 +2292,45 @@ async function createSandboxWithBaseImageResolution( // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { existingEntry, preservedMcpState, liveExists, effectiveToolDisclosure, toolDisclosureMigrationNeeded, toolDisclosureMigrationNote } = toolDisclosureFlow.prepareSandboxToolDisclosure(sandboxName, preparedBuildContext?.rebuildTarget?.fromDockerfile ? preparedBuildContext.stagedDockerfile : fromDockerfile, isRecreateSandbox(createIntent?.recreate), inspectSandboxForCreate, createIntent?.toolDisclosure ?? null); + const recreateTransaction = createIntent?.recreateTransaction + ? sandboxRecreateTransaction.matchingSandboxRecreateTransaction(onboardSession.loadSession(), { + sandboxName, + gatewayName: GATEWAY_NAME, + targetIntentFingerprint: createIntent.recreateTransaction.targetIntentFingerprint, + transactionId: createIntent.recreateTransaction.id, + targetGeneration: createIntent.recreateTransaction.targetGeneration, + }) + : null; + const persistRecreatePhase = ( + phase: Parameters[2], + ): void => { + if (!recreateTransaction) return; + onboardSession.updateSession((current) => { + sandboxRecreateTransaction.advanceSandboxRecreateTransaction( + current, + recreateTransaction.id, + phase, + ); + return current; + }); + }; + if (recreateTransaction) { + const observation = getSandboxRecreateObservation(sandboxName); + const recovery = sandboxRecreateTransaction.planSandboxRecreateRecovery( + recreateTransaction, + observation, + existingEntry, + ); + if (recovery.action === "reject") { + throw new Error(`Cannot resume sandbox '${sandboxName}' recreation: ${recovery.reason}.`); + } + if (recovery.action === "accept_target") { + note(` [resume] Recovering journaled replacement sandbox '${sandboxName}'.`); + return sandboxName; + } + if (recovery.action === "continue_create") persistRecreatePhase("deleted"); + } + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const observabilityDrift = observabilityPolicy.hasRegisteredDcodeObservabilityDrift(liveExists, isManagedDcodeAgent, existingEntry, createIntent?.observabilityEnabled); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. @@ -2594,8 +2636,18 @@ async function createSandboxWithBaseImageResolution( note(` Deleting and recreating sandbox '${sandboxName}'...`); + persistRecreatePhase("deleting"); runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + if (recreateTransaction) { + const afterDelete = getSandboxRecreateObservation(sandboxName); + if (afterDelete.state !== "missing") { + throw new Error( + `Cannot continue sandbox '${sandboxName}' recreation: OpenShell still reports the journaled source after delete.`, + ); + } + persistRecreatePhase("deleted"); + } if (previousEntry?.imageTag) { const rmiResult = dockerRmi(previousEntry.imageTag, { ignoreError: true, @@ -2726,6 +2778,7 @@ async function createSandboxWithBaseImageResolution( }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; + persistRecreatePhase("creating"); const { createResult, dockerGpuCreatePatch, @@ -2867,11 +2920,13 @@ async function createSandboxWithBaseImageResolution( hermesToolGateways, hermesDashboardState: finalHermesDashboardState, dashboardPort: actualDashboardPort, + lifecycleGeneration: recreateTransaction?.targetGeneration, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, }), }, ); + persistRecreatePhase("created"); restoreDefaultAfterRecreate(registry.setDefault, sandboxName, sandboxWasLiveDefault); // #4614: default deferred to finalization // DNS proxy — run a forwarder in the sandbox pod so the isolated @@ -4415,6 +4470,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { hydrateMessagingChannelConfig, messagingChannelConfigsEqual, getSandboxReuseState, + getSandboxRecreateObservation, getDcodeSelectionDrift: (name, selectedProvider, selectedModel, selectedApi) => getDcodeSelectionDrift(name, selectedProvider, selectedModel, selectedApi, { runCaptureOpenshell, diff --git a/src/lib/onboard/checkpoint-replay.test.ts b/src/lib/onboard/checkpoint-replay.test.ts index 639e9eb293e..e2ce29bcc9f 100644 --- a/src/lib/onboard/checkpoint-replay.test.ts +++ b/src/lib/onboard/checkpoint-replay.test.ts @@ -26,6 +26,7 @@ function checkpoint(overrides: Partial = {}): OnboardCheckpoi gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, ...overrides, }; } diff --git a/src/lib/onboard/checkpoint-resume-guard.test.ts b/src/lib/onboard/checkpoint-resume-guard.test.ts index 1ea17ce3006..85b05e5e9ff 100644 --- a/src/lib/onboard/checkpoint-resume-guard.test.ts +++ b/src/lib/onboard/checkpoint-resume-guard.test.ts @@ -39,6 +39,7 @@ const loadedCheckpoint: OnboardCheckpoint = { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; function makeDeps(overrides: Partial): OnboardSessionBootstrapDeps { diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 3909f6a94ff..9a8d88c7bfd 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -119,6 +119,24 @@ runtime mutation | **Credential rotation** — `configRotateToken` in `src/lib/sandbox/config.ts` | A session with `credentialEnv` selects the provider and binding. A non-null different `sandboxName` is rejected, but a legacy/null session name is accepted for the requested sandbox. The new value comes from a named environment variable, stdin, or a secret prompt; it is trimmed, then rejected when empty or still containing internal whitespace. | `saveCredential` first stages the value in the current process. OpenShell provider update is the first external mutation, with provider create as a fallback; audit follows. No sandbox deletion. | The logical binding is unchanged, so session and registry are not rewritten. The raw value exists only in process memory/environment and the gateway provider; audit records action/sandbox/reason without the value. | No rollback after a successful provider update; an audit failure can report failure after the credential is already active. Covered by the rotate-token cases in `test/config-set-nested-ssrf.test.ts`. Gap: a null-name legacy session is not strongly bound to the requested sandbox. | | **Config, policy, resource, port-forward, and runtime setup contributions** — `configSet`; `prepareInitialSandboxCreatePolicy`; `selectResourceProfileForSandbox`; manifest compiler/runtime appliers; dashboard and channel forward helpers | Config uses validated dotpaths and SSRF-safe URL rewriting. Create/rebuild contributions are assembled by `sandbox-create-plan.ts` and `MessagingWorkflowPlanner`: policy presets/keys, resource flags, package/build steps, `hostForward`, runtime node preloads, env aliases, and secret scans. | Config’s first effect is a compare-and-swap sandbox write. Build-time contributions inherit the enclosing create/recreate boundary. Forward helpers can stop an existing forward and start its replacement in place after readiness, without recreating the sandbox. | Durable owners are compact registry messaging/policy/inference metadata, current manifests used for plan rehydration, onboard session, sandbox config/hash, gateway provider state, and shields audit. An interrupted onboarding session records the selected resource values or an explicit OpenShell-default choice; the resolved create intent remains process-local. Logical bindings are serializable; raw provider values are not. | CAS rejects stale config writes; OpenClaw/Hermes commit config and integrity hashes together, while other agents may refresh a path hash afterward. Audit and optional restart are post-commit and forward-only. Forward recovery can re-establish declared forwards. Gaps: no cross-contribution effect transaction/checkpoint. | +## Durable resumed recreate journal + +A resumed same-name replacement writes a secret-free journal before the lower create path can delete the source sandbox. +The journal binds the session, sandbox, selected gateway, source registry row, source OpenShell ID, target intent, and target generation. + +Recovery accepts only these states: + +- The source row and live ID match, so deletion can continue. +- The source row remains and OpenShell reports no sandbox, so creation can continue. +- The registry row has the target generation and OpenShell reports the sandbox ready, so the replacement can be accepted. + +All other combinations stop before the current run reuses, deletes, or creates a sandbox. +The lower create path stamps the target generation into the replacement row. +The handler clears the journal after it records both create and registration receipts. + +This slice covers resumed onboard replacement, including not-ready repair and non-default gateways. +Rebuild and non-resumed re-onboard remain under #6492. + ## Agent-specific differences | Agent | Lifecycle difference | diff --git a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts index 4bb45fd938c..a830f823e64 100644 --- a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts @@ -55,6 +55,7 @@ function crashedCheckpoint(overrides: Partial = {}): OnboardC }, }, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, ...overrides, }; } diff --git a/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts b/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts index 6056233577c..b336f9d48e8 100644 --- a/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts @@ -176,6 +176,7 @@ function withMessagingCheckpoint( gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; session.checkpoint = checkpoint; return session; diff --git a/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts new file mode 100644 index 00000000000..0a7926a09aa --- /dev/null +++ b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, it, vi } from "vitest"; + +import { decisionSelected } from "../../../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../../../state/onboard-checkpoint-migrate"; +import { createSession, type Session } from "../../../state/onboard-session"; +import { fingerprintSandboxRecreateValue } from "../../sandbox-recreate-transaction"; +import { handleSandboxState } from "./sandbox"; +import { baseOptions, createDeps } from "./sandbox-test-fixtures"; + +it("journals not-ready repair on the selected non-default gateway (#6492)", async () => { + const session = createSession({ sandboxName: "saved", agent: "openclaw" }); + session.steps.sandbox.status = "complete"; + session.checkpoint = { + ...deriveCheckpointFromSession(session), + sandboxIdentity: decisionSelected({ name: "saved", agent: "openclaw" }), + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + }; + const sourceEntry = { + name: "saved", + provider: "provider", + model: "model", + endpointUrl: null, + preferredInferenceApi: "openai-completions", + webSearchEnabled: false, + toolDisclosure: "progressive" as const, + fromDockerfile: null, + hermesAuthMethod: null, + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + }; + const phases: Array = []; + const updateSession = vi.fn((mutator: (value: Session) => Session | void) => { + mutator(session); + phases.push(session.checkpoint?.sandboxRecreate?.phase ?? null); + return session; + }); + const getSandboxRecreateObservation = vi.fn( + () => + ({ + state: "not_ready", + liveIdentityFingerprint: fingerprintSandboxRecreateValue("openshell-source-id"), + }) as const, + ); + const createSandbox = vi.fn(async () => "saved"); + const { deps, calls } = createDeps( + { + getSandboxReuseState: () => "not_ready", + getSandboxRecreateObservation, + getSandboxRegistryEntry: () => sourceEntry, + updateSession, + createSandbox, + }, + session, + ); + + await handleSandboxState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "saved", + gatewayName: "nemoclaw-31818", + }); + + expect(calls.repairSandbox).not.toHaveBeenCalled(); + expect(createSandbox).toHaveBeenCalledOnce(); + const createIntent = createSandbox.mock.calls[0]?.at(-1); + expect(createIntent).toMatchObject({ + recreate: true, + recreateTransaction: { + id: expect.any(String), + targetGeneration: expect.any(String), + targetIntentFingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + }); + expect(getSandboxRecreateObservation).toHaveBeenCalledWith("saved"); + expect(phases).toEqual( + expect.arrayContaining(["planned", "registry_committing", "completed", null]), + ); + expect(session.checkpoint?.sandboxRecreate).toBeNull(); +}); diff --git a/src/lib/onboard/machine/handlers/sandbox-test-fixtures.ts b/src/lib/onboard/machine/handlers/sandbox-test-fixtures.ts index 0ebf9e5874e..7aadcc84e5e 100644 --- a/src/lib/onboard/machine/handlers/sandbox-test-fixtures.ts +++ b/src/lib/onboard/machine/handlers/sandbox-test-fixtures.ts @@ -203,6 +203,8 @@ export function createDeps( hydrateMessagingChannelConfig: (config: MessagingChannelConfig | null) => config, messagingChannelConfigsEqual: () => true, getSandboxReuseState: () => "missing", + getSandboxRecreateObservation: () => + ({ state: "missing", liveIdentityFingerprint: null }) as const, getDcodeSelectionDrift: () => ({ changed: false, unknown: false }), hasSandboxGpuDrift: () => false, getSandboxHermesToolGateways: () => [], diff --git a/src/lib/onboard/machine/handlers/sandbox.test.ts b/src/lib/onboard/machine/handlers/sandbox.test.ts index be94acf1a23..b9e3b45944a 100644 --- a/src/lib/onboard/machine/handlers/sandbox.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox.test.ts @@ -161,6 +161,7 @@ describe("handleSandboxState", () => { { name: "my-assistant-brave-search", type: "brave", credentialEnv: "BRAVE_API_KEY" }, ], }, + sandboxRecreate: null, }; const updateSession = vi.fn((mutator: (value: typeof session) => void) => { mutator(session); @@ -608,6 +609,7 @@ describe("handleSandboxState", () => { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; const { deps, calls } = createDeps({ getSandboxReuseState: () => "ready", @@ -649,6 +651,7 @@ describe("handleSandboxState", () => { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; const { deps, calls } = createDeps({ getSandboxReuseState: () => "missing" }); @@ -683,6 +686,7 @@ describe("handleSandboxState", () => { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; const updateSession = vi.fn((mutator: (value: typeof session) => void) => { mutator(session); @@ -725,6 +729,7 @@ describe("handleSandboxState", () => { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; const updateSession = vi.fn((mutator: (value: typeof session) => void) => { mutator(session); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 94350f596bf..016e89a2b68 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -28,6 +28,7 @@ import type { CheckpointProviderBinding, CheckpointResourceProfile, CheckpointSandboxIdentity, + CheckpointSandboxRecreateTransaction, OnboardCheckpoint, } from "../../../state/onboard-checkpoint-types"; import type { @@ -78,10 +79,18 @@ import { isDcodeAgent, } from "../../observability-policy-presets"; import type { SandboxCreateIntent as ResolvedSandboxCreateIntent } from "../../sandbox-create-intent-types"; +import { + advanceSandboxRecreateTransaction, + beginSandboxRecreateTransaction, + clearCompletedSandboxRecreateTransaction, + fingerprintSandboxRecreateValue, + selectedGatewayForSandboxRecreate, +} from "../../sandbox-recreate-transaction"; import { assertBaselineExclusionsMatchCreateIntent, baselineExclusionsForCreate, } from "../../sandbox-registration"; + import { withSandboxPhaseTrace } from "../../tracing"; import type { SandboxCreateIntent } from "../../types"; import { branchTo, type OnboardStateTransitionResult } from "../result"; @@ -179,6 +188,9 @@ export interface SandboxStateOptions< right: MessagingChannelConfig | null, ): boolean; getSandboxReuseState(sandboxName: string | null): string; + getSandboxRecreateObservation?( + sandboxName: string | null, + ): import("../../sandbox-recreate-transaction").SandboxRecreateObservation; hasSandboxGpuDrift(sandboxName: string, config: SandboxGpuConfig): boolean; getSandboxHermesToolGateways(sandboxName: string): unknown; getSandboxRegistryEntry(sandboxName: string): SandboxEntry | null; @@ -370,6 +382,17 @@ type CompleteSandboxCreateIntent = SandboxCreateIntent & { readonly resolved: ResolvedSandboxCreateIntent; }; +type SandboxRecreateRepairMetadata = { + readonly repair: "recorded-sandbox-cleanup"; + readonly sandboxName: string | null; +}; +type SandboxRecreatePreparation = { + readonly transaction: CheckpointSandboxRecreateTransaction | null; + readonly effectiveCreateIntent: CompleteSandboxCreateIntent; + readonly repairMetadata: SandboxRecreateRepairMetadata | null; + readonly removalReceipt: SandboxRemovalReceipt | null; +}; + function observabilityRequestValidationError( issue: ManagedSandboxFeatureIssue | null, ): string | null { @@ -1176,6 +1199,153 @@ class SandboxStateFlow< }; } + private beginSandboxRecreateJournal( + state: SandboxStepState, + sandboxName: string, + createIntent: CompleteSandboxCreateIntent, + ): CheckpointSandboxRecreateTransaction | null { + const existing = state.session?.checkpoint?.sandboxRecreate ?? null; + if (!this.options.resume && !existing) return null; + const gateway = selectedGatewayForSandboxRecreate( + state.session?.checkpoint, + this.options.gatewayName, + ); + if (!gateway) return null; + const sourceEntry = this.deps.getSandboxRegistryEntry(sandboxName); + if (!existing && !sourceEntry) return null; + if (!this.deps.getSandboxRecreateObservation) { + if (existing) throw new Error("Sandbox recreate observation dependency is unavailable."); + return null; + } + const observation = this.deps.getSandboxRecreateObservation(sandboxName); + const targetIntentFingerprint = fingerprintSandboxRecreateValue( + this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), + ); + let transaction: CheckpointSandboxRecreateTransaction | null = null; + this.deps.updateSession((current) => { + transaction = beginSandboxRecreateTransaction(current, { + sandboxName, + gatewayName: gateway.gatewayName, + gatewayPort: gateway.gatewayPort, + sourceEntry, + observation, + targetIntentFingerprint, + }); + return current; + }); + return transaction; + } + + private recordSandboxRecreatePhase( + transaction: CheckpointSandboxRecreateTransaction, + phase: Parameters[2], + ): void { + this.deps.updateSession((current) => { + advanceSandboxRecreateTransaction(current, transaction.id, phase); + return current; + }); + } + + private clearSandboxRecreateJournal(transaction: CheckpointSandboxRecreateTransaction): Session { + return this.deps.updateSession((current) => { + clearCompletedSandboxRecreateTransaction(current, transaction.id); + return current; + }); + } + + private async prepareSandboxRecreate( + state: SandboxStepState, + requestedSandboxName: string, + createIntent: CompleteSandboxCreateIntent, + decision: SandboxCreationDecision, + ): Promise { + const transaction = this.beginSandboxRecreateJournal(state, requestedSandboxName, createIntent); + const repairMetadata: SandboxRecreateRepairMetadata | null = + decision.kind === "repair-and-recreate" + ? { repair: "recorded-sandbox-cleanup", sandboxName: state.sandboxName } + : null; + if (!transaction) { + return { + transaction, + effectiveCreateIntent: createIntent, + repairMetadata, + removalReceipt: await applySandboxResumeDecision(decision, state.sandboxName, this.deps), + }; + } + const effectiveCreateIntent: CompleteSandboxCreateIntent = { + ...createIntent, + recreate: true, + recreateTransaction: { + id: transaction.id, + targetGeneration: transaction.targetGeneration, + targetIntentFingerprint: transaction.targetIntentFingerprint, + }, + }; + if (repairMetadata) { + this.deps.note( + ` [resume] Recorded sandbox '${state.sandboxName}' exists but is not ready; recreating it.`, + ); + await this.deps.recordRepairEvent("state.repair.started", { + state: "sandbox", + metadata: repairMetadata, + }); + } else if (decision.kind === "recreate") { + this.deps.note(decision.note); + } + return { transaction, effectiveCreateIntent, repairMetadata, removalReceipt: null }; + } + + private async recordSandboxRecreateRepairFailure( + transaction: CheckpointSandboxRecreateTransaction | null, + repairMetadata: SandboxRecreateRepairMetadata | null, + error: unknown, + ): Promise { + if (!repairMetadata || !transaction) return; + await this.deps.recordRepairEvent("state.repair.failed", { + state: "sandbox", + error: error instanceof Error ? error.message : String(error), + metadata: repairMetadata, + }); + } + + private async recordSandboxRecreateRepairSuccess( + transaction: CheckpointSandboxRecreateTransaction | null, + repairMetadata: SandboxRecreateRepairMetadata | null, + ): Promise { + if (!repairMetadata || !transaction) return; + await this.deps.recordRepairEvent("state.repair.completed", { + state: "sandbox", + metadata: repairMetadata, + }); + } + + private recordSandboxRecreateRegistryCommit( + transaction: CheckpointSandboxRecreateTransaction | null, + ): void { + if (!transaction || transaction.phase === "completed") return; + this.recordSandboxRecreatePhase(transaction, "registry_committing"); + } + + private recordSandboxCreateEffects( + transaction: CheckpointSandboxRecreateTransaction | null, + sandboxName: string, + createIntent: CompleteSandboxCreateIntent, + ): Session { + const recordedSession = this.deps.updateSession((current) => { + recordCheckpointEffectGroup( + current, + "sandbox_create", + this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), + ); + recordCheckpointEffectGroup(current, "sandbox_register", sandboxName); + if (transaction) { + advanceSandboxRecreateTransaction(current, transaction.id, "completed"); + } + return current; + }); + return transaction ? this.clearSandboxRecreateJournal(transaction) : recordedSession; + } + private async createAndRecordSandbox( initialState: SandboxStepState, requestedSandboxName: string, @@ -1227,11 +1397,8 @@ class SandboxStateFlow< requestedSandboxName, createIntent.resolved.policy.options.baselineExclusions, ); - const removalReceipt = await applySandboxResumeDecision( - decision, - state.sandboxName, - this.deps, - ); + const { transaction, effectiveCreateIntent, repairMetadata, removalReceipt } = + await this.prepareSandboxRecreate(state, requestedSandboxName, createIntent, decision); let rollbackArmed = removalReceipt !== null; const restoreRemovedRegistryEntry = () => { if (!rollbackArmed || !removalReceipt) return; @@ -1269,7 +1436,7 @@ class SandboxStateFlow< resourceProfile, effectiveHermesToolGateways, this.options.hermesAuthMethod, - createIntent, + effectiveCreateIntent, ), ); // createSandbox returns only after the replacement row is registered. @@ -1279,8 +1446,11 @@ class SandboxStateFlow< } catch (error) { restoreRemovedRegistryEntry(); process.removeListener("exit", restoreRemovedRegistryEntry); + await this.recordSandboxRecreateRepairFailure(transaction, repairMetadata, error); throw error; } + await this.recordSandboxRecreateRepairSuccess(transaction, repairMetadata); + this.recordSandboxRecreateRegistryCommit(transaction); // createSandbox() owns the build fingerprint. In particular, reusing an // image must not stamp it with the current version and hide build drift. const { nemoclawVersion: _builtFingerprint, ...agentRegistryFields } = @@ -1310,15 +1480,11 @@ class SandboxStateFlow< hermesToolGateways: effectiveHermesToolGateways, }), ); - const recordedSession = this.deps.updateSession((current) => { - recordCheckpointEffectGroup( - current, - "sandbox_create", - this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), - ); - recordCheckpointEffectGroup(current, "sandbox_register", sandboxName); - return current; - }); + const recordedSession = this.recordSandboxCreateEffects( + transaction, + sandboxName, + createIntent, + ); return { ...state, sandboxName, session: recordedSession }; }; const withGatewayLock = () => diff --git a/src/lib/onboard/not-ready-recreate.ts b/src/lib/onboard/not-ready-recreate.ts index 580c9464986..dc4018b394c 100644 --- a/src/lib/onboard/not-ready-recreate.ts +++ b/src/lib/onboard/not-ready-recreate.ts @@ -112,15 +112,15 @@ export function selectPreUpgradeBackupForCreate(input: PreUpgradeBackupSelectInp // invalid state = registry/gateway inconsistency (a registry entry // exists while the gateway still reports the sandbox // live, or the registry has no entry at all). - // source boundary = pruneStaleSandboxEntry is best-effort and the - // gateway may be mid-recreate, so the two stores can - // disagree at this point. - // source-fix constraint = a real fix needs atomic registry/gateway sync, - // which is out of scope for this PR. + // source boundary = this selector can run before the resumed recreate + // journal starts or from a non-journaled caller. + // source-fix constraint = the resumed recreate journal owns later deletion, + // but this selector has no bound journal observation. // regression test = selectPreUpgradeBackupForCreate returns null when // liveExists=true and when hasExistingRegistryEntry=false // (see not-ready-recreate.test.ts). - // removal condition = drop these guards once registry/gateway sync is atomic. + // removal condition = drop these guards when every caller supplies a + // journal-bound registry and OpenShell observation. if (input.liveExists) { console.debug( ` Registry entry exists for '${input.sandboxName}' but gateway reports sandbox live — skipping pre-upgrade backup select.`, diff --git a/src/lib/onboard/sandbox-lifecycle.test.ts b/src/lib/onboard/sandbox-lifecycle.test.ts index 4fcae628cff..448f695b224 100644 --- a/src/lib/onboard/sandbox-lifecycle.test.ts +++ b/src/lib/onboard/sandbox-lifecycle.test.ts @@ -9,7 +9,10 @@ const registryState = vi.hoisted(() => ({ removeSandbox: vi.fn(), sandbox: null as SandboxEntry | null, })); -const onboardSessionState = vi.hoisted(() => ({ sessionId: "session-owner" as string | null })); +const onboardSessionState = vi.hoisted(() => ({ + sessionId: "session-owner" as string | null, + recreate: null as { sandboxName: string; phase: string } | null, +})); vi.mock("../state/registry", async (importOriginal) => { const actual = await importOriginal(); @@ -21,9 +24,15 @@ vi.mock("../state/registry", async (importOriginal) => { }); vi.mock("../state/onboard-session", () => ({ loadSession: () => - onboardSessionState.sessionId === null ? null : { sessionId: onboardSessionState.sessionId }, + onboardSessionState.sessionId === null + ? null + : { + sessionId: onboardSessionState.sessionId, + checkpoint: onboardSessionState.recreate + ? { sandboxRecreate: onboardSessionState.recreate } + : null, + }, })); - import { createSandboxLifecycleHelpers, removeSandboxUnlessSessionReservation, @@ -33,6 +42,7 @@ describe("sandbox recreate reservation ownership", () => { beforeEach(() => { registryState.removeSandbox.mockReset(); onboardSessionState.sessionId = "session-owner"; + onboardSessionState.recreate = null; }); it("preserves a pending reservation owned by the active session (#6562)", () => { @@ -48,6 +58,13 @@ describe("sandbox recreate reservation ownership", () => { expect(registryState.removeSandbox).not.toHaveBeenCalled(); }); + it("preserves the source registry row while a recreate journal is active (#6492)", () => { + onboardSessionState.recreate = { sandboxName: "alpha", phase: "deleting" }; + removeSandboxUnlessSessionReservation({ name: "alpha", agent: "openclaw" }, "alpha"); + + expect(registryState.removeSandbox).not.toHaveBeenCalled(); + }); + it.each([ { label: "foreign-session", diff --git a/src/lib/onboard/sandbox-lifecycle.ts b/src/lib/onboard/sandbox-lifecycle.ts index 08e9732ddb2..985c0748658 100644 --- a/src/lib/onboard/sandbox-lifecycle.ts +++ b/src/lib/onboard/sandbox-lifecycle.ts @@ -10,6 +10,11 @@ export function removeSandboxUnlessSessionReservation( entry: SandboxEntry | null, sandboxName: string, ): void { + const recreate = onboardSession.loadSession()?.checkpoint?.sandboxRecreate; + if (recreate?.sandboxName === sandboxName && recreate.phase !== "completed") { + return; + } + if (!registry.isPendingReservationForSession(entry, onboardSession.loadSession()?.sessionId)) { registry.removeSandbox(sandboxName); } diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts new file mode 100644 index 00000000000..849b28b0dbd --- /dev/null +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { decisionSelected } from "../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; +import type { + CheckpointSandboxRecreatePhase, + CheckpointSandboxRecreateTransaction, +} from "../state/onboard-checkpoint-types"; +import { createSession } from "../state/onboard-session"; +import type { SandboxEntry } from "../state/registry"; +import { + advanceSandboxRecreateTransaction, + beginSandboxRecreateTransaction, + clearCompletedSandboxRecreateTransaction, + fingerprintSandboxLiveIdentity, + fingerprintSandboxRecreateValue, + fingerprintSandboxRegistryEntry, + matchingSandboxRecreateTransaction, + planSandboxRecreateRecovery, + selectedGatewayForSandboxRecreate, + type SandboxRecreateObservation, +} from "./sandbox-recreate-transaction"; + +const ISO = "2026-07-27T20:00:00.000Z"; +const TX_ID = "11111111-1111-4111-8111-111111111111"; +const TARGET_GENERATION = "22222222-2222-4222-8222-222222222222"; +const SOURCE_ID = fingerprintSandboxRecreateValue("openshell-source-id"); +const TARGET_INTENT = fingerprintSandboxRecreateValue({ + agent: "openclaw", + provider: "nvidia", +}); +const SOURCE_ENTRY: SandboxEntry = { + name: "alpha", + agent: "openclaw", + provider: "nvidia", + model: "model-a", + credentialEnv: "NVIDIA_API_KEY", + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, +}; + +function beginInput(observation: SandboxRecreateObservation) { + return { + sandboxName: "alpha", + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + sourceEntry: SOURCE_ENTRY, + observation, + targetIntentFingerprint: TARGET_INTENT, + now: ISO, + id: TX_ID, + targetGeneration: TARGET_GENERATION, + } as const; +} + +function transactionAt( + phase: CheckpointSandboxRecreatePhase, +): CheckpointSandboxRecreateTransaction { + return { + version: 1, + id: TX_ID, + revision: 3, + sandboxName: "alpha", + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + sourceRegistryFingerprint: fingerprintSandboxRegistryEntry(SOURCE_ENTRY), + sourceLiveIdentityFingerprint: SOURCE_ID, + targetIntentFingerprint: TARGET_INTENT, + targetGeneration: TARGET_GENERATION, + phase, + startedAt: ISO, + updatedAt: ISO, + }; +} + +describe("sandbox recreate journal", () => { + it("binds a secret-free transaction to a non-default gateway before deletion (#6492)", () => { + const session = createSession({ sandboxName: "alpha", agent: "openclaw" }); + const transaction = beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + + expect(transaction).toMatchObject({ + id: TX_ID, + sandboxName: "alpha", + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + targetGeneration: TARGET_GENERATION, + phase: "planned", + }); + expect(session.checkpoint?.sandboxRecreate).toBe(transaction); + const serialized = JSON.stringify(transaction); + expect(serialized).not.toContain("NVIDIA_API_KEY"); + expect(serialized).not.toContain("model-a"); + }); + + it("starts at deleted when the source is already absent", () => { + const session = createSession({ sandboxName: "alpha" }); + + expect( + beginSandboxRecreateTransaction( + session, + beginInput({ state: "missing", liveIdentityFingerprint: null }), + ).phase, + ).toBe("deleted"); + }); + + it("fails closed when a live source has no stable OpenShell identity", () => { + const session = createSession({ sandboxName: "alpha" }); + + expect(() => + beginSandboxRecreateTransaction( + session, + beginInput({ state: "not_ready", liveIdentityFingerprint: null }), + ), + ).toThrow(/did not report a stable sandbox Id/i); + }); + + it("reuses only the same durable target intent", () => { + const session = createSession({ sandboxName: "alpha" }); + const first = beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + + expect( + beginSandboxRecreateTransaction( + session, + beginInput({ state: "missing", liveIdentityFingerprint: null }), + ), + ).toBe(first); + expect(() => + beginSandboxRecreateTransaction(session, { + ...beginInput({ state: "missing", liveIdentityFingerprint: null }), + targetIntentFingerprint: "f".repeat(64), + }), + ).toThrow(/different recreate transaction in progress/i); + }); + + it("advances monotonically and clears only after completion", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + + expect(advanceSandboxRecreateTransaction(session, TX_ID, "deleting", ISO)).toMatchObject({ + phase: "deleting", + revision: 1, + }); + expect(() => advanceSandboxRecreateTransaction(session, TX_ID, "planned", ISO)).toThrow( + /cannot move backward/i, + ); + expect(() => clearCompletedSandboxRecreateTransaction(session, TX_ID)).toThrow(/not complete/i); + advanceSandboxRecreateTransaction(session, TX_ID, "completed", ISO); + clearCompletedSandboxRecreateTransaction(session, TX_ID); + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + + it("requires the exact journal handoff at the lower create boundary", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + + expect( + matchingSandboxRecreateTransaction(session, { + sandboxName: "alpha", + gatewayName: "nemoclaw-31818", + targetIntentFingerprint: TARGET_INTENT, + transactionId: TX_ID, + targetGeneration: TARGET_GENERATION, + }), + ).toEqual(session.checkpoint?.sandboxRecreate); + expect(() => + matchingSandboxRecreateTransaction(session, { + sandboxName: "alpha", + gatewayName: "nemoclaw", + targetIntentFingerprint: TARGET_INTENT, + transactionId: TX_ID, + targetGeneration: TARGET_GENERATION, + }), + ).toThrow(/does not match the requested replacement/i); + }); + + it("selects only the checkpoint-authorized non-default gateway", () => { + const session = createSession({ sandboxName: "alpha", agent: "openclaw" }); + session.checkpoint = { + ...deriveCheckpointFromSession(session), + sandboxIdentity: decisionSelected({ name: "alpha", agent: "openclaw" }), + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + }; + + expect(selectedGatewayForSandboxRecreate(session.checkpoint, "nemoclaw-31818")).toEqual({ + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + }); + expect(selectedGatewayForSandboxRecreate(session.checkpoint, "nemoclaw")).toBeNull(); + }); +}); + +describe("sandbox recreate recovery", () => { + it.each([ + "planned", + "deleting", + ] as const)("continues source deletion from %s when both identities still match", (phase) => { + expect( + planSandboxRecreateRecovery( + transactionAt(phase), + { state: "ready", liveIdentityFingerprint: SOURCE_ID }, + SOURCE_ENTRY, + ), + ).toEqual({ action: "continue_delete" }); + }); + + it.each([ + "planned", + "deleting", + "deleted", + "creating", + ] as const)("continues target creation from %s when the source is durably absent", (phase) => { + expect( + planSandboxRecreateRecovery( + transactionAt(phase), + { state: "missing", liveIdentityFingerprint: null }, + SOURCE_ENTRY, + ), + ).toEqual({ action: "continue_create" }); + }); + + it.each([ + "planned", + "deleting", + "deleted", + "creating", + "created", + "registry_committing", + "completed", + ] as const)("accepts the ready target from %s when its generation matches", (phase) => { + expect( + planSandboxRecreateRecovery( + transactionAt(phase), + { state: "ready", liveIdentityFingerprint: fingerprintSandboxRecreateValue("target-id") }, + { ...SOURCE_ENTRY, lifecycleGeneration: TARGET_GENERATION }, + ), + ).toEqual({ action: "accept_target" }); + }); + + it("rejects a changed source registry row before delete", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("planned"), + { state: "ready", liveIdentityFingerprint: SOURCE_ID }, + { ...SOURCE_ENTRY, model: "changed-out-of-band" }, + ), + ).toMatchObject({ + action: "reject", + reason: expect.stringMatching(/source registry row changed/), + }); + }); + + it("rejects a same-name live sandbox with a different source identity", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("deleting"), + { state: "ready", liveIdentityFingerprint: fingerprintSandboxRecreateValue("other-id") }, + SOURCE_ENTRY, + ), + ).toMatchObject({ action: "reject", reason: expect.stringMatching(/source identity/) }); + }); + + it("rejects a live sandbox that appears before target registration", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("creating"), + { state: "not_ready", liveIdentityFingerprint: fingerprintSandboxRecreateValue("new-id") }, + SOURCE_ENTRY, + ), + ).toMatchObject({ action: "reject", reason: expect.stringMatching(/appeared/) }); + }); + + it("rejects a registered target that is not ready", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("registry_committing"), + { state: "not_ready", liveIdentityFingerprint: fingerprintSandboxRecreateValue("target") }, + { ...SOURCE_ENTRY, lifecycleGeneration: TARGET_GENERATION }, + ), + ).toMatchObject({ action: "reject", reason: expect.stringMatching(/not ready/) }); + }); +}); + +describe("OpenShell live identity", () => { + it("hashes an ANSI-decorated Id without persisting the raw identifier", () => { + const output = "Name: alpha\n\u001b[32mId: openshell-source-id\u001b[0m\nState: Ready\n"; + expect(fingerprintSandboxLiveIdentity(output)).toBe(SOURCE_ID); + }); + + it("returns null when OpenShell omits the Id", () => { + expect(fingerprintSandboxLiveIdentity("Name: alpha\nState: Ready\n")).toBeNull(); + }); +}); diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts new file mode 100644 index 00000000000..ac580b420c9 --- /dev/null +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -0,0 +1,275 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomUUID } from "node:crypto"; + +import { isDecisionSelected } from "../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; +import type { + CheckpointSandboxRecreatePhase, + CheckpointSandboxRecreateTransaction, + OnboardCheckpoint, +} from "../state/onboard-checkpoint-types"; +import type { Session } from "../state/onboard-session"; +import type { SandboxEntry } from "../state/registry"; + +const ORDERED_PHASES: readonly CheckpointSandboxRecreatePhase[] = [ + "planned", + "deleting", + "deleted", + "creating", + "created", + "registry_committing", + "completed", +]; + +function canonicalJsonValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalJsonValue); + if (!value || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonicalJsonValue(entry)]), + ); +} + +export function fingerprintSandboxRecreateValue(value: unknown): string { + const serialized = typeof value === "string" ? value : JSON.stringify(canonicalJsonValue(value)); + return createHash("sha256").update(serialized).digest("hex"); +} + +export function fingerprintSandboxRegistryEntry(entry: SandboxEntry): string { + return fingerprintSandboxRecreateValue(entry); +} + +export function fingerprintSandboxLiveIdentity(getOutput: string): string | null { + const clean = String(getOutput).replace(/\x1b\[[0-9;]*m/g, ""); + const match = clean.match(/^\s*Id:\s+(\S+)\s*$/im); + if (!match?.[1] || match[1].length > 512) return null; + return fingerprintSandboxRecreateValue(match[1]); +} + +export interface SandboxRecreateObservation { + readonly state: "missing" | "not_ready" | "ready"; + readonly liveIdentityFingerprint: string | null; +} + +function baseCheckpoint(session: Session): OnboardCheckpoint { + return session.checkpoint ?? deriveCheckpointFromSession(session); +} + +function activeTransaction(session: Session): CheckpointSandboxRecreateTransaction | null { + return baseCheckpoint(session).sandboxRecreate; +} + +function assertSameTransaction( + transaction: CheckpointSandboxRecreateTransaction, + input: BeginSandboxRecreateTransactionInput, +): void { + if ( + transaction.sandboxName !== input.sandboxName || + transaction.gatewayName !== input.gatewayName || + transaction.gatewayPort !== input.gatewayPort || + transaction.targetIntentFingerprint !== input.targetIntentFingerprint + ) { + throw new Error( + `Sandbox '${input.sandboxName}' has a different recreate transaction in progress; resume or repair that transaction before changing its target.`, + ); + } +} + +export interface BeginSandboxRecreateTransactionInput { + readonly sandboxName: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly sourceEntry: SandboxEntry | null; + readonly observation: SandboxRecreateObservation; + readonly targetIntentFingerprint: string; + readonly now?: string; + readonly id?: string; + readonly targetGeneration?: string; +} + +export function beginSandboxRecreateTransaction( + session: Session, + input: BeginSandboxRecreateTransactionInput, +): CheckpointSandboxRecreateTransaction { + const existing = activeTransaction(session); + if (existing) { + assertSameTransaction(existing, input); + return existing; + } + if (input.observation.state !== "missing" && !input.observation.liveIdentityFingerprint) { + throw new Error( + `Cannot recreate sandbox '${input.sandboxName}': OpenShell did not report a stable sandbox Id.`, + ); + } + const checkpoint = baseCheckpoint(session); + const now = input.now ?? new Date().toISOString(); + if (!input.sourceEntry) { + throw new Error( + `Cannot start sandbox '${input.sandboxName}' recreate transaction without its source registry row.`, + ); + } + const transaction: CheckpointSandboxRecreateTransaction = { + version: 1, + id: input.id ?? randomUUID(), + revision: 0, + sandboxName: input.sandboxName, + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + sourceRegistryFingerprint: fingerprintSandboxRegistryEntry(input.sourceEntry), + sourceLiveIdentityFingerprint: input.observation.liveIdentityFingerprint, + targetIntentFingerprint: input.targetIntentFingerprint, + targetGeneration: input.targetGeneration ?? randomUUID(), + phase: input.observation.state === "missing" ? "deleted" : "planned", + startedAt: now, + updatedAt: now, + }; + session.checkpoint = { + ...checkpoint, + machineState: session.machine.state, + updatedAt: now, + sandboxRecreate: transaction, + }; + return transaction; +} + +function phaseIndex(phase: CheckpointSandboxRecreatePhase): number { + return ORDERED_PHASES.indexOf(phase); +} + +export function advanceSandboxRecreateTransaction( + session: Session, + id: string, + phase: CheckpointSandboxRecreatePhase, + now = new Date().toISOString(), +): CheckpointSandboxRecreateTransaction { + const checkpoint = baseCheckpoint(session); + const current = checkpoint.sandboxRecreate; + if (!current || current.id !== id) { + throw new Error( + "Sandbox recreate transaction ownership changed while applying a lifecycle phase.", + ); + } + if (current.phase === phase) return current; + if (phaseIndex(phase) < phaseIndex(current.phase)) { + throw new Error( + `Sandbox recreate transaction cannot move backward from '${current.phase}' to '${phase}'.`, + ); + } + const next: CheckpointSandboxRecreateTransaction = { + ...current, + revision: current.revision + 1, + phase, + updatedAt: now, + }; + session.checkpoint = { + ...checkpoint, + machineState: session.machine.state, + updatedAt: now, + sandboxRecreate: next, + }; + return next; +} + +export function clearCompletedSandboxRecreateTransaction(session: Session, id: string): void { + const checkpoint = baseCheckpoint(session); + const current = checkpoint.sandboxRecreate; + if (!current || current.id !== id || current.phase !== "completed") { + throw new Error("Sandbox recreate transaction is not complete and cannot be cleared."); + } + const now = new Date().toISOString(); + session.checkpoint = { + ...checkpoint, + machineState: session.machine.state, + updatedAt: now, + sandboxRecreate: null, + }; +} + +export type SandboxRecreateRecoveryPlan = + | { readonly action: "continue_delete" } + | { readonly action: "continue_create" } + | { readonly action: "accept_target" } + | { readonly action: "reject"; readonly reason: string }; + +function reject(reason: string): SandboxRecreateRecoveryPlan { + return { action: "reject", reason }; +} + +export function planSandboxRecreateRecovery( + transaction: CheckpointSandboxRecreateTransaction, + observation: SandboxRecreateObservation, + registryEntry: SandboxEntry | null, +): SandboxRecreateRecoveryPlan { + const targetRegistered = registryEntry?.lifecycleGeneration === transaction.targetGeneration; + if (targetRegistered) { + return observation.state === "ready" + ? { action: "accept_target" } + : reject("the journaled replacement is registered but is not ready"); + } + + const sourceRegistered = + registryEntry !== null && + fingerprintSandboxRegistryEntry(registryEntry) === transaction.sourceRegistryFingerprint; + if (transaction.phase === "completed") { + return reject("the completed transaction no longer matches its replacement registry row"); + } + if (transaction.phase === "planned" || transaction.phase === "deleting") { + if (!sourceRegistered) return reject("the source registry row changed before deletion"); + if (observation.state === "missing") return { action: "continue_create" }; + if ( + !transaction.sourceLiveIdentityFingerprint || + observation.liveIdentityFingerprint !== transaction.sourceLiveIdentityFingerprint + ) { + return reject("the live same-name sandbox no longer has the journaled source identity"); + } + return { action: "continue_delete" }; + } + if (transaction.phase === "deleted" || transaction.phase === "creating") { + if (!sourceRegistered) return reject("the preserved source registry row changed"); + return observation.state === "missing" + ? { action: "continue_create" } + : reject("a live same-name sandbox appeared before replacement registration committed"); + } + return reject("the replacement registration did not commit the journaled generation"); +} + +export function selectedGatewayForSandboxRecreate( + checkpoint: OnboardCheckpoint | null | undefined, + gatewayName: string, +): { gatewayName: string; gatewayPort: number } | null { + if (!checkpoint || !isDecisionSelected(checkpoint.gatewayAuthority)) return null; + const authority = checkpoint.gatewayAuthority.value; + return authority.gatewayName === gatewayName + ? { gatewayName: authority.gatewayName, gatewayPort: authority.gatewayPort } + : null; +} + +export function matchingSandboxRecreateTransaction( + session: Session | null, + input: { + sandboxName: string; + gatewayName: string; + targetIntentFingerprint: string; + transactionId: string; + targetGeneration: string; + }, +): CheckpointSandboxRecreateTransaction { + const transaction = session?.checkpoint?.sandboxRecreate; + if ( + !transaction || + transaction.id !== input.transactionId || + transaction.sandboxName !== input.sandboxName || + transaction.gatewayName !== input.gatewayName || + transaction.targetIntentFingerprint !== input.targetIntentFingerprint || + transaction.targetGeneration !== input.targetGeneration + ) { + throw new Error( + `Sandbox '${input.sandboxName}' recreate journal does not match the requested replacement.`, + ); + } + return transaction; +} diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index 49a8da9ef76..c492f43617b 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -123,6 +123,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { config: { enabled: true, port: 18790, internalPort: 19123, tuiEnabled: true }, }, dashboardPort: 18789, + lifecycleGeneration: "22222222-2222-4222-8222-222222222222", gatewayName: "nemoclaw-19080", gatewayPort: 19080, }); @@ -150,6 +151,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { hermesDashboardInternalPort: 19123, hermesDashboardTui: true, dashboardPort: 18789, + lifecycleGeneration: "22222222-2222-4222-8222-222222222222", gatewayName: "nemoclaw-19080", gatewayPort: 19080, gpuEnabled: true, diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index 2bfe110fba7..649f1a9a429 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -64,6 +64,7 @@ export interface CreatedSandboxRegistryEntryInput { hermesDashboardState: HermesDashboardOnboardState; dashboardPort: number; dashboardRemoteBindPrepared?: boolean; + lifecycleGeneration?: string; gatewayName: string; gatewayPort: number; } @@ -196,6 +197,7 @@ export function buildCreatedSandboxRegistryEntry( ...getHermesDashboardRegistryFields(input.hermesDashboardState), dashboardPort: input.dashboardPort, dashboardRemoteBindPrepared: input.dashboardRemoteBindPrepared === true, + lifecycleGeneration: input.lifecycleGeneration, gatewayName: input.gatewayName, gatewayPort: input.gatewayPort, }; diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index 3b391de8a6c..b9f6f5fdc6a 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -3,7 +3,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; -import { applyReusedSandboxDashboardState } from "./sandbox-reuse"; +import { applyReusedSandboxDashboardState, createSandboxReuseHelpers } from "./sandbox-reuse"; +import { fingerprintSandboxRecreateValue } from "./sandbox-recreate-transaction"; describe("applyReusedSandboxDashboardState", () => { afterEach(() => { @@ -194,3 +195,44 @@ describe("applyReusedSandboxDashboardState", () => { }); }); }); + +describe("createSandboxReuseHelpers", () => { + it("observes state and a stable OpenShell identity together for recreate recovery", () => { + const runCaptureOpenshell = vi.fn((args: string[]) => + args[1] === "get" + ? "Name: alpha\n\u001b[32mId: openshell-source-id\u001b[0m\nState: Ready\n" + : "alpha Ready\n", + ); + const getSandboxStateFromOutputs = vi.fn(() => "ready"); + const helpers = createSandboxReuseHelpers({ + runCaptureOpenshell, + runOpenshell: vi.fn(), + getSandboxStateFromOutputs, + note: vi.fn(), + }); + + expect(helpers.getSandboxRecreateObservation("alpha")).toEqual({ + state: "ready", + liveIdentityFingerprint: fingerprintSandboxRecreateValue("openshell-source-id"), + }); + expect(runCaptureOpenshell).toHaveBeenNthCalledWith(1, ["sandbox", "get", "alpha"], { + ignoreError: true, + }); + expect(getSandboxStateFromOutputs).toHaveBeenCalledWith( + "alpha", + expect.stringContaining("Id: openshell-source-id"), + "alpha Ready\n", + ); + }); + + it("rejects an OpenShell state outside the recovery model", () => { + const helpers = createSandboxReuseHelpers({ + runCaptureOpenshell: vi.fn(() => ""), + runOpenshell: vi.fn(), + getSandboxStateFromOutputs: vi.fn(() => "unknown"), + note: vi.fn(), + }); + + expect(() => helpers.getSandboxRecreateObservation("alpha")).toThrow(/state 'unknown'/); + }); +}); diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index fc261ec5a59..11e8df6b747 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -11,6 +11,10 @@ import { type HermesDashboardOnboardState, } from "./hermes-dashboard"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; +import { + fingerprintSandboxLiveIdentity, + type SandboxRecreateObservation, +} from "./sandbox-recreate-transaction"; export interface SandboxReuseDeps { runCaptureOpenshell(args: string[], opts?: Record): string; @@ -21,6 +25,7 @@ export interface SandboxReuseDeps { export interface SandboxReuseHelpers { getSandboxReuseState(sandboxName: string | null): string; + getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation; repairRecordedSandbox(sandboxName: string | null): void; } @@ -107,13 +112,27 @@ export function applyReusedSandboxDashboardState( } export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseHelpers { - function getSandboxReuseState(sandboxName: string | null): string { - if (!sandboxName) return "missing"; + function getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation { + if (!sandboxName) return { state: "missing", liveIdentityFingerprint: null }; const getOutput = deps.runCaptureOpenshell(["sandbox", "get", sandboxName], { ignoreError: true, }); const listOutput = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); - return deps.getSandboxStateFromOutputs(sandboxName, getOutput, listOutput); + const state = deps.getSandboxStateFromOutputs(sandboxName, getOutput, listOutput); + if (state !== "missing" && state !== "not_ready" && state !== "ready") { + throw new Error( + `Cannot observe sandbox '${sandboxName}' for recreate recovery: OpenShell returned state '${state}'.`, + ); + } + return { + state, + liveIdentityFingerprint: + state === "missing" ? null : fingerprintSandboxLiveIdentity(getOutput), + }; + } + + function getSandboxReuseState(sandboxName: string | null): string { + return getSandboxRecreateObservation(sandboxName).state; } function repairRecordedSandbox(sandboxName: string | null): void { @@ -124,5 +143,5 @@ export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseH registry.removeSandbox(sandboxName); } - return { getSandboxReuseState, repairRecordedSandbox }; + return { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox }; } diff --git a/src/lib/onboard/session-bootstrap.test.ts b/src/lib/onboard/session-bootstrap.test.ts index 9f8cc37fb0b..d07fb352b8b 100644 --- a/src/lib/onboard/session-bootstrap.test.ts +++ b/src/lib/onboard/session-bootstrap.test.ts @@ -385,6 +385,7 @@ describe("prepareOnboardSession", () => { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; session.checkpoint = checkpoint; const { deps } = createDeps(session, { @@ -436,6 +437,7 @@ describe("prepareOnboardSession", () => { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; const { deps } = createDeps(session); diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index 51dee7be488..b60f238fff3 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -72,6 +72,12 @@ export interface SandboxCreateIntent { readonly extraProviders?: readonly string[]; /** Internal OpenClaw resume authority for exact registered provider reuse. */ readonly reuseRegisteredCredentials?: true; + /** Internal durable handoff for one journaled same-name replacement. */ + readonly recreateTransaction?: { + readonly id: string; + readonly targetGeneration: string; + readonly targetIntentFingerprint: string; + }; } export type OnboardOptions = { diff --git a/src/lib/state/onboard-checkpoint-migrate.test.ts b/src/lib/state/onboard-checkpoint-migrate.test.ts index 9a8e3fc18ae..3587583b54a 100644 --- a/src/lib/state/onboard-checkpoint-migrate.test.ts +++ b/src/lib/state/onboard-checkpoint-migrate.test.ts @@ -90,6 +90,7 @@ describe("resolveCheckpointForResume", () => { gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, }; it("returns loaded when the embedded checkpoint is valid", () => { diff --git a/src/lib/state/onboard-checkpoint-migrate.ts b/src/lib/state/onboard-checkpoint-migrate.ts index 99774350ca0..b405fd977f4 100644 --- a/src/lib/state/onboard-checkpoint-migrate.ts +++ b/src/lib/state/onboard-checkpoint-migrate.ts @@ -86,6 +86,7 @@ export function deriveCheckpointFromSession(session: Session): OnboardCheckpoint credentialEnvs: [], registeredProviders: [], }, + sandboxRecreate: null, }; } diff --git a/src/lib/state/onboard-checkpoint-types.ts b/src/lib/state/onboard-checkpoint-types.ts index 837bec3c6e0..9c3e96027e1 100644 --- a/src/lib/state/onboard-checkpoint-types.ts +++ b/src/lib/state/onboard-checkpoint-types.ts @@ -4,7 +4,7 @@ import type { WebSearchConfig } from "../inference/web-search"; import type { OnboardMachineState } from "../onboard/machine/types"; -export const CHECKPOINT_SCHEMA_VERSION = 2 as const; +export const CHECKPOINT_SCHEMA_VERSION = 3 as const; export type CheckpointSchemaVersion = typeof CHECKPOINT_SCHEMA_VERSION; @@ -68,6 +68,36 @@ export interface CheckpointBindings { readonly registeredProviders: readonly CheckpointProviderBinding[]; } +export type CheckpointSandboxRecreatePhase = + | "planned" + | "deleting" + | "deleted" + | "creating" + | "created" + | "registry_committing" + | "completed"; + +/** + * Secret-free journal for one same-name sandbox replacement. The containing + * checkpoint supplies the session identity; the generation stamped into the + * replacement registry row proves which same-name sandbox this run created. + */ +export interface CheckpointSandboxRecreateTransaction { + readonly version: 1; + readonly id: string; + readonly revision: number; + readonly sandboxName: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly sourceRegistryFingerprint: string; + readonly sourceLiveIdentityFingerprint: string | null; + readonly targetIntentFingerprint: string; + readonly targetGeneration: string; + readonly phase: CheckpointSandboxRecreatePhase; + readonly startedAt: string; + readonly updatedAt: string; +} + export interface OnboardCheckpoint { readonly schemaVersion: CheckpointSchemaVersion; readonly sessionId: string; @@ -82,6 +112,7 @@ export interface OnboardCheckpoint { Partial> >; readonly bindings: CheckpointBindings; + readonly sandboxRecreate: CheckpointSandboxRecreateTransaction | null; } export type CheckpointLoadResult = diff --git a/src/lib/state/onboard-checkpoint.test.ts b/src/lib/state/onboard-checkpoint.test.ts index 1ddbdbaa889..7391f1bf769 100644 --- a/src/lib/state/onboard-checkpoint.test.ts +++ b/src/lib/state/onboard-checkpoint.test.ts @@ -14,7 +14,11 @@ import { isDecisionSelected, isDecisionUnset, } from "./onboard-checkpoint-decision"; -import { CHECKPOINT_SCHEMA_VERSION, type OnboardCheckpoint } from "./onboard-checkpoint-types"; +import { + CHECKPOINT_SCHEMA_VERSION, + type CheckpointSandboxRecreateTransaction, + type OnboardCheckpoint, +} from "./onboard-checkpoint-types"; const ISO = "2026-01-01T00:00:00.000Z"; @@ -36,10 +40,47 @@ function baseCheckpoint(overrides: Partial = {}): OnboardChec { name: "web-search-p", type: "brave", credentialEnv: "BRAVE_API_KEY" }, ], }, + sandboxRecreate: null, ...overrides, }; } +function recreateTransaction(): CheckpointSandboxRecreateTransaction { + return { + version: 1, + id: "11111111-1111-4111-8111-111111111111", + revision: 2, + sandboxName: "my-sandbox", + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + sourceRegistryFingerprint: "a".repeat(64), + sourceLiveIdentityFingerprint: "b".repeat(64), + targetIntentFingerprint: "c".repeat(64), + targetGeneration: "22222222-2222-4222-8222-222222222222", + phase: "creating", + startedAt: ISO, + updatedAt: ISO, + }; +} + +function serializedRecreateCheckpoint(): Record { + return serializeCheckpoint( + baseCheckpoint({ + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + sandboxRecreate: recreateTransaction(), + }), + ); +} + describe("checkpoint decision tri-state", () => { it("distinguishes unset, declined, and selected", () => { expect(isDecisionUnset(decisionUnset())).toBe(true); @@ -93,12 +134,23 @@ describe("checkpoint schema inspection", () => { }); }); - it("loads and round-trips a valid v2 checkpoint", () => { + it("loads and round-trips a valid current checkpoint", () => { const checkpoint = baseCheckpoint(); const result = inspectCheckpoint(serializeCheckpoint(checkpoint)); expect(result).toEqual({ status: "loaded", checkpoint }); }); + it("migrates a valid v2 checkpoint with no recreate journal", () => { + const serialized = serializeCheckpoint(baseCheckpoint()); + serialized.schemaVersion = 2; + delete serialized.sandboxRecreate; + + const result = inspectCheckpoint(serialized); + + expect(result).toMatchObject({ status: "migrated", fromVersion: 2 }); + expect(result.status === "migrated" && result.checkpoint.sandboxRecreate).toBeNull(); + }); + it("migrates a valid v1 checkpoint with an unset gateway authority", () => { const serialized = serializeCheckpoint(baseCheckpoint()); serialized.schemaVersion = 1; @@ -136,6 +188,63 @@ describe("checkpoint schema inspection", () => { }); }); + it("round-trips a journal bound to the selected sandbox and non-default gateway", () => { + const checkpoint = baseCheckpoint({ + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + sandboxRecreate: recreateTransaction(), + }); + + expect(inspectCheckpoint(serializeCheckpoint(checkpoint))).toEqual({ + status: "loaded", + checkpoint, + }); + }); + + it.each([ + { + label: "sandbox", + mutate: (serialized: Record) => { + serialized.sandboxIdentity = decisionSelected({ name: "other", agent: "openclaw" }); + }, + }, + { + label: "gateway", + mutate: (serialized: Record) => { + serialized.gatewayAuthority = decisionSelected({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); + }, + }, + ])("rejects a recreate journal copied under a different $label binding", ({ mutate }) => { + const serialized = serializedRecreateCheckpoint(); + mutate(serialized); + + expect(inspectCheckpoint(serialized)).toEqual({ status: "corrupt" }); + }); + + it("rejects malformed recreate journal fingerprints", () => { + const serialized = serializedRecreateCheckpoint(); + (serialized.sandboxRecreate as Record).targetIntentFingerprint = "bad"; + + expect(inspectCheckpoint(serialized)).toEqual({ status: "corrupt" }); + }); + it("rejects a checkpoint whose external authority targets a different port", () => { const serialized = serializeCheckpoint( baseCheckpoint({ diff --git a/src/lib/state/onboard-checkpoint.ts b/src/lib/state/onboard-checkpoint.ts index eb87c63b5d6..3df63cdfdf1 100644 --- a/src/lib/state/onboard-checkpoint.ts +++ b/src/lib/state/onboard-checkpoint.ts @@ -8,7 +8,7 @@ import { DEFAULT_GATEWAY_PORT } from "../core/ports"; import { normalizeWebSearchConfig, type WebSearchConfig } from "../inference/web-search"; import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../name-validation"; import { isOnboardMachineState } from "../onboard/machine/transitions"; -import { parseCheckpointDecision } from "./onboard-checkpoint-decision"; +import { isDecisionSelected, parseCheckpointDecision } from "./onboard-checkpoint-decision"; import { CHECKPOINT_SCHEMA_VERSION, type CheckpointBindings, @@ -22,6 +22,8 @@ import { type CheckpointProviderBinding, type CheckpointResourceProfile, type CheckpointSandboxIdentity, + type CheckpointSandboxRecreatePhase, + type CheckpointSandboxRecreateTransaction, type OnboardCheckpoint, } from "./onboard-checkpoint-types"; @@ -31,6 +33,17 @@ const EFFECT_GROUP_NAMES: readonly CheckpointEffectGroupName[] = [ "sandbox_create", "sandbox_register", ]; +const SANDBOX_RECREATE_PHASES = new Set([ + "planned", + "deleting", + "deleted", + "creating", + "created", + "registry_committing", + "completed", +]); +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const SHA256_PATTERN = /^[a-f0-9]{64}$/; function readString(value: unknown): string | null { return typeof value === "string" ? value : null; @@ -232,6 +245,71 @@ function parseBindings(value: unknown): CheckpointBindings | null { if (credentialEnvs === null || registeredProviders === null) return null; return { credentialEnvs, registeredProviders }; } +function parseSandboxRecreateTransaction( + value: unknown, +): CheckpointSandboxRecreateTransaction | null { + if (!isObjectRecord(value)) return null; + const id = readString(value.id); + const sandboxName = readString(value.sandboxName); + const gatewayName = readString(value.gatewayName); + const gatewayPort = value.gatewayPort; + const sourceRegistryFingerprint = readString(value.sourceRegistryFingerprint); + const sourceLiveIdentityFingerprint = + value.sourceLiveIdentityFingerprint === null + ? null + : readString(value.sourceLiveIdentityFingerprint); + const targetIntentFingerprint = readString(value.targetIntentFingerprint); + const targetGeneration = readString(value.targetGeneration); + const phase = value.phase; + const startedAt = readCanonicalIsoTimestamp(value.startedAt); + const updatedAt = readCanonicalIsoTimestamp(value.updatedAt); + const revision = value.revision; + if ( + value.version !== 1 || + !id || + !UUID_PATTERN.test(id) || + !sandboxName || + sandboxName.length > NAME_MAX_LENGTH || + !NAME_VALID_PATTERN.test(sandboxName) || + !gatewayName || + !Number.isInteger(gatewayPort) || + Number(gatewayPort) < 1 || + Number(gatewayPort) > 65535 || + gatewayName !== + (gatewayPort === DEFAULT_GATEWAY_PORT ? "nemoclaw" : `nemoclaw-${String(gatewayPort)}`) || + !sourceRegistryFingerprint || + !SHA256_PATTERN.test(sourceRegistryFingerprint) || + (sourceLiveIdentityFingerprint !== null && + (!sourceLiveIdentityFingerprint || !SHA256_PATTERN.test(sourceLiveIdentityFingerprint))) || + !targetIntentFingerprint || + !SHA256_PATTERN.test(targetIntentFingerprint) || + !targetGeneration || + !UUID_PATTERN.test(targetGeneration) || + typeof phase !== "string" || + !SANDBOX_RECREATE_PHASES.has(phase as CheckpointSandboxRecreatePhase) || + !Number.isSafeInteger(revision) || + Number(revision) < 0 || + startedAt === null || + updatedAt === null + ) { + return null; + } + return { + version: 1, + id, + revision: Number(revision), + sandboxName, + gatewayName, + gatewayPort: Number(gatewayPort), + sourceRegistryFingerprint, + sourceLiveIdentityFingerprint, + targetIntentFingerprint, + targetGeneration, + phase: phase as CheckpointSandboxRecreatePhase, + startedAt, + updatedAt, + }; +} function requireDecision( raw: unknown, @@ -243,6 +321,7 @@ function requireDecision( function parseSchema( value: Record, gatewayAuthorityRaw: unknown, + sandboxRecreateRaw: unknown, ): OnboardCheckpoint | null { const sessionId = readString(value.sessionId); const machineState = value.machineState; @@ -257,10 +336,23 @@ function parseSchema( const gatewayAuthority = requireDecision(gatewayAuthorityRaw, parseGatewayAuthorityValue); const effectGroups = parseEffectGroups(value.effectGroups); const bindings = parseBindings(value.bindings); + const sandboxRecreate = + sandboxRecreateRaw === null ? null : parseSandboxRecreateTransaction(sandboxRecreateRaw); if (!sandboxIdentity || !webSearch || !messaging || !resourceProfile || !gatewayAuthority) { return null; } - if (!effectGroups || !bindings) return null; + if (!effectGroups || !bindings || (sandboxRecreateRaw !== null && !sandboxRecreate)) return null; + if (sandboxRecreate) { + if ( + !isDecisionSelected(sandboxIdentity) || + sandboxIdentity.value.name !== sandboxRecreate.sandboxName || + !isDecisionSelected(gatewayAuthority) || + gatewayAuthority.value.gatewayName !== sandboxRecreate.gatewayName || + gatewayAuthority.value.gatewayPort !== sandboxRecreate.gatewayPort + ) { + return null; + } + } return { schemaVersion: CHECKPOINT_SCHEMA_VERSION, @@ -274,6 +366,7 @@ function parseSchema( gatewayAuthority, effectGroups, bindings, + sandboxRecreate, }; } @@ -289,11 +382,15 @@ export function inspectCheckpoint(raw: unknown): CheckpointLoadResult { return { status: "unsupported_future", foundVersion: version }; } if (version === CHECKPOINT_SCHEMA_VERSION) { - const checkpoint = parseSchema(raw, raw.gatewayAuthority); + const checkpoint = parseSchema(raw, raw.gatewayAuthority, raw.sandboxRecreate); return checkpoint ? { status: "loaded", checkpoint } : { status: "corrupt" }; } + if (version === 2) { + const checkpoint = parseSchema(raw, raw.gatewayAuthority, null); + return checkpoint ? { status: "migrated", checkpoint, fromVersion: 2 } : { status: "corrupt" }; + } if (version === 1) { - const checkpoint = parseSchema(raw, { kind: "unset" }); + const checkpoint = parseSchema(raw, { kind: "unset" }, null); return checkpoint ? { status: "migrated", checkpoint, fromVersion: 1 } : { status: "corrupt" }; } return { status: "corrupt" }; @@ -312,5 +409,6 @@ export function serializeCheckpoint(checkpoint: OnboardCheckpoint): Record { // Persisted so later lifecycle commands operate on the sandbox's own gateway // instead of the process-global `nemoclaw` singleton — a second sandbox on a // different NEMOCLAW_GATEWAY_PORT no longer recreates/kills the first (#4422). + /** Generation proving which durable same-name recreate registered this row. */ + lifecycleGeneration?: string; gatewayName?: string | null; gatewayPort?: number | null; } From 852158774de9eff1233a31bbc865bc599b560ab6 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 27 Jul 2026 23:18:46 -0700 Subject: [PATCH 2/5] refactor(onboard): extract recreate runtime Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 88 ++++--------------- .../sandbox-recreate-transaction.test.ts | 48 +++++++++- .../onboard/sandbox-recreate-transaction.ts | 67 ++++++++++++++ 3 files changed, 130 insertions(+), 73 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index dcc28a31124..8592439ec1b 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -788,13 +788,8 @@ const { getGatewayReuseSnapshot, selectNamedGatewayForReuseIfNeeded } = cliDisplayName, }); -const { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox } = - sandboxReuse.createSandboxReuseHelpers({ - runCaptureOpenshell, - runOpenshell, - getSandboxStateFromOutputs, - note, - }); +// biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. +const { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox } = sandboxReuse.createSandboxReuseHelpers({ runCaptureOpenshell, runOpenshell, getSandboxStateFromOutputs, note }); const { executeSandboxCommandForVerification, @@ -2292,45 +2287,9 @@ async function createSandboxWithBaseImageResolution( // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { existingEntry, preservedMcpState, liveExists, effectiveToolDisclosure, toolDisclosureMigrationNeeded, toolDisclosureMigrationNote } = toolDisclosureFlow.prepareSandboxToolDisclosure(sandboxName, preparedBuildContext?.rebuildTarget?.fromDockerfile ? preparedBuildContext.stagedDockerfile : fromDockerfile, isRecreateSandbox(createIntent?.recreate), inspectSandboxForCreate, createIntent?.toolDisclosure ?? null); - const recreateTransaction = createIntent?.recreateTransaction - ? sandboxRecreateTransaction.matchingSandboxRecreateTransaction(onboardSession.loadSession(), { - sandboxName, - gatewayName: GATEWAY_NAME, - targetIntentFingerprint: createIntent.recreateTransaction.targetIntentFingerprint, - transactionId: createIntent.recreateTransaction.id, - targetGeneration: createIntent.recreateTransaction.targetGeneration, - }) - : null; - const persistRecreatePhase = ( - phase: Parameters[2], - ): void => { - if (!recreateTransaction) return; - onboardSession.updateSession((current) => { - sandboxRecreateTransaction.advanceSandboxRecreateTransaction( - current, - recreateTransaction.id, - phase, - ); - return current; - }); - }; - if (recreateTransaction) { - const observation = getSandboxRecreateObservation(sandboxName); - const recovery = sandboxRecreateTransaction.planSandboxRecreateRecovery( - recreateTransaction, - observation, - existingEntry, - ); - if (recovery.action === "reject") { - throw new Error(`Cannot resume sandbox '${sandboxName}' recreation: ${recovery.reason}.`); - } - if (recovery.action === "accept_target") { - note(` [resume] Recovering journaled replacement sandbox '${sandboxName}'.`); - return sandboxName; - } - if (recovery.action === "continue_create") persistRecreatePhase("deleted"); - } - + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const recreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); + if (recreateRuntime.acceptedTarget) return sandboxName; // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const observabilityDrift = observabilityPolicy.hasRegisteredDcodeObservabilityDrift(liveExists, isManagedDcodeAgent, existingEntry, createIntent?.observabilityEnabled); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. @@ -2611,18 +2570,13 @@ async function createSandboxWithBaseImageResolution( } const previousEntry: SandboxEntry | null = registry.getSandbox(sandboxName); - baseImageResolutionFlow.captureBaseResolution( - baseImageResolutionContext, - previousEntry?.imageTag, - ); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + baseImageResolutionFlow.captureBaseResolution(baseImageResolutionContext, previousEntry?.imageTag); policyPresetCarry.applyRecreatePolicyCarryForward(sandboxName, isNonInteractive(), note); const noRestorePending = pendingStateRestore === null && pendingStateRestoreBackupPath === null; - if ( - noRestorePending && - !notReadyRecreateInProgress && - !shouldSkipPreRecreateBackup(process.env) - ) { + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (noRestorePending && !notReadyRecreateInProgress && !shouldSkipPreRecreateBackup(process.env)) { note(" Backing up workspace state before recreating sandbox..."); const result = recreateProtection.backup(); if (!result.ok) { @@ -2636,23 +2590,13 @@ async function createSandboxWithBaseImageResolution( note(` Deleting and recreating sandbox '${sandboxName}'...`); - persistRecreatePhase("deleting"); + recreateRuntime.advance("deleting"); runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); - if (recreateTransaction) { - const afterDelete = getSandboxRecreateObservation(sandboxName); - if (afterDelete.state !== "missing") { - throw new Error( - `Cannot continue sandbox '${sandboxName}' recreation: OpenShell still reports the journaled source after delete.`, - ); - } - persistRecreatePhase("deleted"); - } + recreateRuntime.confirmDeleted(); if (previousEntry?.imageTag) { - const rmiResult = dockerRmi(previousEntry.imageTag, { - ignoreError: true, - suppressOutput: true, - }); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const rmiResult = dockerRmi(previousEntry.imageTag, { ignoreError: true, suppressOutput: true }); if (rmiResult.status !== 0) { console.warn(` Warning: failed to remove old sandbox image '${previousEntry.imageTag}'.`); } @@ -2778,7 +2722,7 @@ async function createSandboxWithBaseImageResolution( }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; - persistRecreatePhase("creating"); + recreateRuntime.advance("creating"); const { createResult, dockerGpuCreatePatch, @@ -2920,13 +2864,13 @@ async function createSandboxWithBaseImageResolution( hermesToolGateways, hermesDashboardState: finalHermesDashboardState, dashboardPort: actualDashboardPort, - lifecycleGeneration: recreateTransaction?.targetGeneration, + lifecycleGeneration: recreateRuntime.targetGeneration, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, }), }, ); - persistRecreatePhase("created"); + recreateRuntime.advance("created"); restoreDefaultAfterRecreate(registry.setDefault, sandboxName, sandboxWasLiveDefault); // #4614: default deferred to finalization // DNS proxy — run a forwarder in the sandbox pod so the isolated diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 849b28b0dbd..3524584553f 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -15,13 +15,14 @@ import { advanceSandboxRecreateTransaction, beginSandboxRecreateTransaction, clearCompletedSandboxRecreateTransaction, + createSandboxRecreateRuntime, fingerprintSandboxLiveIdentity, fingerprintSandboxRecreateValue, fingerprintSandboxRegistryEntry, matchingSandboxRecreateTransaction, planSandboxRecreateRecovery, - selectedGatewayForSandboxRecreate, type SandboxRecreateObservation, + selectedGatewayForSandboxRecreate, } from "./sandbox-recreate-transaction"; const ISO = "2026-07-27T20:00:00.000Z"; @@ -188,6 +189,51 @@ describe("sandbox recreate journal", () => { ).toThrow(/does not match the requested replacement/i); }); + it("persists deletion and creation phases through the lower runtime boundary", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + let observation: SandboxRecreateObservation = { + state: "ready", + liveIdentityFingerprint: SOURCE_ID, + }; + const runtime = createSandboxRecreateRuntime( + { + loadSession: () => session, + updateSession: (mutator) => { + mutator(session); + return session; + }, + }, + { + id: TX_ID, + targetGeneration: TARGET_GENERATION, + targetIntentFingerprint: TARGET_INTENT, + }, + "alpha", + "nemoclaw-31818", + SOURCE_ENTRY, + () => observation, + () => undefined, + ); + + runtime.advance("deleting"); + observation = { state: "missing", liveIdentityFingerprint: null }; + runtime.confirmDeleted(); + runtime.advance("creating"); + + expect(runtime).toMatchObject({ + acceptedTarget: false, + targetGeneration: TARGET_GENERATION, + }); + expect(session.checkpoint?.sandboxRecreate).toMatchObject({ + phase: "creating", + revision: 3, + }); + }); + it("selects only the checkpoint-authorized non-default gateway", () => { const session = createSession({ sandboxName: "alpha", agent: "openclaw" }); session.checkpoint = { diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index ac580b420c9..d0f57de4124 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -12,6 +12,7 @@ import type { } from "../state/onboard-checkpoint-types"; import type { Session } from "../state/onboard-session"; import type { SandboxEntry } from "../state/registry"; +import type { SandboxCreateIntent } from "./types"; const ORDERED_PHASES: readonly CheckpointSandboxRecreatePhase[] = [ "planned", @@ -273,3 +274,69 @@ export function matchingSandboxRecreateTransaction( } return transaction; } + +interface SandboxRecreateSessionStore { + loadSession(): Session | null; + updateSession(mutator: (session: Session) => Session | void): Session; +} + +export interface SandboxRecreateRuntime { + readonly acceptedTarget: boolean; + readonly targetGeneration: string | undefined; + advance(phase: CheckpointSandboxRecreatePhase): void; + confirmDeleted(): void; +} + +const NO_SANDBOX_RECREATE: SandboxRecreateRuntime = { + acceptedTarget: false, + targetGeneration: undefined, + advance: () => undefined, + confirmDeleted: () => undefined, +}; + +export function createSandboxRecreateRuntime( + sessionStore: SandboxRecreateSessionStore, + request: SandboxCreateIntent["recreateTransaction"] | undefined, + sandboxName: string, + gatewayName: string, + registryEntry: SandboxEntry | null, + observe: (sandboxName: string) => SandboxRecreateObservation, + note: (message: string) => void, +): SandboxRecreateRuntime { + if (!request) return NO_SANDBOX_RECREATE; + const transaction = matchingSandboxRecreateTransaction(sessionStore.loadSession(), { + sandboxName, + gatewayName, + targetIntentFingerprint: request.targetIntentFingerprint, + transactionId: request.id, + targetGeneration: request.targetGeneration, + }); + const advance = (phase: CheckpointSandboxRecreatePhase): void => { + sessionStore.updateSession((current) => { + advanceSandboxRecreateTransaction(current, transaction.id, phase); + return current; + }); + }; + const recovery = planSandboxRecreateRecovery(transaction, observe(sandboxName), registryEntry); + if (recovery.action === "reject") { + throw new Error(`Cannot resume sandbox '${sandboxName}' recreation: ${recovery.reason}.`); + } + if (recovery.action === "accept_target") { + note(` [resume] Recovering journaled replacement sandbox '${sandboxName}'.`); + } else if (recovery.action === "continue_create") { + advance("deleted"); + } + return { + acceptedTarget: recovery.action === "accept_target", + targetGeneration: transaction.targetGeneration, + advance, + confirmDeleted: () => { + if (observe(sandboxName).state !== "missing") { + throw new Error( + `Cannot continue sandbox '${sandboxName}' recreation: OpenShell still reports the journaled source after delete.`, + ); + } + advance("deleted"); + }, + }; +} From 86254098f61d2be025008243a6cfbbbec91ea55c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 27 Jul 2026 23:52:29 -0700 Subject: [PATCH 3/5] fix(onboard): bind recreate target identity Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 61 ++++----- src/lib/onboard/lifecycle-contracts.md | 6 +- .../onboard/machine/core-flow-phases.test.ts | 1 + .../handlers/sandbox-recreate-journal.test.ts | 5 +- src/lib/onboard/machine/handlers/sandbox.ts | 16 +-- .../sandbox-recreate-transaction.test.ts | 123 +++++++++++++++++- .../onboard/sandbox-recreate-transaction.ts | 103 ++++++++++++++- src/lib/onboard/sandbox-registration.test.ts | 2 + src/lib/onboard/sandbox-registration.ts | 2 + src/lib/onboard/sandbox-reuse.test.ts | 5 +- src/lib/onboard/sandbox-reuse.ts | 11 +- src/lib/state/onboard-checkpoint-types.ts | 1 + src/lib/state/onboard-checkpoint.test.ts | 11 ++ src/lib/state/onboard-checkpoint.ts | 27 ++-- src/lib/state/registry.ts | 6 +- 15 files changed, 299 insertions(+), 81 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 8592439ec1b..e812a455c43 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2289,7 +2289,28 @@ async function createSandboxWithBaseImageResolution( const { existingEntry, preservedMcpState, liveExists, effectiveToolDisclosure, toolDisclosureMigrationNeeded, toolDisclosureMigrationNote } = toolDisclosureFlow.prepareSandboxToolDisclosure(sandboxName, preparedBuildContext?.rebuildTarget?.fromDockerfile ? preparedBuildContext.stagedDockerfile : fromDockerfile, isRecreateSandbox(createIntent?.recreate), inspectSandboxForCreate, createIntent?.toolDisclosure ?? null); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const recreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); - if (recreateRuntime.acceptedTarget) return sandboxName; + const restoreReusedSandboxDashboard = (selectionVerified: boolean): void => { + ({ chatUiUrl } = sandboxReuse.applyReusedSandboxDashboardState({ + sandboxName, + chatUiUrl, + env: process.env, + agent, + model, + provider, + selectionVerified, + sandboxGpuConfig: effectiveSandboxGpuConfig, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + manageDashboard, + ensureDashboardForward, + hermesDashboardForwarding, + updateReusedSandboxMetadata, + })); + }; + if (recreateRuntime.acceptedTarget) { + restoreReusedSandboxDashboard(true); + return sandboxName; + } // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const observabilityDrift = observabilityPolicy.hasRegisteredDcodeObservabilityDrift(liveExists, isManagedDcodeAgent, existingEntry, createIntent?.observabilityEnabled); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. @@ -2436,22 +2457,7 @@ async function createSandboxWithBaseImageResolution( " Pass --recreate-sandbox or set NEMOCLAW_RECREATE_SANDBOX=1 to force recreation.", ); } - ({ chatUiUrl } = sandboxReuse.applyReusedSandboxDashboardState({ - sandboxName, - chatUiUrl, - env: process.env, - agent, - model, - provider, - selectionVerified: !selectionDrift.unknown, - sandboxGpuConfig: effectiveSandboxGpuConfig, - gatewayName: GATEWAY_NAME, - gatewayPort: GATEWAY_PORT, - manageDashboard, - ensureDashboardForward, - hermesDashboardForwarding, - updateReusedSandboxMetadata, - })); + restoreReusedSandboxDashboard(!selectionDrift.unknown); return sandboxName; } } else { @@ -2481,22 +2487,7 @@ async function createSandboxWithBaseImageResolution( if (await promptYesNoOrDefault(" Reuse existing sandbox?", null, true)) { policyPresetCarry.seedReusedSandboxPolicyPresets(sandboxName, isNonInteractive()); upsertMessagingProviders(messagingTokenDefs); - ({ chatUiUrl } = sandboxReuse.applyReusedSandboxDashboardState({ - sandboxName, - chatUiUrl, - env: process.env, - agent, - model, - provider, - selectionVerified: !selectionDrift.unknown, - sandboxGpuConfig: effectiveSandboxGpuConfig, - gatewayName: GATEWAY_NAME, - gatewayPort: GATEWAY_PORT, - manageDashboard, - ensureDashboardForward, - hermesDashboardForwarding, - updateReusedSandboxMetadata, - })); + restoreReusedSandboxDashboard(!selectionDrift.unknown); return sandboxName; } } @@ -2818,6 +2809,7 @@ async function createSandboxWithBaseImageResolution( buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`) ?? resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); + recreateRuntime.recordCreated(); finalizeCreatedSandbox( { sandboxName, @@ -2864,13 +2856,12 @@ async function createSandboxWithBaseImageResolution( hermesToolGateways, hermesDashboardState: finalHermesDashboardState, dashboardPort: actualDashboardPort, - lifecycleGeneration: recreateRuntime.targetGeneration, + ...recreateRuntime.registrationFields, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, }), }, ); - recreateRuntime.advance("created"); restoreDefaultAfterRecreate(registry.setDefault, sandboxName, sandboxWasLiveDefault); // #4614: default deferred to finalization // DNS proxy — run a forwarder in the sandbox pod so the isolated diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 9a8d88c7bfd..e47dcc8fa07 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -122,16 +122,16 @@ runtime mutation ## Durable resumed recreate journal A resumed same-name replacement writes a secret-free journal before the lower create path can delete the source sandbox. -The journal binds the session, sandbox, selected gateway, source registry row, source OpenShell ID, target intent, and target generation. +The journal binds the session, sandbox, selected gateway, source registry row, source OpenShell ID, target intent, target generation, and the replacement OpenShell ID after creation. Recovery accepts only these states: - The source row and live ID match, so deletion can continue. - The source row remains and OpenShell reports no sandbox, so creation can continue. -- The registry row has the target generation and OpenShell reports the sandbox ready, so the replacement can be accepted. +- The journaled replacement ID matches both the registry row and the ready live sandbox, and the registry row has the target generation, so the replacement can be accepted. All other combinations stop before the current run reuses, deletes, or creates a sandbox. -The lower create path stamps the target generation into the replacement row. +The lower create path stamps the target generation and hashed OpenShell ID into the replacement row. The handler clears the journal after it records both create and registration receipts. This slice covers resumed onboard replacement, including not-ready repair and non-default gateways. diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 5fb55eb8f9c..fc384e3b550 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -175,6 +175,7 @@ function createPhases( hydrateMessagingChannelConfig: (config) => config, messagingChannelConfigsEqual: () => true, getSandboxReuseState: () => "missing", + getSandboxRecreateObservation: () => ({ state: "missing", liveIdentityFingerprint: null }), getDcodeSelectionDrift: () => ({ changed: false, unknown: false }), hasSandboxGpuDrift: () => false, getSandboxHermesToolGateways: () => [], diff --git a/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts index 0a7926a09aa..04db4c866b4 100644 --- a/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts @@ -84,8 +84,7 @@ it("journals not-ready repair on the selected non-default gateway (#6492)", asyn }, }); expect(getSandboxRecreateObservation).toHaveBeenCalledWith("saved"); - expect(phases).toEqual( - expect.arrayContaining(["planned", "registry_committing", "completed", null]), - ); + const orderedPhases = phases.filter((phase, index) => index === 0 || phase !== phases[index - 1]); + expect(orderedPhases).toEqual([null, "planned", "registry_committing", "completed", null]); expect(session.checkpoint?.sandboxRecreate).toBeNull(); }); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 016e89a2b68..09ff10ec816 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -84,6 +84,7 @@ import { beginSandboxRecreateTransaction, clearCompletedSandboxRecreateTransaction, fingerprintSandboxRecreateValue, + type SandboxRecreateObservation, selectedGatewayForSandboxRecreate, } from "../../sandbox-recreate-transaction"; import { @@ -188,9 +189,7 @@ export interface SandboxStateOptions< right: MessagingChannelConfig | null, ): boolean; getSandboxReuseState(sandboxName: string | null): string; - getSandboxRecreateObservation?( - sandboxName: string | null, - ): import("../../sandbox-recreate-transaction").SandboxRecreateObservation; + getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation; hasSandboxGpuDrift(sandboxName: string, config: SandboxGpuConfig): boolean; getSandboxHermesToolGateways(sandboxName: string): unknown; getSandboxRegistryEntry(sandboxName: string): SandboxEntry | null; @@ -1213,17 +1212,12 @@ class SandboxStateFlow< if (!gateway) return null; const sourceEntry = this.deps.getSandboxRegistryEntry(sandboxName); if (!existing && !sourceEntry) return null; - if (!this.deps.getSandboxRecreateObservation) { - if (existing) throw new Error("Sandbox recreate observation dependency is unavailable."); - return null; - } const observation = this.deps.getSandboxRecreateObservation(sandboxName); const targetIntentFingerprint = fingerprintSandboxRecreateValue( this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), ); - let transaction: CheckpointSandboxRecreateTransaction | null = null; - this.deps.updateSession((current) => { - transaction = beginSandboxRecreateTransaction(current, { + const updated = this.deps.updateSession((current) => { + beginSandboxRecreateTransaction(current, { sandboxName, gatewayName: gateway.gatewayName, gatewayPort: gateway.gatewayPort, @@ -1233,7 +1227,7 @@ class SandboxStateFlow< }); return current; }); - return transaction; + return updated.checkpoint?.sandboxRecreate ?? null; } private recordSandboxRecreatePhase( diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 3524584553f..f6da928f07d 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -29,6 +29,7 @@ const ISO = "2026-07-27T20:00:00.000Z"; const TX_ID = "11111111-1111-4111-8111-111111111111"; const TARGET_GENERATION = "22222222-2222-4222-8222-222222222222"; const SOURCE_ID = fingerprintSandboxRecreateValue("openshell-source-id"); +const TARGET_ID = fingerprintSandboxRecreateValue("target-id"); const TARGET_INTENT = fingerprintSandboxRecreateValue({ agent: "openclaw", provider: "nvidia", @@ -71,6 +72,7 @@ function transactionAt( sourceLiveIdentityFingerprint: SOURCE_ID, targetIntentFingerprint: TARGET_INTENT, targetGeneration: TARGET_GENERATION, + targetLiveIdentityFingerprint: TARGET_ID, phase, startedAt: ISO, updatedAt: ISO, @@ -223,17 +225,91 @@ describe("sandbox recreate journal", () => { observation = { state: "missing", liveIdentityFingerprint: null }; runtime.confirmDeleted(); runtime.advance("creating"); + observation = { state: "ready", liveIdentityFingerprint: TARGET_ID }; + runtime.recordCreated(); expect(runtime).toMatchObject({ acceptedTarget: false, targetGeneration: TARGET_GENERATION, }); + expect(runtime.registrationFields).toEqual({ + lifecycleGeneration: TARGET_GENERATION, + lifecycleLiveIdentityFingerprint: TARGET_ID, + }); expect(session.checkpoint?.sandboxRecreate).toMatchObject({ - phase: "creating", - revision: 3, + phase: "created", + revision: 4, + targetLiveIdentityFingerprint: TARGET_ID, }); }); + it("recovers or rejects at every resumed-onboard mutation boundary (#6492)", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + let observation: SandboxRecreateObservation = { + state: "ready", + liveIdentityFingerprint: SOURCE_ID, + }; + let registryEntry: SandboxEntry = SOURCE_ENTRY; + const sessionStore = { + loadSession: () => session, + updateSession: (mutator: (current: ReturnType) => void) => { + mutator(session); + return session; + }, + }; + const request = { + id: TX_ID, + targetGeneration: TARGET_GENERATION, + targetIntentFingerprint: TARGET_INTENT, + }; + const restart = () => + createSandboxRecreateRuntime( + sessionStore, + request, + "alpha", + "nemoclaw-31818", + registryEntry, + () => observation, + () => undefined, + ); + + let runtime = restart(); + expect(runtime.acceptedTarget).toBe(false); + + runtime.advance("deleting"); + expect(restart().acceptedTarget).toBe(false); + + observation = { state: "missing", liveIdentityFingerprint: null }; + runtime.confirmDeleted(); + runtime = restart(); + expect(runtime.acceptedTarget).toBe(false); + + runtime.advance("creating"); + expect(restart().acceptedTarget).toBe(false); + + observation = { state: "ready", liveIdentityFingerprint: TARGET_ID }; + runtime.recordCreated(); + expect(() => restart()).toThrow(/registration did not commit/i); + + registryEntry = { + ...SOURCE_ENTRY, + lifecycleGeneration: TARGET_GENERATION, + lifecycleLiveIdentityFingerprint: TARGET_ID, + }; + expect(restart().acceptedTarget).toBe(true); + + advanceSandboxRecreateTransaction(session, TX_ID, "registry_committing", ISO); + expect(restart().acceptedTarget).toBe(true); + advanceSandboxRecreateTransaction(session, TX_ID, "completed", ISO); + expect(restart().acceptedTarget).toBe(true); + clearCompletedSandboxRecreateTransaction(session, TX_ID); + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + it("selects only the checkpoint-authorized non-default gateway", () => { const session = createSession({ sandboxName: "alpha", agent: "openclaw" }); session.checkpoint = { @@ -300,8 +376,12 @@ describe("sandbox recreate recovery", () => { expect( planSandboxRecreateRecovery( transactionAt(phase), - { state: "ready", liveIdentityFingerprint: fingerprintSandboxRecreateValue("target-id") }, - { ...SOURCE_ENTRY, lifecycleGeneration: TARGET_GENERATION }, + { state: "ready", liveIdentityFingerprint: TARGET_ID }, + { + ...SOURCE_ENTRY, + lifecycleGeneration: TARGET_GENERATION, + lifecycleLiveIdentityFingerprint: TARGET_ID, + }, ), ).toEqual({ action: "accept_target" }); }); @@ -343,11 +423,42 @@ describe("sandbox recreate recovery", () => { expect( planSandboxRecreateRecovery( transactionAt("registry_committing"), - { state: "not_ready", liveIdentityFingerprint: fingerprintSandboxRecreateValue("target") }, - { ...SOURCE_ENTRY, lifecycleGeneration: TARGET_GENERATION }, + { state: "not_ready", liveIdentityFingerprint: TARGET_ID }, + { + ...SOURCE_ENTRY, + lifecycleGeneration: TARGET_GENERATION, + lifecycleLiveIdentityFingerprint: TARGET_ID, + }, ), ).toMatchObject({ action: "reject", reason: expect.stringMatching(/not ready/) }); }); + + it("rejects a ready same-name sandbox whose identity differs from the registered target", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("registry_committing"), + { state: "ready", liveIdentityFingerprint: fingerprintSandboxRecreateValue("other-id") }, + { + ...SOURCE_ENTRY, + lifecycleGeneration: TARGET_GENERATION, + lifecycleLiveIdentityFingerprint: TARGET_ID, + }, + ), + ).toMatchObject({ action: "reject", reason: expect.stringMatching(/not the journaled/) }); + }); + + it("rejects a created target whose registry row never committed the generation", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("created"), + { state: "ready", liveIdentityFingerprint: TARGET_ID }, + SOURCE_ENTRY, + ), + ).toMatchObject({ + action: "reject", + reason: expect.stringMatching(/did not commit the journaled generation/), + }); + }); }); describe("OpenShell live identity", () => { diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index d0f57de4124..03cb17c8c48 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -124,6 +124,7 @@ export function beginSandboxRecreateTransaction( sourceLiveIdentityFingerprint: input.observation.liveIdentityFingerprint, targetIntentFingerprint: input.targetIntentFingerprint, targetGeneration: input.targetGeneration ?? randomUUID(), + targetLiveIdentityFingerprint: null, phase: input.observation.state === "missing" ? "deleted" : "planned", startedAt: now, updatedAt: now, @@ -175,6 +176,55 @@ export function advanceSandboxRecreateTransaction( return next; } +export function recordSandboxRecreateTargetCreated( + session: Session, + id: string, + observation: SandboxRecreateObservation, + now = new Date().toISOString(), +): CheckpointSandboxRecreateTransaction { + if (observation.state !== "ready" || !observation.liveIdentityFingerprint) { + throw new Error("The journaled replacement must be ready with a stable OpenShell Id."); + } + const checkpoint = baseCheckpoint(session); + const current = checkpoint.sandboxRecreate; + if (!current || current.id !== id) { + throw new Error( + "Sandbox recreate transaction ownership changed while recording the replacement identity.", + ); + } + if ( + current.targetLiveIdentityFingerprint && + current.targetLiveIdentityFingerprint !== observation.liveIdentityFingerprint + ) { + throw new Error("Sandbox recreate transaction already identifies a different replacement."); + } + if ( + current.phase === "created" && + current.targetLiveIdentityFingerprint === observation.liveIdentityFingerprint + ) { + return current; + } + if (current.phase !== "creating") { + throw new Error( + `Sandbox recreate transaction cannot record its replacement from phase '${current.phase}'.`, + ); + } + const next: CheckpointSandboxRecreateTransaction = { + ...current, + revision: current.revision + 1, + phase: "created", + targetLiveIdentityFingerprint: observation.liveIdentityFingerprint, + updatedAt: now, + }; + session.checkpoint = { + ...checkpoint, + machineState: session.machine.state, + updatedAt: now, + sandboxRecreate: next, + }; + return next; +} + export function clearCompletedSandboxRecreateTransaction(session: Session, id: string): void { const checkpoint = baseCheckpoint(session); const current = checkpoint.sandboxRecreate; @@ -205,11 +255,22 @@ export function planSandboxRecreateRecovery( observation: SandboxRecreateObservation, registryEntry: SandboxEntry | null, ): SandboxRecreateRecoveryPlan { - const targetRegistered = registryEntry?.lifecycleGeneration === transaction.targetGeneration; - if (targetRegistered) { - return observation.state === "ready" - ? { action: "accept_target" } - : reject("the journaled replacement is registered but is not ready"); + if (registryEntry?.lifecycleGeneration === transaction.targetGeneration) { + if (!transaction.targetLiveIdentityFingerprint) { + return reject("the journal did not record the replacement live identity"); + } + if ( + registryEntry.lifecycleLiveIdentityFingerprint !== transaction.targetLiveIdentityFingerprint + ) { + return reject("the replacement registry row does not match the journaled live identity"); + } + if (observation.state !== "ready") { + return reject("the journaled replacement is registered but is not ready"); + } + if (observation.liveIdentityFingerprint !== transaction.targetLiveIdentityFingerprint) { + return reject("the ready same-name sandbox is not the journaled replacement"); + } + return { action: "accept_target" }; } const sourceRegistered = @@ -283,15 +344,22 @@ interface SandboxRecreateSessionStore { export interface SandboxRecreateRuntime { readonly acceptedTarget: boolean; readonly targetGeneration: string | undefined; + readonly registrationFields: Pick< + SandboxEntry, + "lifecycleGeneration" | "lifecycleLiveIdentityFingerprint" + >; advance(phase: CheckpointSandboxRecreatePhase): void; confirmDeleted(): void; + recordCreated(): void; } const NO_SANDBOX_RECREATE: SandboxRecreateRuntime = { acceptedTarget: false, targetGeneration: undefined, + registrationFields: {}, advance: () => undefined, confirmDeleted: () => undefined, + recordCreated: () => undefined, }; export function createSandboxRecreateRuntime( @@ -317,18 +385,30 @@ export function createSandboxRecreateRuntime( return current; }); }; + let targetLiveIdentityFingerprint = transaction.targetLiveIdentityFingerprint; const recovery = planSandboxRecreateRecovery(transaction, observe(sandboxName), registryEntry); if (recovery.action === "reject") { throw new Error(`Cannot resume sandbox '${sandboxName}' recreation: ${recovery.reason}.`); } if (recovery.action === "accept_target") { note(` [resume] Recovering journaled replacement sandbox '${sandboxName}'.`); - } else if (recovery.action === "continue_create") { + } else if ( + recovery.action === "continue_create" && + phaseIndex(transaction.phase) < phaseIndex("deleted") + ) { advance("deleted"); } return { acceptedTarget: recovery.action === "accept_target", targetGeneration: transaction.targetGeneration, + get registrationFields() { + return { + lifecycleGeneration: transaction.targetGeneration, + ...(targetLiveIdentityFingerprint + ? { lifecycleLiveIdentityFingerprint: targetLiveIdentityFingerprint } + : {}), + }; + }, advance, confirmDeleted: () => { if (observe(sandboxName).state !== "missing") { @@ -338,5 +418,16 @@ export function createSandboxRecreateRuntime( } advance("deleted"); }, + recordCreated: () => { + const observation = observe(sandboxName); + sessionStore.updateSession((current) => { + targetLiveIdentityFingerprint = recordSandboxRecreateTargetCreated( + current, + transaction.id, + observation, + ).targetLiveIdentityFingerprint; + return current; + }); + }, }; } diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index c492f43617b..d729e1fa949 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -124,6 +124,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { }, dashboardPort: 18789, lifecycleGeneration: "22222222-2222-4222-8222-222222222222", + lifecycleLiveIdentityFingerprint: "d".repeat(64), gatewayName: "nemoclaw-19080", gatewayPort: 19080, }); @@ -152,6 +153,7 @@ describe("buildCreatedSandboxRegistryEntry", () => { hermesDashboardTui: true, dashboardPort: 18789, lifecycleGeneration: "22222222-2222-4222-8222-222222222222", + lifecycleLiveIdentityFingerprint: "d".repeat(64), gatewayName: "nemoclaw-19080", gatewayPort: 19080, gpuEnabled: true, diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index 649f1a9a429..20f2a82c737 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -65,6 +65,7 @@ export interface CreatedSandboxRegistryEntryInput { dashboardPort: number; dashboardRemoteBindPrepared?: boolean; lifecycleGeneration?: string; + lifecycleLiveIdentityFingerprint?: string; gatewayName: string; gatewayPort: number; } @@ -198,6 +199,7 @@ export function buildCreatedSandboxRegistryEntry( dashboardPort: input.dashboardPort, dashboardRemoteBindPrepared: input.dashboardRemoteBindPrepared === true, lifecycleGeneration: input.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, gatewayName: input.gatewayName, gatewayPort: input.gatewayPort, }; diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index b9f6f5fdc6a..dca7f55cf48 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -3,8 +3,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; -import { applyReusedSandboxDashboardState, createSandboxReuseHelpers } from "./sandbox-reuse"; import { fingerprintSandboxRecreateValue } from "./sandbox-recreate-transaction"; +import { applyReusedSandboxDashboardState, createSandboxReuseHelpers } from "./sandbox-reuse"; describe("applyReusedSandboxDashboardState", () => { afterEach(() => { @@ -225,7 +225,7 @@ describe("createSandboxReuseHelpers", () => { ); }); - it("rejects an OpenShell state outside the recovery model", () => { + it("preserves an unknown reuse state but rejects it for recreate recovery", () => { const helpers = createSandboxReuseHelpers({ runCaptureOpenshell: vi.fn(() => ""), runOpenshell: vi.fn(), @@ -233,6 +233,7 @@ describe("createSandboxReuseHelpers", () => { note: vi.fn(), }); + expect(helpers.getSandboxReuseState("alpha")).toBe("unknown"); expect(() => helpers.getSandboxRecreateObservation("alpha")).toThrow(/state 'unknown'/); }); }); diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index 11e8df6b747..f75a799c75b 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -112,13 +112,18 @@ export function applyReusedSandboxDashboardState( } export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseHelpers { - function getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation { - if (!sandboxName) return { state: "missing", liveIdentityFingerprint: null }; + function readSandboxState(sandboxName: string | null): { state: string; getOutput: string } { + if (!sandboxName) return { state: "missing", getOutput: "" }; const getOutput = deps.runCaptureOpenshell(["sandbox", "get", sandboxName], { ignoreError: true, }); const listOutput = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); const state = deps.getSandboxStateFromOutputs(sandboxName, getOutput, listOutput); + return { state, getOutput }; + } + + function getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation { + const { state, getOutput } = readSandboxState(sandboxName); if (state !== "missing" && state !== "not_ready" && state !== "ready") { throw new Error( `Cannot observe sandbox '${sandboxName}' for recreate recovery: OpenShell returned state '${state}'.`, @@ -132,7 +137,7 @@ export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseH } function getSandboxReuseState(sandboxName: string | null): string { - return getSandboxRecreateObservation(sandboxName).state; + return readSandboxState(sandboxName).state; } function repairRecordedSandbox(sandboxName: string | null): void { diff --git a/src/lib/state/onboard-checkpoint-types.ts b/src/lib/state/onboard-checkpoint-types.ts index 9c3e96027e1..904753187e9 100644 --- a/src/lib/state/onboard-checkpoint-types.ts +++ b/src/lib/state/onboard-checkpoint-types.ts @@ -93,6 +93,7 @@ export interface CheckpointSandboxRecreateTransaction { readonly sourceLiveIdentityFingerprint: string | null; readonly targetIntentFingerprint: string; readonly targetGeneration: string; + readonly targetLiveIdentityFingerprint: string | null; readonly phase: CheckpointSandboxRecreatePhase; readonly startedAt: string; readonly updatedAt: string; diff --git a/src/lib/state/onboard-checkpoint.test.ts b/src/lib/state/onboard-checkpoint.test.ts index 7391f1bf769..b4045564f71 100644 --- a/src/lib/state/onboard-checkpoint.test.ts +++ b/src/lib/state/onboard-checkpoint.test.ts @@ -57,6 +57,7 @@ function recreateTransaction(): CheckpointSandboxRecreateTransaction { sourceLiveIdentityFingerprint: "b".repeat(64), targetIntentFingerprint: "c".repeat(64), targetGeneration: "22222222-2222-4222-8222-222222222222", + targetLiveIdentityFingerprint: null, phase: "creating", startedAt: ISO, updatedAt: ISO, @@ -245,6 +246,16 @@ describe("checkpoint schema inspection", () => { expect(inspectCheckpoint(serialized)).toEqual({ status: "corrupt" }); }); + it.each([ + "sourceLiveIdentityFingerprint", + "targetLiveIdentityFingerprint", + ])("rejects a malformed nullable recreate journal field: %s", (field) => { + const serialized = serializedRecreateCheckpoint(); + (serialized.sandboxRecreate as Record)[field] = 42; + + expect(inspectCheckpoint(serialized)).toEqual({ status: "corrupt" }); + }); + it("rejects a checkpoint whose external authority targets a different port", () => { const serialized = serializeCheckpoint( baseCheckpoint({ diff --git a/src/lib/state/onboard-checkpoint.ts b/src/lib/state/onboard-checkpoint.ts index 3df63cdfdf1..840f9756467 100644 --- a/src/lib/state/onboard-checkpoint.ts +++ b/src/lib/state/onboard-checkpoint.ts @@ -151,6 +151,10 @@ function parseGatewaySupervisor(value: unknown): CheckpointGatewaySupervisor | n return { kind, serviceName, execPath }; } +function canonicalGatewayName(gatewayPort: number): string { + return gatewayPort === DEFAULT_GATEWAY_PORT ? "nemoclaw" : `nemoclaw-${String(gatewayPort)}`; +} + function parseGatewayAuthorityValue(value: unknown): CheckpointGatewayAuthority | null { if (!isObjectRecord(value)) return null; const gatewayName = readString(value.gatewayName); @@ -168,8 +172,7 @@ function parseGatewayAuthorityValue(value: unknown): CheckpointGatewayAuthority ) { return null; } - const canonicalName = - gatewayPort === DEFAULT_GATEWAY_PORT ? "nemoclaw" : `nemoclaw-${String(gatewayPort)}`; + const canonicalName = canonicalGatewayName(Number(gatewayPort)); if (gatewayName !== canonicalName) return null; if (mode !== "nemoclaw-managed" && mode !== "externally-supervised") return null; if (source !== "declared" && source !== "packaged-service" && source !== "standalone") @@ -245,6 +248,12 @@ function parseBindings(value: unknown): CheckpointBindings | null { if (credentialEnvs === null || registeredProviders === null) return null; return { credentialEnvs, registeredProviders }; } + +function readNullableSha256(value: unknown): string | null | undefined { + if (value === null) return null; + return typeof value === "string" && SHA256_PATTERN.test(value) ? value : undefined; +} + function parseSandboxRecreateTransaction( value: unknown, ): CheckpointSandboxRecreateTransaction | null { @@ -254,12 +263,10 @@ function parseSandboxRecreateTransaction( const gatewayName = readString(value.gatewayName); const gatewayPort = value.gatewayPort; const sourceRegistryFingerprint = readString(value.sourceRegistryFingerprint); - const sourceLiveIdentityFingerprint = - value.sourceLiveIdentityFingerprint === null - ? null - : readString(value.sourceLiveIdentityFingerprint); + const sourceLiveIdentityFingerprint = readNullableSha256(value.sourceLiveIdentityFingerprint); const targetIntentFingerprint = readString(value.targetIntentFingerprint); const targetGeneration = readString(value.targetGeneration); + const targetLiveIdentityFingerprint = readNullableSha256(value.targetLiveIdentityFingerprint); const phase = value.phase; const startedAt = readCanonicalIsoTimestamp(value.startedAt); const updatedAt = readCanonicalIsoTimestamp(value.updatedAt); @@ -275,16 +282,15 @@ function parseSandboxRecreateTransaction( !Number.isInteger(gatewayPort) || Number(gatewayPort) < 1 || Number(gatewayPort) > 65535 || - gatewayName !== - (gatewayPort === DEFAULT_GATEWAY_PORT ? "nemoclaw" : `nemoclaw-${String(gatewayPort)}`) || + gatewayName !== canonicalGatewayName(Number(gatewayPort)) || !sourceRegistryFingerprint || !SHA256_PATTERN.test(sourceRegistryFingerprint) || - (sourceLiveIdentityFingerprint !== null && - (!sourceLiveIdentityFingerprint || !SHA256_PATTERN.test(sourceLiveIdentityFingerprint))) || + sourceLiveIdentityFingerprint === undefined || !targetIntentFingerprint || !SHA256_PATTERN.test(targetIntentFingerprint) || !targetGeneration || !UUID_PATTERN.test(targetGeneration) || + targetLiveIdentityFingerprint === undefined || typeof phase !== "string" || !SANDBOX_RECREATE_PHASES.has(phase as CheckpointSandboxRecreatePhase) || !Number.isSafeInteger(revision) || @@ -305,6 +311,7 @@ function parseSandboxRecreateTransaction( sourceLiveIdentityFingerprint, targetIntentFingerprint, targetGeneration, + targetLiveIdentityFingerprint, phase: phase as CheckpointSandboxRecreatePhase, startedAt, updatedAt, diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index c5e71c0742e..a2a9d763ba9 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -187,12 +187,14 @@ export interface SandboxEntry extends Partial { dashboardPort?: number | null; /** Remote dashboard exposure was included in the sandbox's generated config. */ dashboardRemoteBindPrepared?: boolean; + /** Generation proving which durable same-name recreate registered this row. */ + lifecycleGeneration?: string; + /** Hashed OpenShell identity paired with lifecycleGeneration for exact recovery. */ + lifecycleLiveIdentityFingerprint?: string; // OpenShell gateway registration name and host port bound to this sandbox. // Persisted so later lifecycle commands operate on the sandbox's own gateway // instead of the process-global `nemoclaw` singleton — a second sandbox on a // different NEMOCLAW_GATEWAY_PORT no longer recreates/kills the first (#4422). - /** Generation proving which durable same-name recreate registered this row. */ - lifecycleGeneration?: string; gatewayName?: string | null; gatewayPort?: number | null; } From bbee070ace6c74ed77959c73b03d415c1ee47471 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 27 Jul 2026 23:57:15 -0700 Subject: [PATCH 4/5] docs(onboard): clarify recreate fingerprints Signed-off-by: Carlos Villela --- src/lib/onboard/lifecycle-contracts.md | 2 +- src/lib/onboard/sandbox-recreate-transaction.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index e47dcc8fa07..b517a70a82f 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -122,7 +122,7 @@ runtime mutation ## Durable resumed recreate journal A resumed same-name replacement writes a secret-free journal before the lower create path can delete the source sandbox. -The journal binds the session, sandbox, selected gateway, source registry row, source OpenShell ID, target intent, target generation, and the replacement OpenShell ID after creation. +The journal binds the session, sandbox, selected gateway, source registry row, source OpenShell ID fingerprint, target intent, target generation, and the replacement OpenShell ID fingerprint after creation. Recovery accepts only these states: diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index f6da928f07d..2498c0f50ba 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -372,7 +372,7 @@ describe("sandbox recreate recovery", () => { "created", "registry_committing", "completed", - ] as const)("accepts the ready target from %s when its generation matches", (phase) => { + ] as const)("accepts the ready target from %s when its generation and live identity match", (phase) => { expect( planSandboxRecreateRecovery( transactionAt(phase), From 6dc948eb61a485faa428d4e69808671f03e1b0bf Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 28 Jul 2026 00:12:48 -0700 Subject: [PATCH 5/5] fix(onboard): reject recreate gateway mismatch Signed-off-by: Carlos Villela --- .../handlers/sandbox-recreate-journal.test.ts | 97 ++++++++++++++++++- src/lib/onboard/machine/handlers/sandbox.ts | 10 ++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts index 04db4c866b4..838384376ab 100644 --- a/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts @@ -6,7 +6,10 @@ import { expect, it, vi } from "vitest"; import { decisionSelected } from "../../../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../../../state/onboard-checkpoint-migrate"; import { createSession, type Session } from "../../../state/onboard-session"; -import { fingerprintSandboxRecreateValue } from "../../sandbox-recreate-transaction"; +import { + beginSandboxRecreateTransaction, + fingerprintSandboxRecreateValue, +} from "../../sandbox-recreate-transaction"; import { handleSandboxState } from "./sandbox"; import { baseOptions, createDeps } from "./sandbox-test-fixtures"; @@ -88,3 +91,95 @@ it("journals not-ready repair on the selected non-default gateway (#6492)", asyn expect(orderedPhases).toEqual([null, "planned", "registry_committing", "completed", null]); expect(session.checkpoint?.sandboxRecreate).toBeNull(); }); + +it("rejects an active recreate journal on a different gateway authority (#6492)", async () => { + const session = createSession({ sandboxName: "saved", agent: "openclaw" }); + session.steps.sandbox.status = "complete"; + const sourceEntry = { + name: "saved", + provider: "provider", + model: "model", + endpointUrl: null, + preferredInferenceApi: "openai-completions" as const, + webSearchEnabled: false, + toolDisclosure: "progressive" as const, + fromDockerfile: null, + hermesAuthMethod: null, + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + }; + session.checkpoint = { + ...deriveCheckpointFromSession(session), + sandboxIdentity: decisionSelected({ name: "saved", agent: "openclaw" }), + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + }; + beginSandboxRecreateTransaction(session, { + sandboxName: "saved", + gatewayName: "nemoclaw-31818", + gatewayPort: 31818, + sourceEntry, + observation: { + state: "not_ready", + liveIdentityFingerprint: fingerprintSandboxRecreateValue("openshell-source-id"), + }, + targetIntentFingerprint: fingerprintSandboxRecreateValue({ + sandboxName: "saved", + agent: "openclaw", + }), + id: "11111111-1111-4111-8111-111111111111", + targetGeneration: "22222222-2222-4222-8222-222222222222", + now: "2026-07-28T07:00:00.000Z", + }); + session.checkpoint = { + ...session.checkpoint, + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + }; + const currentEntry = { + ...sourceEntry, + gatewayName: "nemoclaw", + gatewayPort: 8080, + }; + const getSandboxRecreateObservation = vi.fn(() => ({ + state: "not_ready" as const, + liveIdentityFingerprint: fingerprintSandboxRecreateValue("openshell-source-id"), + })); + const { deps, calls } = createDeps( + { + getSandboxReuseState: () => "not_ready", + getSandboxRecreateObservation, + getSandboxRegistryEntry: () => currentEntry, + }, + session, + ); + + await expect( + handleSandboxState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "saved", + gatewayName: "nemoclaw", + }), + ).rejects.toThrow(/journaled gateway.*does not match the selected gateway authority/i); + expect(getSandboxRecreateObservation).not.toHaveBeenCalled(); + expect(calls.createSandbox).not.toHaveBeenCalled(); + expect(calls.repairSandbox).not.toHaveBeenCalled(); + expect(calls.removeSandbox).not.toHaveBeenCalled(); +}); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 09ff10ec816..16e428e3e7d 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -1209,6 +1209,16 @@ class SandboxStateFlow< state.session?.checkpoint, this.options.gatewayName, ); + if ( + existing && + (!gateway || + existing.gatewayName !== gateway.gatewayName || + existing.gatewayPort !== gateway.gatewayPort) + ) { + throw new Error( + `Cannot resume sandbox '${existing.sandboxName}' recreation: journaled gateway '${existing.gatewayName}:${String(existing.gatewayPort)}' does not match the selected gateway authority.`, + ); + } if (!gateway) return null; const sourceEntry = this.deps.getSandboxRegistryEntry(sandboxName); if (!existing && !sourceEntry) return null;