diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index d36e879d844..77f9ec2a1ae 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -6,10 +6,10 @@ "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26, "src/lib/actions/sandbox/process-recovery.ts": 26, "src/lib/adapters/docker/index.ts": 45, - "src/lib/adapters/openshell/client.ts": 22, + "src/lib/adapters/openshell/client.ts": 23, "src/lib/adapters/openshell/resolve.ts": 27, - "src/lib/adapters/openshell/runtime.ts": 50, - "src/lib/adapters/openshell/timeouts.ts": 36, + "src/lib/adapters/openshell/runtime.ts": 51, + "src/lib/adapters/openshell/timeouts.ts": 37, "src/lib/agent/defs.ts": 32, "src/lib/cli/branding.ts": 85, "src/lib/cli/nemoclaw-oclif-command.ts": 103, @@ -26,8 +26,8 @@ "src/lib/onboard/gateway-binding.ts": 47, "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, - "src/lib/state/onboard-session.ts": 34, - "src/lib/state/registry.ts": 99, + "src/lib/state/onboard-session.ts": 36, + "src/lib/state/registry.ts": 101, "src/lib/state/state-root.ts": 23, "src/lib/subprocess-env.ts": 23, "src/lib/validation.ts": 25 @@ -42,12 +42,12 @@ "src/lib/actions/sandbox/doctor.ts": 29, "src/lib/actions/sandbox/policy-channel.ts": 28, "src/lib/actions/sandbox/process-recovery.ts": 22, - "src/lib/actions/sandbox/rebuild-pipeline.ts": 27, + "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, "src/lib/actions/sandbox/snapshot.ts": 38, "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 23, - "src/lib/onboard.ts": 224, + "src/lib/onboard.ts": 225, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/sandbox/config.ts": 23, "src/lib/shields/index.ts": 23 @@ -62,9 +62,6 @@ "src/lib/actions/sandbox/connect-boundary-refusal.ts", "src/lib/actions/sandbox/connect-hermes-light-skin.ts", "src/lib/actions/sandbox/connect.ts", - "src/lib/actions/sandbox/destroy-execution.ts", - "src/lib/actions/sandbox/destroy-preflight.ts", - "src/lib/actions/sandbox/destroy.ts", "src/lib/actions/sandbox/gateway-restart.ts", "src/lib/actions/sandbox/gateway-state.ts", "src/lib/actions/sandbox/mcp-bridge-adapter-deepagents-capability.ts", @@ -137,15 +134,13 @@ "src/lib/onboard/setup-inference.ts", "src/lib/openshell-sandbox-list.ts", "src/lib/sandbox/config.ts", - "src/lib/shields/index.ts", - "src/lib/tunnel/allowed-origins.ts", - "src/lib/tunnel/services.ts" + "src/lib/shields/index.ts" ] ], "maxRootFiles": { - "src/lib/onboard": 307, + "src/lib/onboard": 309, "src/lib/actions": 19, - "src/lib/actions/sandbox": 182, + "src/lib/actions/sandbox": 183, "src/lib/state": 38, "src/lib/inference": 62, "scripts": 44 diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index a8edc759f59..ab91c92706a 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -167,6 +167,33 @@ Follow the printed recovery guidance, using `$$nemoclaw recover` The rebuild command preserves manifest-defined Deep Agents state, regenerates `config.toml`, reconstructs managed MCP projection state, and reapplies registered policies while recreating the container. +### Continue an Interrupted Replacement + +Before `rebuild` deletes the existing sandbox, NemoClaw records a replacement journal in the onboarding session. +The journal binds the operation to the sandbox name, recorded OpenShell gateway, source identity, and replacement settings. +It stores fingerprints instead of credential values or raw OpenShell sandbox IDs. + +If `rebuild` stops after recording the journal, rerun the command with the same replacement settings. +The rerun takes one of these actions: + +- It continues deletion when the live sandbox still has the journaled source identity. +- It continues creation when the recorded OpenShell gateway explicitly reports the source sandbox as absent. +- It accepts an existing replacement only when its live identity and sandbox registry generation match the journal. + +An accepted replacement is not deleted again. +The command reports `Sandbox '' already holds the replacement from the interrupted rebuild.` and preserves the state backup path when one exists. +Pass `--verbose` to include the replacement identifier, OpenShell gateway, and journal phase in rebuild diagnostics. + +NemoClaw fails closed when the selected gateway, replacement settings, durable source registry fields, or live source or target identity no longer matches the journal. +The error names the sandbox and the mismatch that stopped recovery. +Do not delete a same-name sandbox to bypass this check. +Inspect the named OpenShell gateway and sandbox, correct the reported drift, and rerun the original command. + +A same-name recreation started by `$$nemoclaw onboard` uses the same replacement journal. +If that recreation is interrupted after the `Journaled replacement` message, rerun the original onboarding command with the same target settings. +The active replacement can continue without adding `--resume`. +Use `--resume` for interrupted onboarding steps that occur before a replacement journal exists. + If an archive command preserves at least one state directory, NemoClaw keeps the usable entries and reports the manifest-defined paths that could not be archived. If a manifest-declared state file fails, NemoClaw stops before deleting the original sandbox even when it preserved state directories, unless you explicitly pass `--force`. If every state directory fails, NemoClaw stops before deleting the original sandbox even when it captured loose files, unless you explicitly pass `--force`. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 5f7c4c7655a..fd20c6cbe69 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -281,6 +281,11 @@ Use `--resume` only for resumable interrupted or failed sessions, not to change During resume, NemoClaw reruns preflight, gateway, provider, and sandbox repair checks even when the saved session has already reached a later nonterminal onboarding phase. If the recorded session conflicts with flags you pass on the recovery run, NemoClaw exits and tells you to either rerun with the original settings or start over. +An active same-name replacement is separate from ordinary onboarding-step resume. +If onboarding printed `Journaled replacement` before it stopped, rerun the original onboarding command with the same target settings. +The replacement can continue without an explicit `--resume` flag. +Refer to [Continue an Interrupted Replacement](../manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes#continue-an-interrupted-replacement) for the identity checks and failure conditions. + Use `--fresh` to discard the saved onboarding session and start the wizard from the beginning. This clears stale or failed session state before NemoClaw creates a new session record. It also bypasses locally recorded sandbox base-image resolution metadata and reruns normal candidate resolution. @@ -640,6 +645,11 @@ NemoClaw aborts the recreate when the backup cannot complete in full, including Set `NEMOCLAW_RECREATE_WITHOUT_BACKUP=1` to skip the pre-recreate backup. The destination sandbox starts with a fresh workspace. +Before deletion, onboarding prints a `Journaled replacement` diagnostic with the replacement identifier, recorded OpenShell gateway, and current phase. +If the process stops after this point, a later same-target onboarding run continues the active replacement without requiring `--resume`. +It accepts a ready same-name replacement only when the live identity and sandbox registry generation match the journal. +It fails closed if the gateway, source, target, durable source registry fields, or replacement settings changed. + For OpenClaw, the backed-up paths include agents, extensions, workspace, skills, hooks, identity, devices, canvas, cron, memory, telegram, wechat, credentials, and `/sandbox/.openclaw/workspace/`. @@ -2792,6 +2802,10 @@ After OpenShell accepts the sandbox deletion, `rebuild` waits until OpenShell ex Only then can NemoClaw perform any required local registry removal and begin creating the replacement. If OpenShell does not confirm absence within the bounded wait, including when gateway transport errors block the probes, `rebuild` exits nonzero before registry removal or replacement creation and preserves both the local registry entry and the state backup. Restore OpenShell connectivity and confirm the sandbox's live state before you retry, and keep the printed backup path for recovery. +Before deletion, rebuild records a replacement journal that binds the operation to the recorded gateway, source identity, and target settings. +Rerunning the same rebuild continues from the recorded boundary or accepts the proven replacement instead of deleting it again. +Use `--verbose` to print the replacement identifier, gateway, and journal phase. +Refer to [Continue an Interrupted Replacement](../manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes#continue-an-interrupted-replacement) for the recovery procedure and fail-closed conditions. When rebuild starts with shields up, NemoClaw opens a 30-minute shields-down window for backup and recreation. A detached auto-lock timer remains active until NemoClaw commits a successful shields-up state, so it can attempt to restore lockdown if the host rebuild process exits unexpectedly. diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 37e35a75d41..256227bae1f 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -101,37 +101,6 @@ export function isMissingSandboxGatewayOutput(output = ""): boolean { ); } -/** - * Strict absence classifier for destructive owner-gateway reconciliation. - * Bare NotFound is not sufficient because OpenShell uses it for missing - * gateways and providers as well as sandboxes. - */ -export function isExplicitMissingSandboxGatewayOutput( - output: string, - sandboxName: string, -): boolean { - const clean = stripAnsi(String(output)).replace(/\r/g, "").trim(); - const exactNoSpec = - /^(?:error:\s*)?status:\s*Internal,\s*message:\s*["']sandbox has no spec["'](?:,\s*details:\s*\[\])?(?:,\s*metadata:\s*MetadataMap\s*\{\s*\})?$/i; - if (exactNoSpec.test(clean)) return true; - // OpenShell can omit the requested name from an owner-scoped lookup. - // Require both exact structured fields so gateway/provider absence and - // transport diagnostics remain ambiguous. - const exactStructuredNotFound = - /^(?:error:\s*)?(?:×\s*)?code:\s*["']Some requested entity was not found["']\s*,\s*message:\s*["']sandbox not found["']$/i; - if (exactStructuredNotFound.test(clean)) return true; - - const escapedName = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - const namedSandbox = `(?:['\"]${escapedName}['\"]|${escapedName})`; - return ( - new RegExp( - `^(?:error:\\s*)?sandbox\\s+${namedSandbox}\\s+(?:(?:is\\s+)?not\\s+(?:found|present)|does\\s+not\\s+exist)[.!]?$`, - "i", - ).test(clean) || - new RegExp(`^(?:error:\\s*)?no\\s+such\\s+sandbox\\s+${namedSandbox}[.!]?$`, "i").test(clean) - ); -} - function formatGatewaySchemaMismatchOutput( issue: OpenShellStateRpcIssue, action: string, diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 911ddc4077d..411b9b9b17f 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -72,6 +72,21 @@ vi.mock("./rebuild-mcp-phase", () => ({ })); import { runRebuildDestroyPhase, waitForRebuildDeleteAbsence } from "./rebuild-destroy-phase"; +import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; + +function stubRecreateJournal(): RebuildRecreateJournal { + return { + id: "journal-1", + acceptedTarget: false, + sourceConfirmedAbsent: false, + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + observeSourceForDelete: vi.fn(() => "source" as const), + confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), + }; +} describe("rebuild destroy phase", () => { beforeEach(() => { @@ -135,6 +150,7 @@ describe("rebuild destroy phase", () => { }, }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log: vi.fn(), bail, @@ -163,6 +179,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "langchain-deepagents-code" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log, bail, @@ -193,6 +210,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log: vi.fn(), bail, @@ -219,6 +237,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log, @@ -254,6 +273,7 @@ describe("rebuild destroy phase", () => { gatewayPort: 19080, }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -308,6 +328,7 @@ describe("rebuild destroy phase", () => { gatewayPort: 8080, }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -353,6 +374,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -396,6 +418,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -447,6 +470,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -565,6 +589,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -609,6 +634,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -657,6 +683,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -703,6 +730,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -838,6 +866,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log: vi.fn(), bail: vi.fn((message: string): never => { @@ -855,7 +884,7 @@ describe("rebuild destroy phase", () => { expect(mocks.listSandboxes).not.toHaveBeenCalled(); }); - it("removes registry state only after the gateway reports the deleted sandbox missing", async () => { + it("keeps the journaled source row after the gateway reports the deleted sandbox missing (#7734)", async () => { const events: string[] = []; let getAttempts = 0; mocks.runOpenshell.mockImplementation(() => { @@ -870,15 +899,13 @@ describe("rebuild destroy phase", () => { ? { status: 0, stdout: "Name: alpha\nPhase: Terminating", stderr: "" } : { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }; }); - mocks.removeSandboxRegistryEntryWithReceipt.mockImplementation(() => { - events.push("remove-registry"); - return null; - }); + const recreateJournal = stubRecreateJournal(); const result = await runRebuildDestroyPhase({ sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal, backupManifest: null, log: vi.fn(), bail: vi.fn((message: string): never => { @@ -889,14 +916,120 @@ describe("rebuild destroy phase", () => { }); expect(result).not.toBeNull(); - expect(events).toEqual(["delete", "get-live", "get-missing", "on-deleted", "remove-registry"]); + expect(result?.removalReceipt).toBeNull(); + expect(events).toEqual(["delete", "get-live", "get-missing", "on-deleted"]); expect(mocks.waitUntil).toHaveBeenCalledOnce(); expect(mocks.captureOpenshell).toHaveBeenNthCalledWith( 1, ["sandbox", "get", "-g", "nemoclaw", "alpha"], expect.objectContaining({ timeout: expect.any(Number) }), ); - expect(mocks.removeSandboxRegistryEntryWithReceipt).toHaveBeenCalledWith("alpha"); + expect(recreateJournal.markDeleting).toHaveBeenCalledOnce(); + expect(recreateJournal.confirmDeleted).toHaveBeenCalledOnce(); + }); + + it("journals the delete boundary before the destructive command (#7734)", async () => { + const order: string[] = []; + const recreateJournal = stubRecreateJournal(); + vi.mocked(recreateJournal.markDeleting).mockImplementation(() => { + order.push("journal:deleting"); + }); + mocks.runOpenshell.mockImplementation((args: string[]) => { + const deleting = args[1] === "delete"; + order.push(...(deleting ? ["openshell:delete"] : [])); + return deleting + ? { status: 0, stdout: "deleted", stderr: "" } + : { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }; + }); + + await runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, + staleRecovery: false, + recreateJournal, + backupManifest: null, + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded: vi.fn(() => true), + onDeleted: vi.fn(), + }); + + expect(order).toEqual(["journal:deleting", "openshell:delete"]); + }); + + it("reattaches MCP providers when the delete boundary cannot be journaled (#7734)", async () => { + const recreateJournal = stubRecreateJournal(); + vi.mocked(recreateJournal.markDeleting).mockImplementation(() => { + throw new Error("session store is unwritable"); + }); + mocks.prepareMcpForRebuild.mockResolvedValue({ + entries: [{ server: "github" }], + detachedProviderEntries: [{ providerName: "nemoclaw-mcp-alpha-github" }], + scrubbedAdapterEntries: [{ server: "github" }], + }); + const relockShieldsIfNeeded = vi.fn(() => true); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, + staleRecovery: false, + recreateJournal, + backupManifest: null, + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded, + onDeleted: vi.fn(), + }), + ).rejects.toThrow("Sandbox deletion could not be journaled"); + + expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith( + "alpha", + [{ providerName: "nemoclaw-mcp-alpha-github" }], + [{ server: "github" }], + ); + expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); + expect(mocks.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.anything(), + ); + }); + + it("stops before inference and registry mutation when absence cannot be journaled (#7734)", async () => { + const recreateJournal = stubRecreateJournal(); + vi.mocked(recreateJournal.confirmDeleted).mockImplementation(() => { + throw new Error("OpenShell still reports the journaled source after delete"); + }); + mocks.runOpenshell.mockReturnValue({ status: 0, stdout: "deleted", stderr: "" }); + const onDeleted = vi.fn(); + const onDeleteStateAmbiguous = vi.fn(); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, + staleRecovery: false, + recreateJournal, + backupManifest: null, + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded: vi.fn(() => true), + onDeleted, + onDeleteStateAmbiguous, + }), + ).rejects.toThrow("Sandbox deletion could not be journaled"); + + expect(onDeleteStateAmbiguous).toHaveBeenCalledOnce(); + expect(onDeleted).not.toHaveBeenCalled(); + expect(mocks.stopNimContainer).not.toHaveBeenCalled(); + expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); + expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled(); }); it("marks accepted deletion as ambiguous when transport failures prevent confirmation", async () => { @@ -915,6 +1048,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: { backupPath: "/tmp/rebuild-backups/alpha/backup" } as never, log: vi.fn(), bail: vi.fn((message: string): never => { diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index ba5eca3049b..9688c4cfa82 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -8,12 +8,11 @@ import { waitUntil } from "../../core/wait"; import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; import * as nim from "../../inference/nim"; import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { isExplicitMissingSandboxGatewayOutput } from "../../onboard/sandbox-recreate-probe"; import { redactFull } from "../../security/redact"; import { parseSandboxPhase } from "../../state/gateway"; import { registryEntryGatewayPort } from "../../state/gateway-registry"; import * as registry from "../../state/registry"; -import { removeSandboxRegistryEntryWithReceipt } from "./destroy"; -import { isExplicitMissingSandboxGatewayOutput } from "./gateway-state"; import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import { type RebuildSandboxEntry, warnUnpreservedUserManagedFiles } from "./rebuild-flow-helpers"; @@ -24,6 +23,10 @@ import { reattachMcpAfterDeleteFailure, } from "./rebuild-mcp-phase"; import { blockRebuildOnPendingBaselineTransition } from "./rebuild-preflight-guards"; +import type { + RebuildRecreateJournal, + RebuildRecreateSourcePresence, +} from "./rebuild-recreate-journal"; export type RebuildDeleteValidationResult = | { ok: true } @@ -33,6 +36,7 @@ export interface RebuildDestroyPhaseInput { sandboxName: string; sandboxEntry: RebuildSandboxEntry; staleRecovery: boolean; + recreateJournal: RebuildRecreateJournal; backupManifest: RebuildBackupManifest; log: RebuildLog; bail: RebuildBail; @@ -225,6 +229,7 @@ export async function runRebuildDestroyPhase( const { sandboxName, staleRecovery, + recreateJournal, backupManifest, log, bail, @@ -314,7 +319,6 @@ export async function runRebuildDestroyPhase( log, }); if (!mcpPreparation) return null; - const rebuildMcpEntries = mcpPreparation.entries; const rebuildDetachedMcpProviderEntries = mcpPreparation.detachedProviderEntries; const rebuildScrubbedMcpAdapterEntries = mcpPreparation.scrubbedAdapterEntries; @@ -386,15 +390,44 @@ export async function runRebuildDestroyPhase( } } - log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); - const deleteResult = runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); - const { alreadyGone } = getSandboxDeleteOutcome(deleteResult); - log(`Delete result: exit=${deleteResult.status}, alreadyGone=${alreadyGone}`); + // MCP adapter entries are already detached and scrubbed here. A journal write + // that fails must reattach them before the rebuild gives up, or the still + // running sandbox is left without its MCP wiring. + let sourcePresence: RebuildRecreateSourcePresence; + try { + sourcePresence = recreateJournal.observeSourceForDelete(); + recreateJournal.markDeleting(); + } catch (error) { + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + ); + relockShieldsIfNeeded(true); + const detail = error instanceof Error ? error.message : String(error); + bail( + mcpRecoveryFailure + ? `Sandbox deletion could not be journaled: ${redactFull(detail)} MCP provider recovery also failed: ${mcpRecoveryFailure}` + : `Sandbox deletion could not be journaled: ${redactFull(detail)}`, + ); + return null; + } + if (sourcePresence === "missing") { + log(`Skipping delete: gateway ${gatewayName} reports '${sandboxName}' already absent`); + } else { + log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); + } + const deleteResult = + sourcePresence === "missing" + ? null + : runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const alreadyGone = deleteResult === null || getSandboxDeleteOutcome(deleteResult).alreadyGone; + if (deleteResult) log(`Delete result: exit=${deleteResult.status}, alreadyGone=${alreadyGone}`); let deletionConfirmed = alreadyGone; - if (deleteResult.status !== 0) { + if (deleteResult && deleteResult.status !== 0) { const reconciledDelete = reconcileFailedSandboxDelete(sandboxName, input.sandboxEntry, log); if (reconciledDelete.state === "deleted") { log("Delete returned nonzero, but exact post-delete state confirms sandbox removal."); @@ -456,29 +489,31 @@ export async function runRebuildDestroyPhase( bail("Sandbox deletion could not be confirmed."); return null; } + try { + recreateJournal.confirmDeleted(); + } catch (error) { + console.error( + " Sandbox delete was accepted, but the replacement journal could not confirm absence.", + ); + if (backupManifest) { + console.error(" State backup is preserved at: " + backupManifest.backupPath); + } + input.onDeleteStateAmbiguous?.(); + const detail = error instanceof Error ? error.message : String(error); + bail(`Sandbox deletion could not be journaled: ${redactFull(detail)}`); + return null; + } stopNimBestEffort(); onDeleted(); - let removalReceipt: registry.SandboxRemovalReceipt | null = null; - const hasBaselineExclusions = (input.sandboxEntry.baselineExclusions?.length ?? 0) > 0; - if (rebuildMcpEntries.length === 0 && !hasBaselineExclusions) { - removalReceipt = removeSandboxRegistryEntryWithReceipt(sandboxName); - } - if (rebuildMcpEntries.length > 0) { - // The registry entry is the durable MCP rebuild transaction. The inner - // onboard run observes that the sandbox is absent, carries the MCP state - // into the replacement registration, and never enters generic live - // recreation. Keeping it here closes every process-death window between - // successful delete and fresh registry registration. - log("Preserving MCP-bearing registry entry across sandbox recreation"); - } - if (hasBaselineExclusions) { - // Baseline exclusions are also registry-only rebuild intent. Keep the row - // until inner onboard snapshots it and replacement registration atomically - // publishes the fresh row. - log("Preserving baseline-exclusion registry entry across sandbox recreation"); - } + const removalReceipt: registry.SandboxRemovalReceipt | null = null; + // The journaled source row is the durable replacement transaction. The inner + // onboard run observes that the sandbox is absent, carries the recorded state + // into the replacement registration, and never enters generic live + // recreation. Keeping it here closes every process-death window between + // successful delete and fresh registry registration. + log("Preserving journaled source registry entry across sandbox recreation"); log( - `Registry after remove: ${JSON.stringify(registry.listSandboxes().sandboxes.map((s: { name: string }) => s.name))}`, + `Registry after delete: ${JSON.stringify(registry.listSandboxes().sandboxes.map((s: { name: string }) => s.name))}`, ); console.log(` ${G}\u2713${R} Old sandbox deleted`); diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index eb951816065..1a1bc41eac5 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -43,6 +43,9 @@ export type RebuildGpuOptOutEntry = { observabilityEnabled?: boolean; policyTier?: string | null; endpointSource?: InferenceEndpointSource | null; + provider?: string | null; + model?: string | null; + preferredInferenceApi?: string | null; }; // Modern source of truth is the persisted `sandboxGpuMode` string ("0" / "1" / @@ -103,6 +106,9 @@ export type RebuildRecreateOnboardOpts = { endpointSource?: InferenceEndpointSource | null; acceptThirdPartySoftware: true; agent: string | null | undefined; + recreateProvider: string | null; + recreateModel: string | null; + recreatePreferredInferenceApi: string | null; fromDockerfile: string | null; sandboxGpu: "enable" | "disable" | null; sandboxGpuDevice: string | null; @@ -110,6 +116,8 @@ export type RebuildRecreateOnboardOpts = { targetGatewayName: string; targetGatewayPort: number; onboardLockAlreadyHeld: true; + /** Target fingerprint of the replacement journal opened before deletion. */ + recreateJournalTargetIntentFingerprint?: string; preparedDcodeRebuild?: PreparedDcodeRebuildHandoff; rebuildRegistryInferenceRoute?: RebuildRouteHandoff; rebuildProviderReconfigure?: RebuildProviderReconfigureHandoff; @@ -177,6 +185,9 @@ export function buildRebuildRecreateOnboardOpts(args: { endpointSource: normalizeInferenceEndpointSource(args.sb?.endpointSource), acceptThirdPartySoftware: args.usageNoticeAccepted, agent: args.rebuildAgent, + recreateProvider: args.sb?.provider ?? null, + recreateModel: args.sb?.model ?? null, + recreatePreferredInferenceApi: args.sb?.preferredInferenceApi ?? null, fromDockerfile: args.storedFromDockerfile, sandboxGpu: gpuOverrides.sandboxGpu, sandboxGpuDevice: gpuOverrides.sandboxGpuDevice, diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index 6edc921d4f3..19eef519066 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -149,12 +149,7 @@ describe("rebuild local-provider recreation", () => { credentialEnv, setup, }) => { - vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ - status: 1, - output: "", - stdout: "", - stderr: "Error: sandbox alpha not found", - }); + let sourceDeleted = false; let harness!: RebuildFlowHarness; let setupResult: SetupResult | undefined; harness = createRebuildFlowHarness({ @@ -174,8 +169,9 @@ describe("rebuild local-provider recreation", () => { }); harness.session.provider = provider; harness.session.model = model; - harness.runOpenshellSpy.mockImplementation((args: string[]) => - args[0] === "sandbox" && args[1] === "get" + harness.runOpenshellSpy.mockImplementation((args: string[]) => { + sourceDeleted ||= args.join(" ") === "sandbox delete -g nemoclaw alpha"; + return args[0] === "sandbox" && args[1] === "get" ? { status: 1, stdout: "", @@ -185,8 +181,15 @@ describe("rebuild local-provider recreation", () => { status: args[0] === "provider" && args[1] === "get" ? 1 : 0, stdout: "", stderr: "", - }, - ); + }; + }); + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + harness.captureOpenshellSpy.mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args.map(String) : []; + return argv.join(" ") === "sandbox get -g nemoclaw alpha" && !sourceDeleted + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } + : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }; + }); await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 55998649437..d7febd9f42c 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -34,6 +34,10 @@ import { revalidatePreparedRecoveryBeforeDelete, } from "./rebuild-prepared-recovery"; import { inspectRebuildGatewayProviderRegistration } from "./rebuild-provider-preflight"; +import { + fingerprintRebuildRecreateTargetIntent, + openRebuildRecreateJournal, +} from "./rebuild-recreate-journal"; import { runRebuildRecreatePhase } from "./rebuild-recreate-phase"; import { createRebuildRegistryRollback } from "./rebuild-registry-rollback"; import { runRebuildRestorePhase } from "./rebuild-restore-phase"; @@ -214,10 +218,37 @@ async function rebuildSandboxUnlocked( return; } + const recreateJournal = openRebuildRecreateJournal({ + target: { + sandboxName, + gatewayName: recreateOptions.targetGatewayName, + gatewayPort: recreateOptions.targetGatewayPort, + }, + agentName: rebuildAgent || "openclaw", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + log, + }); + + // An earlier run of this rebuild already registered and proved the + // replacement. Retire its journal and stop before the destroy phase so a + // restart converges to that sandbox instead of deleting it. + if (recreateJournal.acceptedTarget) { + recreateJournal.completeAcceptedTarget(); + log(`Recovered journaled replacement ${recreateJournal.id} for '${sandboxName}'`); + console.log( + ` Sandbox '${sandboxName}' already holds the replacement from the interrupted rebuild.`, + ); + if (backup.backupManifest) { + console.log(` State backup is preserved at: ${backup.backupManifest.backupPath}`); + } + return; + } + const mcpPreparation = await runRebuildDestroyPhase({ sandboxName, sandboxEntry, staleRecovery, + recreateJournal, backupManifest: backup.backupManifest, force: normalized.force, log, @@ -277,6 +308,7 @@ async function rebuildSandboxUnlocked( durableConfig, resumeConfig, recreateOptions, + recreateJournal, fromDockerfile, rebuildAgent, messagingPlan, diff --git a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts index ff683736b44..705c43e791a 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts @@ -260,9 +260,11 @@ describe("prepared rebuild recovery", () => { ).rejects.toThrow("Recreate failed"); expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + // The journaled source row survives the delete, so no default-sandbox + // transition happened and none has to be reversed (#7734). expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), - { defaultTransition: { from: null, to: "alpha", expectedRevision: 1 } }, + {}, ); expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts new file mode 100644 index 00000000000..ca4cb6fe832 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -0,0 +1,431 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureOpenshell: vi.fn(), + resolveGatewayTeardownAuthority: vi.fn(), +})); + +vi.mock("../../adapters/openshell/runtime", () => ({ + captureOpenshell: mocks.captureOpenshell, + runOpenshell: vi.fn(), +})); + +vi.mock("../../onboard/gateway-teardown-authority", () => ({ + resolveGatewayTeardownAuthority: mocks.resolveGatewayTeardownAuthority, +})); + +import type { Session } from "../../state/onboard-session"; +import * as onboardSession from "../../state/onboard-session"; +import * as registry from "../../state/registry"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { + fingerprintRebuildRecreateTargetIntent, + observeRebuildSandbox, + openRebuildRecreateJournal, +} from "./rebuild-recreate-journal"; + +const SANDBOX_ID = "sbx-0d6f4c2a91"; + +const NON_DEFAULT_TARGET = { + sandboxName: "alpha", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, +}; + +const recreateOptions: RebuildRecreateOnboardOpts = { + resume: true, + nonInteractive: true, + recreateSandbox: true, + authoritativeResumeConfig: true, + acceptThirdPartySoftware: true, + agent: "langchain-deepagents-code", + recreateProvider: "nvidia", + recreateModel: "model-a", + recreatePreferredInferenceApi: "openai", + fromDockerfile: null, + sandboxGpu: null, + sandboxGpuDevice: null, + controlUiPort: null, + targetGatewayName: "nemoclaw-9090", + targetGatewayPort: 9090, + onboardLockAlreadyHeld: true, + autoYes: true, + toolDisclosure: "progressive", + dcodeAutoApprovalMode: "disabled", + dcodeAutoApprovalRequestedExplicitly: false, + observabilityEnabled: true, + observabilityRequestedExplicitly: true, + policyTier: "restricted", + baseImageResolutionHint: null, +}; + +function livePresentProbe(phase = "Ready") { + return { + status: 0, + output: `Name: alpha\nId: ${SANDBOX_ID}\nPhase: ${phase}\n`, + stdout: `Name: alpha\nId: ${SANDBOX_ID}\nPhase: ${phase}\n`, + stderr: "", + }; +} + +function absentProbe() { + return { + status: 1, + output: "", + stdout: "", + stderr: "Error: sandbox alpha not found", + }; +} + +describe("rebuild replacement target fingerprint", () => { + it("carries only validated non-secret rebuild inputs", () => { + const withTransientHandoffs: RebuildRecreateOnboardOpts = { + ...recreateOptions, + autoYes: false, + dcodeAutoApprovalRequestedExplicitly: true, + observabilityRequestedExplicitly: false, + preparedDcodeRebuild: { + stagingDir: "/tmp/rebuild-xyz", + } as unknown as RebuildRecreateOnboardOpts["preparedDcodeRebuild"], + }; + + expect(fingerprintRebuildRecreateTargetIntent(withTransientHandoffs)).toBe( + fingerprintRebuildRecreateTargetIntent(recreateOptions), + ); + }); + + it("changes when a recorded replacement input changes", () => { + for (const drift of [ + { dcodeAutoApprovalMode: "thread-opt-in" }, + { endpointSource: "onboard" }, + { policyTier: "balanced" }, + { recreateProvider: "compatible-endpoint" }, + { recreateModel: "model-b" }, + { recreatePreferredInferenceApi: "anthropic" }, + ] as const) { + expect(fingerprintRebuildRecreateTargetIntent({ ...recreateOptions, ...drift })).not.toBe( + fingerprintRebuildRecreateTargetIntent(recreateOptions), + ); + } + }); + + it("changes when the replacement targets another gateway", () => { + expect( + fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + }), + ).not.toBe(fingerprintRebuildRecreateTargetIntent(recreateOptions)); + }); +}); + +describe("rebuild replacement observation", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("queries only the journaled gateway", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + observeRebuildSandbox(NON_DEFAULT_TARGET); + + expect(mocks.captureOpenshell).toHaveBeenCalledTimes(1); + expect(mocks.captureOpenshell.mock.calls[0]?.[0]).toEqual([ + "sandbox", + "get", + "-g", + "nemoclaw-9090", + "alpha", + ]); + }); + + it("reports explicit absence without an identity", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + expect(observeRebuildSandbox(NON_DEFAULT_TARGET)).toEqual({ + state: "missing", + liveIdentityFingerprint: null, + }); + }); + + it("hashes a live identity instead of recording it", () => { + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + + const observation = observeRebuildSandbox(NON_DEFAULT_TARGET); + + expect(observation.state).toBe("ready"); + expect(observation.liveIdentityFingerprint).toMatch(/^[0-9a-f]{64}$/); + expect(observation.liveIdentityFingerprint).not.toContain(SANDBOX_ID); + }); + + it("separates a live sandbox that is not ready", () => { + mocks.captureOpenshell.mockReturnValue(livePresentProbe("Pending")); + + expect(observeRebuildSandbox(NON_DEFAULT_TARGET).state).toBe("not_ready"); + }); + + it("fails closed when a live sandbox has no stable identity", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nPhase: Ready\n", + stdout: "Name: alpha\nPhase: Ready\n", + stderr: "", + }); + + expect(() => observeRebuildSandbox(NON_DEFAULT_TARGET)).toThrow( + /did not report a stable sandbox Id/, + ); + }); + + it("fails closed when the gateway proves neither presence nor absence", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 1, + output: "", + stdout: "", + stderr: "Error: connection refused", + }); + + expect(() => observeRebuildSandbox(NON_DEFAULT_TARGET)).toThrow( + /neither a live sandbox nor explicit absence/, + ); + }); +}); + +describe("rebuild replacement journal", () => { + let session: Session; + + beforeEach(() => { + vi.clearAllMocks(); + session = onboardSession.createSession({ sandboxName: "alpha" }); + vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); + vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator) => { + session = mutator(session) ?? session; + return session; + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + } as registry.SandboxEntry); + mocks.resolveGatewayTeardownAuthority.mockReturnValue({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + function open() { + return openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + log: vi.fn(), + }); + } + + it("binds a secret-free replacement to the non-default gateway before deletion", () => { + const journal = open(); + + const recorded = session.checkpoint?.sandboxRecreate; + expect(recorded?.phase).toBe("planned"); + expect(recorded?.gatewayName).toBe("nemoclaw-9090"); + expect(recorded?.gatewayPort).toBe(9090); + expect(recorded?.sourceLiveIdentityFingerprint).toMatch(/^[0-9a-f]{64}$/); + expect(recorded?.targetIntentFingerprint).toBe(journal.targetIntentFingerprint); + expect(JSON.stringify(session.checkpoint)).not.toContain(SANDBOX_ID); + }); + + it("selects the exact gateway authority the journal records", () => { + open(); + + expect(mocks.resolveGatewayTeardownAuthority).toHaveBeenCalledWith({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + }); + const authority = session.checkpoint?.gatewayAuthority; + expect(authority?.kind).toBe("selected"); + expect(authority?.kind === "selected" && authority.value.gatewayPort).toBe(9090); + }); + + it("starts at deleted when the source sandbox is already absent", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + open(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + expect(session.checkpoint?.sandboxRecreate?.sourceLiveIdentityFingerprint).toBeNull(); + }); + + it("records the delete boundary before and after the destructive command", () => { + const journal = open(); + + journal.markDeleting(); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + + mocks.captureOpenshell.mockReturnValue(absentProbe()); + journal.confirmDeleted(); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + }); + + it("keeps a proven deletion when a recovery rebuild deletes an already absent source", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + const journal = open(); + + journal.markDeleting(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + }); + + it("stops before the next mutation when the source outlives its delete", () => { + const journal = open(); + journal.markDeleting(); + + expect(() => journal.confirmDeleted()).toThrow( + /OpenShell still reports the journaled source after delete/, + ); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + }); + + it("refuses to resume when a different same-name sandbox holds the name", () => { + open(); + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nId: sbx-replaced-by-another\nPhase: Ready\n", + stdout: "Name: alpha\nId: sbx-replaced-by-another\nPhase: Ready\n", + stderr: "", + }); + + expect(() => open()).toThrow(/no longer has the journaled source identity/); + }); + + it("refuses to resume a journal that targets another replacement", () => { + open(); + + expect(() => + openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + dcodeAutoApprovalMode: "thread-opt-in", + }), + log: vi.fn(), + }), + ).toThrow(/different recreate transaction in progress/); + }); + + it("refuses to resume a journal whose endpoint provenance changed (#7734)", () => { + open(); + + expect(() => + openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + endpointSource: "onboard", + }), + log: vi.fn(), + }), + ).toThrow(/different recreate transaction in progress/); + }); + + it.each([ + { recreateProvider: "compatible-endpoint" }, + { recreateModel: "model-b" }, + { recreatePreferredInferenceApi: "anthropic" }, + ] as const)("refuses to resume a journal whose inference route changed (#7734)", (drift) => { + open(); + + expect(() => + openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + ...drift, + }), + log: vi.fn(), + }), + ).toThrow(/different recreate transaction in progress/); + }); + + it("resumes the same replacement without restarting its generation", () => { + const first = open(); + + const second = open(); + + expect(second.id).toBe(first.id); + expect(second.targetGeneration).toBe(first.targetGeneration); + }); + + function proveReplacement(targetGeneration: string): string { + const identity = session.checkpoint?.sandboxRecreate?.sourceLiveIdentityFingerprint ?? ""; + onboardSession.updateSession((current) => { + const checkpoint = current.checkpoint as NonNullable; + current.checkpoint = { + ...checkpoint, + sandboxRecreate: { + ...(checkpoint.sandboxRecreate as NonNullable), + phase: "registry_committing", + targetLiveIdentityFingerprint: identity, + }, + }; + return current; + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + lifecycleGeneration: targetGeneration, + lifecycleLiveIdentityFingerprint: identity, + } as registry.SandboxEntry); + return identity; + } + + it("reports a registered ready replacement as the proven target (#7734)", () => { + const first = open(); + proveReplacement(first.targetGeneration); + + const resumed = open(); + + expect(first.acceptedTarget).toBe(false); + expect(resumed.acceptedTarget).toBe(true); + expect(resumed.id).toBe(first.id); + }); + + it("retires the journal of a proven replacement instead of deleting it again (#7734)", () => { + const first = open(); + proveReplacement(first.targetGeneration); + const resumed = open(); + + resumed.completeAcceptedTarget(); + + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + + it("refuses to retire a journal whose replacement is not proven (#7734)", () => { + const journal = open(); + + expect(() => journal.completeAcceptedTarget()).toThrow( + /cannot be retired before its replacement is proven/, + ); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("planned"); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts new file mode 100644 index 00000000000..bcfda70d0db --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { checkpointGatewayAuthority } from "../../onboard/gateway-authority-checkpoint"; +import { resolveGatewayTeardownAuthority } from "../../onboard/gateway-teardown-authority"; +import { + observeSandboxOnGateway, + type SandboxRecreateObserver, + type SandboxRecreateTarget, +} from "../../onboard/sandbox-recreate-probe"; +import { + advanceSandboxRecreateTransaction, + beginSandboxRecreateTransaction, + clearCompletedSandboxRecreateTransaction, + fingerprintSandboxRecreateValue, + planSandboxRecreateRecovery, + type SandboxRecreateSourcePresence, + sandboxRecreatePhaseReached, +} from "../../onboard/sandbox-recreate-transaction"; +import { decisionSelected } from "../../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; +import type { CheckpointSandboxRecreatePhase } from "../../state/onboard-checkpoint-types"; +import * as onboardSession from "../../state/onboard-session"; +import * as registry from "../../state/registry"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; + +export type RebuildRecreateJournalTarget = SandboxRecreateTarget; + +export type RebuildSandboxObserver = SandboxRecreateObserver; + +export type RebuildRecreateSourcePresence = SandboxRecreateSourcePresence; + +export interface RebuildRecreateJournal { + readonly id: string; + readonly acceptedTarget: boolean; + readonly sourceConfirmedAbsent: boolean; + readonly targetGeneration: string; + readonly targetIntentFingerprint: string; + markDeleting(): void; + observeSourceForDelete(): RebuildRecreateSourcePresence; + confirmDeleted(): void; + completeAcceptedTarget(): void; +} + +export function fingerprintRebuildRecreateTargetIntent( + options: Pick< + RebuildRecreateOnboardOpts, + | "agent" + | "endpointSource" + | "recreateProvider" + | "recreateModel" + | "recreatePreferredInferenceApi" + | "fromDockerfile" + | "sandboxGpu" + | "sandboxGpuDevice" + | "controlUiPort" + | "targetGatewayName" + | "targetGatewayPort" + | "toolDisclosure" + | "dcodeAutoApprovalMode" + | "observabilityEnabled" + | "policyTier" + >, +): string { + return fingerprintSandboxRecreateValue({ + version: 1, + agent: options.agent ?? null, + endpointSource: options.endpointSource ?? null, + provider: options.recreateProvider, + model: options.recreateModel, + preferredInferenceApi: options.recreatePreferredInferenceApi, + fromDockerfile: options.fromDockerfile, + sandboxGpu: options.sandboxGpu, + sandboxGpuDevice: options.sandboxGpuDevice, + controlUiPort: options.controlUiPort, + gatewayName: options.targetGatewayName, + gatewayPort: options.targetGatewayPort, + toolDisclosure: options.toolDisclosure, + dcodeAutoApprovalMode: options.dcodeAutoApprovalMode, + observabilityEnabled: options.observabilityEnabled, + policyTier: options.policyTier, + }); +} + +export const observeRebuildSandbox = observeSandboxOnGateway; + +export interface OpenRebuildRecreateJournalInput { + readonly target: RebuildRecreateJournalTarget; + readonly agentName: string; + readonly targetIntentFingerprint: string; + readonly log: (message: string) => void; + readonly observe?: RebuildSandboxObserver; +} + +export function openRebuildRecreateJournal( + input: OpenRebuildRecreateJournalInput, +): RebuildRecreateJournal { + const { target, agentName, targetIntentFingerprint, log } = input; + const observe = input.observe ?? observeRebuildSandbox; + const authority = resolveGatewayTeardownAuthority({ + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + }); + const sourceEntry = registry.getSandbox(target.sandboxName); + const observation = observe(target); + const active = onboardSession.loadSession()?.checkpoint?.sandboxRecreate ?? null; + const recovery = active + ? planSandboxRecreateRecovery(active, observation, sourceEntry) + : { action: "continue_delete" as const }; + if (recovery.action === "reject") { + throw new Error( + `Cannot resume sandbox '${target.sandboxName}' replacement: ${recovery.reason}.`, + ); + } + // A registered, ready same-name sandbox carrying the journaled target + // generation and identity is the replacement this rebuild already proved. + // The caller must retire the transaction instead of deleting it again. + const acceptedTarget = recovery.action === "accept_target"; + + const session = onboardSession.updateSession((current) => { + const checkpoint = current.checkpoint ?? deriveCheckpointFromSession(current); + current.checkpoint = { + ...checkpoint, + machineState: current.machine.state, + updatedAt: new Date().toISOString(), + sandboxIdentity: decisionSelected({ name: target.sandboxName, agent: agentName }), + gatewayAuthority: decisionSelected(checkpointGatewayAuthority(authority)), + }; + beginSandboxRecreateTransaction(current, { + sandboxName: target.sandboxName, + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + sourceEntry, + observation, + targetIntentFingerprint, + }); + return current; + }); + + const transaction = session.checkpoint?.sandboxRecreate; + if (!transaction) { + throw new Error( + `Sandbox '${target.sandboxName}' replacement journal could not be recorded before deletion.`, + ); + } + log( + `Journaled replacement ${transaction.id} for '${target.sandboxName}' on ${target.gatewayName}:${String(target.gatewayPort)} at phase '${transaction.phase}'`, + ); + + let phase: CheckpointSandboxRecreatePhase = transaction.phase; + const advance = (next: CheckpointSandboxRecreatePhase): void => { + onboardSession.updateSession((current) => { + phase = advanceSandboxRecreateTransaction(current, transaction.id, next).phase; + return current; + }); + }; + + return { + id: transaction.id, + acceptedTarget, + sourceConfirmedAbsent: recovery.action === "continue_create", + targetGeneration: transaction.targetGeneration, + targetIntentFingerprint: transaction.targetIntentFingerprint, + markDeleting: () => { + if (sandboxRecreatePhaseReached(phase, "deleted")) return; + advance("deleting"); + }, + observeSourceForDelete: () => { + const current = observe(target); + if (current.state === "missing") return "missing"; + if ( + !transaction.sourceLiveIdentityFingerprint || + current.liveIdentityFingerprint !== transaction.sourceLiveIdentityFingerprint + ) { + throw new Error( + `Cannot delete sandbox '${target.sandboxName}': the live same-name sandbox is not the journaled source.`, + ); + } + return "source"; + }, + confirmDeleted: () => { + if (observe(target).state !== "missing") { + throw new Error( + `Cannot continue sandbox '${target.sandboxName}' replacement: OpenShell still reports the journaled source after delete.`, + ); + } + advance("deleted"); + }, + completeAcceptedTarget: () => { + if (!acceptedTarget) { + throw new Error( + `Sandbox '${target.sandboxName}' replacement journal cannot be retired before its replacement is proven.`, + ); + } + for (const next of ["registry_committing", "completed"] as const) { + if (!sandboxRecreatePhaseReached(phase, next)) advance(next); + } + onboardSession.updateSession((current) => { + clearCompletedSandboxRecreateTransaction(current, transaction.id); + return current; + }); + }, + }; +} diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index dde431881ab..190d308898d 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -4,6 +4,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { restoreEnv } from "../../../../test/helpers/env-test-helpers"; +import { decisionSelected } from "../../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; @@ -49,6 +51,9 @@ const recreateOptions: RebuildRecreateOnboardOpts = { authoritativeResumeConfig: true, acceptThirdPartySoftware: true, agent: DCODE_AGENT, + recreateProvider: "nvidia", + recreateModel: "model-a", + recreatePreferredInferenceApi: "openai", fromDockerfile: null, sandboxGpu: null, sandboxGpuDevice: null, @@ -83,6 +88,17 @@ function makeInput(overrides: Partial = {}): RebuildR durableConfig, resumeConfig, recreateOptions, + recreateJournal: { + id: "journal-1", + acceptedTarget: false, + sourceConfirmedAbsent: false, + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + observeSourceForDelete: vi.fn(() => "source" as const), + confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), + }, fromDockerfile: null, rebuildAgent: DCODE_AGENT, messagingPlan: null, @@ -174,6 +190,63 @@ describe("runRebuildRecreatePhase handoff", () => { expect(onboardSession.loadSession()?.observabilityRequestedExplicitly).toBe(false); }); + it("carries the replacement journal and its target fingerprint into inner onboard (#7734)", async () => { + session.checkpoint = { + ...deriveCheckpointFromSession(session), + sandboxIdentity: decisionSelected({ name: "alpha", agent: DCODE_AGENT }), + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + sandboxRecreate: { + version: 1, + id: "journal-1", + revision: 3, + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sourceRegistryFingerprint: "source-registry", + sourceLiveIdentityFingerprint: "source-identity", + targetIntentFingerprint: "intent-1", + targetGeneration: "generation-1", + targetLiveIdentityFingerprint: null, + phase: "deleted", + startedAt: "2026-07-28T00:00:00.000Z", + updatedAt: "2026-07-28T00:00:01.000Z", + }, + }; + const retiredSessionId = session.sessionId; + let observedFingerprint: string | null | undefined; + let observedJournalPhase: string | undefined; + let observedCheckpointSessionId: string | undefined; + const onboardSpy = vi + .spyOn(rebuildOnboardDependencies, "onboard") + .mockImplementation(async (options) => { + observedFingerprint = options.recreateJournalTargetIntentFingerprint; + const carried = onboardSession.loadSession()?.checkpoint; + observedJournalPhase = carried?.sandboxRecreate?.phase; + observedCheckpointSessionId = carried?.sessionId; + }); + + try { + await expect(runRebuildRecreatePhase(makeInput())).resolves.toBe(true); + + expect(observedFingerprint).toBe("intent-1"); + expect(observedJournalPhase).toBe("deleted"); + expect(observedCheckpointSessionId).toBe(onboardSession.loadSession()?.sessionId); + expect(observedCheckpointSessionId).not.toBe(retiredSessionId); + expect(onboardSession.loadSession()?.checkpoint?.effectGroups).toEqual({}); + } finally { + onboardSpy.mockRestore(); + } + }); + it("pins the authoritative restricted tier during recreate and restores ambient policy input", async () => { const previousPolicyTier = process.env.NEMOCLAW_POLICY_TIER; process.env.NEMOCLAW_POLICY_TIER = "open"; diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts index 6b99bfaf3f6..69c403517d7 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-phase.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -7,6 +7,7 @@ import type { SandboxMessagingPlan } from "../../messaging"; import { markLastStartedStepFailed } from "../../onboard/exit-step-failure"; import { applyReasoningEffortEnv } from "../../onboard/reasoning-mode"; import * as shields from "../../shields"; +import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; @@ -29,6 +30,7 @@ import { restoreMcpRegistryForRebuildRetry, } from "./rebuild-mcp-phase"; import { rebuildOnboardDependencies } from "./rebuild-onboard-dependencies"; +import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; import type { RebuildRegistryRollback } from "./rebuild-registry-rollback"; import type { RebuildResumeConfig } from "./rebuild-resume-config"; import { printRebuildShieldsRecovery, type RebuildShieldsWindow } from "./rebuild-shields"; @@ -41,6 +43,7 @@ export interface RebuildRecreatePhaseInput { durableConfig: RebuildDurableConfig; resumeConfig: RebuildResumeConfig; recreateOptions: RebuildRecreateOnboardOpts; + recreateJournal: RebuildRecreateJournal; fromDockerfile: string | null; rebuildAgent: string | null; messagingPlan: SandboxMessagingPlan | null; @@ -75,6 +78,7 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): durableConfig: rebuildDurableConfig, resumeConfig, recreateOptions, + recreateJournal, fromDockerfile: storedFromDockerfile, rebuildAgent, messagingPlan: rebuildMessagingPlan, @@ -104,6 +108,7 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): ); onboardSession.updateSession((s: Session) => { + const journaledCheckpoint = s.checkpoint; Object.assign( s, onboardSession.createSession({ @@ -155,6 +160,16 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): s.toolDisclosure = rebuildDurableConfig.toolDisclosure; s.observabilityEnabled = recreateOptions.observabilityEnabled; s.observabilityRequestedExplicitly = recreateOptions.observabilityRequestedExplicitly; + // The journal outlives this reset, but the retired session owns its effect + // receipts and bindings. Rebind only the values the journal invariant needs. + s.checkpoint = journaledCheckpoint + ? { + ...deriveCheckpointFromSession(s), + sandboxIdentity: journaledCheckpoint.sandboxIdentity, + gatewayAuthority: journaledCheckpoint.gatewayAuthority, + sandboxRecreate: journaledCheckpoint.sandboxRecreate, + } + : null; return s; }); const sessionAfter = onboardSession.loadSession(); @@ -208,7 +223,10 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): const restoreRebuildBaseImageOverride = pinRebuildAgentBaseImageForRecreate(rebuildBaseImagePreflight); try { - await rebuildOnboardDependencies.onboard(recreateOptions); + await rebuildOnboardDependencies.onboard({ + ...recreateOptions, + recreateJournalTargetIntentFingerprint: recreateJournal.targetIntentFingerprint, + }); log("onboard() returned successfully"); } catch (error) { onboardFailed = true; diff --git a/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts b/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts index aa849a4c90d..cff587bff4d 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts @@ -4,11 +4,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { restoreEnv } from "../../../../test/helpers/env-test-helpers"; +import * as shields from "../../shields"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; import { rebuildOnboardDependencies } from "./rebuild-onboard-dependencies"; +import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; import { type RebuildRecreatePhaseInput, runRebuildRecreatePhase } from "./rebuild-recreate-phase"; import type { RebuildResumeConfig } from "./rebuild-resume-config"; @@ -49,6 +51,9 @@ const recreateOptions: RebuildRecreateOnboardOpts = { authoritativeResumeConfig: true, acceptThirdPartySoftware: true, agent: "openclaw", + recreateProvider: "compatible-endpoint", + recreateModel: "mock/deepseek-compatible", + recreatePreferredInferenceApi: "openai-completions", fromDockerfile: null, sandboxGpu: null, sandboxGpuDevice: null, @@ -66,6 +71,18 @@ const recreateOptions: RebuildRecreateOnboardOpts = { baseImageResolutionHint: null, }; +const recreateJournal: RebuildRecreateJournal = { + id: "11111111-1111-4111-8111-111111111111", + acceptedTarget: false, + sourceConfirmedAbsent: true, + targetGeneration: "22222222-2222-4222-8222-222222222222", + targetIntentFingerprint: "rebuild-reasoning-target", + markDeleting: vi.fn(), + observeSourceForDelete: vi.fn((): "missing" => "missing"), + confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), +}; + function makeInput(overrides: Partial = {}): RebuildRecreatePhaseInput { return { sandboxName: SANDBOX_NAME, @@ -75,6 +92,7 @@ function makeInput(overrides: Partial = {}): RebuildR durableConfig, resumeConfig: compatibleResumeConfig, recreateOptions, + recreateJournal, fromDockerfile: null, rebuildAgent: "openclaw", messagingPlan: null, @@ -110,6 +128,7 @@ describe("rebuild recreate compatible-endpoint reasoning handoff (#7940)", () => session = onboardSession.createSession({ sandboxName: SANDBOX_NAME }); vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(shields, "clearShieldsState").mockImplementation(() => undefined); vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator) => { session = mutator(session) ?? session; diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index 83472557089..1fd184b2df7 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -10,6 +10,7 @@ import * as agentDefs from "../../agent/defs"; import * as agentRuntime from "../../agent/runtime"; import * as gatewayRuntime from "../../gateway-runtime-action"; import * as nim from "../../inference/nim"; +import * as gatewayTeardownAuthority from "../../onboard/gateway-teardown-authority"; import * as sessionRecovery from "../../onboard/session-recovery"; import * as sandboxList from "../../openshell-sandbox-list"; import * as sandboxVersion from "../../sandbox/version"; @@ -91,6 +92,18 @@ describe("rebuild resume snapshot repair", () => { spies.push( vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null), vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null), + vi + .spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority") + .mockImplementation(({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + })), vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ recovered: true, before: { state: "healthy_named", status: "", gatewayInfo: "", activeGateway: null }, diff --git a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts index 000da2e2ce2..b1f643b51c8 100644 --- a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts +++ b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts @@ -8,6 +8,12 @@ const phaseMocks = vi.hoisted(() => ({ runDestroy: vi.fn(), runPreflight: vi.fn(), runShields: vi.fn(), + openRecreateJournal: vi.fn(), +})); + +vi.mock("./rebuild-recreate-journal", () => ({ + fingerprintRebuildRecreateTargetIntent: () => "intent-1", + openRebuildRecreateJournal: phaseMocks.openRecreateJournal, })); vi.mock("./rebuild-backup-phase", () => ({ @@ -44,7 +50,11 @@ describe("rebuild shields relock guard", () => { phaseMocks.runPreflight.mockResolvedValue({ sandboxEntry: { name: "alpha", customPolicies: [] }, targetConfig: { durableConfig: { webSearchConfig: null } }, - recreateOptions: { observabilityEnabled: false }, + recreateOptions: { + observabilityEnabled: false, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + }, liveState: { staleRecovery: false, staleRegistrySnapshot: null }, recoveryManifest: null, dcodePreflight: { @@ -64,6 +74,13 @@ describe("rebuild shields relock guard", () => { phaseMocks.runBackup.mockImplementation(() => { throw new Error("unexpected backup exception"); }); + phaseMocks.openRecreateJournal.mockReturnValue({ + id: "journal-1", + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + confirmDeleted: vi.fn(), + }); }); it("relocks shields when an unexpected rebuild phase exception escapes after auto-unlock (#6245)", async () => { @@ -114,7 +131,11 @@ describe("rebuild shields relock guard", () => { }, }, targetConfig: { durableConfig: { webSearchConfig: null } }, - recreateOptions: { observabilityEnabled: false }, + recreateOptions: { + observabilityEnabled: false, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + }, liveState: { staleRecovery: false, staleRegistrySnapshot: null }, recoveryManifest: null, dcodePreflight: { cleanup: cleanupDcodePreflight }, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f8d97673ddb..7b0e01c02b1 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -596,6 +596,7 @@ import { filterEnabledChannelsByAgent } from "./onboard/messaging-state"; import { getValidatedMessagingTokenByEnvKey } from "./onboard/messaging-token"; import * as ollamaFlow from "./onboard/ollama-probe-failure"; import { runOllamaStartupOrGate } from "./onboard/ollama-startup"; +import * as recreateJournal from "./onboard/onboard-recreate-journal"; import type { DockerDriverBinaryOverrides, OpenShellInstallDeps, @@ -785,7 +786,7 @@ const { getGatewayReuseSnapshot, selectNamedGatewayForReuseIfNeeded } = }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. -const { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox } = sandboxReuse.createSandboxReuseHelpers({ runCaptureOpenshell, runOpenshell, getSandboxStateFromOutputs, note }); +const { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox } = sandboxReuse.createSandboxReuseHelpers({ runCaptureOpenshell, runOpenshell, getSandboxStateFromOutputs, note, getGatewayName: () => GATEWAY_NAME }); const { executeSandboxCommandForVerification, @@ -2287,7 +2288,7 @@ async function createSandboxWithBaseImageResolution( // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { existingEntry, preservedMcpState, liveExists, effectiveToolDisclosure, toolDisclosureMigrationNeeded, toolDisclosureMigrationNote } = toolDisclosureFlow.prepareSandboxToolDisclosure(sandboxName, preparedBuildContext?.rebuildTarget?.fromDockerfile ? preparedBuildContext.stagedDockerfile : fromDockerfile, isRecreateSandbox(createIntent?.recreate), inspectSandboxForCreate, createIntent?.toolDisclosure ?? null); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const recreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); + let recreateRuntime: import("./onboard/sandbox-recreate-transaction").SandboxRecreateRuntime | recreateJournal.OwnedSandboxRecreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); const restoreReusedSandboxDashboard = (selectionVerified: boolean): void => { ({ chatUiUrl } = sandboxReuse.applyReusedSandboxDashboardState({ sandboxName, @@ -2549,16 +2550,13 @@ async function createSandboxWithBaseImageResolution( if (preservedMcpState) { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const explicitObservability = observabilityCommandFlag.explicitObservabilityFlag(createIntent?.observabilityEnabled === true, createIntent?.observabilityRequestedExplicitly === true); - console.error( - ` Sandbox '${sandboxName}' has managed MCP servers. Refusing the generic onboard recreation path.`, - ); - console.error( - ` Run \`${cliName()} ${sandboxName} rebuild --yes --tool-disclosure ${effectiveToolDisclosure}${explicitObservability ? ` ${explicitObservability}` : ""}${dcodeAutoApprovalPlan.rebuildFlag}\` so MCP providers and adapter state are preserved transactionally.`, - ); + for (const hint of recreateJournal.managedMcpRecreateRefusalHints({ sandboxName, cliName: cliName(), toolDisclosure: effectiveToolDisclosure, rebuildFlag: dcodeAutoApprovalPlan.rebuildFlag, observabilityFlag: observabilityCommandFlag.explicitObservabilityFlag(createIntent?.observabilityEnabled === true, createIntent?.observabilityRequestedExplicitly === true) })) console.error(hint); process.exit(1); } - + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (!createIntent?.recreateTransaction) recreateRuntime = recreateJournal.openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName, probeTarget.gatewayName), intent: { agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null } }); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (recreateRuntime.acceptedTarget) { if ("complete" in recreateRuntime) recreateRuntime.complete(); restoreReusedSandboxDashboard(true); return sandboxName; } const previousEntry: SandboxEntry | null = registry.getSandbox(sandboxName); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. baseImageResolutionFlow.captureBaseResolution(baseImageResolutionContext, previousEntry?.imageTag); @@ -2580,9 +2578,8 @@ async function createSandboxWithBaseImageResolution( note(` Deleting and recreating sandbox '${sandboxName}'...`); - recreateRuntime.advance("deleting"); - runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); - runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (recreateRuntime.beginDelete() === "source") { runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", "-g", recreateRuntime.journaledGatewayName ?? GATEWAY_NAME, sandboxName], { ignoreError: true }); } recreateRuntime.confirmDeleted(); if (previousEntry?.imageTag) { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. @@ -2861,6 +2858,7 @@ async function createSandboxWithBaseImageResolution( }), }, ); + if ("complete" in recreateRuntime) recreateRuntime.complete(); restoreDefaultAfterRecreate(registry.setDefault, sandboxName, sandboxWasLiveDefault); // #4614: default deferred to finalization // DNS proxy — run a forwarder in the sandbox pod so the isolated @@ -4380,6 +4378,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { authoritativeResumeConfig: opts.authoritativeResumeConfig === true, authoritativePolicyTier: opts.authoritativeResumeConfig === true ? (opts.policyTier ?? null) : undefined, + recreateJournalTargetIntentFingerprint: opts.recreateJournalTargetIntentFingerprint ?? null, resumeAgentChanged, requestedObservabilityEnabled: runtimeControlRequests.requestedObservabilityEnabled, requestedDcodeAutoApprovalMode: runtimeControlRequests.requestedDcodeAutoApprovalMode, diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index 77f509d9c83..1c3a6b3c554 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -9,6 +9,14 @@ import { recreateOpenShellDockerSandboxWithGpu, } from "./docker-gpu-patch"; +// The recreate path probes sandbox DNS through a real `docker run` when these +// stay unstubbed, which makes the rollback assertions depend on a live Docker +// daemon and registry reachability. +const offlineDnsDeps = { + detectSandboxFallbackDns: () => null, + probeContainerDns: () => ({ ok: true }), +}; + function inspectFixture(): DockerContainerInspect { return { Id: "old-container-id", @@ -88,6 +96,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, errorPhaseDebouncePolls: 1, }, ), @@ -141,6 +150,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ), ).toThrow(/Could not start GPU-enabled sandbox container/); @@ -187,6 +197,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { dockerRm: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ), ).toThrow(/Could not move original sandbox container aside/); @@ -229,6 +240,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ); } catch (error) { @@ -274,6 +286,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ); } catch (error) { @@ -326,6 +339,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, errorPhaseDebouncePolls: 1, }, ), @@ -365,6 +379,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, errorPhaseDebouncePolls: 1, }, ), diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 962e1871dba..110069f12ad 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -66,6 +66,7 @@ export interface SandboxOnboardFlowPhaseOptions< gatewayName: string; authoritativeResumeConfig?: boolean; authoritativePolicyTier?: string | null; + recreateJournalTargetIntentFingerprint?: string | null; resumeAgentChanged: boolean; requestedObservabilityEnabled?: boolean | null; requestedDcodeAutoApprovalMode?: DcodeAutoApprovalMode | null; @@ -212,6 +213,7 @@ export function createSandboxOnboardFlowPhase< gatewayName: options.gatewayName, authoritativeResumeConfig: options.authoritativeResumeConfig, authoritativePolicyTier: options.authoritativePolicyTier, + recreateJournalTargetIntentFingerprint: options.recreateJournalTargetIntentFingerprint, endpointSource: endpointProvenance.endpointSource, resumeAgentChanged: options.resumeAgentChanged, requestedObservabilityEnabled: options.requestedObservabilityEnabled, diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 16e428e3e7d..5fb54256ba5 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -132,6 +132,8 @@ export interface SandboxStateOptions< authoritativePolicyTier?: string | null; /** Endpoint source to preserve during an authoritative rebuild. */ endpointSource?: InferenceEndpointSource | null; + /** Internal rebuild target fingerprint recorded by the journal opened before deletion. */ + recreateJournalTargetIntentFingerprint?: string | null; resumeAgentChanged: boolean; requestedObservabilityEnabled?: boolean | null; requestedDcodeAutoApprovalMode?: DcodeAutoApprovalMode | null; @@ -1223,9 +1225,6 @@ class SandboxStateFlow< const sourceEntry = this.deps.getSandboxRegistryEntry(sandboxName); if (!existing && !sourceEntry) return null; const observation = this.deps.getSandboxRecreateObservation(sandboxName); - const targetIntentFingerprint = fingerprintSandboxRecreateValue( - this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), - ); const updated = this.deps.updateSession((current) => { beginSandboxRecreateTransaction(current, { sandboxName, @@ -1233,13 +1232,27 @@ class SandboxStateFlow< gatewayPort: gateway.gatewayPort, sourceEntry, observation, - targetIntentFingerprint, + targetIntentFingerprint: this.sandboxRecreateTargetIntentFingerprint( + sandboxName, + createIntent, + ), }); return current; }); return updated.checkpoint?.sandboxRecreate ?? null; } + private sandboxRecreateTargetIntentFingerprint( + sandboxName: string, + createIntent: CompleteSandboxCreateIntent, + ): string { + const journaled = this.options.recreateJournalTargetIntentFingerprint; + if (journaled) return journaled; + return fingerprintSandboxRecreateValue( + this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), + ); + } + private recordSandboxRecreatePhase( transaction: CheckpointSandboxRecreateTransaction, phase: Parameters[2], diff --git a/src/lib/onboard/onboard-recreate-journal.test.ts b/src/lib/onboard/onboard-recreate-journal.test.ts new file mode 100644 index 00000000000..04a218be278 --- /dev/null +++ b/src/lib/onboard/onboard-recreate-journal.test.ts @@ -0,0 +1,347 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureOpenshell: vi.fn(), + resolveGatewayTeardownAuthority: vi.fn(), +})); + +vi.mock("../adapters/openshell/runtime", () => ({ + captureOpenshell: mocks.captureOpenshell, + runOpenshell: vi.fn(), +})); + +vi.mock("./gateway-teardown-authority", () => ({ + resolveGatewayTeardownAuthority: mocks.resolveGatewayTeardownAuthority, +})); + +import type { Session } from "../state/onboard-session"; +import * as onboardSession from "../state/onboard-session"; +import * as registry from "../state/registry"; +import { + fingerprintOnboardRecreateTargetIntent, + type OnboardRecreateTargetIntent, + openOnboardRecreateJournal, +} from "./onboard-recreate-journal"; + +const BASE_INTENT: OnboardRecreateTargetIntent = { + agent: "openclaw", + fromDockerfile: null, + provider: "nvidia-prod", + model: "gpt-5.4", + preferredInferenceApi: "openai-completions", + sandboxGpuConfig: { sandboxGpuEnabled: false, mode: "0" }, + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + toolDisclosure: "progressive", + dcodeAutoApprovalMode: null, + observabilityEnabled: false, + policyTier: "restricted", +}; + +describe("non-resumed replacement target fingerprint (#7735)", () => { + it("is stable for the same replacement intent", () => { + expect(fingerprintOnboardRecreateTargetIntent({ ...BASE_INTENT })).toBe( + fingerprintOnboardRecreateTargetIntent(BASE_INTENT), + ); + }); + + it("changes when a recorded replacement input changes", () => { + for (const drift of [ + { observabilityEnabled: true }, + { toolDisclosure: "direct" }, + { sandboxGpuConfig: { sandboxGpuEnabled: true, mode: "all" } }, + { dcodeAutoApprovalMode: "thread-opt-in" }, + { policyTier: "balanced" }, + ]) { + expect(fingerprintOnboardRecreateTargetIntent({ ...BASE_INTENT, ...drift })).not.toBe( + fingerprintOnboardRecreateTargetIntent(BASE_INTENT), + ); + } + }); + + it("changes when the replacement targets another gateway", () => { + expect( + fingerprintOnboardRecreateTargetIntent({ + ...BASE_INTENT, + gatewayName: "nemoclaw", + gatewayPort: 8080, + }), + ).not.toBe(fingerprintOnboardRecreateTargetIntent(BASE_INTENT)); + }); +}); + +const SANDBOX_ID = "sbx-71c9a4e08b"; + +const NON_DEFAULT_TARGET = { + sandboxName: "alpha", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, +}; + +function livePresentProbe(phase = "Ready") { + const rendered = `Name: alpha\nId: ${SANDBOX_ID}\nPhase: ${phase}\n`; + return { status: 0, output: rendered, stdout: rendered, stderr: "" }; +} + +function replacementProbe(phase = "Ready") { + const rendered = `Name: alpha\nId: sbx-2f80d5a613\nPhase: ${phase}\n`; + return { status: 0, output: rendered, stdout: rendered, stderr: "" }; +} + +function absentProbe() { + return { + status: 1, + output: "", + stdout: "", + stderr: "Error: sandbox alpha not found", + }; +} + +describe("non-resumed onboard replacement journal (#7735)", () => { + let session: Session; + + beforeEach(() => { + vi.clearAllMocks(); + session = onboardSession.createSession({ sandboxName: "alpha" }); + vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); + vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator) => { + session = mutator(session) ?? session; + return session; + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + } as registry.SandboxEntry); + mocks.resolveGatewayTeardownAuthority.mockReturnValue({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + function open(intent: OnboardRecreateTargetIntent = BASE_INTENT) { + return openOnboardRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "openclaw", + intent, + note: vi.fn(), + }); + } + + it("journals an explicit recreation before the delete command runs", () => { + open(); + + const recorded = session.checkpoint?.sandboxRecreate; + expect(recorded?.phase).toBe("planned"); + expect(recorded?.sandboxName).toBe("alpha"); + expect(recorded?.targetIntentFingerprint).toBe( + fingerprintOnboardRecreateTargetIntent(BASE_INTENT), + ); + expect(recorded?.sourceLiveIdentityFingerprint).toMatch(/^[0-9a-f]{64}$/); + expect(JSON.stringify(session.checkpoint)).not.toContain(SANDBOX_ID); + }); + + it("binds the journal to the selected sandbox identity and gateway authority", () => { + open(); + + expect(mocks.resolveGatewayTeardownAuthority).toHaveBeenCalledWith({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + }); + const identity = session.checkpoint?.sandboxIdentity; + expect(identity?.kind === "selected" && identity.value).toEqual({ + name: "alpha", + agent: "openclaw", + }); + const authority = session.checkpoint?.gatewayAuthority; + expect(authority?.kind === "selected" && authority.value.gatewayPort).toBe(9090); + expect(session.checkpoint?.sandboxRecreate?.gatewayName).toBe("nemoclaw-9090"); + expect(session.checkpoint?.sandboxRecreate?.gatewayPort).toBe(9090); + }); + + it("queries only the journaled gateway so a sibling gateway is never reached", () => { + open(); + + for (const call of mocks.captureOpenshell.mock.calls) { + expect(call[0]).toEqual(["sandbox", "get", "-g", "nemoclaw-9090", "alpha"]); + } + expect(mocks.captureOpenshell).toHaveBeenCalled(); + }); + + it("journals a not-ready repair before the delete boundary", () => { + mocks.captureOpenshell.mockReturnValue(livePresentProbe("Pending")); + + open(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("planned"); + }); + + it("records the delete boundary through the returned runtime", () => { + const runtime = open(); + + runtime.advance("deleting"); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + + mocks.captureOpenshell.mockReturnValue(absentProbe()); + runtime.confirmDeleted(); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + }); + + it("stops before the next mutation when the source outlives its delete", () => { + const runtime = open(); + runtime.advance("deleting"); + + expect(() => runtime.confirmDeleted()).toThrow( + /OpenShell still reports the journaled source after delete/, + ); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + }); + + it("retires its own transaction once the replacement registry row commits", () => { + const runtime = open(); + runtime.advance("deleting"); + mocks.captureOpenshell.mockReturnValue(absentProbe()); + runtime.confirmDeleted(); + runtime.advance("creating"); + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + runtime.recordCreated(); + + runtime.complete(); + + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + + it("accepts the proven replacement instead of deleting it again (#7734)", () => { + const first = open(); + first.advance("deleting"); + mocks.captureOpenshell.mockReturnValue(absentProbe()); + first.confirmDeleted(); + first.advance("creating"); + mocks.captureOpenshell.mockReturnValue(replacementProbe()); + first.recordCreated(); + first.advance("registry_committing"); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + ...first.registrationFields, + } as registry.SandboxEntry); + + const resumed = open(); + + expect(resumed.acceptedTarget).toBe(true); + resumed.complete(); + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + + it("retires a replacement whose registration already reached completion", () => { + const first = open(); + first.advance("deleting"); + mocks.captureOpenshell.mockReturnValue(absentProbe()); + first.confirmDeleted(); + first.advance("creating"); + mocks.captureOpenshell.mockReturnValue(replacementProbe()); + first.recordCreated(); + first.advance("completed"); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + ...first.registrationFields, + } as registry.SandboxEntry); + + const resumed = open(); + + expect(resumed.acceptedTarget).toBe(true); + expect(() => resumed.complete()).not.toThrow(); + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + + it("resumes the same replacement after a restart without a resume flag", () => { + const first = open(); + first.advance("deleting"); + const firstId = session.checkpoint?.sandboxRecreate?.id; + + open(); + + expect(session.checkpoint?.sandboxRecreate?.id).toBe(firstId); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + }); + + it("refuses a replacement whose target intent changed mid-transaction", () => { + open(); + + expect(() => open({ ...BASE_INTENT, observabilityEnabled: true })).toThrow( + /different recreate transaction in progress/, + ); + }); + + it("refuses to continue when the live source identity no longer matches", () => { + open(); + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nId: sbx-000000000\nPhase: Ready\n", + stdout: "Name: alpha\nId: sbx-000000000\nPhase: Ready\n", + stderr: "", + }); + + expect(() => open()).toThrow(/no longer has the journaled source identity/); + }); + + it("refuses to journal a replacement without its source registry row", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue(null); + + expect(() => open()).toThrow(/without its source registry row/); + expect(session.checkpoint?.sandboxRecreate ?? null).toBeNull(); + }); + + it("fails closed when the gateway reports neither a live sandbox nor explicit absence", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 1, + output: "", + stdout: "", + stderr: "Error: connection refused", + }); + + expect(() => open()).toThrow(/neither a live sandbox nor explicit absence/); + expect(session.checkpoint?.sandboxRecreate ?? null).toBeNull(); + }); + + it("fails closed when the live sandbox has no stable OpenShell Id", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nPhase: Ready\n", + stdout: "Name: alpha\nPhase: Ready\n", + stderr: "", + }); + + expect(() => open()).toThrow(/did not report a stable sandbox Id/); + expect(session.checkpoint?.sandboxRecreate ?? null).toBeNull(); + }); + + it("opens at the deleted phase when the source sandbox is already gone", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + open(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + expect(session.checkpoint?.sandboxRecreate?.sourceLiveIdentityFingerprint).toBeNull(); + }); +}); diff --git a/src/lib/onboard/onboard-recreate-journal.ts b/src/lib/onboard/onboard-recreate-journal.ts new file mode 100644 index 00000000000..5c1a2cbc6d2 --- /dev/null +++ b/src/lib/onboard/onboard-recreate-journal.ts @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { decisionSelected } from "../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; +import * as onboardSession from "../state/onboard-session"; +import * as registry from "../state/registry"; +import { checkpointGatewayAuthority } from "./gateway-authority-checkpoint"; +import { resolveGatewayTeardownAuthority } from "./gateway-teardown-authority"; +import { + observeSandboxOnGateway, + type SandboxRecreateObserver, + type SandboxRecreateTarget, +} from "./sandbox-recreate-probe"; +import { + advanceSandboxRecreateTransaction, + beginSandboxRecreateTransaction, + clearCompletedSandboxRecreateTransaction, + createSandboxRecreateRuntime, + fingerprintSandboxRecreateValue, + planSandboxRecreateRecovery, + type SandboxRecreateRuntime, + sandboxRecreatePhaseReached, +} from "./sandbox-recreate-transaction"; + +export interface OnboardRecreateTargetIntent { + readonly agent: string | null; + readonly fromDockerfile: string | null; + readonly provider: string | null; + readonly model: string | null; + readonly preferredInferenceApi: string | null; + readonly sandboxGpuConfig: unknown; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly toolDisclosure: string; + readonly dcodeAutoApprovalMode: string | null; + readonly observabilityEnabled: boolean; + readonly policyTier: string | null; +} + +export function fingerprintOnboardRecreateTargetIntent( + intent: OnboardRecreateTargetIntent, +): string { + return fingerprintSandboxRecreateValue({ version: 1, ...intent }); +} + +export interface ManagedMcpRecreateRefusal { + readonly sandboxName: string; + readonly cliName: string; + readonly toolDisclosure: string; + readonly rebuildFlag: string; + readonly observabilityFlag: string | null; +} + +export function managedMcpRecreateRefusalHints(input: ManagedMcpRecreateRefusal): string[] { + const observability = input.observabilityFlag ? ` ${input.observabilityFlag}` : ""; + return [ + ` Sandbox '${input.sandboxName}' has managed MCP servers. Refusing the generic onboard recreation path.`, + ` Run \`${input.cliName} ${input.sandboxName} rebuild --yes --tool-disclosure ${input.toolDisclosure}${observability}${input.rebuildFlag}\` so MCP providers and adapter state are preserved transactionally.`, + ]; +} + +export interface OpenOnboardRecreateJournalInput { + readonly target: SandboxRecreateTarget; + readonly agentName: string; + readonly intent: OnboardRecreateTargetIntent; + readonly note: (message: string) => void; + readonly observe?: SandboxRecreateObserver; +} + +export type OwnedSandboxRecreateRuntime = SandboxRecreateRuntime & { complete(): void }; + +export function openOnboardRecreateJournal( + input: OpenOnboardRecreateJournalInput, +): OwnedSandboxRecreateRuntime { + const { target, agentName, note } = input; + const targetIntentFingerprint = fingerprintOnboardRecreateTargetIntent(input.intent); + const observe = input.observe ?? observeSandboxOnGateway; + const authority = resolveGatewayTeardownAuthority({ + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + }); + const sourceEntry = registry.getSandbox(target.sandboxName); + if (!sourceEntry) { + throw new Error( + `Cannot start sandbox '${target.sandboxName}' recreate transaction without its source registry row.`, + ); + } + const observation = observe(target); + const active = onboardSession.loadSession()?.checkpoint?.sandboxRecreate ?? null; + if (active) { + const recovery = planSandboxRecreateRecovery(active, observation, sourceEntry); + if (recovery.action === "reject") { + throw new Error( + `Cannot resume sandbox '${target.sandboxName}' replacement: ${recovery.reason}.`, + ); + } + } + + const session = onboardSession.updateSession((current) => { + const checkpoint = current.checkpoint ?? deriveCheckpointFromSession(current); + current.checkpoint = { + ...checkpoint, + machineState: current.machine.state, + updatedAt: new Date().toISOString(), + sandboxIdentity: decisionSelected({ name: target.sandboxName, agent: agentName }), + gatewayAuthority: decisionSelected(checkpointGatewayAuthority(authority)), + }; + beginSandboxRecreateTransaction(current, { + sandboxName: target.sandboxName, + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + sourceEntry, + observation, + targetIntentFingerprint, + }); + return current; + }); + + const transaction = session.checkpoint?.sandboxRecreate; + if (!transaction) { + throw new Error( + `Sandbox '${target.sandboxName}' replacement journal could not be recorded before deletion.`, + ); + } + note( + ` Journaled replacement ${transaction.id} for '${target.sandboxName}' on ${target.gatewayName}:${String(target.gatewayPort)} at phase '${transaction.phase}'.`, + ); + + const runtime = createSandboxRecreateRuntime( + // The journal just committed is authoritative for this run; do not re-read + // it through a store that may not yet observe the write. + { loadSession: () => session, updateSession: onboardSession.updateSession }, + { + id: transaction.id, + targetGeneration: transaction.targetGeneration, + targetIntentFingerprint: transaction.targetIntentFingerprint, + }, + target.sandboxName, + target.gatewayName, + sourceEntry, + (sandboxName, gatewayName) => observe({ ...target, sandboxName, gatewayName }), + note, + ); + + return { + ...runtime, + get registrationFields() { + return runtime.registrationFields; + }, + // This journal has no outer owner, so it retires its own transaction once + // the replacement registry row commits. + complete: () => { + onboardSession.updateSession((current) => { + for (const next of ["registry_committing", "completed"] as const) { + const phase = current.checkpoint?.sandboxRecreate?.phase; + if (phase && sandboxRecreatePhaseReached(phase, next)) continue; + advanceSandboxRecreateTransaction(current, transaction.id, next); + } + clearCompletedSandboxRecreateTransaction(current, transaction.id); + return current; + }); + }, + }; +} diff --git a/src/lib/onboard/sandbox-recreate-probe.ts b/src/lib/onboard/sandbox-recreate-probe.ts new file mode 100644 index 00000000000..eee61c9d3d8 --- /dev/null +++ b/src/lib/onboard/sandbox-recreate-probe.ts @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { stripAnsi } from "../adapters/openshell/client"; +import { captureOpenshell } from "../adapters/openshell/runtime"; +import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; +import { parseSandboxPhase } from "../state/gateway"; +import { + fingerprintSandboxLiveIdentity, + type SandboxRecreateObservation, +} from "./sandbox-recreate-transaction"; + +export interface SandboxRecreateTarget { + readonly sandboxName: string; + readonly gatewayName: string; + readonly gatewayPort: number; +} + +export type SandboxRecreateObserver = (target: SandboxRecreateTarget) => SandboxRecreateObservation; + +/** + * Strict absence classifier for destructive owner-gateway reconciliation. + * Bare NotFound is not sufficient because OpenShell uses it for missing + * gateways and providers as well as sandboxes. + */ +export function isExplicitMissingSandboxGatewayOutput( + output: string, + sandboxName: string, +): boolean { + const clean = stripAnsi(String(output)).replace(/\r/g, "").trim(); + const exactNoSpec = + /^(?:error:\s*)?status:\s*Internal,\s*message:\s*["']sandbox has no spec["'](?:,\s*details:\s*\[\])?(?:,\s*metadata:\s*MetadataMap\s*\{\s*\})?$/i; + if (exactNoSpec.test(clean)) return true; + // OpenShell can omit the requested name from an owner-scoped lookup. + // Require both exact structured fields so gateway/provider absence and + // transport diagnostics remain ambiguous. + const exactStructuredNotFound = + /^(?:error:\s*)?(?:×\s*)?code:\s*["']Some requested entity was not found["']\s*,\s*message:\s*["']sandbox not found["']$/i; + if (exactStructuredNotFound.test(clean)) return true; + + const escapedName = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const namedSandbox = `(?:['\"]${escapedName}['\"]|${escapedName})`; + return ( + new RegExp( + `^(?:error:\\s*)?sandbox\\s+${namedSandbox}\\s+(?:(?:is\\s+)?not\\s+(?:found|present)|does\\s+not\\s+exist)[.!]?$`, + "i", + ).test(clean) || + new RegExp(`^(?:error:\\s*)?no\\s+such\\s+sandbox\\s+${namedSandbox}[.!]?$`, "i").test(clean) + ); +} + +export function observeSandboxOnGateway(target: SandboxRecreateTarget): SandboxRecreateObservation { + const probe = captureOpenshell(["sandbox", "get", "-g", target.gatewayName, target.sandboxName], { + ignoreError: true, + includeStderr: true, + includeStreams: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const stdout = String(probe.stdout ?? (probe.status === 0 ? probe.output : "")).trim(); + const combined = `${stdout}\n${String(probe.stderr ?? probe.output ?? "")}`.trim(); + const failedCleanly = + !probe.error && !probe.signal && probe.status !== null && probe.status !== 0; + if (failedCleanly && isExplicitMissingSandboxGatewayOutput(combined, target.sandboxName)) { + return { state: "missing", liveIdentityFingerprint: null }; + } + if (probe.status === 0 && stdout.length > 0) { + const liveIdentityFingerprint = fingerprintSandboxLiveIdentity(stdout); + if (!liveIdentityFingerprint) { + throw new Error( + `Cannot journal sandbox '${target.sandboxName}' replacement: OpenShell did not report a stable sandbox Id on gateway '${target.gatewayName}'.`, + ); + } + const phase = parseSandboxPhase(combined); + return { + state: phase === "Ready" || phase === "Running" ? "ready" : "not_ready", + liveIdentityFingerprint, + }; + } + throw new Error( + `Cannot journal sandbox '${target.sandboxName}' replacement: gateway '${target.gatewayName}' reported neither a live sandbox nor explicit absence.`, + ); +} diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 2498c0f50ba..604d55655dc 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { decisionSelected } from "../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; @@ -30,6 +33,7 @@ const TX_ID = "11111111-1111-4111-8111-111111111111"; const TARGET_GENERATION = "22222222-2222-4222-8222-222222222222"; const SOURCE_ID = fingerprintSandboxRecreateValue("openshell-source-id"); const TARGET_ID = fingerprintSandboxRecreateValue("target-id"); +const FOREIGN_ID = fingerprintSandboxRecreateValue("foreign-openshell-source-id"); const TARGET_INTENT = fingerprintSandboxRecreateValue({ agent: "openclaw", provider: "nvidia", @@ -243,6 +247,99 @@ describe("sandbox recreate journal", () => { }); }); + it("proves the journaled source at the delete edge before onboarding removes it", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + let observation: SandboxRecreateObservation = { + state: "ready", + liveIdentityFingerprint: SOURCE_ID, + }; + const runtime = createSandboxRecreateRuntime( + { + loadSession: () => session, + updateSession: (mutator) => { + mutator(session); + return session; + }, + }, + { + id: TX_ID, + targetGeneration: TARGET_GENERATION, + targetIntentFingerprint: TARGET_INTENT, + }, + "alpha", + "nemoclaw-31818", + SOURCE_ENTRY, + () => observation, + () => undefined, + ); + + observation = { state: "ready", liveIdentityFingerprint: FOREIGN_ID }; + expect(() => runtime.beginDelete()).toThrow(/not the journaled source/i); + expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "planned", revision: 0 }); + + observation = { state: "ready", liveIdentityFingerprint: SOURCE_ID }; + expect(runtime.beginDelete()).toBe("source"); + expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "deleting" }); + + observation = { state: "missing", liveIdentityFingerprint: null }; + expect(runtime.beginDelete()).toBe("missing"); + expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "deleting" }); + }); + + it("scopes the delete edge to the journaled gateway, not the ambient one", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + const probedGateways: string[] = []; + const sessionStore = { + loadSession: () => session, + updateSession: (mutator: (current: typeof session) => void) => { + mutator(session); + return session; + }, + }; + const request = { + id: TX_ID, + targetGeneration: TARGET_GENERATION, + targetIntentFingerprint: TARGET_INTENT, + }; + const observe = (_sandboxName: string, gatewayName: string): SandboxRecreateObservation => { + probedGateways.push(gatewayName); + return { state: "ready", liveIdentityFingerprint: SOURCE_ID }; + }; + + const runtime = createSandboxRecreateRuntime( + sessionStore, + request, + "alpha", + "nemoclaw-31818", + SOURCE_ENTRY, + observe, + () => undefined, + ); + runtime.beginDelete(); + + expect(runtime.journaledGatewayName).toBe("nemoclaw-31818"); + expect(new Set(probedGateways)).toEqual(new Set(["nemoclaw-31818"])); + expect(() => + createSandboxRecreateRuntime( + sessionStore, + request, + "alpha", + "nemoclaw", + SOURCE_ENTRY, + observe, + () => undefined, + ), + ).toThrow(/does not match the requested replacement/i); + }); + it("recovers or rejects at every resumed-onboard mutation boundary (#6492)", () => { const session = createSession({ sandboxName: "alpha" }); beginSandboxRecreateTransaction( @@ -391,7 +488,7 @@ describe("sandbox recreate recovery", () => { planSandboxRecreateRecovery( transactionAt("planned"), { state: "ready", liveIdentityFingerprint: SOURCE_ID }, - { ...SOURCE_ENTRY, model: "changed-out-of-band" }, + { ...SOURCE_ENTRY, imageTag: "changed-out-of-band" }, ), ).toMatchObject({ action: "reject", @@ -399,6 +496,23 @@ describe("sandbox recreate recovery", () => { }); }); + it("keeps the preserved source row after the replacement reserves its inference route", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("deleted"), + { state: "missing", liveIdentityFingerprint: null }, + { + ...SOURCE_ENTRY, + pendingRouteReservation: true, + reservationSessionId: "session-9", + model: "model-b", + endpointUrl: "https://api.example.test/v1", + gatewayPort: undefined, + }, + ), + ).toEqual({ action: "continue_create" }); + }); + it("rejects a same-name live sandbox with a different source identity", () => { expect( planSandboxRecreateRecovery( @@ -461,6 +575,78 @@ describe("sandbox recreate recovery", () => { }); }); +describe("source registry fingerprint", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + it("survives the route reservation the replacement onboard writes (#1904)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recreate-journal-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("../state/registry"); + registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + agentVersion: "2026.3.11", + createdAt: ISO, + imageTag: "nemoclaw/openclaw:2026.3.11", + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://api.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-responses", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + const journaled = fingerprintSandboxRegistryEntry( + registry.getSandbox("alpha") as SandboxEntry, + ); + + expect( + registry.reserveSandboxInferenceRoute("alpha", { + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://api.example.test/v1", + endpointSource: "onboard", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-responses", + gatewayName: "nemoclaw", + reservationSessionId: "session-9", + }), + ).toBe(true); + expect(fingerprintSandboxRegistryEntry(registry.getSandbox("alpha") as SandboxEntry)).toBe( + journaled, + ); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it("changes when the row records another sandbox", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recreate-journal-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("../state/registry"); + registry.registerSandbox({ name: "alpha", agent: "openclaw", createdAt: ISO }); + const journaled = fingerprintSandboxRegistryEntry( + registry.getSandbox("alpha") as SandboxEntry, + ); + + registry.updateSandbox("alpha", { createdAt: "2026-07-28T20:00:00.000Z" }); + + expect( + fingerprintSandboxRegistryEntry(registry.getSandbox("alpha") as SandboxEntry), + ).not.toBe(journaled); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); +}); + describe("OpenShell live identity", () => { it("hashes an ANSI-decorated Id without persisting the raw identifier", () => { const output = "Name: alpha\n\u001b[32mId: openshell-source-id\u001b[0m\nState: Ready\n"; diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index 03cb17c8c48..dd61db06b78 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -40,8 +40,23 @@ export function fingerprintSandboxRecreateValue(value: unknown): string { return createHash("sha256").update(serialized).digest("hex"); } +const ROUTE_RESERVATION_FIELDS: readonly (keyof SandboxEntry)[] = [ + "pendingRouteReservation", + "reservationSessionId", + "provider", + "model", + "endpointUrl", + "endpointSource", + "credentialEnv", + "preferredInferenceApi", + "gatewayName", + "gatewayPort", +]; + export function fingerprintSandboxRegistryEntry(entry: SandboxEntry): string { - return fingerprintSandboxRecreateValue(entry); + const durable: Record = { ...entry }; + for (const field of ROUTE_RESERVATION_FIELDS) delete durable[field]; + return fingerprintSandboxRecreateValue(durable); } export function fingerprintSandboxLiveIdentity(getOutput: string): string | null { @@ -142,6 +157,13 @@ function phaseIndex(phase: CheckpointSandboxRecreatePhase): number { return ORDERED_PHASES.indexOf(phase); } +export function sandboxRecreatePhaseReached( + phase: CheckpointSandboxRecreatePhase, + target: CheckpointSandboxRecreatePhase, +): boolean { + return phaseIndex(phase) >= phaseIndex(target); +} + export function advanceSandboxRecreateTransaction( session: Session, id: string, @@ -341,14 +363,18 @@ interface SandboxRecreateSessionStore { updateSession(mutator: (session: Session) => Session | void): Session; } +export type SandboxRecreateSourcePresence = "missing" | "source"; + export interface SandboxRecreateRuntime { readonly acceptedTarget: boolean; readonly targetGeneration: string | undefined; + readonly journaledGatewayName: string | null; readonly registrationFields: Pick< SandboxEntry, "lifecycleGeneration" | "lifecycleLiveIdentityFingerprint" >; advance(phase: CheckpointSandboxRecreatePhase): void; + beginDelete(): SandboxRecreateSourcePresence; confirmDeleted(): void; recordCreated(): void; } @@ -356,8 +382,10 @@ export interface SandboxRecreateRuntime { const NO_SANDBOX_RECREATE: SandboxRecreateRuntime = { acceptedTarget: false, targetGeneration: undefined, + journaledGatewayName: null, registrationFields: {}, advance: () => undefined, + beginDelete: () => "source", confirmDeleted: () => undefined, recordCreated: () => undefined, }; @@ -368,7 +396,7 @@ export function createSandboxRecreateRuntime( sandboxName: string, gatewayName: string, registryEntry: SandboxEntry | null, - observe: (sandboxName: string) => SandboxRecreateObservation, + observe: (sandboxName: string, gatewayName: string) => SandboxRecreateObservation, note: (message: string) => void, ): SandboxRecreateRuntime { if (!request) return NO_SANDBOX_RECREATE; @@ -379,14 +407,19 @@ export function createSandboxRecreateRuntime( transactionId: request.id, targetGeneration: request.targetGeneration, }); - const advance = (phase: CheckpointSandboxRecreatePhase): void => { + let phase: CheckpointSandboxRecreatePhase = transaction.phase; + const advance = (next: CheckpointSandboxRecreatePhase): void => { sessionStore.updateSession((current) => { - advanceSandboxRecreateTransaction(current, transaction.id, phase); + phase = advanceSandboxRecreateTransaction(current, transaction.id, next).phase; return current; }); }; let targetLiveIdentityFingerprint = transaction.targetLiveIdentityFingerprint; - const recovery = planSandboxRecreateRecovery(transaction, observe(sandboxName), registryEntry); + const recovery = planSandboxRecreateRecovery( + transaction, + observe(sandboxName, transaction.gatewayName), + registryEntry, + ); if (recovery.action === "reject") { throw new Error(`Cannot resume sandbox '${sandboxName}' recreation: ${recovery.reason}.`); } @@ -401,6 +434,7 @@ export function createSandboxRecreateRuntime( return { acceptedTarget: recovery.action === "accept_target", targetGeneration: transaction.targetGeneration, + journaledGatewayName: transaction.gatewayName, get registrationFields() { return { lifecycleGeneration: transaction.targetGeneration, @@ -410,8 +444,23 @@ export function createSandboxRecreateRuntime( }; }, advance, + beginDelete: () => { + const live = observe(sandboxName, transaction.gatewayName); + if (live.state !== "missing") { + if ( + !transaction.sourceLiveIdentityFingerprint || + live.liveIdentityFingerprint !== transaction.sourceLiveIdentityFingerprint + ) { + throw new Error( + `Cannot delete sandbox '${sandboxName}': the live same-name sandbox is not the journaled source.`, + ); + } + } + if (!sandboxRecreatePhaseReached(phase, "deleted")) advance("deleting"); + return live.state === "missing" ? "missing" : "source"; + }, confirmDeleted: () => { - if (observe(sandboxName).state !== "missing") { + if (observe(sandboxName, transaction.gatewayName).state !== "missing") { throw new Error( `Cannot continue sandbox '${sandboxName}' recreation: OpenShell still reports the journaled source after delete.`, ); @@ -419,7 +468,7 @@ export function createSandboxRecreateRuntime( advance("deleted"); }, recordCreated: () => { - const observation = observe(sandboxName); + const observation = observe(sandboxName, transaction.gatewayName); sessionStore.updateSession((current) => { targetLiveIdentityFingerprint = recordSandboxRecreateTargetCreated( current, diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index dca7f55cf48..b2529e1f799 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -225,6 +225,32 @@ describe("createSandboxReuseHelpers", () => { ); }); + it("observes a resumed replacement on the gateway its journal records (#7734)", () => { + const runCaptureOpenshell = vi.fn((args: string[]) => + args[1] === "get" ? "Name: alpha\nId: openshell-source-id\nState: Ready\n" : "alpha Ready\n", + ); + const helpers = createSandboxReuseHelpers({ + runCaptureOpenshell, + runOpenshell: vi.fn(), + getSandboxStateFromOutputs: vi.fn(() => "ready"), + note: vi.fn(), + getGatewayName: () => "nemoclaw", + }); + + helpers.getSandboxRecreateObservation("alpha", "nemoclaw-9090"); + + expect(runCaptureOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "get", "-g", "nemoclaw-9090", "alpha"], + { ignoreError: true }, + ); + expect(runCaptureOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "list", "-g", "nemoclaw-9090"], + { ignoreError: true }, + ); + }); + it("preserves an unknown reuse state but rejects it for recreate recovery", () => { const helpers = createSandboxReuseHelpers({ runCaptureOpenshell: vi.fn(() => ""), diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index f75a799c75b..be6d510c42a 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -21,11 +21,16 @@ export interface SandboxReuseDeps { runOpenshell(args: string[], opts?: Record): unknown; getSandboxStateFromOutputs(sandboxName: string, getOutput: string, listOutput: string): string; note(message: string): void; + // Read at call time: onboarding rebinds the gateway after preflight resolves it. + getGatewayName?(): string; } export interface SandboxReuseHelpers { getSandboxReuseState(sandboxName: string | null): string; - getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation; + getSandboxRecreateObservation( + sandboxName: string | null, + gatewayName?: string, + ): SandboxRecreateObservation; repairRecordedSandbox(sandboxName: string | null): void; } @@ -112,18 +117,34 @@ export function applyReusedSandboxDashboardState( } export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseHelpers { - function readSandboxState(sandboxName: string | null): { state: string; getOutput: string } { + // A recorded gateway wins over the ambient one: a resumed replacement must + // prove presence or absence on the gateway its journal names. + function gatewayArgs(recordedGatewayName?: string): string[] { + const gatewayName = recordedGatewayName ?? deps.getGatewayName?.(); + return gatewayName ? ["-g", gatewayName] : []; + } + + function readSandboxState( + sandboxName: string | null, + recordedGatewayName?: string, + ): { state: string; getOutput: string } { if (!sandboxName) return { state: "missing", getOutput: "" }; - const getOutput = deps.runCaptureOpenshell(["sandbox", "get", sandboxName], { + const args = gatewayArgs(recordedGatewayName); + const getOutput = deps.runCaptureOpenshell(["sandbox", "get", ...args, sandboxName], { + ignoreError: true, + }); + const listOutput = deps.runCaptureOpenshell(["sandbox", "list", ...args], { ignoreError: true, }); - const listOutput = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); const state = deps.getSandboxStateFromOutputs(sandboxName, getOutput, listOutput); return { state, getOutput }; } - function getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation { - const { state, getOutput } = readSandboxState(sandboxName); + function getSandboxRecreateObservation( + sandboxName: string | null, + recordedGatewayName?: string, + ): SandboxRecreateObservation { + const { state, getOutput } = readSandboxState(sandboxName, recordedGatewayName); if (state !== "missing" && state !== "not_ready" && state !== "ready") { throw new Error( `Cannot observe sandbox '${sandboxName}' for recreate recovery: OpenShell returned state '${state}'.`, @@ -144,7 +165,7 @@ export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseH if (!sandboxName) return; deps.note(` [resume] Cleaning up recorded sandbox '${sandboxName}' before recreating it.`); bestEffortForwardStop(deps.runOpenshell, DASHBOARD_PORT); - deps.runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + deps.runOpenshell(["sandbox", "delete", ...gatewayArgs(), sandboxName], { ignoreError: true }); registry.removeSandbox(sandboxName); } diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index b60f238fff3..5227b6d44c6 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -92,6 +92,8 @@ export type OnboardOptions = { targetGatewayPort?: number | null; /** Internal rebuild handoff: the outer destructive lifecycle owns the onboard lock. */ onboardLockAlreadyHeld?: boolean; + /** Internal rebuild handoff: target fingerprint of the journal opened before deletion. */ + recreateJournalTargetIntentFingerprint?: string | null; /** Internal one-shot handoff for a prevalidated managed DCode replacement. */ preparedDcodeRebuild?: import("./prepared-dcode-rebuild").PreparedDcodeRebuildHandoff; /** Internal authoritative registry route captured before rebuild deletion. */ diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 730057aa899..e5e41405810 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -159,9 +159,26 @@ function mockOnboardRunCapture(command, options = {}) { return mockSandboxExecCurl(command, options); } +function mockStandaloneGatewayTeardownAuthority() { + const authority = require( + path.resolve(__dirname, "../../src/lib/onboard/gateway-teardown-authority.ts"), + ); + authority.resolveGatewayTeardownAuthority = ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); +} + module.exports = { isOpenClawSecurityInventoryProbe, mockOnboardRunCapture, mockSandboxExecCurl, + mockStandaloneGatewayTeardownAuthority, normalizeCommand, }; diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index eabf2c1fb90..972081810e9 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -53,6 +53,9 @@ const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); const rebuildManagedImage = requireDist("./rebuild-managed-image-preflight.js"); const rebuildMessagingConflict = requireDist("./rebuild-messaging-conflict-preflight.js"); const rebuildRoutePreflight = requireDist("./rebuild-preflight-guards.js"); +const gatewayTeardownAuthority = requireDist( + "../../onboard/gateway-teardown-authority.js", +) as typeof import("../../src/lib/onboard/gateway-teardown-authority"); const shields = requireDist("../../shields/index.js"); type RebuildFlowStep = { @@ -227,12 +230,32 @@ function createStep(status: string): RebuildFlowStep { return { status, startedAt: null, completedAt: null, error: null }; } +function sourceSandboxGateway(argv: string[], verb: string): string | null { + const gatewayFlag = argv.indexOf("-g"); + return argv[0] === "sandbox" && argv[1] === verb && argv.at(-1) === "alpha" && gatewayFlag > 0 + ? (argv[gatewayFlag + 1] ?? null) + : null; +} + function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSession { return { + sessionId: "rebuild-flow-session", + updatedAt: "2026-06-01T00:00:00.000Z", sandboxName: "alpha", + agent: null, provider: "ollama-local", model: "nvidia/nemotron", credentialEnv: null, + checkpoint: null, + webSearchConfig: null, + resourceProfile: null, + messagingPlan: null, + sandboxPromptProgress: { + sandboxName: true, + webSearch: false, + messaging: false, + resourceProfile: false, + }, metadata: {}, hermesToolGateways: [], lastStepStarted: null, @@ -312,6 +335,18 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + vi.spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority").mockImplementation( + ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { status: 0, output: overrides.sandboxListOutput ?? "alpha Ready" }, }); @@ -581,8 +616,13 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .spyOn(openshellRuntime, "captureOpenshell") .mockImplementation((args: unknown, options?: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; - return overrides.captureOpenshell - ? overrides.captureOpenshell(argv, options as Record | undefined) + if (overrides.captureOpenshell) { + return overrides.captureOpenshell(argv, options as Record | undefined); + } + const probedGateway = sourceSandboxGateway(argv, "get"); + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + return probedGateway && !deletedSourceGateways.has(probedGateway) + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, output: "", @@ -590,8 +630,14 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): stderr: "Error: sandbox alpha not found", }; }); + const deletedSourceGateways = new Set(); const runOpenshellSpy = vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((args) => { const argv = args as string[]; + const deleteGateway = sourceSandboxGateway(argv, "delete"); + if (deleteGateway) { + deletedSourceGateways.add(deleteGateway); + return { status: 0, output: "" }; + } if ( argv.join(" ") === "sandbox get alpha" || argv.join(" ") === "sandbox get -g nemoclaw alpha" diff --git a/test/helpers/rebuild-flow-lifecycle-cases.ts b/test/helpers/rebuild-flow-lifecycle-cases.ts index 24fb71533b7..a91e725a38b 100644 --- a/test/helpers/rebuild-flow-lifecycle-cases.ts +++ b/test/helpers/rebuild-flow-lifecycle-cases.ts @@ -113,7 +113,7 @@ export function registerRebuildFlowLifecycleTests(): void { expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Preserving MCP-bearing registry entry across sandbox recreation", + "Preserving journaled source registry entry across sandbox recreation", ); expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "npm"); expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "bad"); @@ -178,7 +178,7 @@ export function registerRebuildFlowLifecycleTests(): void { expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.onboardSpy).toHaveBeenCalledOnce(); expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Preserving baseline-exclusion registry entry across sandbox recreation", + "Preserving journaled source registry entry across sandbox recreation", ); expect(harness.restoreSandboxEntrySpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); @@ -257,7 +257,7 @@ network_policies: expect(harness.restoreSandboxEntrySpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Preserving baseline-exclusion registry entry across sandbox recreation", + "Preserving journaled source registry entry across sandbox recreation", ); }); @@ -267,7 +267,7 @@ network_policies: const probeSequence = [ { event: "stale-live", - result: { status: 0, output: "Sandbox: alpha\nPhase: Ready" }, + result: { status: 0, output: "Sandbox: alpha\nId: sbx-0d6f4c2a91\nPhase: Ready" }, }, { event: "absent", @@ -290,12 +290,14 @@ network_policies: harness.rebuildSandbox("alpha", ["--yes", "--verbose"], { throwOnError: true }), ).resolves.toBeUndefined(); - expect(events).toEqual(["stale-live", "absent", "onboard"]); + // Open the journal against the live source, wait for absence, then prove + // absence once more before the journal records the deleted phase (#7734). + expect(events).toEqual(["stale-live", "absent", "absent", "onboard"]); expect( harness.captureOpenshellSpy.mock.calls.filter( ([args]) => Array.isArray(args) && args.join(" ") === "sandbox get -g nemoclaw alpha", ), - ).toHaveLength(2); + ).toHaveLength(3); }); it("accepts the agent version cached by the confirmation probe before lock acquisition", async () => { @@ -402,18 +404,18 @@ network_policies: } }); - it("relocks as absent when registry cleanup throws after confirmed delete", async () => { + it("relocks as absent and keeps the journaled row when replacement creation fails (#7734)", async () => { const harness = createRebuildFlowHarness({ - removeSandboxRegistryEntryWithReceipt: () => { - throw new Error("registry cleanup after delete failed"); + onboard: () => { + throw new Error("recreate failed"); }, }); await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("registry cleanup after delete failed"); + ).rejects.toThrow("Recreate failed"); - expect(harness.onboardSpy).not.toHaveBeenCalled(); + expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.relockSpy).toHaveBeenLastCalledWith( "alpha", expect.any(Object), diff --git a/test/helpers/rebuild-flow-recovery-cases.ts b/test/helpers/rebuild-flow-recovery-cases.ts index 1aba1431732..f0b96afff9f 100644 --- a/test/helpers/rebuild-flow-recovery-cases.ts +++ b/test/helpers/rebuild-flow-recovery-cases.ts @@ -8,6 +8,7 @@ import { makeActiveTeamsMessagingPlan, makePreparedRecoveryManifest, } from "../../src/lib/actions/sandbox/rebuild-flow-test-fixtures"; +import { fingerprintSandboxLiveIdentity } from "../../src/lib/onboard/sandbox-recreate-transaction"; import { expectNoSandboxDelete } from "./rebuild-delete-assertions"; import { createRebuildFlowHarness, installRebuildFlowTestHooks } from "./rebuild-flow-test-harness"; @@ -201,28 +202,16 @@ export function registerRebuildFlowRecoveryTests(): void { expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), - { - defaultTransition: { - from: null, - to: "alpha", - expectedRevision: 11, - }, - }, + {}, ); expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); }); - it("preserves an explicit same-fallback default choice during prepared rollback", async () => { + it("keeps an explicit default choice made while the replacement was in flight (#7734)", async () => { let harness!: ReturnType; harness = createRebuildFlowHarness({ defaultSandbox: "alpha", defaultSelectionRevision: 10, - removalReceipt: { - entry: { name: "alpha", agentVersion: "0.1.0" }, - wasDefault: true, - fallbackDefault: "beta", - postRemovalDefaultSelectionRevision: 11, - }, onboard: () => { expect(harness.setDefault("beta")).toBe(true); throw new Error("recreate failed after explicit default choice"); @@ -238,35 +227,23 @@ export function registerRebuildFlowRecoveryTests(): void { expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha" }), - { - defaultTransition: { - from: "beta", - to: "alpha", - expectedRevision: 11, - }, - }, + {}, ); expect(harness.getDefaultSelectionState()).toEqual({ defaultSandbox: "beta", - defaultSelectionRevision: 12, + defaultSelectionRevision: 11, }); }); - it("preserves replacement registry metadata after a custom removal receipt", async () => { + it("keeps the default selection untouched across a journaled replacement (#7734)", async () => { let harness!: ReturnType; harness = createRebuildFlowHarness({ defaultSandbox: "alpha", defaultSelectionRevision: 10, - removeSandboxRegistryEntryWithReceipt: () => ({ - entry: { name: "alpha", model: "old-model" }, - wasDefault: true, - fallbackDefault: "beta", - postRemovalDefaultSelectionRevision: 11, - }), onboard: () => { expect(harness.getDefaultSelectionState()).toEqual({ - defaultSandbox: "beta", - defaultSelectionRevision: 11, + defaultSandbox: "alpha", + defaultSelectionRevision: 10, }); harness.registerSandboxEntry("alpha"); throw new Error("recreate failed after replacement registration"); @@ -277,14 +254,210 @@ export function registerRebuildFlowRecoveryTests(): void { harness.rebuildSandbox("alpha", ["--yes", "--verbose"], { throwOnError: true }), ).rejects.toThrow("Recreate failed"); - expect(harness.restoreSandboxEntryIfMissingSpy).toHaveReturnedWith(false); + expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); + expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); expect(harness.getDefaultSelectionState()).toEqual({ - defaultSandbox: "beta", - defaultSelectionRevision: 11, + defaultSandbox: "alpha", + defaultSelectionRevision: 10, + }); + }); + + const REPLACEMENT_PROBE = "Name: alpha\nId: sbx-replacement\nPhase: Ready\n"; + const FOREIGN_PROBE = "Name: alpha\nId: sbx-foreign\nPhase: Ready\n"; + const REPLACEMENT_IDENTITY = fingerprintSandboxLiveIdentity(REPLACEMENT_PROBE); + const POST_DELETE_PHASES = ["creating", "created", "registry_committing", "completed"] as const; + const provenReplacement = { + sandboxEntry: { + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: REPLACEMENT_IDENTITY, + }, + }; + + // Journal a replacement through the real pipeline, then die at the given + // post-delete phase so the restart reads persisted state, not a hand-built + // checkpoint. + async function interruptAfterCreate(phase: string): Promise { + const interrupted = createRebuildFlowHarness({ + ...provenReplacement, + onboard: (session) => { + Object.assign( + (session.checkpoint as { sandboxRecreate: Record }).sandboxRecreate, + { + phase, + targetGeneration: "generation-1", + targetLiveIdentityFingerprint: REPLACEMENT_IDENTITY, + }, + ); + throw new Error("interrupted after replacement creation"); + }, + }); + await expect( + interrupted.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recreate failed"); + return interrupted.session.checkpoint; + } + + function restartRebuild(probe: string, checkpoint: unknown) { + const restarted = createRebuildFlowHarness({ + ...provenReplacement, + captureOpenshell: (argv) => + argv[0] === "sandbox" && argv[1] === "get" + ? { status: 0, output: probe, stdout: probe, stderr: "" } + : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }, + }); + restarted.session.checkpoint = checkpoint; + // Both harnesses share one spy per mocked module function, so the + // interrupted run's calls have to be dropped before the restart. + restarted.runOpenshellSpy.mockClear(); + restarted.onboardSpy.mockClear(); + return restarted; + } + + it.each( + POST_DELETE_PHASES, + )("keeps a registered replacement when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartRebuild(REPLACEMENT_PROBE, await interruptAfterCreate(phase)); + + await restarted.rebuildSandbox("alpha", ["--yes"]); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + expect( + (restarted.session.checkpoint as { sandboxRecreate: unknown }).sandboxRecreate, + ).toBeNull(); + }); + + it.each( + POST_DELETE_PHASES, + )("refuses a foreign same-name sandbox when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartRebuild(FOREIGN_PROBE, await interruptAfterCreate(phase)); + + await expect(restarted.rebuildSandbox("alpha", ["--yes"])).rejects.toThrow( + /not the journaled replacement/, + ); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + }); + + const SOURCE_PROBE = "Name: alpha\nId: sbx-source\nPhase: Ready\n"; + const MISSING_SOURCE = { + status: 1, + output: "", + stdout: "", + stderr: "Error: sandbox alpha not found", + }; + const PRE_CREATE_PHASES = ["planned", "deleting", "deleted"] as const; + const LIVE_SOURCE_PHASES = ["planned", "deleting"] as const; + + function sandboxGetProbes(probes: readonly (string | null)[]) { + let gets = 0; + return (argv: string[]) => { + if (argv[0] !== "sandbox" || argv[1] !== "get") return MISSING_SOURCE; + const probe = probes[Math.min(gets++, probes.length - 1)]; + return probe ? { status: 0, output: probe, stdout: probe, stderr: "" } : MISSING_SOURCE; + }; + } + + async function interruptBeforeCreate(phase: string): Promise { + const interrupted = createRebuildFlowHarness({ + captureOpenshell: sandboxGetProbes([SOURCE_PROBE, null]), + onboard: (session) => { + Object.assign( + (session.checkpoint as { sandboxRecreate: Record }).sandboxRecreate, + { phase }, + ); + throw new Error("interrupted before replacement creation"); + }, + }); + await expect( + interrupted.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recreate failed"); + return interrupted.session.checkpoint; + } + + function restartFromJournaledSource(probes: readonly (string | null)[], checkpoint: unknown) { + const restarted = createRebuildFlowHarness({ + captureOpenshell: sandboxGetProbes(probes), }); - expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Recreate failed: kept the replacement registry metadata already present", + restarted.session.checkpoint = checkpoint; + restarted.runOpenshellSpy.mockClear(); + restarted.onboardSpy.mockClear(); + return restarted; + } + + it.each( + LIVE_SOURCE_PHASES, + )("deletes the journaled source when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartFromJournaledSource( + [SOURCE_PROBE, SOURCE_PROBE, null], + await interruptBeforeCreate(phase), ); + + await restarted.rebuildSandbox("alpha", ["--yes"]); + + expect(restarted.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(restarted.onboardSpy).toHaveBeenCalled(); + }); + + it.each( + PRE_CREATE_PHASES, + )("creates the replacement without a second delete when a rebuild restarts from '%s' with the source already absent (#7734)", async (phase) => { + const restarted = restartFromJournaledSource([null], await interruptBeforeCreate(phase)); + + await restarted.rebuildSandbox("alpha", ["--yes"]); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).toHaveBeenCalled(); + }); + + it.each( + LIVE_SOURCE_PHASES, + )("stops before deletion when a same-name sandbox appears after a '%s' restart probe (#7734)", async (phase) => { + const restarted = restartFromJournaledSource( + [null, FOREIGN_PROBE], + await interruptBeforeCreate(phase), + ); + + await expect( + restarted.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow(/the live same-name sandbox is not the journaled source/); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + }); + + it.each( + LIVE_SOURCE_PHASES, + )("refuses a changed same-name sandbox when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartFromJournaledSource( + [FOREIGN_PROBE, null], + await interruptBeforeCreate(phase), + ); + + await expect(restarted.rebuildSandbox("alpha", ["--yes"])).rejects.toThrow( + /no longer has the journaled source identity/, + ); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + }); + + it("refuses a live same-name sandbox when a rebuild restarts from 'deleted' (#7734)", async () => { + const restarted = restartFromJournaledSource( + [SOURCE_PROBE, null], + await interruptBeforeCreate("deleted"), + ); + + await expect(restarted.rebuildSandbox("alpha", ["--yes"])).rejects.toThrow( + /appeared before replacement registration committed/, + ); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); }); it("performs exactly one prepared-recovery rollback when MCP state is present", async () => { @@ -343,13 +516,7 @@ export function registerRebuildFlowRecoveryTests(): void { expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha", toolDisclosure: "progressive" }), - { - defaultTransition: { - from: null, - to: "alpha", - expectedRevision: 11, - }, - }, + {}, ); expect(harness.errorSpy).toHaveBeenCalledWith( expect.stringContaining("onboard --resume --name alpha --tool-disclosure direct"), diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index b7adc2391f2..b12300593a1 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -33,6 +33,9 @@ const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); const dockerInspect = requireDist("../../adapters/docker/inspect.js"); const sandboxList = requireDist("../../openshell-sandbox-list.js"); const resolve = requireDist("../../adapters/openshell/resolve.js"); +const gatewayTeardownAuthority = requireDist( + "../../onboard/gateway-teardown-authority.js", +) as typeof import("../../src/lib/onboard/gateway-teardown-authority"); const agentDefs = requireDist("../../agent/defs.js"); const agentRuntime = requireDist("../../agent/runtime.js"); const { rebuildOnboardDependencies } = requireDist("./rebuild-onboard-dependencies.js"); @@ -61,6 +64,13 @@ const mcpBridge = requireDist("./mcp-bridge.js"); const messaging = requireDist("../../messaging/index.js"); const shields = requireDist("../../shields/index.js"); +function sourceSandboxGateway(argv: string[], verb: string): string | null { + const gatewayFlag = argv.indexOf("-g"); + return argv[0] === "sandbox" && argv[1] === verb && argv.at(-1) === "alpha" && gatewayFlag > 0 + ? (argv[gatewayFlag + 1] ?? null) + : null; +} + export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): RebuildFlowHarness { delete require.cache[requireDist.resolve(rebuildModulePath)]; @@ -86,6 +96,18 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + vi.spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority").mockImplementation( + ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { status: 0, @@ -424,12 +446,18 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): failedFiles: [], })), ); + const deletedSourceGateways = new Set(); const runOpenshellSpy = vi .spyOn(openshellRuntime, "runOpenshell") .mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; const overrideResult = overrides.runOpenshell?.(argv); if (overrideResult) return overrideResult; + const deleteGateway = sourceSandboxGateway(argv, "delete"); + if (deleteGateway) { + deletedSourceGateways.add(deleteGateway); + return { status: 0, output: "" }; + } if ( argv.join(" ") === "sandbox get alpha" || argv.join(" ") === "sandbox get -g nemoclaw alpha" @@ -447,8 +475,13 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .spyOn(openshellRuntime, "captureOpenshell") .mockImplementation((args: unknown, options?: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; - return overrides.captureOpenshell - ? overrides.captureOpenshell(argv, options as Record | undefined) + if (overrides.captureOpenshell) { + return overrides.captureOpenshell(argv, options as Record | undefined); + } + const probedGateway = sourceSandboxGateway(argv, "get"); + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + return probedGateway && !deletedSourceGateways.has(probedGateway) + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, output: "", stderr: "Error: sandbox alpha not found" }; }); const defaultRemovalReceipt = { diff --git a/test/helpers/rebuild-flow-test-support.ts b/test/helpers/rebuild-flow-test-support.ts index 4082a0adf21..e1ea84f8ad7 100644 --- a/test/helpers/rebuild-flow-test-support.ts +++ b/test/helpers/rebuild-flow-test-support.ts @@ -180,10 +180,23 @@ function createStep(status: string): RebuildFlowStep { } export function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSession { return { + sessionId: "rebuild-flow-session", + updatedAt: "2026-06-01T00:00:00.000Z", sandboxName: "alpha", + agent: null, provider: "ollama-local", model: "nvidia/nemotron", credentialEnv: null, + checkpoint: null, + webSearchConfig: null, + resourceProfile: null, + messagingPlan: null, + sandboxPromptProgress: { + sandboxName: true, + webSearch: false, + messaging: false, + resourceProfile: false, + }, metadata: {}, hermesToolGateways: [], lastStepStarted: null, diff --git a/test/mcp-destroy-lifecycle.test.ts b/test/mcp-destroy-lifecycle.test.ts index b47fae403ac..a4ce5170023 100644 --- a/test/mcp-destroy-lifecycle.test.ts +++ b/test/mcp-destroy-lifecycle.test.ts @@ -91,8 +91,23 @@ vi.mock("../src/lib/inference/nim", () => ({ import * as bridge from "../src/lib/actions/sandbox/mcp-bridge"; import { isAgentMcpAdapter } from "../src/lib/actions/sandbox/mcp-bridge-contracts"; import { runRebuildDestroyPhase } from "../src/lib/actions/sandbox/rebuild-destroy-phase"; +import type { RebuildRecreateJournal } from "../src/lib/actions/sandbox/rebuild-recreate-journal"; import * as registry from "../src/lib/state/registry"; +function stubRecreateJournal(): RebuildRecreateJournal { + return { + id: "journal-1", + acceptedTarget: false, + sourceConfirmedAbsent: false, + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + observeSourceForDelete: vi.fn(() => "source" as const), + confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), + }; +} + const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.85"); const bridgeEntries: Record<"github" | "slack", McpBridgeEntry> = { @@ -872,6 +887,7 @@ describe("authenticated MCP sandbox destroy lifecycle", () => { sandboxName: "alpha", sandboxEntry: before ?? { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -948,6 +964,7 @@ describe("authenticated MCP sandbox destroy lifecycle", () => { sandboxName: "alpha", sandboxEntry: beforeRegistry ?? { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), diff --git a/test/onboard-custom-dockerfile.test.ts b/test/onboard-custom-dockerfile.test.ts index bcd6b641d59..1a3c464cc87 100644 --- a/test/onboard-custom-dockerfile.test.ts +++ b/test/onboard-custom-dockerfile.test.ts @@ -232,7 +232,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -374,7 +374,7 @@ const destructive = []; const sandboxLive = process.env.SANDBOX_LIVE === "1"; const capture = (command) => { const text = Array.isArray(command) ? command.join(" ") : String(command); - if (/sandbox get my-assistant/.test(text)) return sandboxLive ? "my-assistant Ready" : ""; + if (/sandbox get(?: -g \S+)? my-assistant/.test(text)) return sandboxLive ? "my-assistant Ready" : ""; if (/sandbox list/.test(text)) return sandboxLive ? "my-assistant Ready" : ""; if (/forward list/.test(text)) return ""; return ""; diff --git a/test/onboard-extra-provider-reconciliation.test.ts b/test/onboard-extra-provider-reconciliation.test.ts index f0c6ce4d98c..266b781b127 100644 --- a/test/onboard-extra-provider-reconciliation.test.ts +++ b/test/onboard-extra-provider-reconciliation.test.ts @@ -77,7 +77,7 @@ runner.run = (command, opts = {}) => { }; runner.runCapture = (command) => { const normalized = _n(command); - if (normalized.includes("sandbox get my-assistant")) return ""; + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; if (normalized.includes("sandbox list")) return "my-assistant Ready"; const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index fe64a99582a..240bba273f9 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -37,6 +37,7 @@ describe("createSandbox installer restore intent", () => { const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); @@ -46,6 +47,7 @@ const { EventEmitter } = require("node:events"); const PRE_UPGRADE_BACKUP = "/tmp/fake-pre-upgrade-backup"; const events = []; let sandboxDeleted = false; +let sandboxRecreated = false; runner.run = (command) => { const cmd = _n(command); events.push({ kind: "run", cmd }); @@ -54,9 +56,9 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -107,6 +109,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + sandboxRecreated = true; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -245,6 +248,7 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); @@ -257,7 +261,7 @@ runner.run = (command) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return "my-assistant NotReady"; // Keep dashboard allocation inside this restore-intent fixture; host port // occupancy is unrelated to the not-ready decision under test. diff --git a/test/onboard-messaging.test.ts b/test/onboard-messaging.test.ts index 682dd8f5db9..74541303f3e 100644 --- a/test/onboard-messaging.test.ts +++ b/test/onboard-messaging.test.ts @@ -82,7 +82,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("provider get")) return "Provider: discord-bridge"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; @@ -352,7 +352,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; { @@ -533,7 +533,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -693,7 +693,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -842,7 +842,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -998,7 +998,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -1211,7 +1211,7 @@ runner.run = (command, opts = {}) => { }; runner.runCapture = (command) => { // Existing sandbox that is ready - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return "my-assistant"; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; // All messaging providers already exist in gateway if (_n(command).includes("provider get")) return "Provider: exists"; @@ -1313,7 +1313,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -1442,7 +1442,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); diff --git a/test/onboard-reservation-recreate.test.ts b/test/onboard-reservation-recreate.test.ts index 72b8d78885b..e27b22b5d5a 100644 --- a/test/onboard-reservation-recreate.test.ts +++ b/test/onboard-reservation-recreate.test.ts @@ -50,6 +50,7 @@ describe("onboard sandbox recreate reservation safety", () => { const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const onboardSession = require(${onboardSessionPath}); @@ -58,6 +59,7 @@ const { EventEmitter } = require("node:events"); const events = []; let sandboxDeleted = false; +let sandboxRecreated = false; runner.run = (command) => { const cmd = _n(command); events.push({ kind: "run", cmd }); @@ -66,9 +68,9 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -101,6 +103,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + sandboxRecreated = true; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index 5d1d23ef073..c1081a89b47 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -33,7 +33,7 @@ describe("onboard helpers", () => { const registry = require(${registryPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.runCapture = (command) => (_n(command).includes("sandbox get my-assistant") ? "" : ""); +runner.runCapture = (command) => (_n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? "" : ""); registry.registerSandbox({ name: "my-assistant" }); @@ -105,7 +105,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -526,7 +526,7 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -628,7 +628,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -730,7 +730,7 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; // Custom port: dashboard readiness curl uses 19000 (DASHBOARD_PORT from env) { diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index b7d28aaeeda..11b35a58b6e 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -30,11 +30,14 @@ describe("onboard helpers", () => { const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const childProcess = require("node:child_process"); runner.run = (command) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); if (_n(command).includes("sandbox delete")) { throw new Error("unexpected sandbox delete"); } @@ -42,8 +45,8 @@ runner.run = (command) => { }; runner.runCapture = (command) => { // Existing sandbox that is NOT ready - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant NotReady"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant NotReady"; return ""; }; registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressive" }); @@ -108,19 +111,22 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); const commands = []; let registeredSandbox = null; runner.run = (command, opts = {}) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -140,6 +146,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -218,7 +225,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -226,13 +235,14 @@ const { EventEmitter } = require("node:events"); const events = []; runner.run = (command) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); events.push({ kind: "run", cmd: _n(command) }); return { status: 0 }; }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; - if (cmd.includes("sandbox list")) return "my-assistant Ready"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -274,6 +284,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -365,7 +376,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -373,13 +386,14 @@ const { EventEmitter } = require("node:events"); const events = []; runner.run = (command) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); events.push({ kind: "run", cmd: _n(command) }); return { status: 0 }; }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; - if (cmd.includes("sandbox list")) return "my-assistant Ready"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -408,6 +422,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -486,7 +501,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -495,6 +512,7 @@ const { EventEmitter } = require("node:events"); const events = []; let sandboxDeleted = false; runner.run = (command) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); const cmd = _n(command); events.push({ kind: "run", cmd }); if (cmd.includes("sandbox delete")) sandboxDeleted = true; @@ -502,9 +520,9 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -547,6 +565,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -631,7 +650,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const onboardSession = require(${sessionModulePath}); const childProcess = require("node:child_process"); @@ -639,12 +660,13 @@ const { EventEmitter } = require("node:events"); const commands = []; runner.run = (command, opts = {}) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -673,6 +695,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -749,7 +772,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -759,6 +784,7 @@ const path = require("node:path"); const commands = []; runner.run = (command, opts = {}) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); const commandString = Array.isArray(command) ? command.join(" ") : String(command); if (_n(command).includes("sandbox download")) { const parts = commandString.match(/'([^']*)'/g) || []; @@ -783,8 +809,8 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; @@ -794,6 +820,7 @@ registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressiv credentials.prompt = async () => "y"; childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -878,7 +905,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -888,6 +917,7 @@ const path = require("node:path"); const commands = []; runner.run = (command, opts = {}) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); const commandString = Array.isArray(command) ? command.join(" ") : String(command); if (_n(command).includes("sandbox download")) { const parts = commandString.match(/'([^']*)'/g) || []; @@ -912,8 +942,8 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -936,6 +966,7 @@ preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => "y"; childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -1020,7 +1051,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1029,15 +1062,16 @@ const { EventEmitter } = require("node:events"); const commands = []; let sandboxDeleted = false; runner.run = (command, opts = {}) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); if (_n(command).includes("sandbox delete")) sandboxDeleted = true; return { status: 0 }; }; runner.runCapture = (command) => { // Existing sandbox that is NOT ready initially, becomes Ready after recreation - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -1073,7 +1107,7 @@ const fakeSpawn = (...args) => { }); return child; }; -childProcess.spawn = fakeSpawn; +childProcess.spawn = (...args) => { _deleted = false; return fakeSpawn(...args); }; // Also patch spawn inside the compiled sandbox-create-stream module. // It imports spawn at load time from "node:child_process", so patching the @@ -1157,7 +1191,9 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); @@ -1169,11 +1205,12 @@ const commands = []; let sandboxListCalls = 0; const keepAlive = setInterval(() => {}, 1000); runner.run = (command, opts = {}) => { + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) { sandboxListCalls += 1; return sandboxListCalls >= 2 ? "my-assistant Ready" : "my-assistant Pending"; @@ -1203,6 +1240,7 @@ process.kill = (pid, signal) => { }; childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index e45a9b5b17e..61baa83310a 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -104,8 +104,10 @@ runner.runCapture = (command) => { "Endpoint: https://inference.local/v1", ].join("\n"); } - if (normalized.includes("sandbox get " + sandboxName)) { - return scenario === "reuse" ? sandboxName : ""; + if (normalized.includes("sandbox get") && normalized.includes(sandboxName)) { + return scenario === "reuse" + ? [sandboxName, "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) + : ""; } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; if (normalized.includes("forward list")) return sandboxName + " 127.0.0.1 18789 12345 running"; diff --git a/test/onboard.test.ts b/test/onboard.test.ts index b929f1ef725..0fc4b36c01c 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -767,7 +767,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; diff --git a/test/rebuild-stale-recovery.test.ts b/test/rebuild-stale-recovery.test.ts index a1683a22f3c..138bee2db30 100644 --- a/test/rebuild-stale-recovery.test.ts +++ b/test/rebuild-stale-recovery.test.ts @@ -275,7 +275,9 @@ describe("stale sandbox rebuild recovery (#4497)", () => { targetGatewayPort: 8080, }), ); - expect(harness.removeSandboxRegistryEntryWithReceiptSpy).toHaveBeenCalledOnce(); + // The journaled source row is the durable replacement contract, so it is + // preserved until replacement registration commits (#7734). + expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntrySpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); }); @@ -365,10 +367,12 @@ describe("stale sandbox rebuild recovery (#4497)", () => { expect(output).not.toContain("Backing up sandbox state"); expect(output).toContain("Creating new sandbox with current image"); expect(output).toContain("Recovery recreate failed"); - // The preserved entry must survive the failed recreate. Its obsolete image - // tag is intentionally cleared so a leftover image remains eligible for GC. + // The preserved entry must survive the failed recreate carrying no obsolete + // image tag, so a leftover image remains eligible for GC. The journaled row + // is now preserved in place rather than removed and restored, so the field + // is absent instead of explicitly null (#7734). const registry = readRegistry(fixture); expect(registry.defaultSandbox).toBe(fixture.sandboxName); - expect(registry.sandboxes[fixture.sandboxName].imageTag).toBe(null); + expect(registry.sandboxes[fixture.sandboxName].imageTag ?? null).toBe(null); }); }); diff --git a/test/repro-2201.test.ts b/test/repro-2201.test.ts index 1766f1d5973..3bbafd10aaa 100644 --- a/test/repro-2201.test.ts +++ b/test/repro-2201.test.ts @@ -244,7 +244,7 @@ if (a[0]==="sandbox" && a[1]==="ssh-config") { process.stdout.write("${sshConfig if (a[0]==="sandbox" && a[1]==="delete") { fs.writeFileSync(deleteMarker, "deleted\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="get") { if (fs.existsSync(deleteMarker)) { process.stderr.write("Error: sandbox ${sandboxName} not found\\n"); process.exit(1); } - process.stdout.write("Name: ${sandboxName}\\nPhase: Ready\\n"); + process.stdout.write("Name: ${sandboxName}\\nId: sbx-2201a7c3f5\\nPhase: Ready\\n"); process.exit(0); } process.exit(0); diff --git a/test/shellquote-sandbox.test.ts b/test/shellquote-sandbox.test.ts index 3dee1080caa..277bebe7882 100644 --- a/test/shellquote-sandbox.test.ts +++ b/test/shellquote-sandbox.test.ts @@ -95,7 +95,7 @@ runner.runFile = (file, args = [], opts = {}) => { }; runner.runCapture = (command) => { const text = asText(command); - if (text.includes("sandbox get my-assistant")) return ""; + if (text.includes("sandbox get") && text.includes("my-assistant")) return ""; if (text.includes("sandbox list")) return "my-assistant Ready"; if (text.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; if (text.includes("sandbox exec") && text.includes("http://localhost:") && text.includes("/health")) return "200";