Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
083fa1d
fix(rebuild): journal same-name sandbox replacement
laitingsheng Jul 29, 2026
dc778c1
fix(onboard): journal non-resumed same-name recreation
laitingsheng Jul 29, 2026
a2254bd
test(onboard): match gateway-scoped sandbox probes in onboard stubs
laitingsheng Jul 29, 2026
158139d
fix(onboard): retire the self-opened same-name replacement journal
laitingsheng Jul 29, 2026
0713167
test(onboard): track sandbox deletion without new test branches
laitingsheng Jul 29, 2026
3109fe7
test(rebuild): order the delete boundary without a test branch
laitingsheng Jul 29, 2026
de718e2
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 29, 2026
4c1c550
fix(rebuild): converge a restart onto its proven replacement
laitingsheng Jul 29, 2026
3d15a73
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 29, 2026
49baa53
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 29, 2026
a1af774
test(rebuild): keep the replacement journal cases branch-free
laitingsheng Jul 29, 2026
e8e8a28
test(rebuild): cover restart from every post-delete journal phase
laitingsheng Jul 29, 2026
3b2dc85
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 30, 2026
b5adecd
fix(onboard): keep the journaled source row across its route reservation
laitingsheng Jul 30, 2026
9b01609
fix(onboard): stop deleting a same-name replacement the journal alrea…
laitingsheng Jul 30, 2026
ba01650
ci: split the rebuild phase entries in the allowed-cycle list
laitingsheng Jul 30, 2026
63b198b
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 30, 2026
015a86f
fix(rebuild): journal endpoint provenance in the replacement target i…
laitingsheng Jul 30, 2026
9bcf05d
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 30, 2026
0555bf4
test(rebuild): cover restarts from pre-creation recreate journal phases
laitingsheng Jul 30, 2026
4f1ae33
fix(rebuild): prove the journaled source before a restart deletes it
laitingsheng Jul 30, 2026
e2192d6
test(rebuild): model a live source in the rebuild flow harness defaults
laitingsheng Jul 30, 2026
d2732d9
fix(onboard): prove the journaled source before recreation deletes it
laitingsheng Jul 30, 2026
d9b8648
test(rebuild): scope the harness source mock to its sandbox and gateway
laitingsheng Jul 30, 2026
a40043c
fix(rebuild): bind the inference route to the replacement target fing…
laitingsheng Jul 30, 2026
603857d
fix(onboard): delete the recreate source through its journaled gateway
laitingsheng Jul 30, 2026
09810f1
merge: resolve conflicts with main
github-actions[bot] Jul 30, 2026
49de233
merge: refresh PR #7788 from upstream/main
senthilr-nv Jul 31, 2026
304893a
test(rebuild): align refreshed journal fixtures
senthilr-nv Jul 31, 2026
ff814ae
merge: refresh PR #7788 from upstream/main
senthilr-nv Jul 31, 2026
9e19967
test(rebuild): type refreshed authority mocks
senthilr-nv Jul 31, 2026
e3f5982
docs(rebuild): explain interrupted replacement recovery
senthilr-nv Jul 31, 2026
09d2c9a
docs(rebuild): correct recovery error guidance
senthilr-nv Jul 31, 2026
dfb1463
test(rebuild): isolate resume authority fixture
senthilr-nv Jul 31, 2026
6406eb9
Merge remote-tracking branch 'upstream/main' into codex/pr7788-refres…
senthilr-nv Jul 31, 2026
f2465b5
merge: resolve conflicts with main
github-actions[bot] Jul 31, 2026
440880f
Merge remote-tracking branch 'upstream/main' into codex/pr7788-refres…
senthilr-nv Jul 31, 2026
dbd695b
merge: integrate remote branch refresh
senthilr-nv Jul 31, 2026
9b42d7c
Merge branch 'main' into fix/rebuild-recreate-journal
senthilr-nv Jul 31, 2026
12a41cc
merge: refresh PR #7788 from upstream/main
senthilr-nv Jul 31, 2026
3e6c3d0
merge: integrate remote branch refresh
senthilr-nv Jul 31, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 11 additions & 17 deletions ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
"src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26,
"src/lib/actions/sandbox/process-recovery.ts": 26,
"src/lib/adapters/docker/index.ts": 45,
"src/lib/adapters/openshell/client.ts": 22,
"src/lib/adapters/openshell/client.ts": 23,
"src/lib/adapters/openshell/resolve.ts": 28,
"src/lib/adapters/openshell/runtime.ts": 50,
"src/lib/adapters/openshell/timeouts.ts": 36,
"src/lib/adapters/openshell/runtime.ts": 51,
"src/lib/adapters/openshell/timeouts.ts": 37,
"src/lib/agent/defs.ts": 32,
"src/lib/cli/branding.ts": 85,
"src/lib/cli/nemoclaw-oclif-command.ts": 103,
Expand All @@ -26,8 +26,8 @@
"src/lib/onboard/gateway-binding.ts": 47,
"src/lib/runner.ts": 89,
"src/lib/security/redact.ts": 51,
"src/lib/state/onboard-session.ts": 34,
"src/lib/state/registry.ts": 99,
"src/lib/state/onboard-session.ts": 36,
"src/lib/state/registry.ts": 101,
"src/lib/state/state-root.ts": 23,
"src/lib/subprocess-env.ts": 23,
"src/lib/validation.ts": 25
Expand All @@ -42,12 +42,12 @@
"src/lib/actions/sandbox/doctor.ts": 29,
"src/lib/actions/sandbox/policy-channel.ts": 28,
"src/lib/actions/sandbox/process-recovery.ts": 22,
"src/lib/actions/sandbox/rebuild-pipeline.ts": 27,
"src/lib/actions/sandbox/rebuild-pipeline.ts": 28,
"src/lib/actions/sandbox/snapshot.ts": 38,
"src/lib/actions/uninstall/run-plan.ts": 25,
"src/lib/inference/onboard-probes.ts": 21,
"src/lib/inference/vllm.ts": 23,
"src/lib/onboard.ts": 224,
"src/lib/onboard.ts": 225,
"src/lib/onboard/machine/handlers/sandbox.ts": 21,
"src/lib/sandbox/config.ts": 23,
"src/lib/shields/index.ts": 23
Expand All @@ -62,9 +62,6 @@
"src/lib/actions/sandbox/connect-boundary-refusal.ts",
"src/lib/actions/sandbox/connect-hermes-light-skin.ts",
"src/lib/actions/sandbox/connect.ts",
"src/lib/actions/sandbox/destroy-execution.ts",
"src/lib/actions/sandbox/destroy-preflight.ts",
"src/lib/actions/sandbox/destroy.ts",
"src/lib/actions/sandbox/gateway-restart.ts",
"src/lib/actions/sandbox/gateway-state.ts",
"src/lib/actions/sandbox/mcp-bridge-adapter-deepagents-capability.ts",
Expand Down Expand Up @@ -115,8 +112,7 @@
"src/lib/actions/sandbox/rebuild-pipeline.ts",
"src/lib/actions/sandbox/rebuild-post-restore-phase.ts",
"src/lib/actions/sandbox/rebuild-preflight-phase.ts",
"src/lib/actions/sandbox/rebuild-preflight-target-phase.ts",
"src/lib/actions/sandbox/rebuild-recreate-phase.ts",
"src/lib/actions/sandbox/rebuild-preflight-target-phase.ts", "src/lib/actions/sandbox/rebuild-recreate-phase.ts",
"src/lib/actions/sandbox/rebuild-shields-phase.ts",
"src/lib/actions/sandbox/rebuild-shields.ts",
"src/lib/actions/sandbox/rebuild-target-config.ts",
Expand All @@ -137,15 +133,13 @@
"src/lib/onboard/setup-inference.ts",
"src/lib/openshell-sandbox-list.ts",
"src/lib/sandbox/config.ts",
"src/lib/shields/index.ts",
"src/lib/tunnel/allowed-origins.ts",
"src/lib/tunnel/services.ts"
"src/lib/shields/index.ts"
]
],
"maxRootFiles": {
"src/lib/onboard": 307,
"src/lib/onboard": 309,
"src/lib/actions": 19,
"src/lib/actions/sandbox": 182,
"src/lib/actions/sandbox": 183,
"src/lib/state": 38,
"src/lib/inference": 62,
"scripts": 44
Expand Down
31 changes: 0 additions & 31 deletions src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,37 +101,6 @@ export function isMissingSandboxGatewayOutput(output = ""): boolean {
);
}

/**
* Strict absence classifier for destructive owner-gateway reconciliation.
* Bare NotFound is not sufficient because OpenShell uses it for missing
* gateways and providers as well as sandboxes.
*/
export function isExplicitMissingSandboxGatewayOutput(
output: string,
sandboxName: string,
): boolean {
const clean = stripAnsi(String(output)).replace(/\r/g, "").trim();
const exactNoSpec =
/^(?:error:\s*)?status:\s*Internal,\s*message:\s*["']sandbox has no spec["'](?:,\s*details:\s*\[\])?(?:,\s*metadata:\s*MetadataMap\s*\{\s*\})?$/i;
if (exactNoSpec.test(clean)) return true;
// OpenShell can omit the requested name from an owner-scoped lookup.
// Require both exact structured fields so gateway/provider absence and
// transport diagnostics remain ambiguous.
const exactStructuredNotFound =
/^(?:error:\s*)?(?:×\s*)?code:\s*["']Some requested entity was not found["']\s*,\s*message:\s*["']sandbox not found["']$/i;
if (exactStructuredNotFound.test(clean)) return true;

const escapedName = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const namedSandbox = `(?:['\"]${escapedName}['\"]|${escapedName})`;
return (
new RegExp(
`^(?:error:\\s*)?sandbox\\s+${namedSandbox}\\s+(?:(?:is\\s+)?not\\s+(?:found|present)|does\\s+not\\s+exist)[.!]?$`,
"i",
).test(clean) ||
new RegExp(`^(?:error:\\s*)?no\\s+such\\s+sandbox\\s+${namedSandbox}[.!]?$`, "i").test(clean)
);
}

function formatGatewaySchemaMismatchOutput(
issue: OpenShellStateRpcIssue,
action: string,
Expand Down
145 changes: 138 additions & 7 deletions src/lib/actions/sandbox/rebuild-destroy-phase.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,19 @@ vi.mock("./rebuild-mcp-phase", () => ({
}));

import { runRebuildDestroyPhase, waitForRebuildDeleteAbsence } from "./rebuild-destroy-phase";
import type { RebuildRecreateJournal } from "./rebuild-recreate-journal";

function stubRecreateJournal(): RebuildRecreateJournal {
return {
id: "journal-1",
acceptedTarget: false,
targetGeneration: "generation-1",
targetIntentFingerprint: "intent-1",
markDeleting: vi.fn(),
confirmDeleted: vi.fn(),
completeAcceptedTarget: vi.fn(),
};
}

describe("rebuild destroy phase", () => {
beforeEach(() => {
Expand Down Expand Up @@ -134,6 +147,7 @@ describe("rebuild destroy phase", () => {
},
},
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
log: vi.fn(),
bail,
Expand Down Expand Up @@ -162,6 +176,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "langchain-deepagents-code" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
log,
bail,
Expand Down Expand Up @@ -192,6 +207,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log,
Expand Down Expand Up @@ -227,6 +243,7 @@ describe("rebuild destroy phase", () => {
gatewayPort: 19080,
},
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -281,6 +298,7 @@ describe("rebuild destroy phase", () => {
gatewayPort: 8080,
},
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -326,6 +344,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -369,6 +388,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -420,6 +440,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -538,6 +559,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -582,6 +604,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -630,6 +653,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -676,6 +700,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
force: true,
log: vi.fn(),
Expand Down Expand Up @@ -811,6 +836,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: null,
log: vi.fn(),
bail: vi.fn((message: string): never => {
Expand All @@ -828,7 +854,7 @@ describe("rebuild destroy phase", () => {
expect(mocks.listSandboxes).not.toHaveBeenCalled();
});

it("removes registry state only after the gateway reports the deleted sandbox missing", async () => {
it("keeps the journaled source row after the gateway reports the deleted sandbox missing (#7734)", async () => {
const events: string[] = [];
let getAttempts = 0;
mocks.runOpenshell.mockImplementation(() => {
Expand All @@ -843,15 +869,13 @@ describe("rebuild destroy phase", () => {
? { status: 0, stdout: "Name: alpha\nPhase: Terminating", stderr: "" }
: { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" };
});
mocks.removeSandboxRegistryEntryWithReceipt.mockImplementation(() => {
events.push("remove-registry");
return null;
});
const recreateJournal = stubRecreateJournal();

const result = await runRebuildDestroyPhase({
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal,
backupManifest: null,
log: vi.fn(),
bail: vi.fn((message: string): never => {
Expand All @@ -862,14 +886,120 @@ describe("rebuild destroy phase", () => {
});

expect(result).not.toBeNull();
expect(events).toEqual(["delete", "get-live", "get-missing", "on-deleted", "remove-registry"]);
expect(result?.removalReceipt).toBeNull();
expect(events).toEqual(["delete", "get-live", "get-missing", "on-deleted"]);
expect(mocks.waitUntil).toHaveBeenCalledOnce();
expect(mocks.captureOpenshell).toHaveBeenNthCalledWith(
1,
["sandbox", "get", "-g", "nemoclaw", "alpha"],
expect.objectContaining({ timeout: expect.any(Number) }),
);
expect(mocks.removeSandboxRegistryEntryWithReceipt).toHaveBeenCalledWith("alpha");
expect(recreateJournal.markDeleting).toHaveBeenCalledOnce();
expect(recreateJournal.confirmDeleted).toHaveBeenCalledOnce();
});

it("journals the delete boundary before the destructive command (#7734)", async () => {
const order: string[] = [];
const recreateJournal = stubRecreateJournal();
vi.mocked(recreateJournal.markDeleting).mockImplementation(() => {
order.push("journal:deleting");
});
mocks.runOpenshell.mockImplementation((args: string[]) => {
const deleting = args[1] === "delete";
order.push(...(deleting ? ["openshell:delete"] : []));
return deleting
? { status: 0, stdout: "deleted", stderr: "" }
: { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" };
});

await runRebuildDestroyPhase({
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal,
backupManifest: null,
log: vi.fn(),
bail: vi.fn((message: string): never => {
throw new Error(message);
}),
relockShieldsIfNeeded: vi.fn(() => true),
onDeleted: vi.fn(),
});

expect(order).toEqual(["journal:deleting", "openshell:delete"]);
});

it("reattaches MCP providers when the delete boundary cannot be journaled (#7734)", async () => {
const recreateJournal = stubRecreateJournal();
vi.mocked(recreateJournal.markDeleting).mockImplementation(() => {
throw new Error("session store is unwritable");
});
mocks.prepareMcpForRebuild.mockResolvedValue({
entries: [{ server: "github" }],
detachedProviderEntries: [{ providerName: "nemoclaw-mcp-alpha-github" }],
scrubbedAdapterEntries: [{ server: "github" }],
});
const relockShieldsIfNeeded = vi.fn(() => true);

await expect(
runRebuildDestroyPhase({
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal,
backupManifest: null,
log: vi.fn(),
bail: vi.fn((message: string): never => {
throw new Error(message);
}),
relockShieldsIfNeeded,
onDeleted: vi.fn(),
}),
).rejects.toThrow("Sandbox deletion could not be journaled");

expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith(
"alpha",
[{ providerName: "nemoclaw-mcp-alpha-github" }],
[{ server: "github" }],
);
expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true);
expect(mocks.runOpenshell).not.toHaveBeenCalledWith(
["sandbox", "delete", "-g", "nemoclaw", "alpha"],
expect.anything(),
);
});

it("stops before inference and registry mutation when absence cannot be journaled (#7734)", async () => {
const recreateJournal = stubRecreateJournal();
vi.mocked(recreateJournal.confirmDeleted).mockImplementation(() => {
throw new Error("OpenShell still reports the journaled source after delete");
});
mocks.runOpenshell.mockReturnValue({ status: 0, stdout: "deleted", stderr: "" });
const onDeleted = vi.fn();
const onDeleteStateAmbiguous = vi.fn();

await expect(
runRebuildDestroyPhase({
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal,
backupManifest: null,
log: vi.fn(),
bail: vi.fn((message: string): never => {
throw new Error(message);
}),
relockShieldsIfNeeded: vi.fn(() => true),
onDeleted,
onDeleteStateAmbiguous,
}),
).rejects.toThrow("Sandbox deletion could not be journaled");

expect(onDeleteStateAmbiguous).toHaveBeenCalledOnce();
expect(onDeleted).not.toHaveBeenCalled();
expect(mocks.stopNimContainer).not.toHaveBeenCalled();
expect(mocks.stopNimContainerByName).not.toHaveBeenCalled();
expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled();
});

it("marks accepted deletion as ambiguous when transport failures prevent confirmation", async () => {
Expand All @@ -888,6 +1018,7 @@ describe("rebuild destroy phase", () => {
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" },
staleRecovery: false,
recreateJournal: stubRecreateJournal(),
backupManifest: { backupPath: "/tmp/rebuild-backups/alpha/backup" } as never,
log: vi.fn(),
bail: vi.fn((message: string): never => {
Expand Down
Loading
Loading