Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
083fa1d
fix(rebuild): journal same-name sandbox replacement
laitingsheng Jul 29, 2026
dc778c1
fix(onboard): journal non-resumed same-name recreation
laitingsheng Jul 29, 2026
a2254bd
test(onboard): match gateway-scoped sandbox probes in onboard stubs
laitingsheng Jul 29, 2026
158139d
fix(onboard): retire the self-opened same-name replacement journal
laitingsheng Jul 29, 2026
0713167
test(onboard): track sandbox deletion without new test branches
laitingsheng Jul 29, 2026
3109fe7
test(rebuild): order the delete boundary without a test branch
laitingsheng Jul 29, 2026
de718e2
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 29, 2026
4c1c550
fix(rebuild): converge a restart onto its proven replacement
laitingsheng Jul 29, 2026
3d15a73
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 29, 2026
49baa53
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 29, 2026
a1af774
test(rebuild): keep the replacement journal cases branch-free
laitingsheng Jul 29, 2026
e8e8a28
test(rebuild): cover restart from every post-delete journal phase
laitingsheng Jul 29, 2026
3b2dc85
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 30, 2026
b5adecd
fix(onboard): keep the journaled source row across its route reservation
laitingsheng Jul 30, 2026
9b01609
fix(onboard): stop deleting a same-name replacement the journal alrea…
laitingsheng Jul 30, 2026
ba01650
ci: split the rebuild phase entries in the allowed-cycle list
laitingsheng Jul 30, 2026
63b198b
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 30, 2026
015a86f
fix(rebuild): journal endpoint provenance in the replacement target i…
laitingsheng Jul 30, 2026
9bcf05d
Merge remote-tracking branch 'origin/main' into fix/rebuild-recreate-…
laitingsheng Jul 30, 2026
0555bf4
test(rebuild): cover restarts from pre-creation recreate journal phases
laitingsheng Jul 30, 2026
4f1ae33
fix(rebuild): prove the journaled source before a restart deletes it
laitingsheng Jul 30, 2026
e2192d6
test(rebuild): model a live source in the rebuild flow harness defaults
laitingsheng Jul 30, 2026
d2732d9
fix(onboard): prove the journaled source before recreation deletes it
laitingsheng Jul 30, 2026
d9b8648
test(rebuild): scope the harness source mock to its sandbox and gateway
laitingsheng Jul 30, 2026
a40043c
fix(rebuild): bind the inference route to the replacement target fing…
laitingsheng Jul 30, 2026
603857d
fix(onboard): delete the recreate source through its journaled gateway
laitingsheng Jul 30, 2026
09810f1
merge: resolve conflicts with main
github-actions[bot] Jul 30, 2026
49de233
merge: refresh PR #7788 from upstream/main
senthilr-nv Jul 31, 2026
304893a
test(rebuild): align refreshed journal fixtures
senthilr-nv Jul 31, 2026
ff814ae
merge: refresh PR #7788 from upstream/main
senthilr-nv Jul 31, 2026
9e19967
test(rebuild): type refreshed authority mocks
senthilr-nv Jul 31, 2026
e3f5982
docs(rebuild): explain interrupted replacement recovery
senthilr-nv Jul 31, 2026
09d2c9a
docs(rebuild): correct recovery error guidance
senthilr-nv Jul 31, 2026
dfb1463
test(rebuild): isolate resume authority fixture
senthilr-nv Jul 31, 2026
6406eb9
Merge remote-tracking branch 'upstream/main' into codex/pr7788-refres…
senthilr-nv Jul 31, 2026
f2465b5
merge: resolve conflicts with main
github-actions[bot] Jul 31, 2026
440880f
Merge remote-tracking branch 'upstream/main' into codex/pr7788-refres…
senthilr-nv Jul 31, 2026
dbd695b
merge: integrate remote branch refresh
senthilr-nv Jul 31, 2026
9b42d7c
Merge branch 'main' into fix/rebuild-recreate-journal
senthilr-nv Jul 31, 2026
12a41cc
merge: refresh PR #7788 from upstream/main
senthilr-nv Jul 31, 2026
3e6c3d0
merge: integrate remote branch refresh
senthilr-nv Jul 31, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 10 additions & 15 deletions ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
"src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26,
"src/lib/actions/sandbox/process-recovery.ts": 26,
"src/lib/adapters/docker/index.ts": 45,
"src/lib/adapters/openshell/client.ts": 22,
"src/lib/adapters/openshell/client.ts": 23,
"src/lib/adapters/openshell/resolve.ts": 27,
"src/lib/adapters/openshell/runtime.ts": 50,
"src/lib/adapters/openshell/timeouts.ts": 36,
"src/lib/adapters/openshell/runtime.ts": 51,
"src/lib/adapters/openshell/timeouts.ts": 37,
"src/lib/agent/defs.ts": 32,
"src/lib/cli/branding.ts": 85,
"src/lib/cli/nemoclaw-oclif-command.ts": 103,
Expand All @@ -26,8 +26,8 @@
"src/lib/onboard/gateway-binding.ts": 47,
"src/lib/runner.ts": 89,
"src/lib/security/redact.ts": 51,
"src/lib/state/onboard-session.ts": 34,
"src/lib/state/registry.ts": 99,
"src/lib/state/onboard-session.ts": 36,
"src/lib/state/registry.ts": 101,
"src/lib/state/state-root.ts": 23,
"src/lib/subprocess-env.ts": 23,
"src/lib/validation.ts": 25
Expand All @@ -42,12 +42,12 @@
"src/lib/actions/sandbox/doctor.ts": 29,
"src/lib/actions/sandbox/policy-channel.ts": 28,
"src/lib/actions/sandbox/process-recovery.ts": 22,
"src/lib/actions/sandbox/rebuild-pipeline.ts": 27,
"src/lib/actions/sandbox/rebuild-pipeline.ts": 28,
"src/lib/actions/sandbox/snapshot.ts": 38,
"src/lib/actions/uninstall/run-plan.ts": 25,
"src/lib/inference/onboard-probes.ts": 21,
"src/lib/inference/vllm.ts": 23,
"src/lib/onboard.ts": 224,
"src/lib/onboard.ts": 225,
"src/lib/onboard/machine/handlers/sandbox.ts": 21,
"src/lib/sandbox/config.ts": 23,
"src/lib/shields/index.ts": 23
Expand All @@ -62,9 +62,6 @@
"src/lib/actions/sandbox/connect-boundary-refusal.ts",
"src/lib/actions/sandbox/connect-hermes-light-skin.ts",
"src/lib/actions/sandbox/connect.ts",
"src/lib/actions/sandbox/destroy-execution.ts",
"src/lib/actions/sandbox/destroy-preflight.ts",
"src/lib/actions/sandbox/destroy.ts",
"src/lib/actions/sandbox/gateway-restart.ts",
"src/lib/actions/sandbox/gateway-state.ts",
"src/lib/actions/sandbox/mcp-bridge-adapter-deepagents-capability.ts",
Expand Down Expand Up @@ -137,15 +134,13 @@
"src/lib/onboard/setup-inference.ts",
"src/lib/openshell-sandbox-list.ts",
"src/lib/sandbox/config.ts",
"src/lib/shields/index.ts",
"src/lib/tunnel/allowed-origins.ts",
"src/lib/tunnel/services.ts"
"src/lib/shields/index.ts"
]
],
"maxRootFiles": {
"src/lib/onboard": 307,
"src/lib/onboard": 309,
"src/lib/actions": 19,
"src/lib/actions/sandbox": 182,
"src/lib/actions/sandbox": 183,
"src/lib/state": 38,
"src/lib/inference": 62,
"scripts": 44
Expand Down
27 changes: 27 additions & 0 deletions docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,33 @@ Follow the printed recovery guidance, using `$$nemoclaw <sandbox-name> recover`
The rebuild command preserves manifest-defined Deep Agents state, regenerates `config.toml`, reconstructs managed MCP projection state, and reapplies registered policies while recreating the container.
</AgentOnly>

### Continue an Interrupted Replacement

Before `rebuild` deletes the existing sandbox, NemoClaw records a replacement journal in the onboarding session.
The journal binds the operation to the sandbox name, recorded OpenShell gateway, source identity, and replacement settings.
It stores fingerprints instead of credential values or raw OpenShell sandbox IDs.

If `rebuild` stops after recording the journal, rerun the command with the same replacement settings.
The rerun takes one of these actions:

- It continues deletion when the live sandbox still has the journaled source identity.
- It continues creation when the recorded OpenShell gateway explicitly reports the source sandbox as absent.
- It accepts an existing replacement only when its live identity and sandbox registry generation match the journal.

An accepted replacement is not deleted again.
The command reports `Sandbox '<name>' already holds the replacement from the interrupted rebuild.` and preserves the state backup path when one exists.
Pass `--verbose` to include the replacement identifier, OpenShell gateway, and journal phase in rebuild diagnostics.

NemoClaw fails closed when the selected gateway, replacement settings, durable source registry fields, or live source or target identity no longer matches the journal.
The error names the sandbox and the mismatch that stopped recovery.
Do not delete a same-name sandbox to bypass this check.
Inspect the named OpenShell gateway and sandbox, correct the reported drift, and rerun the original command.

A same-name recreation started by `$$nemoclaw onboard` uses the same replacement journal.
If that recreation is interrupted after the `Journaled replacement` message, rerun the original onboarding command with the same target settings.
The active replacement can continue without adding `--resume`.
Use `--resume` for interrupted onboarding steps that occur before a replacement journal exists.

If an archive command preserves at least one state directory, NemoClaw keeps the usable entries and reports the manifest-defined paths that could not be archived.
If a manifest-declared state file fails, NemoClaw stops before deleting the original sandbox even when it preserved state directories, unless you explicitly pass `--force`.
If every state directory fails, NemoClaw stops before deleting the original sandbox even when it captured loose files, unless you explicitly pass `--force`.
Expand Down
14 changes: 14 additions & 0 deletions docs/reference/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,11 @@ Use `--resume` only for resumable interrupted or failed sessions, not to change
During resume, NemoClaw reruns preflight, gateway, provider, and sandbox repair checks even when the saved session has already reached a later nonterminal onboarding phase.
If the recorded session conflicts with flags you pass on the recovery run, NemoClaw exits and tells you to either rerun with the original settings or start over.

An active same-name replacement is separate from ordinary onboarding-step resume.
If onboarding printed `Journaled replacement` before it stopped, rerun the original onboarding command with the same target settings.
The replacement can continue without an explicit `--resume` flag.
Refer to [Continue an Interrupted Replacement](../manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes#continue-an-interrupted-replacement) for the identity checks and failure conditions.

Use `--fresh` to discard the saved onboarding session and start the wizard from the beginning.
This clears stale or failed session state before NemoClaw creates a new session record.
It also bypasses locally recorded sandbox base-image resolution metadata and reruns normal candidate resolution.
Expand Down Expand Up @@ -640,6 +645,11 @@ NemoClaw aborts the recreate when the backup cannot complete in full, including
Set `NEMOCLAW_RECREATE_WITHOUT_BACKUP=1` to skip the pre-recreate backup.
The destination sandbox starts with a fresh workspace.

Before deletion, onboarding prints a `Journaled replacement` diagnostic with the replacement identifier, recorded OpenShell gateway, and current phase.
If the process stops after this point, a later same-target onboarding run continues the active replacement without requiring `--resume`.
It accepts a ready same-name replacement only when the live identity and sandbox registry generation match the journal.
It fails closed if the gateway, source, target, durable source registry fields, or replacement settings changed.

<AgentOnly variant="openclaw">

For OpenClaw, the backed-up paths include agents, extensions, workspace, skills, hooks, identity, devices, canvas, cron, memory, telegram, wechat, credentials, and `/sandbox/.openclaw/workspace/`.
Expand Down Expand Up @@ -2792,6 +2802,10 @@ After OpenShell accepts the sandbox deletion, `rebuild` waits until OpenShell ex
Only then can NemoClaw perform any required local registry removal and begin creating the replacement.
If OpenShell does not confirm absence within the bounded wait, including when gateway transport errors block the probes, `rebuild` exits nonzero before registry removal or replacement creation and preserves both the local registry entry and the state backup.
Restore OpenShell connectivity and confirm the sandbox's live state before you retry, and keep the printed backup path for recovery.
Before deletion, rebuild records a replacement journal that binds the operation to the recorded gateway, source identity, and target settings.
Rerunning the same rebuild continues from the recorded boundary or accepts the proven replacement instead of deleting it again.
Use `--verbose` to print the replacement identifier, gateway, and journal phase.
Refer to [Continue an Interrupted Replacement](../manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes#continue-an-interrupted-replacement) for the recovery procedure and fail-closed conditions.
When rebuild starts with shields up, NemoClaw opens a 30-minute shields-down window for backup and recreation.
A detached auto-lock timer remains active until NemoClaw commits a successful shields-up state, so it can attempt to restore lockdown if the host rebuild process exits unexpectedly.

Expand Down
31 changes: 0 additions & 31 deletions src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,37 +101,6 @@ export function isMissingSandboxGatewayOutput(output = ""): boolean {
);
}

/**
* Strict absence classifier for destructive owner-gateway reconciliation.
* Bare NotFound is not sufficient because OpenShell uses it for missing
* gateways and providers as well as sandboxes.
*/
export function isExplicitMissingSandboxGatewayOutput(
output: string,
sandboxName: string,
): boolean {
const clean = stripAnsi(String(output)).replace(/\r/g, "").trim();
const exactNoSpec =
/^(?:error:\s*)?status:\s*Internal,\s*message:\s*["']sandbox has no spec["'](?:,\s*details:\s*\[\])?(?:,\s*metadata:\s*MetadataMap\s*\{\s*\})?$/i;
if (exactNoSpec.test(clean)) return true;
// OpenShell can omit the requested name from an owner-scoped lookup.
// Require both exact structured fields so gateway/provider absence and
// transport diagnostics remain ambiguous.
const exactStructuredNotFound =
/^(?:error:\s*)?(?:×\s*)?code:\s*["']Some requested entity was not found["']\s*,\s*message:\s*["']sandbox not found["']$/i;
if (exactStructuredNotFound.test(clean)) return true;

const escapedName = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const namedSandbox = `(?:['\"]${escapedName}['\"]|${escapedName})`;
return (
new RegExp(
`^(?:error:\\s*)?sandbox\\s+${namedSandbox}\\s+(?:(?:is\\s+)?not\\s+(?:found|present)|does\\s+not\\s+exist)[.!]?$`,
"i",
).test(clean) ||
new RegExp(`^(?:error:\\s*)?no\\s+such\\s+sandbox\\s+${namedSandbox}[.!]?$`, "i").test(clean)
);
}

function formatGatewaySchemaMismatchOutput(
issue: OpenShellStateRpcIssue,
action: string,
Expand Down
Loading
Loading