Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
db52f20
fix(recovery): retry blank OpenShell registration follow-up
jyaunches Jul 29, 2026
24681d5
test(e2e): prove gateway health after restart
jyaunches Jul 30, 2026
ae26a2b
test(recovery): cover repeated blank follow-ups
jyaunches Jul 30, 2026
6609e6a
fix(recovery): retry opaque registration follow-ups
jyaunches Jul 30, 2026
31c30de
fix(recovery): retry informational registration follow-ups
jyaunches Jul 30, 2026
0e531dd
fix(recovery): retry exact registration error metadata
jyaunches Jul 30, 2026
9faffab
fix(recovery): preserve supervisor during container handoff
prekshivyas Jul 30, 2026
b2b66c5
test(recovery): cover supervisor handoff lifecycle
jyaunches Jul 30, 2026
b8e273b
test(recovery): clarify supervisor handoff lifecycle
prekshivyas Jul 30, 2026
25c3d99
Merge branch 'main' into codex/fix-sandbox-survival-status
cv Jul 30, 2026
297ade9
fix(recovery): relaunch legacy supervisor in place
prekshivyas Jul 30, 2026
e45d4d8
Merge remote-tracking branch 'origin/codex/fix-sandbox-survival-statu…
prekshivyas Jul 30, 2026
ab4fca5
fix(recovery): harden supervisor launch cleanup
sandl99 Jul 30, 2026
49cf4c1
fix(recovery): stabilize restored dashboard forward
sandl99 Jul 30, 2026
5b295a5
test(recovery): keep forward retry fixture linear
sandl99 Jul 30, 2026
f1b0c95
merge(main): simplify sandbox supervisor recovery
sandl99 Jul 30, 2026
062820e
fix(recovery): explain temporary cleanup
sandl99 Jul 30, 2026
0358523
test(e2e): recover sandbox before survival checks
sandl99 Jul 30, 2026
0e1f363
test(e2e): focus sandbox survival regression
sandl99 Jul 30, 2026
14a03d5
fix(recovery): preserve supervisor during handoff
sandl99 Jul 30, 2026
2edb387
docs(recovery): clarify handoff state
sandl99 Jul 30, 2026
b06f526
fix(recovery): defer handoff commit until ready
sandl99 Jul 30, 2026
9de564f
fix(recovery): preserve sandbox state across restart
sandl99 Jul 30, 2026
3ac8315
refactor(recovery): trim handoff scaffolding
sandl99 Jul 30, 2026
ff87797
merge: resolve recovery handoff conflict
sandl99 Jul 30, 2026
1e8b1dd
merge: resolve conflicts with main
github-actions[bot] Jul 30, 2026
691389b
refactor(recovery): finish main conflict cleanup
sandl99 Jul 30, 2026
f0a952b
Merge remote-tracking branch 'origin/codex/fix-sandbox-survival-statu…
sandl99 Jul 30, 2026
eaee157
Merge remote-tracking branch 'origin/main' into codex/pr7856-linear
sandl99 Jul 30, 2026
8a60ab9
fix(recovery): stop original before replacement
sandl99 Jul 30, 2026
8bb87f9
fix(recovery): restart legacy supervisor in place
sandl99 Jul 30, 2026
0fc1c1b
docs(recovery): clarify writable state preservation
sandl99 Jul 30, 2026
0b64a9c
test(e2e): capture sandbox survival diagnostics
sandl99 Jul 31, 2026
d1a9dd2
fix(onboard): persist OpenClaw startup command
sandl99 Jul 31, 2026
ac61a67
fix(onboard): recognize default OpenClaw startup
sandl99 Jul 31, 2026
6ab61e8
test(onboard): model startup container recreation
sandl99 Jul 31, 2026
8bf8033
Merge branch 'main' into codex/fix-sandbox-survival-status
prekshivyas Jul 31, 2026
01d1b41
test(e2e): prove Docker restart delivery
prekshivyas Jul 31, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions src/lib/actions/sandbox/process-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,62 @@ describe("recreated sandbox OpenShell readiness", () => {
expect(sleeps).toEqual([3]);
});

it("retries repeated blank OpenShell failures after an exact re-registration state", () => {
const captureOpenshellImpl = vi
.fn()
.mockReturnValueOnce({
status: 1,
output: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR.trim(),
stdout: "",
stderr: OPENSHELL_TRANSIENT_ERROR_PHASE_STDERR,
})
.mockReturnValueOnce({ status: 1, output: "", stdout: "", stderr: "" })
.mockReturnValueOnce({ status: 1, output: "", stdout: "", stderr: "" })
.mockReturnValueOnce({ status: 0, output: "", stdout: "", stderr: "" });
const beforeProbe = vi.fn(() => true);
const sleeps: number[] = [];
let nowMs = 0;

expect(
waitForRecreatedSandboxOpenShellReady("recreated-box", {
beforeProbe,
captureOpenshellImpl,
intervalSeconds: 3,
nowImpl: () => nowMs,
sleepImpl: (seconds) => {
sleeps.push(seconds);
nowMs += seconds * 1000;
},
timeoutSeconds: 10,
}),
).toBe(true);
expect(beforeProbe).toHaveBeenCalledTimes(4);
expect(captureOpenshellImpl).toHaveBeenCalledTimes(4);
expect(sleeps).toEqual([3, 3, 3]);
});

it("keeps an isolated blank OpenShell failure terminal", () => {
const captureOpenshellImpl = vi.fn(() => ({
status: 1,
output: "",
stdout: "",
stderr: "",
}));
const sleeps: number[] = [];

expect(
waitForRecreatedSandboxOpenShellReady("recreated-box", {
beforeProbe: () => true,
captureOpenshellImpl,
intervalSeconds: 3,
sleepImpl: (seconds) => sleeps.push(seconds),
timeoutSeconds: 30,
}),
).toBe(false);
expect(captureOpenshellImpl).toHaveBeenCalledOnce();
expect(sleeps).toEqual([]);
});

it("rides out a transient Error phase past the old 30s budget by default (#7227)", () => {
// No timeoutSeconds option and no env override: the default recovery budget
// must be large enough (120s, aligned with connect's readiness wait) to keep
Expand Down
25 changes: 21 additions & 4 deletions src/lib/actions/sandbox/process-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,7 @@ function waitForRecreatedSandboxOpenShellReadyResult(
? Math.max(1, Math.floor(timeoutSeconds / intervalSeconds) + 1)
: Math.max(1, Math.floor(timeoutSeconds) + 1);
let lastOpenshellError: string | undefined;
let sawRetryableReRegistrationState = false;

for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const preGuardRemainingMs = deadlineMs - now();
Expand Down Expand Up @@ -785,10 +786,26 @@ function waitForRecreatedSandboxOpenShellReadyResult(
// mutation outcome to reconcile. Treat that exact timeout as inconclusive
// and retry behind the pinned managed-health guard on the next iteration.
// All other unexpected OpenShell failures remain definitive.
if (
!isRetryableOpenshellReRegistrationState(result, sandboxName) &&
!isCommandTimeout(result)
) {
const retryableReRegistrationState = isRetryableOpenshellReRegistrationState(
result,
sandboxName,
);
if (retryableReRegistrationState) {
sawRetryableReRegistrationState = true;
}
// OpenShell 0.0.85 can follow its exact phase/session transition response
// with one or more blank exit-1 results while the replacement registration
// settles. A blank failure is inconclusive only after this loop observed a
// known retryable state and while the pinned managed-health guard continues
// to pass. Remove this follow-up when supported OpenShell versions keep
// returning a structured transition response until exec is available.
const retryableEmptyFollowUp =
sawRetryableReRegistrationState &&
result.status === 1 &&
!result.error &&
String(result.stdout ?? "").trim() === "" &&
String(result.stderr ?? "").trim() === "";
if (!retryableReRegistrationState && !retryableEmptyFollowUp && !isCommandTimeout(result)) {
return {
failure: "openshell-readiness-failure",
openshellError: lastOpenshellError,
Expand Down
7 changes: 5 additions & 2 deletions test/e2e/live/sandbox-survival.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
*
* Preserves the real boundaries: install.sh/onboard, Docker, OpenShell
* gateway stop/start, NemoClaw registry/list/status, sandbox SSH/exec, durable
* /sandbox/.openclaw state markers, and inference.local chat completion before
* and after gateway restart.
* /sandbox/.openclaw state markers, OpenClaw gateway health after restart, and
* inference.local chat completion before and after gateway restart.
*/

import fs from "node:fs";
Expand Down Expand Up @@ -115,6 +115,7 @@ test(
"NemoClaw registry, nemoclaw list/status, and openshell sandbox list discover the sandbox",
"OpenShell version supports gateway resume and state persistence",
"sandbox exec/SSH-equivalent access works before and after gateway restart",
"OpenClaw gateway health is restored after the OpenShell gateway restart",
"inference.local returns a live PONG before and after gateway restart",
"markers under /sandbox/.openclaw survive the gateway stop/start cycle",
"final destroy removes the sandbox from NemoClaw registry/list state",
Expand Down Expand Up @@ -324,6 +325,7 @@ test(
artifactName: "post-restart-nemoclaw-status",
timeoutMs: 120_000,
});
await stateValidation.from("cloud-openclaw-ready", instance);
await expectSandboxExecAlive(SANDBOX_NAME, execShell, "post-restart-sandbox-exec-alive");
await stateValidation.expectSandboxMarkers(instance, markers, "post-restart-marker-read");
await stateValidation.expectSandboxDirectoryPopulated(
Expand Down Expand Up @@ -360,6 +362,7 @@ test(
assertions: {
installCompleted: install.exitCode === 0,
registryListedBeforeRestart: true,
openClawGatewayHealthyAfterRestart: true,
inferenceLocalBeforeRestart: true,
markersPersistedAfterRestart: true,
inferenceLocalAfterRestart: true,
Expand Down
Loading