Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
db52f20
fix(recovery): retry blank OpenShell registration follow-up
jyaunches Jul 29, 2026
24681d5
test(e2e): prove gateway health after restart
jyaunches Jul 30, 2026
ae26a2b
test(recovery): cover repeated blank follow-ups
jyaunches Jul 30, 2026
6609e6a
fix(recovery): retry opaque registration follow-ups
jyaunches Jul 30, 2026
31c30de
fix(recovery): retry informational registration follow-ups
jyaunches Jul 30, 2026
0e531dd
fix(recovery): retry exact registration error metadata
jyaunches Jul 30, 2026
9faffab
fix(recovery): preserve supervisor during container handoff
prekshivyas Jul 30, 2026
b2b66c5
test(recovery): cover supervisor handoff lifecycle
jyaunches Jul 30, 2026
b8e273b
test(recovery): clarify supervisor handoff lifecycle
prekshivyas Jul 30, 2026
25c3d99
Merge branch 'main' into codex/fix-sandbox-survival-status
cv Jul 30, 2026
297ade9
fix(recovery): relaunch legacy supervisor in place
prekshivyas Jul 30, 2026
e45d4d8
Merge remote-tracking branch 'origin/codex/fix-sandbox-survival-statu…
prekshivyas Jul 30, 2026
ab4fca5
fix(recovery): harden supervisor launch cleanup
sandl99 Jul 30, 2026
49cf4c1
fix(recovery): stabilize restored dashboard forward
sandl99 Jul 30, 2026
5b295a5
test(recovery): keep forward retry fixture linear
sandl99 Jul 30, 2026
f1b0c95
merge(main): simplify sandbox supervisor recovery
sandl99 Jul 30, 2026
062820e
fix(recovery): explain temporary cleanup
sandl99 Jul 30, 2026
0358523
test(e2e): recover sandbox before survival checks
sandl99 Jul 30, 2026
0e1f363
test(e2e): focus sandbox survival regression
sandl99 Jul 30, 2026
14a03d5
fix(recovery): preserve supervisor during handoff
sandl99 Jul 30, 2026
2edb387
docs(recovery): clarify handoff state
sandl99 Jul 30, 2026
b06f526
fix(recovery): defer handoff commit until ready
sandl99 Jul 30, 2026
9de564f
fix(recovery): preserve sandbox state across restart
sandl99 Jul 30, 2026
3ac8315
refactor(recovery): trim handoff scaffolding
sandl99 Jul 30, 2026
ff87797
merge: resolve recovery handoff conflict
sandl99 Jul 30, 2026
1e8b1dd
merge: resolve conflicts with main
github-actions[bot] Jul 30, 2026
691389b
refactor(recovery): finish main conflict cleanup
sandl99 Jul 30, 2026
f0a952b
Merge remote-tracking branch 'origin/codex/fix-sandbox-survival-statu…
sandl99 Jul 30, 2026
eaee157
Merge remote-tracking branch 'origin/main' into codex/pr7856-linear
sandl99 Jul 30, 2026
8a60ab9
fix(recovery): stop original before replacement
sandl99 Jul 30, 2026
8bb87f9
fix(recovery): restart legacy supervisor in place
sandl99 Jul 30, 2026
0fc1c1b
docs(recovery): clarify writable state preservation
sandl99 Jul 30, 2026
0b64a9c
test(e2e): capture sandbox survival diagnostics
sandl99 Jul 31, 2026
d1a9dd2
fix(onboard): persist OpenClaw startup command
sandl99 Jul 31, 2026
ac61a67
fix(onboard): recognize default OpenClaw startup
sandl99 Jul 31, 2026
6ab61e8
test(onboard): model startup container recreation
sandl99 Jul 31, 2026
8bf8033
Merge branch 'main' into codex/fix-sandbox-survival-status
prekshivyas Jul 31, 2026
01d1b41
test(e2e): prove Docker restart delivery
prekshivyas Jul 31, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/lib/onboard/docker-startup-command-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ export function resolveDockerStartupCommandPatch(
if (dockerDriverGateway !== true) {
return { persistStartupCommand: false, requiredUlimits: null };
}
const agentName = agent?.name;
const agentName = agent?.name ?? "openclaw";
return {
persistStartupCommand: agentName === "hermes" || agentName === DCODE_AGENT_NAME,
persistStartupCommand:
agentName === "openclaw" || agentName === "hermes" || agentName === DCODE_AGENT_NAME,
requiredUlimits: agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null,
};
}
9 changes: 7 additions & 2 deletions src/lib/onboard/sandbox-create-step.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,12 @@ describe("runSandboxCreateStep", () => {
});
});

it("persists the Hermes startup command for Docker-driver container restarts", async () => {
it.each([
{ label: "OpenClaw", agent: null },
{ label: "Hermes", agent: { name: "hermes" } as SandboxCreateStepContext["agent"] },
])("persists the $label startup command for Docker-driver container restarts", async ({
agent,
}) => {
const launch = makeLaunch({
sandboxStartupCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"],
});
Expand All @@ -143,7 +148,7 @@ describe("runSandboxCreateStep", () => {

await runSandboxCreateStep(
makeContext({
agent: { name: "hermes" } as SandboxCreateStepContext["agent"],
agent,
prebuild: {
buildCtx: "/tmp/ctx",
buildId: "b1",
Expand Down
107 changes: 106 additions & 1 deletion test/e2e/live/sandbox-survival.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@ import {
cleanupWhenCommandAvailable,
cleanupWhenOpenShellAvailable,
} from "../fixtures/cleanup-resources.ts";
import { assertExitZero, resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts";
import {
assertExitZero,
type HostCliClient,
resultText,
sandboxAccessEnv,
} from "../fixtures/clients/index.ts";
import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
Expand All @@ -29,6 +34,86 @@ const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-survival";
const MIN_OPENSHELL_VERSION = "0.0.24";
const MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b";

const SURVIVAL_DIAGNOSTICS_SCRIPT = String.raw`
set +e
sandbox_name="$1"

printf '%s\n' '== OpenShell sandbox status =='
openshell sandbox get "$sandbox_name" 2>&1
printf '%s\n' '== OpenShell forwards =='
openshell forward list 2>&1
printf '%s\n' '== OpenShell gateway service =='
systemctl --user status nemoclaw-openshell-gateway --no-pager -l 2>&1
printf '%s\n' '== OpenShell gateway journal =='
journalctl --user -u nemoclaw-openshell-gateway -n 200 --no-pager 2>&1

container_ids="$(docker ps -aq \
--filter label=openshell.ai/managed-by=openshell \
--filter "label=openshell.ai/sandbox-name=$sandbox_name")"
printf '%s\n' '== matching containers =='
if [ -n "$container_ids" ]; then
docker ps -a --no-trunc \
--filter label=openshell.ai/managed-by=openshell \
--filter "label=openshell.ai/sandbox-name=$sandbox_name" \
--format '{{.ID}} {{.Names}} {{.Status}}'
else
printf '%s\n' 'none'
fi

for container_id in $container_ids; do
printf '%s\n' "== container $container_id inspect =="
docker inspect "$container_id" 2>&1 | node -e '
const fs = require("node:fs");
const row = JSON.parse(fs.readFileSync(0, "utf8"))[0] || {};
const prefix = "OPENSHELL_SANDBOX_COMMAND=";
const matches = (row.Config?.Env || []).filter((entry) => entry.startsWith(prefix));
const command = matches.length === 1 ? matches[0].slice(prefix.length) : "";
const tokens = command.trim().split(/\s+/).filter(Boolean);
process.stdout.write(JSON.stringify({
name: row.Name || "",
configUser: row.Config?.User || "",
state: {
status: row.State?.Status || "",
running: Boolean(row.State?.Running),
restarting: Boolean(row.State?.Restarting),
pid: row.State?.Pid || 0,
exitCode: row.State?.ExitCode ?? null,
error: row.State?.Error || "",
startedAt: row.State?.StartedAt || "",
finishedAt: row.State?.FinishedAt || "",
health: row.State?.Health?.Status || "",
},
restartPolicy: row.HostConfig?.RestartPolicy?.Name || "",
startupCommandCount: matches.length,
startupCommandIsSleepInfinity: tokens.length === 2
&& tokens[0] === "sleep" && tokens[1] === "infinity",
startupCommandEndsWithNemoclawStart: tokens.length > 0
&& ["nemoclaw-start", "/usr/local/bin/nemoclaw-start"].includes(tokens.at(-1)),
}) + "\n");
'
printf '%s\n' "== container $container_id host process tree =="
docker top "$container_id" -eo pid,ppid,user,stat,comm 2>&1
printf '%s\n' "== container $container_id runtime state =="
docker exec "$container_id" sh -lc '
printf "%s\n" "== pid 1 =="
cat /proc/1/comm 2>/dev/null || true
printf "\n%s\n" "== process tree =="
ps -eo user=,pid=,ppid=,stat=,comm= 2>&1 || true
printf "%s\n" "== direct gateway health =="
curl -q --noproxy "*" -sS -o /dev/null -w "HTTP %{http_code}\n" \
--connect-timeout 2 --max-time 5 http://127.0.0.1:18789/health 2>&1 || true
printf "%s\n" "== managed controller status =="
cat /run/nemoclaw/gateway-control/status 2>&1 || true
printf "%s\n" "== startup log =="
tail -n 300 /tmp/nemoclaw-start.log 2>&1 || true
printf "%s\n" "== gateway log =="
tail -n 300 /tmp/gateway.log 2>&1 || true
' 2>&1
printf '%s\n' "== container $container_id logs =="
docker logs --tail 300 "$container_id" 2>&1
done
`;

function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
Expand Down Expand Up @@ -61,6 +146,23 @@ function installEnv(hostedEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
};
}

async function captureSurvivalDiagnostics(
host: HostCliClient,
stage: string,
redactionValues: string[],
): Promise<void> {
await host.command(
"sh",
["-lc", SURVIVAL_DIAGNOSTICS_SCRIPT, "sandbox-survival-diagnostics", SANDBOX_NAME],
{
artifactName: `sandbox-survival-${stage}-diagnostics`,
env: buildAvailabilityProbeEnv(),
redactionValues,
timeoutMs: 60_000,
},
);
}

async function expectSandboxExecAlive(
sandboxName: string,
exec: (
Expand Down Expand Up @@ -303,10 +405,12 @@ test(
await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read");

progress.phase("restart the gateway and reconnect the sandbox");
await captureSurvivalDiagnostics(host, "before-gateway-restart", [apiKey]);
await lifecycle.restartGatewayRuntime({
delayMs: 5_000,
sandboxName: SANDBOX_NAME,
});
await captureSurvivalDiagnostics(host, "after-gateway-restart", [apiKey]);
await lifecycle.waitForGatewayConnected({
attempts: 60,
intervalMs: 5_000,
Expand All @@ -324,6 +428,7 @@ test(
artifactName: "post-restart-nemoclaw-status",
timeoutMs: 120_000,
});
await stateValidation.from("cloud-openclaw-ready", instance);
await expectSandboxExecAlive(SANDBOX_NAME, execShell, "post-restart-sandbox-exec-alive");
await stateValidation.expectSandboxMarkers(instance, markers, "post-restart-marker-read");
await stateValidation.expectSandboxDirectoryPopulated(
Expand Down
37 changes: 37 additions & 0 deletions test/helpers/onboard-script-mocks.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,30 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [
`printf '%s\\n' "nemoclaw-security-inventory-ok"`,
].join("; ");

const ONBOARD_SANDBOX_OLD_CONTAINER_ID = "a".repeat(64);
const ONBOARD_SANDBOX_NEW_CONTAINER_ID = "b".repeat(64);
const ONBOARD_SANDBOX_INSPECT = {
Id: ONBOARD_SANDBOX_OLD_CONTAINER_ID,
Image: `sha256:${"c".repeat(64)}`,
Name: "/openshell-my-assistant",
Config: {
Image: "openshell/sandbox:test",
Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"],
Labels: {
"openshell.ai/managed-by": "openshell",
"openshell.ai/sandbox-name": "my-assistant",
},
Entrypoint: ["/opt/openshell/bin/openshell-sandbox"],
Cmd: [],
User: "0",
WorkingDir: "/sandbox",
},
HostConfig: {
NetworkMode: "openshell-docker",
RestartPolicy: { Name: "unless-stopped" },
},
};

function isOpenClawSecurityInventoryProbe(command) {
const commandArgs = Array.isArray(command) ? command.map(String) : [];
const dockerArgs = commandArgs[0] === "docker" ? commandArgs.slice(1) : commandArgs;
Expand Down Expand Up @@ -113,6 +137,19 @@ function mockSandboxExecCurl(command, options = {}) {

function mockOnboardRunCapture(command, options = {}) {
const normalized = normalizeCommand(command);
if (
normalized.startsWith("docker ps -a --no-trunc ") &&
normalized.includes("label=openshell.ai/sandbox-name=my-assistant") &&
normalized.endsWith("--format {{.ID}}")
) {
return `${ONBOARD_SANDBOX_OLD_CONTAINER_ID}\n${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`;
}
if (
normalized ===
`docker inspect --type container ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`
) {
return JSON.stringify([ONBOARD_SANDBOX_INSPECT]);
}
if (isOpenClawSecurityInventoryProbe(command)) {
return "nemoclaw-security-inventory-ok";
}
Expand Down
9 changes: 9 additions & 0 deletions test/onboard-sandbox-build.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,15 @@ const { createSandbox } = require(${onboardPath});
),
"expected dashboard forward (loopback or WSL 0.0.0.0)",
);
assert.ok(
payload.commands.some(
(entry: CommandEntry) =>
entry.command.includes("docker run -d") &&
entry.command.includes("OPENSHELL_SANDBOX_COMMAND=") &&
entry.command.includes("nemoclaw-start"),
),
"expected the default OpenClaw startup command to be persisted in the recreated container",
);
});

it("skips OpenClaw sandbox-base resolution for agent-staged Dockerfiles", async () => {
Expand Down
8 changes: 6 additions & 2 deletions test/pr-risk-plan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,8 +274,12 @@ describe("deterministic PR risk plan", () => {
expect(riskPlanRequiredTargetIds(docsAndTestsOnly)).toEqual([]);
});

it("selects post-reboot recovery for status delivery recovery changes (#7824)", () => {
const changedFile = "src/lib/actions/sandbox/status-snapshot.ts";
it.each([
"src/lib/actions/sandbox/status-snapshot.ts",
"src/lib/onboard/docker-driver-sandbox-recovery.ts",
"src/lib/onboard/docker-startup-command-agent.ts",
"src/lib/onboard/sandbox-create-step.ts",
])("selects post-reboot recovery for Docker delivery changes in %s (#7824)", (changedFile) => {
const result = plan(changedFile);
const adjacentStatusFile = plan("src/lib/actions/sandbox/status-text.ts");

Expand Down
13 changes: 9 additions & 4 deletions tools/advisors/risk-plan.mts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ const PR_E2E_TYPED_TARGET_ID_SET = new Set<string>(PR_E2E_TYPED_TARGET_IDS);
const DEEPAGENTS_HEADLESS_INFERENCE_CHECK =
"test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh";
const DEEPAGENTS_CODE_RUNTIME_ROOT = "agents/langchain-deepagents-code/";
const POST_REBOOT_STATUS_RUNTIME = "src/lib/actions/sandbox/status-snapshot.ts";
const POST_REBOOT_DELIVERY_RUNTIME_FILES = new Set([
"src/lib/actions/sandbox/status-snapshot.ts",
"src/lib/onboard/docker-driver-sandbox-recovery.ts",
"src/lib/onboard/docker-startup-command-agent.ts",
"src/lib/onboard/sandbox-create-step.ts",
]);

export type RiskTier = 0 | 1 | 2 | 3;
export type RiskFamilyId =
Expand Down Expand Up @@ -129,9 +134,9 @@ export function focusedPrE2eTargetsForChangedFiles(
(file.startsWith(DEEPAGENTS_CODE_RUNTIME_ROOT) && isRuntimeRelevant(file)),
),
);
const postRebootMatchedFiles = changedFiles.includes(POST_REBOOT_STATUS_RUNTIME)
? [POST_REBOOT_STATUS_RUNTIME]
: [];
const postRebootMatchedFiles = stableUnique(
changedFiles.filter((file) => POST_REBOOT_DELIVERY_RUNTIME_FILES.has(file)),
);
return [
...(deepAgentsMatchedFiles.length > 0
? [
Expand Down
Loading