Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
25bb9d2
fix(sandbox): verify readiness after container start
senthilr-nv Aug 3, 2026
55de826
fix(sandbox): restore startup before readiness
senthilr-nv Aug 3, 2026
96991b3
test(sandbox): cover startup recovery branches
senthilr-nv Aug 3, 2026
43f90ec
merge: update branch from upstream main
senthilr-nv Aug 4, 2026
59e7238
test(sandbox): cover protected start recovery
senthilr-nv Aug 4, 2026
b5db426
fix(sandbox): restore sealed startup state
senthilr-nv Aug 4, 2026
b3730ed
docs(security): clarify sealed credential access
senthilr-nv Aug 4, 2026
3d1aac5
test(sandbox): keep sealed restart setup linear
senthilr-nv Aug 4, 2026
e81c7a9
merge: update branch from upstream main
senthilr-nv Aug 4, 2026
7626d4e
test(sandbox): exercise configured restart paths
senthilr-nv Aug 4, 2026
53d3157
test(security): avoid auth profile path race
senthilr-nv Aug 4, 2026
609b3dd
test(security): keep auth metadata descriptor-bound
senthilr-nv Aug 4, 2026
8161171
test(security): prove sealed credential boundary
senthilr-nv Aug 4, 2026
d87d734
merge: update branch from upstream main
senthilr-nv Aug 4, 2026
4f4a04e
Merge branch 'main' into codex/fix-sandbox-start-runtime-state
cv Aug 4, 2026
e58c09a
Merge branch 'main' into codex/fix-sandbox-start-runtime-state
senthilr-nv Aug 4, 2026
0d2f226
fix(shields): preserve agent startup posture
jyaunches Aug 4, 2026
80c913e
test(sandbox): cover startup restoration rerun
jyaunches Aug 4, 2026
8d02a83
Merge branch 'main' into codex/fix-sandbox-start-runtime-state
cv Aug 4, 2026
7e469a0
docs(security): document sealed startup failures
jyaunches Aug 4, 2026
0a8509d
test(sandbox): cover backup recovery readiness
jyaunches Aug 4, 2026
e9735c3
Merge branch 'main' into codex/fix-sandbox-start-runtime-state
senthilr-nv Aug 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/manage-sandboxes/run-sandboxes.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,11 @@ $$nemoclaw <sandbox-name> start
```

<AgentOnly variant="openclaw,hermes">
NemoClaw restarts the container and repairs the in-sandbox gateway and host forwards.
After Docker reports the existing container as running, NemoClaw checks the selected agent's managed startup state and recovers missing processes when needed.
NemoClaw completes this recovery before its final OpenShell readiness and host-forward checks.
</AgentOnly>
<AgentOnly variant="deepagents">
NemoClaw restarts the container so the managed terminal runtime can run again.
After Docker reports the existing container as running, NemoClaw verifies the managed terminal runtime before the command reports success.
</AgentOnly>
Refer to [`$$nemoclaw <name> stop`](../../reference/commands#$$nemoclaw-name-stop) and [`$$nemoclaw <name> start`](../../reference/commands#$$nemoclaw-name-start) for details.
Use [`$$nemoclaw <name> destroy`](../../reference/commands#$$nemoclaw-name-destroy) when you want to delete the sandbox instead.
Expand Down
4 changes: 4 additions & 0 deletions src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -891,6 +891,10 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void {
}
}

export function restoreSandboxStartupState(sandboxName: string): void {
checkAndRecoverSandboxProcesses(sandboxName, { quiet: true });
}

function restoreInteractiveTerminal(): void {
if (!process.stdin.isTTY) return;

Expand Down
29 changes: 26 additions & 3 deletions src/lib/actions/sandbox/start.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
const verifyGateway = vi.fn<NonNullable<SandboxStartDeps["verifyGateway"]>>(() =>
Promise.resolve(),
);
const restoreStartupState = vi.fn<NonNullable<SandboxStartDeps["restoreStartupState"]>>();
const log = vi.fn<(message: string) => void>();
const runtimeProviders = createRuntimeProviderBundleRegistry([
[
Expand All @@ -62,6 +63,7 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
const deps: SandboxStartDeps = {
getSandbox,
runtimeProviders,
restoreStartupState,
verifyGateway,
log,
...overrides,
Expand All @@ -74,21 +76,28 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
isDockerRuntimeDown,
log,
printDockerRuntimeDownGuidance,
verifyGateway,
recoverDockerDriverSandbox,
restoreStartupState,
verifyGateway,
};
}

describe("startSandbox", () => {
it("starts the stopped container and then probes gateway health (#6026)", async () => {
it("restores startup state before probing readiness after a stopped container starts (#8112)", async () => {
const h = harness();

const result = await startSandbox("my-sandbox", h.deps);

expect(result.exitCode).toBe(0);
expect(h.recoverDockerDriverSandbox).toHaveBeenCalledWith("my-sandbox");
expect(h.recoverDockerDriverSandbox).toHaveBeenCalledWith("my-sandbox", {
readiness: "runtime-running",
});
expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox");
expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox");
expect(h.recoverDockerDriverSandbox.mock.invocationCallOrder[0]).toBeLessThan(
h.restoreStartupState.mock.invocationCallOrder[0],
);
expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan(
h.verifyGateway.mock.invocationCallOrder[0],
);
});
Expand Down Expand Up @@ -116,7 +125,11 @@ describe("startSandbox", () => {
const result = await startSandbox("my-sandbox", h.deps);

expect(result.exitCode).toBe(0);
expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox");
expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox");
expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan(
h.verifyGateway.mock.invocationCallOrder[0],
);
const output = h.log.mock.calls.map(([line]) => line).join("\n");
expect(output).toContain("already running");
});
Expand All @@ -139,7 +152,11 @@ describe("startSandbox", () => {
timeout: 30_000,
});
expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled();
expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox");
expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox");
expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan(
h.verifyGateway.mock.invocationCallOrder[0],
);
const output = h.log.mock.calls.map(([line]) => line).join("\n");
expect(output).toContain("unpaused");
});
Expand All @@ -160,6 +177,7 @@ describe("startSandbox", () => {
expect(result.exitCode).toBe(1);
expect(result.message).toContain("openshell-my-sandbox");
expect(result.message).toContain("125");
expect(h.restoreStartupState).not.toHaveBeenCalled();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

Expand Down Expand Up @@ -189,6 +207,7 @@ describe("startSandbox", () => {
retryCommand: "start",
});
expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled();
expect(h.restoreStartupState).not.toHaveBeenCalled();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

Expand All @@ -206,6 +225,7 @@ describe("startSandbox", () => {
expect(result.exitCode).toBe(1);
expect(result.message).toContain("no Docker container labeled");
expect(result.message).toContain("rebuild");
expect(h.restoreStartupState).not.toHaveBeenCalled();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

Expand All @@ -218,6 +238,7 @@ describe("startSandbox", () => {
expect(result.exitCode).toBe(1);
expect(result.message).toContain("not registered");
expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled();
expect(h.restoreStartupState).not.toHaveBeenCalled();
});

it("refuses non-direct drivers instead of guessing at container control (#6026)", async () => {
Expand All @@ -231,6 +252,7 @@ describe("startSandbox", () => {
expect(result.message).toContain("does not authorize 'start' mutation");
expect(h.findLabeledSandboxContainers).not.toHaveBeenCalled();
expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled();
expect(h.restoreStartupState).not.toHaveBeenCalled();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

Expand All @@ -249,6 +271,7 @@ describe("startSandbox", () => {
expect(h.findLabeledSandboxContainers).not.toHaveBeenCalled();
expect(h.dockerUnpause).not.toHaveBeenCalled();
expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled();
expect(h.restoreStartupState).not.toHaveBeenCalled();
expect(h.verifyGateway).not.toHaveBeenCalled();
});

Expand Down
17 changes: 14 additions & 3 deletions src/lib/actions/sandbox/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,24 @@ function verifyGateway(sandboxName: string): Promise<void> {
return connectSandbox(sandboxName, { probeOnly: true });
}

function restoreStartupState(sandboxName: string): void {
const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect");
restoreSandboxStartupState(sandboxName);
}

export interface SandboxStartDeps {
environment?: NodeJS.ProcessEnv;
getSandbox?: typeof registry.getSandbox;
runtimeProviders?: RuntimeProviderBundleRegistry;
restoreStartupState?: (sandboxName: string) => void;
verifyGateway?: (sandboxName: string) => Promise<void>;
log?: (message: string) => void;
}

/**
* Restart a stopped sandbox through the lifecycle facet bound to its durable
* provider identity, then restore gateway health and host forwards.
* provider identity, then restore startup state before verifying readiness and
* host forwards.
*/
export async function startSandbox(
sandboxName: string,
Expand All @@ -55,7 +62,11 @@ export async function startSandbox(
const result = resolved.lifecycle.start(input);
if (result.exitCode !== 0) return result;

log(" Checking gateway health and host forwards…");
await resolved.lifecycle.verifyStarted(input, deps.verifyGateway ?? verifyGateway);
await resolved.lifecycle.verifyStarted(input, async (name) => {
log(" Restoring sandbox startup state…");
(deps.restoreStartupState ?? restoreStartupState)(name);
log(" Checking gateway health and host forwards…");
await (deps.verifyGateway ?? verifyGateway)(name);
});
return { exitCode: 0 };
}
19 changes: 19 additions & 0 deletions src/lib/onboard/docker-driver-sandbox-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,25 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => {
expect(sleep).toHaveBeenCalledWith(1_000);
});

it("hands a running container to lifecycle verification while Docker health is starting (#8112)", () => {
const sleep = vi.fn();
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n", [
"running\tstarting",
]),
dockerStart: fakeStart(0),
readiness: "runtime-running",
sleep,
});

expect(result).toEqual({
recovered: true,
via: "started-stopped-original",
containerName: "openshell-e2e-x",
});
expect(sleep).not.toHaveBeenCalled();
});

it("accepts a running container whose image has no Docker health check", () => {
const sleep = vi.fn();
const result = recoverDockerDriverSandbox("e2e-x", {
Expand Down
34 changes: 26 additions & 8 deletions src/lib/onboard/docker-driver-sandbox-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,13 @@ export interface DockerDriverRecoveryDeps {
sleep?: (ms: number) => void;
/** Injectable clock for deterministic readiness deadlines in tests. */
now?: () => number;
/**
* Readiness boundary for the caller. Recovery keeps Docker health as its
* default authority. The explicit sandbox lifecycle start path stops at a
* running container because its provider-owned verification then proves
* OpenShell registration, the managed agent gateway, and host forwards.
*/
readiness?: "docker-health" | "runtime-running";
}

interface LabeledContainer {
Expand Down Expand Up @@ -242,6 +249,7 @@ interface ContainerReadinessResult {
function waitForRecoveredContainerReady(
containerName: string,
deps: ReturnType<typeof depsWithDefaults>,
readiness: NonNullable<DockerDriverRecoveryDeps["readiness"]>,
): ContainerReadinessResult {
let lastState = "unknown";
const deadlineMs = deps.now() + DOCKER_RECOVERY_READY_TIMEOUT_MS;
Expand All @@ -264,7 +272,10 @@ function waitForRecoveredContainerReady(
? `runtime=${runtimeState || "unknown"}, health=${healthState}`
: `runtime=${runtimeState || "unknown"}`;

if (runtimeState === "running" && (healthState === "healthy" || healthState === "none")) {
if (
runtimeState === "running" &&
(readiness === "runtime-running" || healthState === "healthy" || healthState === "none")
) {
return { ready: true, detail: lastState };
}
// These states cannot become ready without another lifecycle action.
Expand All @@ -284,16 +295,17 @@ function recoveredAfterReadiness(
containerName: string,
via: DockerDriverRecoveryVia,
deps: ReturnType<typeof depsWithDefaults>,
readiness: NonNullable<DockerDriverRecoveryDeps["readiness"]>,
): DockerDriverRecoveryResult {
const readiness = waitForRecoveredContainerReady(containerName, deps);
if (!readiness.ready) {
const result = waitForRecoveredContainerReady(containerName, deps, readiness);
if (!result.ready) {
return {
recovered: false,
via: null,
containerName,
detail:
`docker container ${containerName} did not become ready after recovery ` +
`(${readiness.detail})`,
`(${result.detail})`,
};
}
return {
Expand Down Expand Up @@ -322,6 +334,7 @@ export function recoverDockerDriverSandbox(
deps: DockerDriverRecoveryDeps = {},
): DockerDriverRecoveryResult {
const d = depsWithDefaults(deps);
const readiness = deps.readiness ?? "docker-health";
const containers = findLabeledSandboxContainers(sandboxName, deps);
if (containers.length === 0) {
return {
Expand All @@ -347,9 +360,9 @@ export function recoverDockerDriverSandbox(
detail: `docker unpause ${runningOriginal.name} failed (exit ${unpause.status ?? "unknown"}).`,
};
}
return recoveredAfterReadiness(runningOriginal.name, "unpaused-original", d);
return recoveredAfterReadiness(runningOriginal.name, "unpaused-original", d, readiness);
}
return recoveredAfterReadiness(runningOriginal.name, "started-running-original", d);
return recoveredAfterReadiness(runningOriginal.name, "started-running-original", d, readiness);
}

if (stoppedOriginal) {
Expand All @@ -358,7 +371,12 @@ export function recoverDockerDriverSandbox(
timeout: DOCKER_OPERATION_TIMEOUT_MS,
});
if (result.status === 0) {
return recoveredAfterReadiness(stoppedOriginal.name, "started-stopped-original", d);
return recoveredAfterReadiness(
stoppedOriginal.name,
"started-stopped-original",
d,
readiness,
);
}
return {
recovered: false,
Expand All @@ -385,7 +403,7 @@ export function recoverDockerDriverSandbox(
timeout: DOCKER_OPERATION_TIMEOUT_MS,
});
if (startResult.status === 0) {
return recoveredAfterReadiness(restoreName, "renamed-and-started-backup", d);
return recoveredAfterReadiness(restoreName, "renamed-and-started-backup", d, readiness);
}
return {
recovered: false,
Expand Down
8 changes: 7 additions & 1 deletion src/lib/onboard/runtime-provider/docker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,13 @@ function startDockerSandbox(
return { exitCode: 0 };
}

const recovery = deps.recoverSandbox(input.sandboxName);
// Docker health is an image-level signal, not the lifecycle authority for
// `start`. Once the container is running, verifyStarted performs the
// provider-owned OpenShell, managed gateway, and host-forward recovery.
// Waiting for Docker health here can prevent that repair from running.
const recovery = deps.recoverSandbox(input.sandboxName, {
readiness: "runtime-running",
});
if (!recovery.recovered) {
return {
exitCode: 1,
Expand Down
Loading