Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
1ceef4d
fix(onboard): finalize exact Docker sandbox handoff
apurvvkumaria Aug 23, 2026
1172663
fix(onboard): retain failed handoff recovery
apurvvkumaria Aug 23, 2026
006feaf
test(onboard): require explicit fixture removal
apurvvkumaria Aug 24, 2026
e91f4e9
Merge branch 'main' into codex/issue-9531-final-handoff
apurvvkumaria Aug 24, 2026
2097ebd
Merge branch 'main' into codex/issue-9531-final-handoff
apurvvkumaria Aug 24, 2026
d37fbef
merge: resolve conflicts with main
github-actions[bot] Aug 24, 2026
d5ea62c
Merge branch 'main' into codex/issue-9531-final-handoff
apurvvkumaria Aug 24, 2026
bb5d7a2
merge: resolve conflicts with main
github-actions[bot] Aug 24, 2026
6e6d0d1
fix(onboard): define final handoff commit point
apurvvkumaria Aug 24, 2026
b13eba7
merge: integrate current main
apurvvkumaria Aug 24, 2026
4a93657
Merge branch 'main' into codex/issue-9531-final-handoff
cv Aug 24, 2026
4424e0f
Merge branch 'main' into codex/issue-9531-final-handoff
cv Aug 24, 2026
d36ae6f
Merge branch 'main' into codex/issue-9531-final-handoff
cv Aug 24, 2026
286a408
Merge branch 'main' into codex/issue-9531-final-handoff
cv Aug 24, 2026
29e4885
Merge branch 'main' into codex/issue-9531-final-handoff
cv Aug 24, 2026
377837d
merge: resolve conflicts with main
github-actions[bot] Aug 24, 2026
e79f7eb
Merge branch 'main' into codex/issue-9531-final-handoff
cv Aug 24, 2026
b48b4d3
Merge branch 'main' into codex/issue-9531-final-handoff
cv Aug 24, 2026
ca92a03
merge: integrate current main
ericksoa Aug 25, 2026
c7d1e94
fix(onboard): use canonical sandbox container query
ericksoa Aug 25, 2026
5548731
fix(onboard): restore prior container after handoff failure
ericksoa Aug 25, 2026
2392026
fix(onboard): persist failed handoff recovery
ericksoa Aug 25, 2026
9588a38
test(onboard): model rollback image commands
ericksoa Aug 25, 2026
54ba903
fix(onboard): secure rollback recovery records
ericksoa Aug 25, 2026
cd3669b
fix(recovery): report retained rollback artifacts
ericksoa Aug 25, 2026
c64e1e5
merge: resolve conflicts with main
github-actions[bot] Aug 25, 2026
f79330c
fix(onboard): restore bounded handoff contract
ericksoa Aug 25, 2026
87368cf
fix(recovery): preserve bounded handoff diagnostics
ericksoa Aug 25, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 41 additions & 30 deletions src/lib/actions/sandbox/process-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -506,11 +506,16 @@ function finalRelaunchContainerFailureDetail(
return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward";
}
if (completion.lifecycleReleaseObserved === false) {
return "OpenShell did not release the sandbox name before the final recovery handoff. NemoClaw did not restart the replacement container or start the primary dashboard/API host forward";
return "OpenShell did not retire the previous lifecycle record before the final replacement start. NemoClaw did not start the primary dashboard/API host forward";
}
if (completion.replacementRestarted === false) {
return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward";
}
if (completion.finalHandoffAcknowledged === false) {
return `OpenShell did not acknowledge the final replacement container handoff${
completion.lastSandboxPhase ? `; last sandbox phase was ${completion.lastSandboxPhase}` : ""
}. NemoClaw did not start the primary dashboard/API host forward`;
}
return null;
}

Expand Down Expand Up @@ -759,6 +764,15 @@ function recoverSandboxProcesses(
quiet,
deps: {
runOpenshell,
runCaptureOpenshell: (args, options) =>
captureOpenshell(args, {
ignoreError: true,
includeStderr: true,
killProcessTreeOnTimeout: options?.killProcessTreeOnTimeout === true,
killSignal: options?.killSignal === "SIGKILL" ? "SIGKILL" : undefined,
timeout:
typeof options?.timeout === "number" ? options.timeout : OPENSHELL_PROBE_TIMEOUT_MS,
}).output,
confirmMissingSupervisor: (containerId) =>
isExactlyMissingManagedSupervisor(
requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId),
Expand Down Expand Up @@ -830,32 +844,32 @@ export function restartSandboxGateway(
return withUnsupportedHermesPortableGatewayRestartFence(sandboxName, () => {
return withTimerBoundShieldsMutationLock(sandboxName, "gateway restart", () =>
restartSandboxGatewayWithDeps(sandboxName, {
quiet,
deps: {
getSessionAgent: agentRuntime.getSessionAgent,
getSandbox: registry.getSandbox,
resolveSandboxDashboardPort,
requestGatewaySupervisorAction: executeGatewaySupervisorAction,
executeSandboxExecCommand,
waitForRecoveredSandboxGateway: (name, options) =>
waitForRecoveredSandboxGateway(name, {
...options,
initialManagedHealthPassed: true,
timeoutSeconds: gatewayRecoveryTimeoutSeconds(agentRuntime.getSessionAgent(name)),
managedProbeImpl: (sandboxName) =>
confirmRecoveredSandboxGatewayManaged(sandboxName, {
requestGatewaySupervisorActionImpl:
deps.requestGatewaySupervisorAction ?? executeGatewaySupervisorAction,
}),
quiet,
deps: {
getSessionAgent: agentRuntime.getSessionAgent,
getSandbox: registry.getSandbox,
resolveSandboxDashboardPort,
requestGatewaySupervisorAction: executeGatewaySupervisorAction,
executeSandboxExecCommand,
waitForRecoveredSandboxGateway: (name, options) =>
waitForRecoveredSandboxGateway(name, {
...options,
initialManagedHealthPassed: true,
timeoutSeconds: gatewayRecoveryTimeoutSeconds(agentRuntime.getSessionAgent(name)),
managedProbeImpl: (sandboxName) =>
confirmRecoveredSandboxGatewayManaged(sandboxName, {
requestGatewaySupervisorActionImpl:
deps.requestGatewaySupervisorAction ?? executeGatewaySupervisorAction,
}),
}),
ensureSandboxPortForward,
ensureHermesDashboardPortForwardIfEnabled,
recoverMessagingHostForward,
recoverDeclaredAgentForwardPorts,
printGatewayWedgeDiagnostics,
inspectHermesMcpReconciliationRefusal,
...deps,
},
ensureSandboxPortForward,
ensureHermesDashboardPortForwardIfEnabled,
recoverMessagingHostForward,
recoverDeclaredAgentForwardPorts,
printGatewayWedgeDiagnostics,
inspectHermesMcpReconciliationRefusal,
...deps,
},
}),
);
});
Expand All @@ -879,10 +893,7 @@ const OPENSHELL_RELAY_TARGET_NOT_FOUND = 'message: "No such file or directory (o
const OPENSHELL_RELAY_TARGET_REFUSED = 'message: "Connection refused (os error 111)"';

function normalizeOpenshellStructuredError(value: string): string {
return stripAnsi(value)
.replace(/[×│]/gu, " ")
.replace(/\s+/gu, " ")
.trim();
return stripAnsi(value).replace(/[×│]/gu, " ").replace(/\s+/gu, " ").trim();
}

function hasRetryableOpenshellFailureShape(result: ReturnType<typeof captureOpenshell>): boolean {
Expand Down
41 changes: 36 additions & 5 deletions src/lib/actions/sandbox/supervisor-relaunch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,11 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) {
})),
removeBackup: vi.fn(() => true),
runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })),
runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"),
recreate: vi.fn(() => patchResult()),
finalize: vi.fn(({ supervisorReady }) =>
supervisorReady
? { backupRemoved: true, rolledBack: false }
? { backupRemoved: true, finalHandoffAcknowledged: true, rolledBack: false }
: { backupRemoved: false, rolledBack: true },
),
...overrides,
Expand Down Expand Up @@ -148,6 +149,7 @@ describe("relaunchManagedSupervisorSession", () => {

expect(relaunch?.finalize(true)).toEqual({
backupRemoved: true,
finalHandoffAcknowledged: true,
rolledBack: false,
stateRestored: true,
stateBackupRemoved: true,
Expand All @@ -156,12 +158,15 @@ describe("relaunchManagedSupervisorSession", () => {
expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
expect(deps.finalize).toHaveBeenCalledWith(
{
lifecycleReleaseTimeoutSecs: 900,
finalHandoffTimeoutSecs: 900,
result: expect.objectContaining({ newContainerId: "new-container-id" }),
sandboxName: "alpha",
supervisorReady: true,
},
{ runOpenshell: deps.runOpenshell },
{
runCaptureOpenshell: deps.runCaptureOpenshell,
runOpenshell: deps.runOpenshell,
},
);
});

Expand Down Expand Up @@ -389,12 +394,15 @@ describe("relaunchManagedSupervisorSession", () => {
expect(deps.restartRestoredManagedGateway).toHaveBeenCalledWith("new-container-id");
expect(deps.finalize).toHaveBeenCalledWith(
{
lifecycleReleaseTimeoutSecs: 900,
finalHandoffTimeoutSecs: 900,
result: expect.objectContaining({ newContainerId: "new-container-id" }),
sandboxName: "alpha",
supervisorReady: true,
},
{ runOpenshell: deps.runOpenshell },
{
runCaptureOpenshell: deps.runCaptureOpenshell,
runOpenshell: deps.runOpenshell,
},
);
});

Expand All @@ -405,6 +413,7 @@ describe("relaunchManagedSupervisorSession", () => {

expect(relaunch?.finalize(true)).toMatchObject({ backupRemoved: true, rolledBack: false });
expect(deps.finalize).toHaveBeenCalledWith(expect.objectContaining({ supervisorReady: true }), {
runCaptureOpenshell: deps.runCaptureOpenshell,
runOpenshell: deps.runOpenshell,
sleep,
});
Expand Down Expand Up @@ -492,12 +501,34 @@ describe("relaunchManagedSupervisorSession", () => {

expect(relaunch?.finalize(true)).toEqual({
backupRemoved: true,
finalHandoffAcknowledged: true,
rolledBack: false,
stateRestored: true,
stateBackupRemoved: false,
});
});

it("retains the restored state backup when final handoff is not acknowledged (#9531)", () => {
const deps = baseDeps({
finalize: vi.fn(() => ({
backupRemoved: true,
finalHandoffAcknowledged: false,
lastSandboxPhase: "Deleting",
rolledBack: false,
})),
});
const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });

expect(relaunch?.finalize(true)).toEqual({
backupRemoved: true,
finalHandoffAcknowledged: false,
lastSandboxPhase: "Deleting",
rolledBack: false,
stateRestored: true,
});
expect(deps.removeBackup).not.toHaveBeenCalled();
});

it("returns null when the pinned recreation fails", () => {
const deps = baseDeps({
recreate: vi.fn(() => {
Expand Down
30 changes: 19 additions & 11 deletions src/lib/actions/sandbox/supervisor-relaunch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ export type ManagedSupervisorRelaunchDeps = {
recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand;
finalize?: typeof finalizeDockerGpuPatchBackup;
runOpenshell?: NonNullable<Parameters<typeof finalizeDockerGpuPatchBackup>[1]>["runOpenshell"];
runCaptureOpenshell?: NonNullable<
Parameters<typeof finalizeDockerGpuPatchBackup>[1]
>["runCaptureOpenshell"];
};

function inspectContainer(containerId: string): DockerContainerInspect {
Expand Down Expand Up @@ -287,23 +290,28 @@ export function relaunchManagedSupervisorSession(
return finalizeFailure();
}
const runLifecycleProbe = deps.runOpenshell;
if (!runLifecycleProbe) return finalizeFailure();
const captureLifecycleProbe = deps.runCaptureOpenshell;
if (!runLifecycleProbe || !captureLifecycleProbe) return finalizeFailure();
const lifecycleDeps = {
runCaptureOpenshell: captureLifecycleProbe,
runOpenshell: runLifecycleProbe,
...(deps.sleep ? { sleep: deps.sleep } : {}),
};
const finalized = finalize(
{
result,
supervisorReady: true,
sandboxName,
finalHandoffTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1),
},
lifecycleDeps,
);
const outcome = {
...finalize(
{
result,
supervisorReady: true,
sandboxName,
lifecycleReleaseTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1),
},
lifecycleDeps,
),
...finalized,
stateRestored: true,
stateBackupRemoved: removeSettledStateBackup(),
...(finalized.finalHandoffAcknowledged === true
? { stateBackupRemoved: removeSettledStateBackup() }
: {}),
};
completed = { supervisorReady, outcome };
return outcome;
Expand Down
16 changes: 16 additions & 0 deletions src/lib/adapters/openshell/client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
stripAnsi,
versionGte,
} from "./client";
import { processTreeBoundedOpenshellInvocation } from "./process-tree-timeout";

interface SpawnResultSpec {
status: number | null;
Expand Down Expand Up @@ -51,6 +52,21 @@ function exitWithCode(code: number): never {
}

describe("openshell helpers", () => {
it("wraps a synchronous Linux probe in a process-group timeout (#10238)", () => {
expect(
processTreeBoundedOpenshellInvocation(
"/opt/openshell",
["sandbox", "list"],
{ killProcessTreeOnTimeout: true, timeout: 1000 },
{ platform: "linux", timeoutExecutableExists: () => true },
),
).toEqual({
binary: "/usr/bin/timeout",
args: ["--signal=KILL", "0.75s", "/opt/openshell", "sandbox", "list"],
killSignal: "SIGKILL",
});
});

afterEach(() => {
vi.unstubAllEnvs();
});
Expand Down
12 changes: 9 additions & 3 deletions src/lib/adapters/openshell/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {

import { redirectInheritedChildStdoutToStderr } from "../../cli/stdout-guard";
import { buildSubprocessEnv } from "../../subprocess-env";
import { processTreeBoundedOpenshellInvocation } from "./process-tree-timeout";

export { openshellSandboxSshHost, resolveOpenshellSandboxSshHost } from "./sandbox-ssh-host";

Expand All @@ -28,6 +29,7 @@ interface OpenshellSpawnOptions {
env?: NodeJS.ProcessEnv;
replaceEnv?: boolean;
timeout?: number;
killProcessTreeOnTimeout?: boolean;
ignoreError?: boolean;
spawnSyncImpl?: OpenshellSpawnSync;
errorLine?: (message: string) => void;
Expand All @@ -53,6 +55,7 @@ export interface RunOpenshellOptions extends OpenshellSpawnOptions {
export interface CaptureOpenshellOptions extends OpenshellSpawnOptions {
includeStderr?: boolean;
includeStreams?: boolean;
killSignal?: SpawnSyncOptions["killSignal"];
maxBuffer?: number;
}

Expand Down Expand Up @@ -207,14 +210,15 @@ export function runOpenshellCommand(
opts: RunOpenshellOptions = {},
): SpawnSyncReturns<string> {
const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync;
const result = spawnSyncImpl(binary, args, {
const bounded = processTreeBoundedOpenshellInvocation(binary, args, opts);
const result = spawnSyncImpl(bounded.binary, bounded.args, {
cwd: opts.cwd,
env: openshellSpawnEnv(opts),
encoding: "utf-8",
stdio: redirectInheritedChildStdoutToStderr(opts.stdio ?? "inherit"),
input: opts.input,
timeout: opts.timeout,
killSignal: opts.killSignal,
killSignal: bounded.killSignal,
maxBuffer: opts.maxBuffer,
});
if (result.error) {
Expand All @@ -236,12 +240,14 @@ export function captureOpenshellCommand(
opts: CaptureOpenshellOptions = {},
): CaptureOpenshellResult {
const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync;
const result = spawnSyncImpl(binary, args, {
const bounded = processTreeBoundedOpenshellInvocation(binary, args, opts);
const result = spawnSyncImpl(bounded.binary, bounded.args, {
cwd: opts.cwd,
env: openshellSpawnEnv(opts),
encoding: "utf-8",
stdio: ["ignore", "pipe", "pipe"],
timeout: opts.timeout,
killSignal: bounded.killSignal,
maxBuffer: opts.maxBuffer,
});
if (result.error) {
Expand Down
51 changes: 51 additions & 0 deletions src/lib/adapters/openshell/process-tree-timeout.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import type { SpawnSyncOptions } from "node:child_process";
import fs from "node:fs";

type ProcessTreeTimeoutDeps = {
platform?: NodeJS.Platform;
timeoutExecutableExists?: (pathname: string) => boolean;
};

export type ProcessTreeTimeoutOptions = {
killProcessTreeOnTimeout?: boolean;
killSignal?: SpawnSyncOptions["killSignal"];
timeout?: number;
};

/**
* #10238 ownership decision: NemoClaw owns the wall-clock bound at its
* synchronous OpenShell adapter boundary. This remains opt-in for lifecycle
* probes; ordinary commands keep their existing execution behavior. Linux uses
* a process-group timeout so descendants cannot retain captured pipes after the
* direct CLI is killed. Real DGX Spark confirmation remains tracked by #10238.
*/
export function processTreeBoundedOpenshellInvocation(
binary: string,
args: readonly string[],
opts: ProcessTreeTimeoutOptions,
deps: ProcessTreeTimeoutDeps = {},
): { binary: string; args: string[]; killSignal?: SpawnSyncOptions["killSignal"] } {
if (opts.killProcessTreeOnTimeout !== true) {
return { binary, args: [...args], killSignal: opts.killSignal };
}
const timeoutMs = Number(opts.timeout);
const platform = deps.platform ?? process.platform;
const timeoutExecutableExists = deps.timeoutExecutableExists ?? fs.existsSync;
if (
platform !== "linux" ||
!timeoutExecutableExists("/usr/bin/timeout") ||
!Number.isFinite(timeoutMs) ||
timeoutMs <= 0
) {
return { binary, args: [...args], killSignal: "SIGKILL" };
}
const groupTimeoutSeconds = Math.max(1, Math.floor(timeoutMs) - 250) / 1000;
return {
binary: "/usr/bin/timeout",
args: ["--signal=KILL", `${String(groupTimeoutSeconds)}s`, binary, ...args],
killSignal: "SIGKILL",
};
}
Loading
Loading