Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
4be72a7
test(recovery): reproduce final handoff capture failure
rsliter Aug 26, 2026
dfd1139
fix(recovery): capture final handoff lifecycle state
rsliter Aug 26, 2026
f6b26e0
Merge branch 'main' into codex/fix-10153-final-handoff
prekshivyas Aug 26, 2026
1176511
test(onboard): align rollback lifecycle capture fixture
prekshivyas Aug 26, 2026
cd12de1
Merge remote-tracking branch 'origin/main' into pr10339-local-fix
prekshivyas Aug 26, 2026
9c40562
Merge commit '1527587930720b17e6ba8fbbdade0599842fa0fb' into pr10339-…
prekshivyas Aug 26, 2026
c1e53de
test(e2e): parse legacy handoff receipt after progress
prekshivyas Aug 26, 2026
9ce71f5
Merge origin/main into codex/fix-10153-final-handoff
prekshivyas Aug 26, 2026
1093a53
fix(recovery): retry incomplete supervisor discovery
prekshivyas Aug 26, 2026
80e8f0b
fix(recovery): wait for lifecycle name absence
prekshivyas Aug 26, 2026
70a94ef
Merge remote-tracking branch 'origin/main' into pr10339-local-fix
prekshivyas Aug 26, 2026
403b25d
test(recovery): cover lifecycle deletion sequence
prekshivyas Aug 26, 2026
92b5c5f
test(recovery): cover lifecycle deletion sequence
prekshivyas Aug 26, 2026
5ab9022
Merge branch 'codex/fix-10153-final-handoff' of github.com:NVIDIA/Nem…
prekshivyas Aug 27, 2026
8faae46
Merge remote-tracking branch 'origin/main' into pr10339-local-fix
prekshivyas Aug 27, 2026
dd50adb
Merge branch 'main' into codex/fix-10153-final-handoff
ericksoa Aug 27, 2026
da0926f
Merge remote-tracking branch 'origin/main' into pr10339-main-update
prekshivyas Aug 27, 2026
faa54cc
test(onboard): capture lifecycle fixture output
prekshivyas Aug 27, 2026
3eb72e9
test(onboard): keep messaging fixture within budget
prekshivyas Aug 27, 2026
6015a93
test(onboard): cover messaging lifecycle capture
prekshivyas Aug 27, 2026
e68696f
fix(recovery): fence final handoff through OpenShell
prekshivyas Aug 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 89 additions & 47 deletions src/lib/onboard/docker-gpu-patch-finalize.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,12 @@ function exactDeferredCreateResult(): DockerGpuPatchResult {

function readyHandoffDeps() {
return {
runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"),
runOpenshell: vi.fn((args: readonly string[]) =>
args[1] === "list"
? { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" }
: { status: 0 },
),
runCaptureOpenshell: vi
.fn()
.mockReturnValueOnce("beta 2026-08-23 10:00:00 Ready\n")
.mockReturnValue("alpha 2026-08-23 10:00:02 Ready\n"),
runOpenshell: vi.fn(() => ({ status: 0 })),
sleep: vi.fn(),
};
}

Expand Down Expand Up @@ -124,21 +124,19 @@ describe("finalizeDockerGpuPatchBackup", () => {
events.push("start replacement");
return { status: 0 };
});
const runCaptureOpenshell = vi.fn(() => {
events.push("observe ready");
return "alpha 2026-08-23 10:00:00 Ready\n";
});
const runOpenshellResults = {
exec: { event: "exec ready", result: { status: 0 } },
list: {
event: "observe lifecycle release",
result: { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" },
},
} as const;
const runOpenshell = vi.fn((args: readonly string[]) => {
const response = runOpenshellResults[args[1] === "list" ? "list" : "exec"];
events.push(response.event);
return response.result;
const runCaptureOpenshell = vi
.fn()
.mockImplementationOnce(() => {
events.push("observe lifecycle release");
return "beta 2026-08-23 10:00:00 Ready\n";
})
.mockImplementation(() => {
events.push("observe ready");
return "alpha 2026-08-23 10:00:02 Ready\n";
});
const runOpenshell = vi.fn(() => {
events.push("exec ready");
return { status: 0 };
});
const dockerResults = {
ps: {
Expand Down Expand Up @@ -236,15 +234,14 @@ describe("finalizeDockerGpuPatchBackup", () => {
events.push("start replacement");
return { status: 0 };
}),
runCaptureOpenshell: vi.fn(() => {
events.push("observe deleting");
return "alpha 2026-08-23 10:00:00 Deleting\n";
}),
runOpenshell: vi.fn((args: readonly string[]) =>
args[1] === "list"
? { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" }
: { status: 1 },
),
runCaptureOpenshell: vi
.fn()
.mockReturnValueOnce("beta 2026-08-23 10:00:00 Ready\n")
.mockImplementation(() => {
events.push("observe deleting");
return "alpha 2026-08-23 10:00:02 Deleting\n";
}),
runOpenshell: vi.fn(() => ({ status: 1 })),
dockerRun: vi.fn(() => ({ status: 0, stdout: `${result.newContainerId}\n` })),
sleep,
},
Expand Down Expand Up @@ -290,12 +287,11 @@ describe("finalizeDockerGpuPatchBackup", () => {
dockerRm: vi.fn(() => ({ status: 0 })),
dockerStart: vi.fn(() => ({ status: 0 })),
dockerRun,
runCaptureOpenshell: vi.fn(() => "restored-name 2026-08-23 01:40:35 Ready\n"),
runOpenshell: vi.fn((args: readonly string[]) =>
args[1] === "list"
? { status: 0, stdout: "restored-name 2026-08-23 01:40:35 Error\n" }
: { status: 0 },
),
runCaptureOpenshell: vi
.fn()
.mockReturnValueOnce("restored-name 2026-08-23 01:40:35 Error\n")
.mockReturnValue("restored-name 2026-08-23 01:40:37 Ready\n"),
runOpenshell: vi.fn(() => ({ status: 0 })),
sleep: vi.fn(),
},
);
Expand Down Expand Up @@ -363,12 +359,11 @@ describe("finalizeDockerGpuPatchBackup", () => {
dockerRm: vi.fn(() => ({ status: 0 })),
dockerStart: vi.fn(() => ({ status: 0 })),
dockerRun,
runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"),
runOpenshell: vi.fn((args: readonly string[]) =>
args[1] === "list"
? { status: 0, stdout: "beta 2026-08-23 01:40:35 Ready\n" }
: { status: 0 },
),
runCaptureOpenshell: vi
.fn()
.mockReturnValueOnce("beta 2026-08-23 01:40:35 Ready\n")
.mockReturnValue("alpha 2026-08-23 01:40:37 Ready\n"),
runOpenshell: vi.fn(() => ({ status: 0 })),
sleep: vi.fn(),
},
);
Expand All @@ -381,6 +376,56 @@ describe("finalizeDockerGpuPatchBackup", () => {
});
});

it("captures the retiring lifecycle row before restarting the exact replacement (#10153)", () => {
const result = exactDeferredCreateResult();
const dockerRunResults = {
inspect: { status: 0, stdout: "true\n" },
ps: { status: 0, stdout: `${result.newContainerId}\n` },
} as const;
const dockerStart = vi.fn(() => ({ status: 0 }));
const runCaptureOpenshell = vi
.fn()
.mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n")
.mockReturnValueOnce("alpha 2026-08-23 10:00:02 Ready\n");
const runOpenshell = vi.fn(() => ({ status: 0 }));

const outcome = finalizeDockerGpuPatchBackup(
{
result,
supervisorReady: true,
sandboxName: "alpha",
finalHandoffTimeoutSecs: 1,
},
{
dockerStop: vi.fn(() => ({ status: 0 })),
dockerRm: vi.fn(() => ({ status: 0 })),
dockerRun: vi.fn(
(args: readonly string[]) =>
dockerRunResults[String(args[0]) as keyof typeof dockerRunResults],
),
dockerStart,
runCaptureOpenshell,
runOpenshell,
sleep: vi.fn(),
},
);

expect(outcome).toMatchObject({
backupRemoved: true,
lifecycleReleaseObserved: true,
replacementRestarted: true,
finalHandoffAcknowledged: true,
});
expect(dockerStart).toHaveBeenCalledWith(
result.newContainerId,
expect.objectContaining({ ignoreError: true }),
);
expect(runOpenshell).not.toHaveBeenCalledWith(
["sandbox", "list"],
expect.any(Object),
);
});

it.each([
["a failed query", { status: 1, stderr: "daemon unavailable" }],
["no labeled replacement", { status: 0, stdout: "" }],
Expand All @@ -405,11 +450,8 @@ describe("finalizeDockerGpuPatchBackup", () => {
dockerRm: vi.fn(() => ({ status: 0 })),
dockerRun: vi.fn(() => query),
dockerStart,
runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"),
runOpenshell: vi.fn(() => ({
status: 0,
stdout: "alpha 2026-08-23 01:40:35 Error\n",
})),
runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Error\n"),
runOpenshell: vi.fn(() => ({ status: 0 })),
sleep: vi.fn(),
},
);
Expand Down
2 changes: 1 addition & 1 deletion src/lib/onboard/docker-gpu-patch-finalize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ export function finalizeDockerGpuPatchBackup(
options.sandboxName,
options.finalHandoffTimeoutSecs,
{
runOpenshell: deps.runOpenshell,
runCaptureOpenshell: deps.runCaptureOpenshell,
sleep: deps.sleep,
soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) =>
isExactOpenShellReplacement(
Expand Down
10 changes: 8 additions & 2 deletions src/lib/onboard/docker-gpu-patch-recreate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ describe("Docker GPU recreate orchestration", () => {
const runOpenshell = vi.fn((args: readonly string[]) =>
args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 },
);
const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n");
const runCaptureOpenshell = vi
.fn()
.mockReturnValueOnce("No sandboxes found.\n")
.mockReturnValue("alpha 2026-08-23 10:00:02 Ready\n");

const result = recreateOpenShellDockerSandboxWithGpu(
{ sandboxName: "alpha", timeoutSecs: 1 },
Expand Down Expand Up @@ -125,7 +128,10 @@ describe("Docker GPU recreate orchestration", () => {
dockerStop: vi.fn(() => ({ status: 0 })),
dockerRm: vi.fn(() => ({ status: 0 })),
dockerStart: vi.fn(() => ({ status: 0 })),
runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Deleting\n"),
runCaptureOpenshell: vi
.fn()
.mockReturnValueOnce("No sandboxes found.\n")
.mockReturnValue("alpha 2026-08-23 10:00:02 Deleting\n"),
runOpenshell: vi.fn((args: readonly string[]) =>
args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 },
),
Expand Down
6 changes: 5 additions & 1 deletion src/lib/onboard/docker-gpu-patch-rollback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,11 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => {
runOpenshell: vi.fn((args: readonly string[]) =>
args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 },
),
runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"),
runCaptureOpenshell: vi.fn(() =>
!restoredPresent && !startTargets.includes(retryId)
? "No sandboxes found.\n"
: "alpha 2026-08-23 10:00:02 Ready\n",
),
sleep: vi.fn(),
homedir: () => tmpDir,
now: () => new Date("2026-07-03T00:00:00Z"),
Expand Down
33 changes: 22 additions & 11 deletions src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,19 @@ import {
describe("Docker GPU final lifecycle release", () => {
it("requires corroboration for a retiring lifecycle row (#9962)", () => {
const corroborate = vi.fn(() => true);
const runOpenshell = vi.fn(() => ({
status: 0,
stdout: "alpha 2026-08-23 01:40:35 Deleting\n",
}));
const runCaptureOpenshell = vi.fn(
() => "alpha 2026-08-23 01:40:35 Deleting\n",
);

expect(
waitForOpenShellSandboxLifecycleRelease("alpha", 1, {
runOpenshell,
runCaptureOpenshell,
sleep: vi.fn(),
soleLabeledReplacementCorroboratesRetiringPhase: corroborate,
}),
).toBe(true);
expect(corroborate).toHaveBeenCalledOnce();
expect(runOpenshell).toHaveBeenCalledWith(
expect(runCaptureOpenshell).toHaveBeenCalledWith(
["sandbox", "list"],
expect.objectContaining({
killProcessTreeOnTimeout: true,
Expand All @@ -38,17 +37,29 @@ describe("Docker GPU final lifecycle release", () => {
});

it("does not accept an uncorroborated retiring lifecycle row (#9962)", () => {
const runOpenshell = vi.fn(() => ({
status: 0,
stdout: "alpha 2026-08-23 01:40:35 Error\n",
}));
const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 01:40:35 Error\n");

expect(
waitForOpenShellSandboxLifecycleRelease("alpha", 1, {
runOpenshell,
runCaptureOpenshell,
sleep: vi.fn(),
}),
).toBe(false);
});

it("does not release the lifecycle when the captured sandbox list is unavailable (#10153)", () => {
const corroborate = vi.fn(() => true);

expect(
waitForOpenShellSandboxLifecycleRelease("alpha", 1, {
runCaptureOpenshell: vi.fn(() => {
throw new Error("sandbox list transport unavailable");
}),
sleep: vi.fn(),
soleLabeledReplacementCorroboratesRetiringPhase: corroborate,
}),
).toBe(false);
expect(corroborate).not.toHaveBeenCalled();
});

it("does not report reconnect without an OpenShell execution boundary (#9531)", () => {
Expand Down
28 changes: 18 additions & 10 deletions src/lib/onboard/docker-gpu-supervisor-reconnect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ export type DockerGpuSupervisorReconnectDeps = {

type DockerLifecycleReleaseDeps = Pick<
DockerGpuSupervisorReconnectDeps,
"runOpenshell" | "sleep"
"runCaptureOpenshell" | "sleep"
> & {
/** Corroborates a retiring lifecycle row with the stopped exact replacement. */
soleLabeledReplacementCorroboratesRetiringPhase?: (remainingMs: number) => boolean;
Expand Down Expand Up @@ -114,22 +114,30 @@ export function waitForOpenShellSandboxLifecycleRelease(
timeoutSecs: number,
deps: DockerLifecycleReleaseDeps,
): boolean {
if (!deps.runOpenshell) return false;
if (!deps.runCaptureOpenshell) return false;
const sleep = deps.sleep ?? defaultSleep;
const deadline = Date.now() + Math.max(1, Math.round(timeoutSecs)) * 1000;
const maxAttempts = Math.max(1, Math.ceil(Math.max(1, Math.round(timeoutSecs)) / 2) + 1);

for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) break;
const result = deps.runOpenshell(["sandbox", "list"], {
ignoreError: true,
...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS,
suppressOutput: true,
timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingMs),
});
if (hasZeroDockerExitStatus(result)) {
const output = String(result.stdout ?? "").trim();
let output = "";
try {
// The streaming runner does not return sandbox-list stdout. Capture the
// row so a successful command cannot hide a retiring lifecycle phase.
output = deps
.runCaptureOpenshell(["sandbox", "list"], {
ignoreError: true,
...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS,
suppressOutput: true,
timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingMs),
})
.trim();
} catch {
output = "";
}
if (output) {
const entries = parseLiveSandboxEntries(output);
const sandboxPresent = entries.some((entry) => entry.name === sandboxName);
const retiring = entries.some(
Expand Down
Loading
Loading