From 2f14e703f093c05a2c0425eadebced6730be8d4e Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 13 Jul 2026 22:40:44 -0700 Subject: [PATCH 1/5] test(e2e): fix deterministic nightly failures --- test/e2e/live/openshell-version-pin.test.ts | 36 ++++++++++++++++----- test/e2e/live/shields-config.test.ts | 15 +++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/test/e2e/live/openshell-version-pin.test.ts b/test/e2e/live/openshell-version-pin.test.ts index fa4fe691d1e..860465e9db8 100644 --- a/test/e2e/live/openshell-version-pin.test.ts +++ b/test/e2e/live/openshell-version-pin.test.ts @@ -250,14 +250,39 @@ esac`, ); } -// tar stub: write the corresponding binary into the -C outdir. Each binary -// reports the replacement version + carries the messaging-rewrite and MCP-L7 -// capability markers so the post-install feature probes pass. +// tar stub: model archive listing, inspection, and extraction. Each extracted +// binary reports the replacement version + carries the messaging-rewrite and +// MCP-L7 capability markers so the post-install feature probes pass. function createFakeTar(binDir: string, replacementVersion: string): void { writeExecutable( path.join(binDir, "tar"), `#!/usr/bin/env bash set -euo pipefail +mode="\${1:-}" +archive="\${2:-}" +case "$(basename "$archive")" in + openshell-gateway-*) name="openshell-gateway" ;; + openshell-sandbox-*) name="openshell-sandbox" ;; + openshell-*) name="openshell" ;; + *) exit 2 ;; +esac + +case "$mode" in + -tzf) + printf '%s\\n' "$name" + exit 0 + ;; + -tvzf) + printf '%s\\n' "-rwxr-xr-x 0/0 0 2026-01-01 00:00 $name" + exit 0 + ;; + xzf|-xzf) + ;; + *) + exit 2 + ;; +esac + outdir="" prev="" for arg in "$@"; do @@ -268,11 +293,6 @@ for arg in "$@"; do prev="$arg" done [ -n "$outdir" ] || exit 1 -case "$*" in - *openshell-gateway*) name="openshell-gateway" ;; - *openshell-sandbox*) name="openshell-sandbox" ;; - *) name="openshell" ;; -esac cat > "$outdir/$name" <<'EOS' #!/usr/bin/env bash if [ "\${1:-}" = "--version" ]; then echo "openshell ${replacementVersion}"; exit 0; fi diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index 85b4df8a19d..0bd30a7428e 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -434,6 +434,13 @@ test("shields-config: live shields up/down locks config and detects drift", { }); expect(shieldsUp.exitCode, resultText(shieldsUp)).toBe(0); expect(resultText(shieldsUp)).toContain("Lockdown active"); + // Keep fixture teardown out of an artificial mutable window if a later + // assertion aborts before the explicit final restore below. + cleanup.trackDisposable(`restore shields for ${SANDBOX_NAME} before destroy`, async () => { + await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { + artifactName: "cleanup-shields-up-before-destroy", + }).catch(() => undefined); + }); const configUp = await statPath(sandbox, CONFIG_PATH, "phase-3-config-perms-up"); expect(configUp.mode).toMatch(/^4[0-4][0-4]$/); @@ -684,6 +691,14 @@ test("shields-config: live shields up/down locks config and detects drift", { expect(doubleDown.exitCode, resultText(doubleDown)).not.toBe(0); expect(resultText(doubleDown)).toContain("already unlocked"); + // The duplicate-down assertion deliberately leaves its first timer active; + // restore the target's normal locked posture before generic destruction. + const finalUp = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { + artifactName: "phase-11-restore-shields-up", + }); + expect(finalUp.exitCode, resultText(finalUp)).toBe(0); + expect(resultText(finalUp)).toContain("Lockdown active"); + await artifacts.target.complete({ id: "shields-config", sandboxName: SANDBOX_NAME, From 15ccb25621aa6e8a7345c8419e49db96764f585c Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 13 Jul 2026 23:06:11 -0700 Subject: [PATCH 2/5] test(e2e): diagnose Hermes rebuild runner loss Signed-off-by: Charan Jagwani --- test/e2e/live/rebuild-hermes-progress.ts | 130 ++++++++++++++++++ test/e2e/live/rebuild-hermes.test.ts | 17 +++ test/e2e/mock-parity.json | 10 +- .../support/rebuild-hermes-progress.test.ts | 77 +++++++++++ 4 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 test/e2e/live/rebuild-hermes-progress.ts create mode 100644 test/e2e/support/rebuild-hermes-progress.test.ts diff --git a/test/e2e/live/rebuild-hermes-progress.ts b/test/e2e/live/rebuild-hermes-progress.ts new file mode 100644 index 00000000000..f04eb960697 --- /dev/null +++ b/test/e2e/live/rebuild-hermes-progress.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; + +import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { ShellProbeOutputEvent } from "../fixtures/shell-probe.ts"; + +interface RebuildHermesResourceSnapshot { + freeMemoryBytes: number; + processRssBytes: number; + totalMemoryBytes: number; + workspaceFreeBytes: number; + loadAverage1m: number; +} + +interface TimerHandle { + unref?: () => void; +} + +export interface RebuildHermesProgressOptions { + heartbeatIntervalMs?: number; + now?: () => number; + setTimer?: (callback: () => void, intervalMs: number) => TimerHandle; + clearTimer?: (timer: TimerHandle) => void; + logLine?: (line: string) => void; + sampleResources?: () => RebuildHermesResourceSnapshot; +} + +export interface RebuildHermesProgress { + onOutput: (event: ShellProbeOutputEvent) => void; + phase: (label: string) => void; + stop: () => void; +} + +const DEFAULT_HEARTBEAT_INTERVAL_MS = 60_000; + +function formatGiB(bytes: number): string { + return `${(bytes / 1024 ** 3).toFixed(1)} GiB`; +} + +function defaultResourceSnapshot(): RebuildHermesResourceSnapshot { + const workspace = fs.statfsSync(REPO_ROOT); + return { + freeMemoryBytes: os.freemem(), + processRssBytes: process.memoryUsage().rss, + totalMemoryBytes: os.totalmem(), + workspaceFreeBytes: workspace.bavail * workspace.bsize, + loadAverage1m: os.loadavg()[0] ?? 0, + }; +} + +function formatResources(sampleResources: () => RebuildHermesResourceSnapshot): string { + try { + const snapshot = sampleResources(); + return [ + `memory free ${formatGiB(snapshot.freeMemoryBytes)}/${formatGiB(snapshot.totalMemoryBytes)}`, + `test RSS ${formatGiB(snapshot.processRssBytes)}`, + `workspace free ${formatGiB(snapshot.workspaceFreeBytes)}`, + `load 1m ${snapshot.loadAverage1m.toFixed(2)}`, + ].join("; "); + } catch { + return "host resources unavailable"; + } +} + +/** + * Keep the long Hermes scenario visible without forwarding command output, + * which may contain credentials. The timestamp-only output observer records + * liveness while resource snapshots make hosted-runner loss diagnosable. + */ +export function startRebuildHermesProgress( + initialPhase: string, + options: RebuildHermesProgressOptions = {}, +): RebuildHermesProgress { + const now = options.now ?? Date.now; + const setTimer = + options.setTimer ?? ((callback, intervalMs) => setInterval(callback, intervalMs)); + const clearTimer = options.clearTimer ?? ((timer) => clearInterval(timer as NodeJS.Timeout)); + const logLine = options.logLine ?? ((line) => process.stdout.write(`${line}\n`)); + const sampleResources = options.sampleResources ?? defaultResourceSnapshot; + let phaseLabel = initialPhase; + let phaseStartedAt = now(); + let lastOutputAt: number | null = null; + let stopped = false; + + const logBestEffort = (state: "started" | "running" | "finished") => { + try { + const current = now(); + const elapsedSeconds = Math.max(0, Math.floor((current - phaseStartedAt) / 1_000)); + const outputAge = + lastOutputAt === null + ? "no child output observed" + : `last child output ${Math.max(0, Math.floor((current - lastOutputAt) / 1_000))}s ago`; + logLine( + `[rebuild-hermes] ${phaseLabel} ${state} (${elapsedSeconds}s elapsed; ${outputAge}; ${formatResources(sampleResources)})`, + ); + } catch { + // Diagnostics must not change the live test result. + } + }; + + logBestEffort("started"); + const timer = setTimer( + () => logBestEffort("running"), + options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS, + ); + timer.unref?.(); + + return { + onOutput(event) { + lastOutputAt = event.atMs; + }, + phase(label) { + if (stopped) return; + logBestEffort("finished"); + phaseLabel = label; + phaseStartedAt = now(); + lastOutputAt = null; + logBestEffort("started"); + }, + stop() { + if (stopped) return; + stopped = true; + clearTimer(timer); + logBestEffort("finished"); + }, + }; +} diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index df5fccf886d..29cac591a75 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -23,6 +23,7 @@ import { REPO_ROOT } from "../fixtures/paths.ts"; import { listCredentialLeakPaths } from "../fixtures/phases/state-validation.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { buildRebuildHermesChildEnv } from "./rebuild-hermes-env.ts"; +import { startRebuildHermesProgress } from "./rebuild-hermes-progress.ts"; // The migrated scope is the legacy non-interactive shell regression: install.sh, // Docker base-image builds, OpenShell provider/sandbox commands, direct Hermes @@ -411,6 +412,8 @@ test(STALE_BASE_REBUILD const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const redactionValues = [apiKey, DISCORD_FAKE_TOKEN]; const expectedVersion = expectedHermesVersion(); + const progress = startRebuildHermesProgress("setup"); + cleanup.trackDisposable("stop Hermes rebuild progress", progress.stop); const registrySnapshot = snapshotFile(REGISTRY_FILE); const sessionSnapshot = snapshotFile(SESSION_FILE); @@ -489,13 +492,16 @@ test(STALE_BASE_REBUILD cleanup.trackDisposable(`destroy Hermes rebuild sandbox ${SANDBOX_NAME}`, () => cleanupHermesNemoClawSandbox(host, apiKey), ); + cleanup.trackDisposable("mark Hermes rebuild cleanup progress", () => progress.phase("cleanup")); + progress.phase("phase 1 install"); const install = await host.command("bash", ["install.sh", "--non-interactive"], { artifactName: "phase-1-install-hermes", cwd: REPO_ROOT, env: testEnv(apiKey), redactionValues, timeoutMs: INSTALL_TIMEOUT_MS, + onOutput: progress.onOutput, }); expectExitZero(install, "NemoClaw install.sh"); @@ -550,6 +556,7 @@ test(STALE_BASE_REBUILD timeoutMs: OPENSHELL_TIMEOUT_MS, }); + progress.phase("phase 2 old base build"); const buildOldBase = await host.command( "docker", [ @@ -575,6 +582,7 @@ test(STALE_BASE_REBUILD env: testEnv(apiKey), redactionValues, timeoutMs: DOCKER_BUILD_TIMEOUT_MS, + onOutput: progress.onOutput, }, ); expectExitZero(buildOldBase, `docker build old Hermes base ${OLD_HERMES_VERSION}`); @@ -622,6 +630,7 @@ test(STALE_BASE_REBUILD ); expectExitZero(provider, "OpenShell Discord provider create/update"); + progress.phase("phase 3 old sandbox create"); const createOldSandbox = await host.command( "openshell", [ @@ -644,6 +653,7 @@ test(STALE_BASE_REBUILD env: testEnv(apiKey), redactionValues, timeoutMs: SANDBOX_CREATE_TIMEOUT_MS, + onOutput: progress.onOutput, }, ); expectExitZero(createOldSandbox, "create old Hermes sandbox"); @@ -652,6 +662,7 @@ test(STALE_BASE_REBUILD } await waitForSandboxReady(host, apiKey); + progress.phase("phase 4 seed rebuild state"); const writeMarker = await host.command( "openshell", [ @@ -715,6 +726,7 @@ test(STALE_BASE_REBUILD switch (STALE_BASE_REBUILD) { case false: { + progress.phase("phase 5 current base build"); const buildCurrentBase = await host.command( "docker", [ @@ -730,26 +742,31 @@ test(STALE_BASE_REBUILD env: testEnv(apiKey), redactionValues, timeoutMs: DOCKER_BUILD_TIMEOUT_MS, + onOutput: progress.onOutput, }, ); expectExitZero(buildCurrentBase, "docker build current Hermes base image"); break; } case true: + progress.phase("phase 5 stale base setup"); await artifacts.writeText( "phase-5-stale-base-note.txt", `Left ${CURRENT_BASE_TAG} pointing at ${OLD_HERMES_VERSION}; rebuild must refresh the base cache.\n`, ); } + progress.phase("phase 6 nemoclaw rebuild"); const rebuild = await host.command("nemoclaw", [SANDBOX_NAME, "rebuild", "--yes", "--verbose"], { artifactName: "phase-6-nemoclaw-rebuild-hermes", env: testEnv(apiKey, { NEMOCLAW_REBUILD_VERBOSE: "1" }), redactionValues, timeoutMs: REBUILD_TIMEOUT_MS, + onOutput: progress.onOutput, }); expectExitZero(rebuild, "nemoclaw rebuild Hermes sandbox"); + progress.phase("phase 7 verification"); const restoredMarker = await host.command( "openshell", ["sandbox", "exec", "--name", SANDBOX_NAME, "--", "cat", MARKER_FILE], diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 5be7cf5bb3a..45fb7a71fa0 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -370,6 +370,13 @@ "test/e2e/support/e2e-clients.test.ts" ] }, + { + "live": "test/e2e/live/openshell-version-pin.test.ts", + "fast": [ + "test/onboard-openshell-version.test.ts", + "test/runner.test.ts" + ] + }, { "live": "test/e2e/live/overlayfs-autofix.test.ts", "fast": [ @@ -382,7 +389,8 @@ "live": "test/e2e/live/rebuild-hermes.test.ts", "fast": [ "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" + "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/rebuild-hermes-progress.test.ts" ] }, { diff --git a/test/e2e/support/rebuild-hermes-progress.test.ts b/test/e2e/support/rebuild-hermes-progress.test.ts new file mode 100644 index 00000000000..eca79c1fb6e --- /dev/null +++ b/test/e2e/support/rebuild-hermes-progress.test.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + type RebuildHermesProgressOptions, + startRebuildHermesProgress, +} from "../live/rebuild-hermes-progress.ts"; + +function progressHarness() { + const state = { + clearCalls: 0, + clockMs: 1_000, + lines: [] as string[], + timerCallback: null as (() => void) | null, + }; + const options: RebuildHermesProgressOptions = { + heartbeatIntervalMs: 60_000, + now: () => state.clockMs, + setTimer: (callback) => { + state.timerCallback = callback; + return { unref() {} }; + }, + clearTimer: () => { + state.clearCalls += 1; + }, + logLine: (line) => state.lines.push(line), + sampleResources: () => ({ + freeMemoryBytes: 8 * 1024 ** 3, + processRssBytes: 0.5 * 1024 ** 3, + totalMemoryBytes: 16 * 1024 ** 3, + workspaceFreeBytes: 6 * 1024 ** 3, + loadAverage1m: 2.5, + }), + }; + return { options, state }; +} + +describe("Hermes rebuild live progress", () => { + it("streams timestamp-only phase and resource heartbeats through cleanup", () => { + const { options, state } = progressHarness(); + const progress = startRebuildHermesProgress("phase 6 nemoclaw rebuild", options); + + progress.onOutput({ stream: "stderr", atMs: 21_000 }); + state.clockMs = 61_000; + state.timerCallback?.(); + progress.phase("cleanup"); + progress.stop(); + progress.stop(); + + expect(state.clearCalls).toBe(1); + expect(state.lines).toEqual([ + "[rebuild-hermes] phase 6 nemoclaw rebuild started (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[rebuild-hermes] phase 6 nemoclaw rebuild running (60s elapsed; last child output 40s ago; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[rebuild-hermes] phase 6 nemoclaw rebuild finished (60s elapsed; last child output 40s ago; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[rebuild-hermes] cleanup started (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[rebuild-hermes] cleanup finished (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + ]); + }); + + it("keeps diagnostics best-effort when host sampling and output fail", () => { + const { options, state } = progressHarness(); + options.logLine = vi.fn(() => { + throw new Error("closed output"); + }); + options.sampleResources = () => { + throw new Error("statfs unavailable"); + }; + + expect(() => { + const progress = startRebuildHermesProgress("phase 2 old base build", options); + progress.phase("cleanup"); + progress.stop(); + }).not.toThrow(); + expect(state.clearCalls).toBe(1); + }); +}); From cad33aea608860988b1d78983b0131c18254e27a Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 13 Jul 2026 23:15:18 -0700 Subject: [PATCH 3/5] test(e2e): report shields restore failures Signed-off-by: Charan Jagwani --- test/e2e/live/shields-config.test.ts | 7 +++++-- .../e2e/support/e2e-cleanup-resources.test.ts | 19 +++++++++++++++++++ .../support/rebuild-hermes-progress.test.ts | 3 +++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index 0bd30a7428e..23a0afb80e9 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -437,9 +437,12 @@ test("shields-config: live shields up/down locks config and detects drift", { // Keep fixture teardown out of an artificial mutable window if a later // assertion aborts before the explicit final restore below. cleanup.trackDisposable(`restore shields for ${SANDBOX_NAME} before destroy`, async () => { - await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { + const restore = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { artifactName: "cleanup-shields-up-before-destroy", - }).catch(() => undefined); + }); + if (restore.exitCode !== 0 || !resultText(restore).includes("Lockdown active")) { + throw new Error(`failed to restore shields before destroy: ${resultText(restore)}`); + } }); const configUp = await statPath(sandbox, CONFIG_PATH, "phase-3-config-perms-up"); diff --git a/test/e2e/support/e2e-cleanup-resources.test.ts b/test/e2e/support/e2e-cleanup-resources.test.ts index 689e1c84e16..234ba1ead03 100644 --- a/test/e2e/support/e2e-cleanup-resources.test.ts +++ b/test/e2e/support/e2e-cleanup-resources.test.ts @@ -154,6 +154,25 @@ describe("cleanup resources", () => { }); }); + it("reports failed shields restoration before continuing sandbox cleanup", async () => { + const calls: string[] = []; + const cleanup = new CleanupRegistry(); + cleanup.trackDisposable("destroy shields sandbox", () => { + calls.push("destroy"); + }); + cleanup.trackDisposable("restore shields before destroy", () => { + calls.push("restore"); + throw new Error("shields up exited 1"); + }); + + const result = await cleanup.runAll(); + expect(calls).toEqual(["restore", "destroy"]); + expect(result).toEqual({ + passed: ["destroy shields sandbox"], + failures: [{ name: "restore shields before destroy", message: "shields up exited 1" }], + }); + }); + it("accepts successful or explicitly absent cleanup results and rejects other failures (#6352)", () => { expect(() => assertCleanupSucceededOrAbsent( diff --git a/test/e2e/support/rebuild-hermes-progress.test.ts b/test/e2e/support/rebuild-hermes-progress.test.ts index eca79c1fb6e..12ac2f0d337 100644 --- a/test/e2e/support/rebuild-hermes-progress.test.ts +++ b/test/e2e/support/rebuild-hermes-progress.test.ts @@ -46,9 +46,12 @@ describe("Hermes rebuild live progress", () => { state.timerCallback?.(); progress.phase("cleanup"); progress.stop(); + const linesAfterStop = state.lines.length; progress.stop(); + progress.phase("after stop"); expect(state.clearCalls).toBe(1); + expect(state.lines).toHaveLength(linesAfterStop); expect(state.lines).toEqual([ "[rebuild-hermes] phase 6 nemoclaw rebuild started (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", "[rebuild-hermes] phase 6 nemoclaw rebuild running (60s elapsed; last child output 40s ago; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", From df674065a425b4e827e02602ad6d164c741f788b Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 13 Jul 2026 23:22:45 -0700 Subject: [PATCH 4/5] test(e2e): keep shields cleanup linear Signed-off-by: Charan Jagwani --- test/e2e/live/shields-config.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index 23a0afb80e9..b4647fcd75a 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -440,9 +440,8 @@ test("shields-config: live shields up/down locks config and detects drift", { const restore = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { artifactName: "cleanup-shields-up-before-destroy", }); - if (restore.exitCode !== 0 || !resultText(restore).includes("Lockdown active")) { - throw new Error(`failed to restore shields before destroy: ${resultText(restore)}`); - } + expect(restore.exitCode, resultText(restore)).toBe(0); + expect(resultText(restore)).toContain("Lockdown active"); }); const configUp = await statPath(sandbox, CONFIG_PATH, "phase-3-config-perms-up"); From 8082d9e2cfcceb0b6bb97149e2d6e09335569ede Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 13 Jul 2026 23:45:05 -0700 Subject: [PATCH 5/5] fix(onboard): preserve Docker patch phase diagnostics Signed-off-by: Charan Jagwani --- ...u-patch-diagnostics-classification.test.ts | 7 +++++++ src/lib/onboard/docker-gpu-patch.ts | 19 ++----------------- src/lib/runtime-recovery.test.ts | 2 ++ src/lib/runtime-recovery.ts | 1 + 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics-classification.test.ts b/src/lib/onboard/docker-gpu-patch-diagnostics-classification.test.ts index faba1386fdb..d11fd247254 100644 --- a/src/lib/onboard/docker-gpu-patch-diagnostics-classification.test.ts +++ b/src/lib/onboard/docker-gpu-patch-diagnostics-classification.test.ts @@ -81,6 +81,13 @@ describe("Docker GPU patch diagnostics", () => { "Ready", "alpha Ready 1m ago", ], + [ + "reads the trailing phase from the modern name-created-phase layout", + "Name: alpha\nPhase: Error\nReason: ContainerCannotRun\n", + "NAME CREATED PHASE\nalpha 2026-07-14 05:11:40 Provisioning\n", + "Provisioning", + "alpha 2026-07-14 05:11:40 Provisioning", + ], [ "keeps the get-derived phase when the sandbox row is absent from list output", "Name: alpha\nPhase: Terminated\n", diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index ba3b154fd04..eb9dfa9fda9 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { dockerCapture } from "../adapters/docker"; +import { parseLiveSandboxEntries } from "../runtime-recovery"; import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import type { @@ -304,15 +305,6 @@ function stripAnsi(value: string): string { return value.replace(ANSI_RE, ""); } -function parseSandboxRowForName(output: string, sandboxName: string): string[] | null { - if (typeof output !== "string") return null; - for (const line of stripAnsi(output).split("\n")) { - const cols = line.trim().split(/\s+/); - if (cols[0] === sandboxName) return cols; - } - return null; -} - function findSandboxListLine(output: string, sandboxName: string): string | null { if (typeof output !== "string") return null; for (const line of stripAnsi(output).split("\n")) { @@ -328,14 +320,7 @@ function parseSandboxPhaseFromGetOutput(output: string): string | null { } function parseSandboxPhaseFromListOutput(output: string, sandboxName: string): string | null { - const cols = parseSandboxRowForName(output, sandboxName); - if (!cols) return null; - return ( - cols.find((col) => SANDBOX_FAILURE_PHASE_TOKENS.has(col)) ?? - cols.find((col) => SANDBOX_LIVE_PHASE_TOKENS.has(col)) ?? - cols[1] ?? - null - ); + return parseLiveSandboxEntries(output).find((entry) => entry.name === sandboxName)?.phase ?? null; } function isFailurePhase(phase: string | null | undefined): boolean { diff --git a/src/lib/runtime-recovery.test.ts b/src/lib/runtime-recovery.test.ts index b74e6311788..bfd281465f2 100644 --- a/src/lib/runtime-recovery.test.ts +++ b/src/lib/runtime-recovery.test.ts @@ -39,12 +39,14 @@ describe("runtime recovery helpers", () => { "alpha 2026-06-25 09:40:49 Failed", "beta 2026-06-25 09:41:00 CrashLoopBackOff", "gamma 2026-06-25 09:42:00 Creating", + "delta 2026-06-25 09:43:00 Evicted", ].join("\n"), ), ).toEqual([ { name: "alpha", phase: "Failed" }, { name: "beta", phase: "CrashLoopBackOff" }, { name: "gamma", phase: "Creating" }, + { name: "delta", phase: "Evicted" }, ]); }); diff --git a/src/lib/runtime-recovery.ts b/src/lib/runtime-recovery.ts index bdfc7000eb6..179d9de5198 100644 --- a/src/lib/runtime-recovery.ts +++ b/src/lib/runtime-recovery.ts @@ -26,6 +26,7 @@ const LIVE_SANDBOX_DISPLAY_PHASES = new Set([ "Failed", "CrashLoopBackOff", "ImagePullBackOff", + "Evicted", "Unknown", ]);