From 4e960f2c9518d8898f2b2ac9ac9dd38090504542 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 16:01:17 +0700 Subject: [PATCH 1/8] fix(e2e): load private file helper as ESM Signed-off-by: San Dang --- test/e2e-private-file.test.ts | 13 +++++++++++-- test/e2e/risk-signal-reporter.ts | 2 +- test/pr-risk-plan.test.ts | 2 +- tools/e2e/live-test-outcome.mts | 2 +- tools/e2e/pr-e2e-gate.mts | 2 +- tools/e2e/{private-file.ts => private-file.mts} | 0 tools/e2e/runner-pressure.mts | 2 +- 7 files changed, 16 insertions(+), 7 deletions(-) rename tools/e2e/{private-file.ts => private-file.mts} (100%) diff --git a/test/e2e-private-file.test.ts b/test/e2e-private-file.test.ts index cfdfdcb898c..2cb8f19180f 100644 --- a/test/e2e-private-file.test.ts +++ b/test/e2e-private-file.test.ts @@ -12,9 +12,18 @@ import { appendPrivateRegularFile, readPrivateRegularFile, writePrivateRegularFile, -} from "../tools/e2e/private-file.ts"; +} from "../tools/e2e/private-file.mts"; describe("private E2E controller files", () => { + it("loads the private-file helper through the live tsx entrypoint", () => { + const result = spawnSync("npx", ["tsx", "tools/e2e/live-test-outcome.mts"], { + encoding: "utf8", + timeout: 10_000, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + }); + it("writes private regular files without following links or truncating hardlink targets", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-file-")); const regular = path.join(directory, "regular.json"); @@ -50,7 +59,7 @@ describe("private E2E controller files", () => { const fifo = path.join(directory, "state.json"); try { execFileSync("mkfifo", [fifo]); - const moduleUrl = pathToFileURL(path.resolve("tools/e2e/private-file.ts")).href; + const moduleUrl = pathToFileURL(path.resolve("tools/e2e/private-file.mts")).href; const read = spawnSync( process.execPath, [ diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index 696ba2b8dbc..3465426e0c6 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -13,7 +13,7 @@ import { type LiveTestOutcome, writeLiveTestOutcome, } from "../../tools/e2e/live-test-outcome.mts"; -import { readPrivateRegularFile, writePrivateRegularFile } from "../../tools/e2e/private-file.ts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "../../tools/e2e/private-file.mts"; import type { E2eRiskSignal } from "../../tools/e2e/risk-signal.ts"; export const RISK_SIGNAL_FILE = "risk-signal.json"; diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index daa119e1401..c62703916ce 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -234,7 +234,7 @@ describe("deterministic PR risk plan", () => { "tools/e2e/pr-e2e-gate.mts", "tools/e2e/pr-e2e-required.mts", "tools/e2e/risk-signal.ts", - "tools/e2e/private-file.ts", + "tools/e2e/private-file.mts", "tools/e2e/workflow-plan.mts", "tools/e2e/workflow-boundary.mts", "tools/e2e/job-map.txt", diff --git a/tools/e2e/live-test-outcome.mts b/tools/e2e/live-test-outcome.mts index 00817d14586..631e2ae253f 100644 --- a/tools/e2e/live-test-outcome.mts +++ b/tools/e2e/live-test-outcome.mts @@ -3,7 +3,7 @@ import path from "node:path"; -import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.mts"; export const LIVE_TEST_OUTCOME_FILE = "live-test-outcome.json"; const OUTCOME_FILE_MAX_BYTES = 128; diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index 94aed23c185..dfd00d3995a 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -23,7 +23,7 @@ import { riskPlanRequiredTargetIds, } from "../advisors/risk-plan.mts"; import { SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; -import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.mts"; import type { E2eRiskSignal } from "./risk-signal.ts"; import { decideRetry, diff --git a/tools/e2e/private-file.ts b/tools/e2e/private-file.mts similarity index 100% rename from tools/e2e/private-file.ts rename to tools/e2e/private-file.mts diff --git a/tools/e2e/runner-pressure.mts b/tools/e2e/runner-pressure.mts index 69782aecd5c..874965b8319 100644 --- a/tools/e2e/runner-pressure.mts +++ b/tools/e2e/runner-pressure.mts @@ -43,7 +43,7 @@ import { appendPrivateRegularFile, readPrivateRegularFile, writePrivateRegularFile, -} from "./private-file.ts"; +} from "./private-file.mts"; import { assertPhaseLabel, classifyFailure, From 2a2df02de09680981218f7f7e2ea75ea1e0b5217 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 16:19:59 +0700 Subject: [PATCH 2/8] test(e2e): keep live entrypoint check in support lane Signed-off-by: San Dang --- test/e2e-private-file.test.ts | 9 --------- test/e2e/support/live-test-outcome-invocation.test.ts | 10 ++++++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/test/e2e-private-file.test.ts b/test/e2e-private-file.test.ts index 2cb8f19180f..4288cc9fe4c 100644 --- a/test/e2e-private-file.test.ts +++ b/test/e2e-private-file.test.ts @@ -15,15 +15,6 @@ import { } from "../tools/e2e/private-file.mts"; describe("private E2E controller files", () => { - it("loads the private-file helper through the live tsx entrypoint", () => { - const result = spawnSync("npx", ["tsx", "tools/e2e/live-test-outcome.mts"], { - encoding: "utf8", - timeout: 10_000, - }); - - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - }); - it("writes private regular files without following links or truncating hardlink targets", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-file-")); const regular = path.join(directory, "regular.json"); diff --git a/test/e2e/support/live-test-outcome-invocation.test.ts b/test/e2e/support/live-test-outcome-invocation.test.ts index 95c9328a14a..d5794d4c8da 100644 --- a/test/e2e/support/live-test-outcome-invocation.test.ts +++ b/test/e2e/support/live-test-outcome-invocation.test.ts @@ -24,6 +24,16 @@ const FIXTURE = "test/e2e/support/fixtures/live-test-outcome.fixture.test.ts"; const CLASSIFIER = path.join(ROOT, "tools/e2e/runner-pressure.mts"); describe("live-test outcome invocation contract (#7146)", () => { + it("loads the private-file helper through the live tsx entrypoint", () => { + const result = spawnSync("npx", ["tsx", "tools/e2e/live-test-outcome.mts"], { + cwd: ROOT, + encoding: "utf8", + timeout: 10_000, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + }); + it.each([ "assertion", "timeout", From bc33a856312cc146217d962cae87c72ae7c5e979 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 16:47:25 +0530 Subject: [PATCH 3/8] fix(e2e): keep Hermes heartbeats off Docker Signed-off-by: San Dang --- test/e2e/live/rebuild-hermes-progress.ts | 5 ++- .../support/rebuild-hermes-progress.test.ts | 32 +++++++++++++++++ tools/e2e/runner-pressure.mts | 36 +++++++++++++------ 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/test/e2e/live/rebuild-hermes-progress.ts b/test/e2e/live/rebuild-hermes-progress.ts index 8b8219c1c85..e4f0de190bd 100644 --- a/test/e2e/live/rebuild-hermes-progress.ts +++ b/test/e2e/live/rebuild-hermes-progress.ts @@ -99,7 +99,10 @@ export function startRebuildHermesProgress( const sampleResources = options.sampleResources ?? defaultResourceSnapshot; const sampleResourceEvidence = options.sampleResourceEvidence ?? - ((phase) => renderSnapshotLine(collectResourceSnapshot(resourcePhaseLabel(phase)))); + ((phase) => + renderSnapshotLine( + collectResourceSnapshot(resourcePhaseLabel(phase), { includeDocker: false }), + )); const recordResourceBaseline = options.recordResourceBaseline ?? ((phase) => { diff --git a/test/e2e/support/rebuild-hermes-progress.test.ts b/test/e2e/support/rebuild-hermes-progress.test.ts index ea0f40e88a5..50bc91ebae8 100644 --- a/test/e2e/support/rebuild-hermes-progress.test.ts +++ b/test/e2e/support/rebuild-hermes-progress.test.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, it, vi } from "vitest"; import { type RebuildHermesProgressOptions, @@ -40,6 +44,34 @@ function progressHarness() { } describe("Hermes rebuild live progress", () => { + it("keeps long-build heartbeats from invoking Docker inspection", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-progress-")); + const dockerMarker = path.join(directory, "docker-invoked"); + const fakeDocker = path.join(directory, "docker"); + fs.writeFileSync(fakeDocker, `#!/bin/sh\ntouch ${JSON.stringify(dockerMarker)}\n`, { + mode: 0o755, + }); + vi.stubEnv("PATH", `${directory}${path.delimiter}${process.env.PATH ?? ""}`); + + try { + const { options, state } = progressHarness(); + delete options.sampleResourceEvidence; + const progress = startRebuildHermesProgress("phase 1 install", options); + progress.stop(); + + expect(fs.existsSync(dockerMarker)).toBe(false); + const snapshotLine = state.lines.find((line) => line.startsWith("E2E_RESOURCE_SNAPSHOT ")); + expect(snapshotLine).toBeDefined(); + expect(JSON.parse(snapshotLine!.slice("E2E_RESOURCE_SNAPSHOT ".length))).toMatchObject({ + containers: [], + dockerDisk: null, + }); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it("streams timestamp-only phase and resource heartbeats through cleanup", () => { const { options, state } = progressHarness(); const progress = startRebuildHermesProgress("phase 6 nemoclaw rebuild", options); diff --git a/tools/e2e/runner-pressure.mts b/tools/e2e/runner-pressure.mts index 874965b8319..bafc70a460d 100644 --- a/tools/e2e/runner-pressure.mts +++ b/tools/e2e/runner-pressure.mts @@ -106,7 +106,14 @@ function collectDisk(): ResourceSnapshot["disk"] { } } -export function collectResourceSnapshot(phase: string): ResourceSnapshot { +export interface CollectResourceSnapshotOptions { + includeDocker?: boolean; +} + +export function collectResourceSnapshot( + phase: string, + options: CollectResourceSnapshotOptions = {}, +): ResourceSnapshot { const meminfoText = readTextOrNull("/proc/meminfo"); const loadText = readTextOrNull("/proc/loadavg"); const current = readTextOrNull(`${CGROUP_ROOT}/memory.current`); @@ -116,8 +123,16 @@ export function collectResourceSnapshot(phase: string): ResourceSnapshot { const memoryPressure = readTextOrNull(`${CGROUP_ROOT}/memory.pressure`); const ioPressure = readTextOrNull(`${CGROUP_ROOT}/io.pressure`); const psText = runOrNull("ps", ["-eo", "rss="]); - const statsText = runOrNull("docker", ["stats", "--no-stream", "--format", "{{json .}}"]); - const dfText = runOrNull("docker", ["system", "df", "--format", "{{json .}}"]); + // Docker inspection can block behind a large BuildKit export and add load to + // the daemon. Long-build heartbeats keep the host evidence below while the + // workflow baseline and terminal classifier retain Docker-specific probes. + const includeDocker = options.includeDocker ?? true; + const statsText = includeDocker + ? runOrNull("docker", ["stats", "--no-stream", "--format", "{{json .}}"]) + : null; + const dfText = includeDocker + ? runOrNull("docker", ["system", "df", "--format", "{{json .}}"]) + : null; return { phase, at: new Date().toISOString(), @@ -181,9 +196,13 @@ function assertEvidencePath(path: string | undefined, variableName: string): str /** Append one phase baseline without replacing the immutable workflow baseline. */ export function appendResourcePhaseBaseline(path: string, phase: string): void { const validatedPath = assertEvidencePath(path, "E2E_RESOURCE_PHASE_BASELINES_FILE"); - appendPrivateRegularFile(validatedPath, `${renderBaselineLine(collectResourceBaseline(phase))}\n`, { - maxBytes: PHASE_BASELINES_FILE_MAX_BYTES, - }); + appendPrivateRegularFile( + validatedPath, + `${renderBaselineLine(collectResourceBaseline(phase))}\n`, + { + maxBytes: PHASE_BASELINES_FILE_MAX_BYTES, + }, + ); } /** Create the trusted evidence files before PR-controlled live tests execute. */ @@ -201,10 +220,7 @@ function runInitializeEvidence(): void { process.env.E2E_TERMINAL_CLASSIFICATION_FILE, "E2E_TERMINAL_CLASSIFICATION_FILE", ); - writePrivateRegularFile( - baselinePath, - `${renderBaselineLine(collectResourceBaseline(phase))}\n`, - ); + writePrivateRegularFile(baselinePath, `${renderBaselineLine(collectResourceBaseline(phase))}\n`); writePrivateRegularFile(phaseBaselinesPath, ""); writePrivateRegularFile(classificationPath, ""); } From 47eecfe93f47f4d4097b8285c9817b7fc891909c Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 17:16:17 +0530 Subject: [PATCH 4/8] test(e2e): harden fake Docker marker Signed-off-by: San Dang --- test/e2e/support/rebuild-hermes-progress.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/support/rebuild-hermes-progress.test.ts b/test/e2e/support/rebuild-hermes-progress.test.ts index 50bc91ebae8..423c7f46b74 100644 --- a/test/e2e/support/rebuild-hermes-progress.test.ts +++ b/test/e2e/support/rebuild-hermes-progress.test.ts @@ -48,7 +48,7 @@ describe("Hermes rebuild live progress", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-progress-")); const dockerMarker = path.join(directory, "docker-invoked"); const fakeDocker = path.join(directory, "docker"); - fs.writeFileSync(fakeDocker, `#!/bin/sh\ntouch ${JSON.stringify(dockerMarker)}\n`, { + fs.writeFileSync(fakeDocker, '#!/bin/sh\n: > "${0%/*}/docker-invoked"\n', { mode: 0o755, }); vi.stubEnv("PATH", `${directory}${path.delimiter}${process.env.PATH ?? ""}`); From 24ba4cf8411ebd936bc741c443bc2a4fea72e1ab Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 17:48:12 +0530 Subject: [PATCH 5/8] fix(e2e): seed old Hermes through pinned runtime Signed-off-by: San Dang --- test/e2e/live/rebuild-hermes.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index 04a37c9636a..3cc4665f635 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -48,6 +48,7 @@ const OLD_HERMES_TARBALL_SHA256 = "c0a554050a50ee9a62f3fa5cd288a167ba5640c42d647d100cdea084b7294143"; const OLD_HERMES_NPM_INTEGRITY = "sha512-kkHSw8iprp0JWAOf3ZZF0OHzRBj3E/BbG/QV0O4lwonxuY7AWhSepOhzSMlWo21VbQ/fTLwFkr/q3cIjDZDLBA=="; +const OLD_HERMES_PYTHON = "/opt/hermes/.venv/bin/python"; const STALE_BASE_REBUILD = process.env.NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E === "1"; const TEST_SANDBOX_PREFIX = STALE_BASE_REBUILD ? "e2e-rebuild-hermes-base" : "e2e-rebuild-hermes"; const SANDBOX_NAME = @@ -750,8 +751,8 @@ test(STALE_BASE_REBUILD "sh", "-lc", [ - "hermes kanban init", - `hermes kanban create ${shellQuote(KANBAN_TASK_TITLE)} --initial-status blocked --json`, + `${shellQuote(OLD_HERMES_PYTHON)} -m hermes_cli.main kanban init`, + `${shellQuote(OLD_HERMES_PYTHON)} -m hermes_cli.main kanban create ${shellQuote(KANBAN_TASK_TITLE)} --initial-status blocked --json`, `mkdir -p ${shellQuote(path.dirname(EXCLUDED_KANBAN_FILE))}`, `printf '%s' ${shellQuote(MARKER_CONTENT)} > ${shellQuote(EXCLUDED_KANBAN_FILE)}`, ].join(" && "), From 37ae75877844ae0ab312a3b39a93174670720119 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 18:13:40 +0530 Subject: [PATCH 6/8] fix(e2e): apply Hermes policy to rebuild fixture Signed-off-by: San Dang --- test/e2e/live/rebuild-hermes.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index 3cc4665f635..6f646d12833 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -7,6 +7,7 @@ import os from "node:os"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { prepareInitialSandboxCreatePolicy } from "../../../src/lib/onboard/initial-policy"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { assertExitZero as expectExitZero } from "../fixtures/clients/command.ts"; @@ -41,6 +42,7 @@ import { buildRebuildHermesTimingSummary, describeRunnerClass } from "./rebuild- // Vitest. const HERMES_MANIFEST = path.join(REPO_ROOT, "agents", "hermes", "manifest.yaml"); +const HERMES_POLICY = path.join(REPO_ROOT, "agents", "hermes", "policy-additions.yaml"); const OLD_HERMES_VERSION = "v2026.5.16"; const OLD_HERMES_REGISTRY_VERSION = OLD_HERMES_VERSION.slice(1); const OLD_HERMES_SEMVER = "0.14.0"; @@ -48,7 +50,6 @@ const OLD_HERMES_TARBALL_SHA256 = "c0a554050a50ee9a62f3fa5cd288a167ba5640c42d647d100cdea084b7294143"; const OLD_HERMES_NPM_INTEGRITY = "sha512-kkHSw8iprp0JWAOf3ZZF0OHzRBj3E/BbG/QV0O4lwonxuY7AWhSepOhzSMlWo21VbQ/fTLwFkr/q3cIjDZDLBA=="; -const OLD_HERMES_PYTHON = "/opt/hermes/.venv/bin/python"; const STALE_BASE_REBUILD = process.env.NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E === "1"; const TEST_SANDBOX_PREFIX = STALE_BASE_REBUILD ? "e2e-rebuild-hermes-base" : "e2e-rebuild-hermes"; const SANDBOX_NAME = @@ -655,6 +656,9 @@ test(STALE_BASE_REBUILD const oldDockerfileDir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-rebuild-hermes-")); const oldDockerfile = path.join(oldDockerfileDir, "Dockerfile"); fs.writeFileSync(oldDockerfile, oldHermesDockerfile(), "utf8"); + const oldSandboxPolicy = prepareInitialSandboxCreatePolicy(HERMES_POLICY, ["discord"], { + agentName: "hermes", + }); try { const provider = await host.command( "bash", @@ -688,6 +692,8 @@ test(STALE_BASE_REBUILD SANDBOX_NAME, "--from", oldDockerfile, + "--policy", + oldSandboxPolicy.policyPath, "--gateway", "nemoclaw", "--provider", @@ -708,6 +714,7 @@ test(STALE_BASE_REBUILD expectExitZero(createOldSandbox, "create old Hermes sandbox"); oldSandboxImageState = rebuildHermesRegistryImageState(resultText(createOldSandbox)); } finally { + oldSandboxPolicy.cleanup?.(); fs.rmSync(oldDockerfileDir, { recursive: true, force: true }); } const seededOldSandboxImageState = @@ -751,8 +758,8 @@ test(STALE_BASE_REBUILD "sh", "-lc", [ - `${shellQuote(OLD_HERMES_PYTHON)} -m hermes_cli.main kanban init`, - `${shellQuote(OLD_HERMES_PYTHON)} -m hermes_cli.main kanban create ${shellQuote(KANBAN_TASK_TITLE)} --initial-status blocked --json`, + "hermes kanban init", + `hermes kanban create ${shellQuote(KANBAN_TASK_TITLE)} --initial-status blocked --json`, `mkdir -p ${shellQuote(path.dirname(EXCLUDED_KANBAN_FILE))}`, `printf '%s' ${shellQuote(MARKER_CONTENT)} > ${shellQuote(EXCLUDED_KANBAN_FILE)}`, ].join(" && "), From efec55b1a5397818343d764aeba257f29f3dde68 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 18:28:08 +0530 Subject: [PATCH 7/8] fix(e2e): use old Hermes kanban syntax Signed-off-by: San Dang --- test/e2e/live/rebuild-hermes.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index 6f646d12833..92b53e9db16 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -759,7 +759,7 @@ test(STALE_BASE_REBUILD "-lc", [ "hermes kanban init", - `hermes kanban create ${shellQuote(KANBAN_TASK_TITLE)} --initial-status blocked --json`, + `hermes kanban create ${shellQuote(KANBAN_TASK_TITLE)} --triage --json`, `mkdir -p ${shellQuote(path.dirname(EXCLUDED_KANBAN_FILE))}`, `printf '%s' ${shellQuote(MARKER_CONTENT)} > ${shellQuote(EXCLUDED_KANBAN_FILE)}`, ].join(" && "), From b4c0214f80a4202638ffc15367505b16c852ae5c Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 20 Jul 2026 20:32:09 +0530 Subject: [PATCH 8/8] fix(e2e): retry Hermes post-restore supervisor churn Signed-off-by: San Dang --- .../hermes-secret-boundary-recovery.test.ts | 15 ++++++ .../hermes-secret-boundary-recovery.ts | 20 +++++++ .../rebuild-hermes-post-restore.test.ts | 32 ++++++++++- .../sandbox/rebuild-hermes-post-restore.ts | 53 +++++++++++++++---- 4 files changed, 109 insertions(+), 11 deletions(-) diff --git a/src/lib/actions/sandbox/hermes-secret-boundary-recovery.test.ts b/src/lib/actions/sandbox/hermes-secret-boundary-recovery.test.ts index f431abf628e..32aa9292b43 100644 --- a/src/lib/actions/sandbox/hermes-secret-boundary-recovery.test.ts +++ b/src/lib/actions/sandbox/hermes-secret-boundary-recovery.test.ts @@ -119,6 +119,21 @@ describe("enforceHermesSecretBoundaryOnRunningGateway", () => { expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Re-image the sandbox")); }); + it("classifies an exact staged supervisor failure as transient churn (#7229)", () => { + mockSandboxAgent("hermes"); + const stderr = "SUPERVISOR_UNAVAILABLE\nNEMOCLAW_CONTROL_STAGE=preflight\n"; + const exec = vi.fn(() => makeExecResult("", stderr, 1)); + + const result = enforceHermesSecretBoundaryOnRunningGateway(SANDBOX, HERMES_AGENT, exec); + + expect(result).toEqual({ refused: true, reason: "supervisor-churn", stderr }); + expect(consoleErrorSpy).toHaveBeenCalledWith(" SUPERVISOR_UNAVAILABLE"); + expect(consoleErrorSpy).toHaveBeenCalledWith(" NEMOCLAW_CONTROL_STAGE=preflight"); + expect(consoleErrorSpy).not.toHaveBeenCalledWith( + expect.stringContaining("did not complete cleanly"), + ); + }); + it("distinguishes unrecognized validator output from infrastructure failures", () => { mockSandboxAgent("hermes"); const exec = vi.fn(() => makeExecResult("unexpected output\n", "validator failed\n", 1)); diff --git a/src/lib/actions/sandbox/hermes-secret-boundary-recovery.ts b/src/lib/actions/sandbox/hermes-secret-boundary-recovery.ts index 387aa9001a0..fdda1b98e75 100644 --- a/src/lib/actions/sandbox/hermes-secret-boundary-recovery.ts +++ b/src/lib/actions/sandbox/hermes-secret-boundary-recovery.ts @@ -11,6 +11,7 @@ export type SecretBoundaryRefusalReason = | "exec-failed" | "validator-missing" | "unexpected-marker" + | "supervisor-churn" | "agent-missing"; export type HermesSecretBoundaryEnforcement = @@ -34,6 +35,21 @@ function printValidatorStderr(stderr: string): void { } } +const MANAGED_CONTROL_STAGE_RE = /^NEMOCLAW_CONTROL_STAGE=[a-z][a-z-]*$/; + +function isStagedSupervisorUnavailable(result: SandboxCommandResult): boolean { + if (result.status !== 1 || result.stdout.trim() !== "") return false; + const lines = result.stderr + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + return ( + lines.length === 2 && + lines[0] === "SUPERVISOR_UNAVAILABLE" && + MANAGED_CONTROL_STAGE_RE.test(lines[1]) + ); +} + /** * Re-run the Hermes env-file secret boundary through the authenticated PID 1 * control path before a healthy recover returns. PID 1 owns the exact gateway @@ -91,6 +107,10 @@ export function enforceHermesSecretBoundaryOnRunningGateway( ); return { refused: true, reason: "validator-missing", stderr: result.stderr }; } + if (isStagedSupervisorUnavailable(result)) { + printValidatorStderr(result.stderr); + return { refused: true, reason: "supervisor-churn", stderr: result.stderr }; + } printValidatorStderr(result.stderr); console.error(""); console.error( diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts index e6aba209276..c0400914586 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts @@ -1,17 +1,47 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createRebuildFlowHarness, resetRebuildFlowTestEnvironment, restoreRebuildFlowTestEnvironment, } from "../../../../test/helpers/rebuild-flow-harness"; +import { ensureHermesGatewayAfterStateRestore } from "./rebuild-hermes-post-restore"; describe("Hermes rebuild post-restore verification", () => { beforeEach(resetRebuildFlowTestEnvironment); afterEach(restoreRebuildFlowTestEnvironment); + it("retries exact managed-supervisor churn before accepting restored Hermes state (#7229)", () => { + const checkAndRecoverSandboxProcesses = vi + .fn() + .mockReturnValueOnce({ + checked: true, + wasRunning: true, + recovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "supervisor-churn", + }) + .mockReturnValueOnce({ + checked: true, + wasRunning: true, + recovered: false, + }); + const sleep = vi.fn(); + + expect( + ensureHermesGatewayAfterStateRestore("alpha", "hermes", { + checkAndRecoverSandboxProcesses, + sleepSeconds: sleep, + }), + ).toBe("healthy"); + + expect(checkAndRecoverSandboxProcesses).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledOnce(); + expect(sleep).toHaveBeenCalledWith(3); + }); + it("fails instead of reporting readiness when restored state leaves the gateway down (#7084)", async () => { const mcpEntry = { server: "blender", diff --git a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts index 5362f592a7c..31902f9c52c 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-post-restore.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { CLI_NAME } from "../../cli/branding"; +import { sleepSeconds } from "../../core/wait"; import * as processRecovery from "./process-recovery"; export type HermesPostRestoreGatewayState = @@ -16,6 +17,7 @@ type GatewayRecoveryObservation = { recovered: boolean; forwardRecoveryFailed?: boolean; secretBoundaryRefused?: boolean; + secretBoundaryReason?: string; mcpReconciliationRefused?: boolean; }; @@ -24,6 +26,36 @@ interface HermesPostRestoreGatewayDeps { sandboxName: string, options: { quiet: boolean }, ) => GatewayRecoveryObservation; + sleepSeconds?: (seconds: number) => void; +} + +const POST_RESTORE_SUPERVISOR_ATTEMPTS = 3; +const POST_RESTORE_SUPERVISOR_RETRY_SECONDS = 3; + +function isTransientManagedSupervisorChurn(observation: GatewayRecoveryObservation): boolean { + // State restoration can make PID 1 replace Hermes between the healthy HTTP + // probe and its validator-enforced recovery request. Retry only the exact + // controller classification; validator and integrity output stays terminal. + return ( + observation.secretBoundaryRefused === true && + observation.secretBoundaryReason === "supervisor-churn" + ); +} + +function classifyGatewayObservation( + observation: GatewayRecoveryObservation, +): HermesPostRestoreGatewayState { + if ( + !observation.checked || + observation.forwardRecoveryFailed === true || + observation.secretBoundaryRefused === true || + observation.mcpReconciliationRefused === true + ) { + return "unverified"; + } + if (observation.wasRunning === true) return "healthy"; + if (observation.recovered) return "recovered"; + return "unverified"; } /** @@ -41,17 +73,18 @@ export function ensureHermesGatewayAfterStateRestore( if (agentName !== "hermes") return "not-applicable"; const checkAndRecover = deps.checkAndRecoverSandboxProcesses ?? processRecovery.checkAndRecoverSandboxProcesses; - const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true }); - if ( - !observation.checked || - observation.forwardRecoveryFailed === true || - observation.secretBoundaryRefused === true || - observation.mcpReconciliationRefused === true - ) { - return "unverified"; + const wait = deps.sleepSeconds ?? sleepSeconds; + for (let attempt = 1; attempt <= POST_RESTORE_SUPERVISOR_ATTEMPTS; attempt += 1) { + const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true }); + if ( + attempt < POST_RESTORE_SUPERVISOR_ATTEMPTS && + isTransientManagedSupervisorChurn(observation) + ) { + wait(POST_RESTORE_SUPERVISOR_RETRY_SECONDS); + continue; + } + return classifyGatewayObservation(observation); } - if (observation.wasRunning === true) return "healthy"; - if (observation.recovered) return "recovered"; return "unverified"; }