Skip to content
15 changes: 15 additions & 0 deletions src/lib/actions/sandbox/hermes-secret-boundary-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,21 @@ describe("enforceHermesSecretBoundaryOnRunningGateway", () => {
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Re-image the sandbox"));
});

it("classifies an exact staged supervisor failure as transient churn (#7229)", () => {
mockSandboxAgent("hermes");
const stderr = "SUPERVISOR_UNAVAILABLE\nNEMOCLAW_CONTROL_STAGE=preflight\n";
const exec = vi.fn(() => makeExecResult("", stderr, 1));

const result = enforceHermesSecretBoundaryOnRunningGateway(SANDBOX, HERMES_AGENT, exec);

expect(result).toEqual({ refused: true, reason: "supervisor-churn", stderr });
expect(consoleErrorSpy).toHaveBeenCalledWith(" SUPERVISOR_UNAVAILABLE");
expect(consoleErrorSpy).toHaveBeenCalledWith(" NEMOCLAW_CONTROL_STAGE=preflight");
expect(consoleErrorSpy).not.toHaveBeenCalledWith(
expect.stringContaining("did not complete cleanly"),
);
});

it("distinguishes unrecognized validator output from infrastructure failures", () => {
mockSandboxAgent("hermes");
const exec = vi.fn(() => makeExecResult("unexpected output\n", "validator failed\n", 1));
Expand Down
20 changes: 20 additions & 0 deletions src/lib/actions/sandbox/hermes-secret-boundary-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export type SecretBoundaryRefusalReason =
| "exec-failed"
| "validator-missing"
| "unexpected-marker"
| "supervisor-churn"
| "agent-missing";

export type HermesSecretBoundaryEnforcement =
Expand All @@ -34,6 +35,21 @@ function printValidatorStderr(stderr: string): void {
}
}

const MANAGED_CONTROL_STAGE_RE = /^NEMOCLAW_CONTROL_STAGE=[a-z][a-z-]*$/;

function isStagedSupervisorUnavailable(result: SandboxCommandResult): boolean {
if (result.status !== 1 || result.stdout.trim() !== "") return false;
const lines = result.stderr
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
return (
lines.length === 2 &&
lines[0] === "SUPERVISOR_UNAVAILABLE" &&
MANAGED_CONTROL_STAGE_RE.test(lines[1])
);
}

/**
* Re-run the Hermes env-file secret boundary through the authenticated PID 1
* control path before a healthy recover returns. PID 1 owns the exact gateway
Expand Down Expand Up @@ -91,6 +107,10 @@ export function enforceHermesSecretBoundaryOnRunningGateway(
);
return { refused: true, reason: "validator-missing", stderr: result.stderr };
}
if (isStagedSupervisorUnavailable(result)) {
printValidatorStderr(result.stderr);
return { refused: true, reason: "supervisor-churn", stderr: result.stderr };
}
printValidatorStderr(result.stderr);
console.error("");
console.error(
Expand Down
32 changes: 31 additions & 1 deletion src/lib/actions/sandbox/rebuild-hermes-post-restore.test.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,47 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
createRebuildFlowHarness,
resetRebuildFlowTestEnvironment,
restoreRebuildFlowTestEnvironment,
} from "../../../../test/helpers/rebuild-flow-harness";
import { ensureHermesGatewayAfterStateRestore } from "./rebuild-hermes-post-restore";

describe("Hermes rebuild post-restore verification", () => {
beforeEach(resetRebuildFlowTestEnvironment);
afterEach(restoreRebuildFlowTestEnvironment);

it("retries exact managed-supervisor churn before accepting restored Hermes state (#7229)", () => {
const checkAndRecoverSandboxProcesses = vi
.fn()
.mockReturnValueOnce({
checked: true,
wasRunning: true,
recovered: false,
secretBoundaryRefused: true,
secretBoundaryReason: "supervisor-churn",
})
.mockReturnValueOnce({
checked: true,
wasRunning: true,
recovered: false,
});
const sleep = vi.fn();

expect(
ensureHermesGatewayAfterStateRestore("alpha", "hermes", {
checkAndRecoverSandboxProcesses,
sleepSeconds: sleep,
}),
).toBe("healthy");

expect(checkAndRecoverSandboxProcesses).toHaveBeenCalledTimes(2);
expect(sleep).toHaveBeenCalledOnce();
expect(sleep).toHaveBeenCalledWith(3);
});

it("fails instead of reporting readiness when restored state leaves the gateway down (#7084)", async () => {
const mcpEntry = {
server: "blender",
Expand Down
53 changes: 43 additions & 10 deletions src/lib/actions/sandbox/rebuild-hermes-post-restore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

import { CLI_NAME } from "../../cli/branding";
import { sleepSeconds } from "../../core/wait";
import * as processRecovery from "./process-recovery";

export type HermesPostRestoreGatewayState =
Expand All @@ -16,6 +17,7 @@ type GatewayRecoveryObservation = {
recovered: boolean;
forwardRecoveryFailed?: boolean;
secretBoundaryRefused?: boolean;
secretBoundaryReason?: string;
mcpReconciliationRefused?: boolean;
};

Expand All @@ -24,6 +26,36 @@ interface HermesPostRestoreGatewayDeps {
sandboxName: string,
options: { quiet: boolean },
) => GatewayRecoveryObservation;
sleepSeconds?: (seconds: number) => void;
}

const POST_RESTORE_SUPERVISOR_ATTEMPTS = 3;
const POST_RESTORE_SUPERVISOR_RETRY_SECONDS = 3;

function isTransientManagedSupervisorChurn(observation: GatewayRecoveryObservation): boolean {
// State restoration can make PID 1 replace Hermes between the healthy HTTP
// probe and its validator-enforced recovery request. Retry only the exact
// controller classification; validator and integrity output stays terminal.
return (
observation.secretBoundaryRefused === true &&
observation.secretBoundaryReason === "supervisor-churn"
);
}

function classifyGatewayObservation(
observation: GatewayRecoveryObservation,
): HermesPostRestoreGatewayState {
if (
!observation.checked ||
observation.forwardRecoveryFailed === true ||
observation.secretBoundaryRefused === true ||
observation.mcpReconciliationRefused === true
) {
return "unverified";
}
if (observation.wasRunning === true) return "healthy";
if (observation.recovered) return "recovered";
return "unverified";
}

/**
Expand All @@ -41,17 +73,18 @@ export function ensureHermesGatewayAfterStateRestore(
if (agentName !== "hermes") return "not-applicable";
const checkAndRecover =
deps.checkAndRecoverSandboxProcesses ?? processRecovery.checkAndRecoverSandboxProcesses;
const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true });
if (
!observation.checked ||
observation.forwardRecoveryFailed === true ||
observation.secretBoundaryRefused === true ||
observation.mcpReconciliationRefused === true
) {
return "unverified";
const wait = deps.sleepSeconds ?? sleepSeconds;
for (let attempt = 1; attempt <= POST_RESTORE_SUPERVISOR_ATTEMPTS; attempt += 1) {
const observation: GatewayRecoveryObservation = checkAndRecover(sandboxName, { quiet: true });
if (
attempt < POST_RESTORE_SUPERVISOR_ATTEMPTS &&
isTransientManagedSupervisorChurn(observation)
) {
wait(POST_RESTORE_SUPERVISOR_RETRY_SECONDS);
continue;
}
return classifyGatewayObservation(observation);
}
if (observation.wasRunning === true) return "healthy";
if (observation.recovered) return "recovered";
return "unverified";
}

Expand Down
5 changes: 4 additions & 1 deletion test/e2e/live/rebuild-hermes-progress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,10 @@ export function startRebuildHermesProgress(
const sampleResources = options.sampleResources ?? defaultResourceSnapshot;
const sampleResourceEvidence =
options.sampleResourceEvidence ??
((phase) => renderSnapshotLine(collectResourceSnapshot(resourcePhaseLabel(phase))));
((phase) =>
renderSnapshotLine(
collectResourceSnapshot(resourcePhaseLabel(phase), { includeDocker: false }),
));
const recordResourceBaseline =
options.recordResourceBaseline ??
((phase) => {
Expand Down
10 changes: 9 additions & 1 deletion test/e2e/live/rebuild-hermes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import os from "node:os";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { shellQuote } from "../../../src/lib/core/shell-quote";
import { prepareInitialSandboxCreatePolicy } from "../../../src/lib/onboard/initial-policy";
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts";
import { assertExitZero as expectExitZero } from "../fixtures/clients/command.ts";
Expand Down Expand Up @@ -41,6 +42,7 @@ import { buildRebuildHermesTimingSummary, describeRunnerClass } from "./rebuild-
// Vitest.

const HERMES_MANIFEST = path.join(REPO_ROOT, "agents", "hermes", "manifest.yaml");
const HERMES_POLICY = path.join(REPO_ROOT, "agents", "hermes", "policy-additions.yaml");
const OLD_HERMES_VERSION = "v2026.5.16";
const OLD_HERMES_REGISTRY_VERSION = OLD_HERMES_VERSION.slice(1);
const OLD_HERMES_SEMVER = "0.14.0";
Expand Down Expand Up @@ -654,6 +656,9 @@ test(STALE_BASE_REBUILD
const oldDockerfileDir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-rebuild-hermes-"));
const oldDockerfile = path.join(oldDockerfileDir, "Dockerfile");
fs.writeFileSync(oldDockerfile, oldHermesDockerfile(), "utf8");
const oldSandboxPolicy = prepareInitialSandboxCreatePolicy(HERMES_POLICY, ["discord"], {
agentName: "hermes",
});
try {
const provider = await host.command(
"bash",
Expand Down Expand Up @@ -687,6 +692,8 @@ test(STALE_BASE_REBUILD
SANDBOX_NAME,
"--from",
oldDockerfile,
"--policy",
oldSandboxPolicy.policyPath,
"--gateway",
"nemoclaw",
"--provider",
Expand All @@ -707,6 +714,7 @@ test(STALE_BASE_REBUILD
expectExitZero(createOldSandbox, "create old Hermes sandbox");
oldSandboxImageState = rebuildHermesRegistryImageState(resultText(createOldSandbox));
} finally {
oldSandboxPolicy.cleanup?.();
fs.rmSync(oldDockerfileDir, { recursive: true, force: true });
}
const seededOldSandboxImageState =
Expand Down Expand Up @@ -751,7 +759,7 @@ test(STALE_BASE_REBUILD
"-lc",
[
"hermes kanban init",
`hermes kanban create ${shellQuote(KANBAN_TASK_TITLE)} --initial-status blocked --json`,
`hermes kanban create ${shellQuote(KANBAN_TASK_TITLE)} --triage --json`,
`mkdir -p ${shellQuote(path.dirname(EXCLUDED_KANBAN_FILE))}`,
`printf '%s' ${shellQuote(MARKER_CONTENT)} > ${shellQuote(EXCLUDED_KANBAN_FILE)}`,
].join(" && "),
Expand Down
10 changes: 10 additions & 0 deletions test/e2e/support/live-test-outcome-invocation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ const FIXTURE = "test/e2e/support/fixtures/live-test-outcome.fixture.test.ts";
const CLASSIFIER = path.join(ROOT, "tools/e2e/runner-pressure.mts");

describe("live-test outcome invocation contract (#7146)", () => {
it("loads the private-file helper through the live tsx entrypoint", () => {
const result = spawnSync("npx", ["tsx", "tools/e2e/live-test-outcome.mts"], {
cwd: ROOT,
encoding: "utf8",
timeout: 10_000,
});

expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0);
});

it.each([
"assertion",
"timeout",
Expand Down
32 changes: 32 additions & 0 deletions test/e2e/support/rebuild-hermes-progress.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { describe, expect, it, vi } from "vitest";
import {
type RebuildHermesProgressOptions,
Expand Down Expand Up @@ -40,6 +44,34 @@ function progressHarness() {
}

describe("Hermes rebuild live progress", () => {
it("keeps long-build heartbeats from invoking Docker inspection", () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-rebuild-progress-"));
const dockerMarker = path.join(directory, "docker-invoked");
const fakeDocker = path.join(directory, "docker");
fs.writeFileSync(fakeDocker, '#!/bin/sh\n: > "${0%/*}/docker-invoked"\n', {
mode: 0o755,
});
vi.stubEnv("PATH", `${directory}${path.delimiter}${process.env.PATH ?? ""}`);

try {
const { options, state } = progressHarness();
delete options.sampleResourceEvidence;
const progress = startRebuildHermesProgress("phase 1 install", options);
progress.stop();

expect(fs.existsSync(dockerMarker)).toBe(false);
const snapshotLine = state.lines.find((line) => line.startsWith("E2E_RESOURCE_SNAPSHOT "));
expect(snapshotLine).toBeDefined();
expect(JSON.parse(snapshotLine!.slice("E2E_RESOURCE_SNAPSHOT ".length))).toMatchObject({
containers: [],
dockerDisk: null,
});
} finally {
vi.unstubAllEnvs();
fs.rmSync(directory, { recursive: true, force: true });
}
});

it("streams timestamp-only phase and resource heartbeats through cleanup", () => {
const { options, state } = progressHarness();
const progress = startRebuildHermesProgress("phase 6 nemoclaw rebuild", options);
Expand Down
36 changes: 26 additions & 10 deletions tools/e2e/runner-pressure.mts
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,14 @@ function collectDisk(): ResourceSnapshot["disk"] {
}
}

export function collectResourceSnapshot(phase: string): ResourceSnapshot {
export interface CollectResourceSnapshotOptions {
includeDocker?: boolean;
}

export function collectResourceSnapshot(
phase: string,
options: CollectResourceSnapshotOptions = {},
): ResourceSnapshot {
const meminfoText = readTextOrNull("/proc/meminfo");
const loadText = readTextOrNull("/proc/loadavg");
const current = readTextOrNull(`${CGROUP_ROOT}/memory.current`);
Expand All @@ -116,8 +123,16 @@ export function collectResourceSnapshot(phase: string): ResourceSnapshot {
const memoryPressure = readTextOrNull(`${CGROUP_ROOT}/memory.pressure`);
const ioPressure = readTextOrNull(`${CGROUP_ROOT}/io.pressure`);
const psText = runOrNull("ps", ["-eo", "rss="]);
const statsText = runOrNull("docker", ["stats", "--no-stream", "--format", "{{json .}}"]);
const dfText = runOrNull("docker", ["system", "df", "--format", "{{json .}}"]);
// Docker inspection can block behind a large BuildKit export and add load to
// the daemon. Long-build heartbeats keep the host evidence below while the
// workflow baseline and terminal classifier retain Docker-specific probes.
const includeDocker = options.includeDocker ?? true;
const statsText = includeDocker
? runOrNull("docker", ["stats", "--no-stream", "--format", "{{json .}}"])
: null;
const dfText = includeDocker
? runOrNull("docker", ["system", "df", "--format", "{{json .}}"])
: null;
return {
phase,
at: new Date().toISOString(),
Expand Down Expand Up @@ -181,9 +196,13 @@ function assertEvidencePath(path: string | undefined, variableName: string): str
/** Append one phase baseline without replacing the immutable workflow baseline. */
export function appendResourcePhaseBaseline(path: string, phase: string): void {
const validatedPath = assertEvidencePath(path, "E2E_RESOURCE_PHASE_BASELINES_FILE");
appendPrivateRegularFile(validatedPath, `${renderBaselineLine(collectResourceBaseline(phase))}\n`, {
maxBytes: PHASE_BASELINES_FILE_MAX_BYTES,
});
appendPrivateRegularFile(
validatedPath,
`${renderBaselineLine(collectResourceBaseline(phase))}\n`,
{
maxBytes: PHASE_BASELINES_FILE_MAX_BYTES,
},
);
}

/** Create the trusted evidence files before PR-controlled live tests execute. */
Expand All @@ -201,10 +220,7 @@ function runInitializeEvidence(): void {
process.env.E2E_TERMINAL_CLASSIFICATION_FILE,
"E2E_TERMINAL_CLASSIFICATION_FILE",
);
writePrivateRegularFile(
baselinePath,
`${renderBaselineLine(collectResourceBaseline(phase))}\n`,
);
writePrivateRegularFile(baselinePath, `${renderBaselineLine(collectResourceBaseline(phase))}\n`);
writePrivateRegularFile(phaseBaselinesPath, "");
writePrivateRegularFile(classificationPath, "");
}
Expand Down
Loading