Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2298,7 +2298,28 @@ jobs:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
mkdir -p "$E2E_ARTIFACT_DIR"
baseline_file="$E2E_ARTIFACT_DIR/runner-pressure-baseline.jsonl"
phase_baselines_file="$E2E_ARTIFACT_DIR/runner-pressure-phase-baselines.jsonl"
classification_file="$E2E_ARTIFACT_DIR/runner-pressure-classification.jsonl"
test_outcome_file="$E2E_ARTIFACT_DIR/live-test-outcome.json"
export DOCKER_OOM_CONTAINER="$NEMOCLAW_SANDBOX_NAME"
export E2E_PHASE="rebuild-hermes.workflow"
export E2E_RESOURCE_BASELINE_FILE="$baseline_file"
export E2E_RESOURCE_PHASE_BASELINES_FILE="$phase_baselines_file"
export E2E_TERMINAL_CLASSIFICATION_FILE="$classification_file"
export E2E_TEST_OUTCOME_FILE="$test_outcome_file"
npx tsx tools/e2e/runner-pressure.mts snapshot
npx tsx tools/e2e/runner-pressure.mts initialize-evidence
set +e
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/rebuild-hermes.test.ts
test_status="$?"
set -e
if [ "$test_status" -ne 0 ]; then
npx tsx tools/e2e/runner-pressure.mts classify
npx tsx tools/e2e/runner-pressure.mts validate-classification
fi
exit "$test_status"

- name: Upload Hermes rebuild artifacts
if: always()
Expand Down Expand Up @@ -2351,7 +2372,28 @@ jobs:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
mkdir -p "$E2E_ARTIFACT_DIR"
baseline_file="$E2E_ARTIFACT_DIR/runner-pressure-baseline.jsonl"
phase_baselines_file="$E2E_ARTIFACT_DIR/runner-pressure-phase-baselines.jsonl"
classification_file="$E2E_ARTIFACT_DIR/runner-pressure-classification.jsonl"
test_outcome_file="$E2E_ARTIFACT_DIR/live-test-outcome.json"
export DOCKER_OOM_CONTAINER="$NEMOCLAW_SANDBOX_NAME"
export E2E_PHASE="rebuild-hermes-stale-base.workflow"
export E2E_RESOURCE_BASELINE_FILE="$baseline_file"
export E2E_RESOURCE_PHASE_BASELINES_FILE="$phase_baselines_file"
export E2E_TERMINAL_CLASSIFICATION_FILE="$classification_file"
export E2E_TEST_OUTCOME_FILE="$test_outcome_file"
npx tsx tools/e2e/runner-pressure.mts snapshot
npx tsx tools/e2e/runner-pressure.mts initialize-evidence
set +e
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/rebuild-hermes.test.ts
test_status="$?"
set -e
if [ "$test_status" -ne 0 ]; then
npx tsx tools/e2e/runner-pressure.mts classify
npx tsx tools/e2e/runner-pressure.mts validate-classification
fi
exit "$test_status"

- name: Upload Hermes stale-base rebuild artifacts
if: always()
Expand Down
13 changes: 11 additions & 2 deletions test/e2e-private-file.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ import path from "node:path";
import { pathToFileURL } from "node:url";

import { describe, expect, it } from "vitest";
import { readPrivateRegularFile, writePrivateRegularFile } from "../tools/e2e/private-file.ts";
import {
appendPrivateRegularFile,
readPrivateRegularFile,
writePrivateRegularFile,
} from "../tools/e2e/private-file.ts";

describe("private E2E controller files", () => {
it("writes private regular files without following links or truncating hardlink targets", () => {
Expand All @@ -22,14 +26,19 @@ describe("private E2E controller files", () => {
expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("regular\n");
expect(fs.statSync(regular).mode & 0o777).toBe(0o600);
writePrivateRegularFile(regular, "updated\n");
expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("updated\n");
appendPrivateRegularFile(regular, "appended\n", { maxBytes: 64 });
expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("updated\nappended\n");

fs.writeFileSync(target, "protected\n");
fs.symlinkSync(target, symlink);
fs.linkSync(target, hardlink);

expect(() => writePrivateRegularFile(symlink, "replaced\n")).toThrow();
expect(() => writePrivateRegularFile(hardlink, "replaced\n")).toThrow(/private regular/u);
expect(() => appendPrivateRegularFile(symlink, "replaced\n", { maxBytes: 64 })).toThrow();
expect(() => appendPrivateRegularFile(hardlink, "replaced\n", { maxBytes: 64 })).toThrow(
/private regular/u,
);
expect(fs.readFileSync(target, "utf8")).toBe("protected\n");
} finally {
fs.rmSync(directory, { recursive: true, force: true });
Expand Down
77 changes: 77 additions & 0 deletions test/e2e-risk-signal-reporter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,18 @@ import path from "node:path";

import { describe, expect, it } from "vitest";
import type { TestModule } from "vitest/node";
import {
classifyLiveTestOutcome,
configuredLiveTestOutcomeFile,
isVitestTimeoutError,
LIVE_TEST_OUTCOME_FILE,
parseLiveTestOutcome,
readLiveTestOutcome,
writeLiveTestOutcome,
} from "../tools/e2e/live-test-outcome.mts";
import {
configuredEnvironment,
outcomeForRun,
RISK_SIGNAL_FILE,
type RiskSignalEnvironment,
writeRiskSignal,
Expand All @@ -28,6 +38,16 @@ function moduleWithStates(states: Array<"passed" | "failed" | "skipped" | "pendi
} as unknown as TestModule;
}

function moduleWithFailedError(error: unknown): TestModule {
return {
children: {
*allTests() {
yield { result: () => ({ state: "failed", errors: [error] }) };
},
},
} as unknown as TestModule;
}

function environment(artifactDir: string): RiskSignalEnvironment {
return {
artifactDir,
Expand Down Expand Up @@ -141,3 +161,60 @@ describe("E2E risk signal reporter", () => {
}
});
});

describe("trusted live-test outcome reporter (#7146)", () => {
it("pins the configured outcome file to the E2E artifact directory", () => {
const artifactDir = "/tmp/nemoclaw-live-outcome";
const expected = path.join(artifactDir, LIVE_TEST_OUTCOME_FILE);
expect(
configuredLiveTestOutcomeFile({
E2E_ARTIFACT_DIR: artifactDir,
E2E_TEST_OUTCOME_FILE: expected,
}),
).toBe(expected);
expect(() =>
configuredLiveTestOutcomeFile({
E2E_ARTIFACT_DIR: artifactDir,
E2E_TEST_OUTCOME_FILE: "/tmp/outside.json",
}),
).toThrow(/must name live-test-outcome\.json/u);
});

it("derives assertion and timeout only from the trusted Vitest result objects", () => {
const timeout = new Error(
'Test timed out in 30000ms.\nIf this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout".',
);
expect(isVitestTimeoutError(timeout)).toBe(true);
expect(
outcomeForRun([moduleWithFailedError(new Error("expected true to be false"))], [], "failed"),
).toBe("assertion");
expect(outcomeForRun([moduleWithFailedError(timeout)], [], "failed")).toBe("timeout");
expect(
classifyLiveTestOutcome({
failedTests: 0,
unhandledErrors: [],
testErrors: [],
runReason: "interrupted",
processTimedOut: true,
}),
).toBe("timeout");
});

it.each([
"assertion",
"timeout",
] as const)("writes and strictly reads a private %s artifact", (outcome) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-live-outcome-"));
const file = path.join(dir, LIVE_TEST_OUTCOME_FILE);
try {
writeLiveTestOutcome(file, outcome);
expect(readLiveTestOutcome(file)).toBe(outcome);
expect(fs.statSync(file).mode & 0o777).toBe(0o600);
expect(() => parseLiveTestOutcome('{"v":1,"outcome":"assertion","token":"secret"}')).toThrow(
/unsupported shape/u,
);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
});
12 changes: 6 additions & 6 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,12 @@ revision, or closed PR can leave the controller green while coordination is
failed or cancelled and the native job is non-passing. Only a successful native
`E2E / PR Gate` for the current head and base satisfies the required check. An
eligible prerequisite-CI failure records the versioned retry reason
`prerequisite-ci`. A selected child records `child-cancelled` only when the
workflow is cancelled or a complete job listing shows that every non-passing
job was cancelled. Assertion failures and other selected-E2E outcomes do not
receive a retry reason. An unexpected controller error still fails the
controller workflow and fails coordination closed, which prevents the native
job from passing.
`prerequisite-ci`. A selected child records `child-cancelled` only when a
trusted hosted-runner-loss marker is present and no terminal classification
was produced; cancellation alone is not retryable. Assertion failures and
other selected-E2E outcomes do not receive a retry reason. An unexpected
controller error still fails the controller workflow and fails coordination
closed, which prevents the native job from passing.

On open, synchronization, reopen, transition out of draft, or base retarget,
`.github/workflows/pr-e2e-gate.yaml` reserves `E2E / PR Gate Coordination` for
Expand Down
36 changes: 36 additions & 0 deletions test/e2e/live/rebuild-hermes-progress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@
import fs from "node:fs";
import os from "node:os";

import {
appendResourcePhaseBaseline,
collectResourceSnapshot,
} from "../../../tools/e2e/runner-pressure.mts";
import { renderSnapshotLine } from "../../../tools/e2e/runner-pressure-core.mts";
import { REPO_ROOT } from "../fixtures/paths.ts";
import type { ShellProbeOutputEvent } from "../fixtures/shell-probe.ts";
import type { RebuildHermesTimeline, RebuildHermesTimingPhase } from "./rebuild-hermes-timing.ts";
Expand All @@ -27,6 +32,8 @@ export interface RebuildHermesProgressOptions {
clearTimer?: (timer: TimerHandle) => void;
logLine?: (line: string) => void;
sampleResources?: () => RebuildHermesResourceSnapshot;
sampleResourceEvidence?: (phase: string) => string;
recordResourceBaseline?: (phase: string) => void;
}

export interface RebuildHermesProgress {
Expand Down Expand Up @@ -67,6 +74,14 @@ function formatResources(sampleResources: () => RebuildHermesResourceSnapshot):
}
}

function resourcePhaseLabel(label: string): string {
const suffix = label
.toLowerCase()
.replace(/[^a-z0-9]+/gu, "-")
.replace(/^-|-$/gu, "");
return `rebuild-hermes.${suffix}`;
}

/**
* Keep the long Hermes scenario visible without forwarding command output,
* which may contain credentials. The timestamp-only output observer records
Expand All @@ -82,6 +97,15 @@ export function startRebuildHermesProgress(
const clearTimer = options.clearTimer ?? ((timer) => clearInterval(timer as NodeJS.Timeout));
const logLine = options.logLine ?? ((line) => process.stdout.write(`${line}\n`));
const sampleResources = options.sampleResources ?? defaultResourceSnapshot;
const sampleResourceEvidence =
options.sampleResourceEvidence ??
((phase) => renderSnapshotLine(collectResourceSnapshot(resourcePhaseLabel(phase))));
const recordResourceBaseline =
options.recordResourceBaseline ??
((phase) => {
const baselinePath = process.env.E2E_RESOURCE_PHASE_BASELINES_FILE;
if (baselinePath) appendResourcePhaseBaseline(baselinePath, resourcePhaseLabel(phase));
});
const overallStartedAt = now();
const completedPhases: RebuildHermesTimingPhase[] = [];
let phaseLabel = initialPhase;
Expand All @@ -94,6 +118,15 @@ export function startRebuildHermesProgress(
completedPhases.push({ label: phaseLabel, elapsedMs: completedAt - phaseStartedAt });
};

const recordBaselineBestEffort = () => {
try {
recordResourceBaseline(phaseLabel);
} catch {
// Diagnostics must not change the live result. Classification retains
// the immutable pre-test baseline when the phase ledger is unavailable.
}
};

const logBestEffort = (state: "started" | "running" | "finished") => {
try {
const current = now();
Expand All @@ -105,11 +138,13 @@ export function startRebuildHermesProgress(
logLine(
`[rebuild-hermes] ${phaseLabel} ${state} (${elapsedSeconds}s elapsed; ${outputAge}; ${formatResources(sampleResources)})`,
);
logLine(sampleResourceEvidence(phaseLabel));
} catch {
// Diagnostics must not change the live test result.
}
};

recordBaselineBestEffort();
logBestEffort("started");
const timer = setTimer(
() => logBestEffort("running"),
Expand All @@ -128,6 +163,7 @@ export function startRebuildHermesProgress(
phaseLabel = label;
phaseStartedAt = now();
lastOutputAt = null;
recordBaselineBestEffort();
logBestEffort("started");
},
stop() {
Expand Down
60 changes: 58 additions & 2 deletions test/e2e/risk-signal-reporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ import path from "node:path";

import type { TestModule } from "vitest/node";
import type { Reporter, TestRunEndReason } from "vitest/reporters";
import {
classifyLiveTestOutcome,
configuredLiveTestOutcomeFile,
type LiveTestOutcome,
writeLiveTestOutcome,
} from "../../tools/e2e/live-test-outcome.mts";
import { readPrivateRegularFile, writePrivateRegularFile } from "../../tools/e2e/private-file.ts";
import type { E2eRiskSignal } from "../../tools/e2e/risk-signal.ts";

Expand Down Expand Up @@ -81,6 +87,33 @@ function counts(testModules: ReadonlyArray<TestModule>) {
return result;
}

function failedTestErrors(testModules: ReadonlyArray<TestModule>): unknown[] {
const errors: unknown[] = [];
for (const module of testModules) {
for (const test of module.children.allTests()) {
const result = test.result();
if (result.state === "failed") errors.push(...result.errors);
}
}
return errors;
}

export function outcomeForRun(
testModules: ReadonlyArray<TestModule>,
unhandledErrors: ReadonlyArray<unknown>,
runReason: TestRunEndReason,
processTimedOut = false,
): LiveTestOutcome {
const summary = counts(testModules);
return classifyLiveTestOutcome({
failedTests: summary.failed,
unhandledErrors,
testErrors: failedTestErrors(testModules),
runReason,
processTimedOut,
});
}

function mergeSignal(previous: E2eRiskSignal | null, current: E2eRiskSignal): E2eRiskSignal {
if (!previous) return current;
if (
Expand Down Expand Up @@ -145,17 +178,40 @@ export function writeRiskSignal(

export default class E2eRiskSignalReporter implements Reporter {
private readonly environment: RiskSignalEnvironment | null;
private readonly outcomeFile: string | null;
private processTimedOut = false;

constructor() {
this.environment = configuredEnvironment(process.env);
this.outcomeFile = configuredLiveTestOutcomeFile(process.env);
}

onTestRunStart(): void {
this.processTimedOut = false;
if (!this.outcomeFile) return;
fs.mkdirSync(path.dirname(this.outcomeFile), { recursive: true });
writeLiveTestOutcome(this.outcomeFile, "none");
}

onProcessTimeout(): void {
this.processTimedOut = true;
if (!this.outcomeFile) return;
writeLiveTestOutcome(this.outcomeFile, "timeout");
}

onTestRunEnd(
testModules: ReadonlyArray<TestModule>,
unhandledErrors: ReadonlyArray<unknown>,
reason: TestRunEndReason,
): void {
if (!this.environment) return;
writeRiskSignal(this.environment, testModules, unhandledErrors, reason);
if (this.environment) {
writeRiskSignal(this.environment, testModules, unhandledErrors, reason);
}
if (this.outcomeFile) {
writeLiveTestOutcome(
this.outcomeFile,
outcomeForRun(testModules, unhandledErrors, reason, this.processTimedOut),
);
}
}
}
Loading