diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts index 5fb36af300..5a2777ea25 100644 --- a/test/e2e-risk-signal-reporter.test.ts +++ b/test/e2e-risk-signal-reporter.test.ts @@ -5,8 +5,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; -import type { TestModule } from "vitest/node"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { TestModule, Vitest } from "vitest/node"; import { classifyLiveTestOutcome, configuredLiveTestOutcomeFile, @@ -18,12 +18,17 @@ import { } from "../tools/e2e/live-test-outcome.mts"; import { configuredEnvironment, + default as E2eRiskSignalReporter, outcomeForRun, RISK_SIGNAL_FILE, type RiskSignalEnvironment, writeRiskSignal, } from "./e2e/risk-signal-reporter.ts"; +vi.mock("node:child_process", () => ({ + execFileSync: vi.fn(() => "a".repeat(40)), +})); + const EXPECTED_SHA = "a".repeat(40); const PLAN_HASH = "b".repeat(64); const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; @@ -32,7 +37,26 @@ function moduleWithStates(states: Array<"passed" | "failed" | "skipped" | "pendi return { children: { *allTests() { - for (const state of states) yield { result: () => ({ state }) }; + for (const [index, state] of states.entries()) { + yield { fullName: `test ${index}`, result: () => ({ state }) }; + } + }, + }, + } as unknown as TestModule; +} + +function moduleWithNamedStates( + tests: Array<{ + fullName: string; + state: "passed" | "failed" | "skipped" | "pending"; + }>, +): TestModule { + return { + children: { + *allTests() { + for (const { fullName, state } of tests) { + yield { fullName, result: () => ({ state }) }; + } }, }, } as unknown as TestModule; @@ -42,7 +66,7 @@ function moduleWithFailedError(error: unknown): TestModule { return { children: { *allTests() { - yield { result: () => ({ state: "failed", errors: [error] }) }; + yield { fullName: "failed test", result: () => ({ state: "failed", errors: [error] }) }; }, }, } as unknown as TestModule; @@ -61,6 +85,10 @@ function environment(artifactDir: string): RiskSignalEnvironment { } describe("E2E risk signal reporter", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("stays disabled when no expected commit is configured", () => { expect(configuredEnvironment({})).toBeNull(); }); @@ -127,6 +155,93 @@ describe("E2E risk signal reporter", () => { } }); + it("applies the configured name pattern through the reporter lifecycle", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); + try { + vi.stubEnv("E2E_ARTIFACT_DIR", dir); + vi.stubEnv("E2E_TARGET_ID", "network-policy"); + vi.stubEnv("NEMOCLAW_E2E_EXPECTED_SHA", EXPECTED_SHA); + vi.stubEnv("NEMOCLAW_E2E_PLAN_HASH", PLAN_HASH); + vi.stubEnv("NEMOCLAW_E2E_CORRELATION_ID", CORRELATION_ID); + vi.stubEnv("NEMOCLAW_E2E_SHARD", "live-probes"); + + const reporter = new E2eRiskSignalReporter(); + reporter.onInit({ + config: { testNamePattern: /^network-policy:.+probes$/u }, + } as Vitest); + reporter.onTestRunEnd( + [ + moduleWithNamedStates([ + { + fullName: "network-policy: restricted sandbox enforces live allow/deny policy probes", + state: "passed", + }, + { + fullName: + "network-policy: default restricted OpenClaw onboard leaves policy-list with zero active presets", + state: "skipped", + }, + ]), + ], + [], + "passed", + ); + + const signal = JSON.parse( + fs.readFileSync(path.join(dir, RISK_SIGNAL_FILE), "utf8"), + ) as Record; + expect(signal).toMatchObject({ passed: 1, failed: 0, skipped: 0, pending: 0 }); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("counts a selected test that skips", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); + try { + const signal = writeRiskSignal( + environment(dir), + [ + moduleWithNamedStates([ + { + fullName: "network-policy: restricted sandbox enforces live allow/deny policy probes", + state: "skipped", + }, + { + fullName: + "network-policy: default restricted OpenClaw onboard leaves policy-list with zero active presets", + state: "skipped", + }, + ]), + ], + [], + "passed", + /^network-policy:.+probes$/u, + ); + + expect(signal).toMatchObject({ passed: 0, failed: 0, skipped: 1, pending: 0 }); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("emits no passing evidence when the name pattern matches no tests", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); + try { + const signal = writeRiskSignal( + environment(dir), + [moduleWithNamedStates([{ fullName: "selected elsewhere", state: "skipped" }])], + [], + "passed", + /^missing test$/u, + ); + + expect(signal).toMatchObject({ passed: 0, failed: 0, skipped: 0, pending: 0 }); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("refuses a symlinked prior signal without modifying its target", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); const target = path.join(dir, "target.json"); diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index c823930a1a..3b7d605442 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -5,7 +5,7 @@ import { execFileSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; -import type { TestModule } from "vitest/node"; +import type { TestModule, Vitest } from "vitest/node"; import type { Reporter, TestRunEndReason } from "vitest/reporters"; import { classifyLiveTestOutcome, @@ -79,10 +79,18 @@ export function configuredEnvironment( return { ...values, testedSha }; } -function counts(testModules: ReadonlyArray) { +function matchesNamePattern(fullName: string, pattern: RegExp | undefined): boolean { + if (!pattern) return true; + const stablePattern = new RegExp(pattern.source, pattern.flags); + // Vitest joins suite names with spaces when it applies testNamePattern. + return stablePattern.test(fullName.replaceAll(" > ", " ")); +} + +function counts(testModules: ReadonlyArray, testNamePattern?: RegExp) { const result = { passed: 0, failed: 0, skipped: 0, pending: 0 }; for (const module of testModules) { for (const test of module.children.allTests()) { + if (!matchesNamePattern(test.fullName, testNamePattern)) continue; result[test.result().state] += 1; } } @@ -158,6 +166,7 @@ export function writeRiskSignal( testModules: ReadonlyArray, unhandledErrors: ReadonlyArray, runReason: TestRunEndReason, + testNamePattern?: RegExp, ): E2eRiskSignal { const signal: E2eRiskSignal = { version: 1, @@ -167,7 +176,7 @@ export function writeRiskSignal( testedSha: environment.testedSha, planHash: environment.planHash, correlationId: environment.correlationId, - ...counts(testModules), + ...counts(testModules, testNamePattern), unhandledErrors: unhandledErrors.length, runReason, }; @@ -181,6 +190,7 @@ export function writeRiskSignal( export default class E2eRiskSignalReporter implements Reporter { private readonly environment: RiskSignalEnvironment | null; private readonly outcomeFile: string | null; + private testNamePattern: RegExp | undefined; private processTimedOut = false; constructor() { @@ -188,6 +198,10 @@ export default class E2eRiskSignalReporter implements Reporter { this.outcomeFile = configuredLiveTestOutcomeFile(process.env); } + onInit(vitest: Vitest): void { + this.testNamePattern = vitest.config.testNamePattern; + } + onTestRunStart(): void { this.processTimedOut = false; if (!this.outcomeFile) return; @@ -207,7 +221,7 @@ export default class E2eRiskSignalReporter implements Reporter { reason: TestRunEndReason, ): void { if (this.environment) { - writeRiskSignal(this.environment, testModules, unhandledErrors, reason); + writeRiskSignal(this.environment, testModules, unhandledErrors, reason, this.testNamePattern); } if (this.outcomeFile) { writeLiveTestOutcome(