From 7908854077b09a301d1503433c4d745aadcb54ed Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 2 Aug 2026 17:45:40 +0800 Subject: [PATCH 1/7] fix(review): read why the collocated test was not green, do not assert it A mutant or hunk whose own collocated test was not green in the unmutated baseline is held `inconclusive` -- correctly, since the other probes passing shows only that THEY do not cover it. But both guards then named a cause they had not measured: "likely a compile or import error in the probe tree". There are two ways a probe is not green and they are different failures with different fixes. Measured on PR #8368: `AuthDialog.test.tsx` compiled fine, collected 26 tests and failed exactly one, and all three mutants in its source were held with the import-error wording -- sending a reader after a problem that was never there. The baseline had already classified it `gated` (a real assertion failure) rather than `inconclusive` (collected nothing); the guards just did not look. Both now build their detail from one exported function that reads the verdict off the baseline, so the two sentences cannot drift apart again and what the tests pin is the string the report carries. A probe with no baseline entry takes the collected-nothing wording: absent is an evidentiary hole, never the claim that its tests failed. --- .../src/commands/review/test-efficacy.test.ts | 60 +++++++++++++++++++ .../cli/src/commands/review/test-efficacy.ts | 35 ++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/review/test-efficacy.test.ts b/packages/cli/src/commands/review/test-efficacy.test.ts index 67e3276418f..8649e2f80dd 100644 --- a/packages/cli/src/commands/review/test-efficacy.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.test.ts @@ -19,6 +19,7 @@ import { parseAddedLines, hasCollocatedNewTest, collocatedProbe, + collocatedNotGreenDetail, fitsAnotherMutantRun, probeCleanupFailureDetail, findVitestBin, @@ -1313,6 +1314,65 @@ describe('collocatedProbe', () => { }); }); +describe('collocatedNotGreenDetail', () => { + const perFile = [ + { file: 'packages/cli/src/red.test.ts', verdict: 'gated' as const }, + { + file: 'packages/cli/src/empty.test.ts', + verdict: 'inconclusive' as const, + }, + ]; + + it('says the test was red when the baseline collected it and it failed', () => { + // The case that produced this function: on PR #8368 AuthDialog.test.tsx + // compiled and ran 26 tests, one of which failed. Reporting that as an + // import error sends the reader to a file that imports fine. + const detail = collocatedNotGreenDetail( + 'mutant', + 'packages/cli/src/red.test.ts', + perFile, + ); + expect(detail).toContain('was RED there'); + expect(detail).not.toContain('compile or import error'); + }); + + it('says nothing was collected when the baseline collected nothing', () => { + expect( + collocatedNotGreenDetail( + 'hunk', + 'packages/cli/src/empty.test.ts', + perFile, + ), + ).toContain('collected no tests there'); + }); + + it('takes the collected-nothing wording for a probe the baseline never reported', () => { + // Absent is an evidentiary hole, never the claim that its tests failed. + const detail = collocatedNotGreenDetail( + 'mutant', + 'packages/cli/src/absent.test.ts', + perFile, + ); + expect(detail).toContain('collected no tests there'); + expect(detail).not.toContain('RED'); + }); + + it('names the probe and what the passing probes cannot show, per kind', () => { + expect( + collocatedNotGreenDetail( + 'mutant', + 'packages/cli/src/red.test.ts', + perFile, + ), + ).toBe( + "this mutant's collocated test packages/cli/src/red.test.ts did not run green in the unmutated baseline — it was RED there, so the remaining probes passing cannot show the statement is uncovered", + ); + expect( + collocatedNotGreenDetail('hunk', 'packages/cli/src/red.test.ts', perFile), + ).toContain('cannot show the hunk is uncovered'); + }); +}); + describe('classifyMutantRun', () => { // Verdicts flow through the SAME per-file classifier the revert probe uses, // so these fixtures are the vitest-JSON shapes classifyProbeRun already diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index 84f2213e021..93785eb491c 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -817,6 +817,35 @@ export function collocatedProbe( }); } +/** + * The whole sentence a mutant or hunk is held `inconclusive` with when its own + * collocated test was not green in the unmutated baseline — the ONLY wording + * either guard has, so what this returns is what the report says. + * + * It reads the reason off the baseline rather than asserting one. The two + * reasons are different failures with different fixes, and the guards used to + * name the wrong one flatly: measured on PR #8368, `AuthDialog.test.tsx` + * compiled, collected 26 tests and failed exactly one, and all three mutants in + * its source were held with "likely a compile or import error in the probe + * tree" — sending a reader after an import problem that was never there. + * + * A probe with no baseline entry at all takes the collected-nothing wording: + * the run said nothing about it, which is the same evidentiary hole and never + * the claim that its tests failed. + */ +export function collocatedNotGreenDetail( + kind: 'mutant' | 'hunk', + probe: string, + perFile: Array<{ file: string; verdict: ProbeVerdict }>, +): string { + const reason = + perFile.find((p) => p.file === probe)?.verdict === 'gated' + ? 'it was RED there' + : 'it collected no tests there (a compile or import error in the probe tree, or every test skipped)'; + const what = kind === 'mutant' ? 'the statement' : 'the hunk'; + return `this ${kind}'s collocated test ${probe} did not run green in the unmutated baseline — ${reason}, so the remaining probes passing cannot show ${what} is uncovered`; +} + /** * Does the diff add or change a test collocated with this production file? * The repo convention is `file.test.ts` beside `file.ts`. Used only to ORDER @@ -1940,6 +1969,8 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { const greenProbes = baseline.perFile .filter((r) => r.verdict === 'inert') .map((r) => r.file); + const notGreen = (kind: 'mutant' | 'hunk', probe: string) => + collocatedNotGreenDetail(kind, probe, baseline.perFile); if (greenProbes.length === 0) { mutantsSkippedForBaseline = candidates.length; // Their own reason, not the budget's: the mutants ran zero suites in @@ -2019,7 +2050,7 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { mutantResults.push({ ...c, verdict: 'inconclusive' as const, - detail: `this mutant's collocated test ${own} did not run green in the unmutated baseline (likely a compile or import error in the probe tree), so the remaining probes passing cannot show the statement is uncovered`, + detail: notGreen('mutant', own), }); continue; } @@ -2064,7 +2095,7 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { hunkResults.push({ ...meta, verdict: 'inconclusive' as const, - detail: `this hunk's collocated test ${own} did not run green in the unmutated baseline (likely a compile or import error in the probe tree), so the remaining probes passing cannot show the hunk is uncovered`, + detail: notGreen('hunk', own), }); continue; } From 50ffbf45e144629272f61f71c47966f36be5b40d Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 2 Aug 2026 20:09:11 +0800 Subject: [PATCH 2/7] fix(review): carry why a probe was inconclusive, and decide the hold once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from review, both landing on this PR's own thesis. The else branch told every non-`gated` probe the same story -- "collected no tests there (a compile or import error…)" -- but `classifyProbeRun` reaches `inconclusive` three different ways, and one of them is the runner producing no parseable output at all. Nothing is known about collection there, so naming a compile error is the same invented cause this PR was opened to remove, one layer down: a reader sent hunting for an import problem while the runner itself is what fell over. The classifier now tags each branch with a machine-readable `reason` -- it already had the three cases -- and the explanation reads it. An entry that names no reason gets the disjunction; a probe with no entry at all is reported as not measured, which is a different thing from measured-as-empty. The rule itself was stated twice, once per loop, and the duplication had drifted twice already: the hunk loop had the guard first and eight mutant survivors shipped through the gap before it was copied across, then the shared wording was corrected in one place and hand-copied to the other. Both loops now call one exported decision, `heldForRedCollocatedTest`, which is where the tests point. Three mutations, each reddening only its own case: collapsing `no-output` into the compile-error wording, treating an absent probe as an inconclusive entry, and a guard that never holds. --- .../src/commands/review/test-efficacy.test.ts | 107 +++++++++++++++-- .../cli/src/commands/review/test-efficacy.ts | 110 +++++++++++++++--- 2 files changed, 192 insertions(+), 25 deletions(-) diff --git a/packages/cli/src/commands/review/test-efficacy.test.ts b/packages/cli/src/commands/review/test-efficacy.test.ts index 8649e2f80dd..b5cbfad1f8d 100644 --- a/packages/cli/src/commands/review/test-efficacy.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.test.ts @@ -20,6 +20,7 @@ import { hasCollocatedNewTest, collocatedProbe, collocatedNotGreenDetail, + heldForRedCollocatedTest, fitsAnotherMutantRun, probeCleanupFailureDetail, findVitestBin, @@ -1336,27 +1337,59 @@ describe('collocatedNotGreenDetail', () => { expect(detail).not.toContain('compile or import error'); }); - it('says nothing was collected when the baseline collected nothing', () => { - expect( - collocatedNotGreenDetail( - 'hunk', - 'packages/cli/src/empty.test.ts', - perFile, - ), - ).toContain('collected no tests there'); + it('gives the disjunction for an inconclusive entry that names no reason', () => { + // An older artifact, or a future branch that forgets to tag itself: say + // what is not known instead of picking one of its arms. + const detail = collocatedNotGreenDetail( + 'hunk', + 'packages/cli/src/empty.test.ts', + perFile, + ); + expect(detail).toContain('did not come back green there'); + expect(detail).toContain('no parseable output'); }); - it('takes the collected-nothing wording for a probe the baseline never reported', () => { - // Absent is an evidentiary hole, never the claim that its tests failed. + it('says the baseline never reported a probe it has no entry for', () => { + // Absent is an evidentiary hole, never the claim that its tests failed — + // and not the claim that it collected nothing either, which is a + // measurement the baseline never took. const detail = collocatedNotGreenDetail( 'mutant', 'packages/cli/src/absent.test.ts', perFile, ); - expect(detail).toContain('collected no tests there'); + expect(detail).toContain('did not report it'); expect(detail).not.toContain('RED'); }); + it.each([ + ['no-output', 'produced no parseable output', 'nothing at all is known'], + ['no-tests', 'collected no tests there', 'compile or import error'], + ['all-skipped', 'executed none of them', 'collected tests there'], + ])( + 'names %s as the reason rather than guessing one', + (reason, phrase, alsoPhrase) => { + const detail = collocatedNotGreenDetail( + 'mutant', + 'packages/cli/src/x.test.ts', + [ + { + file: 'packages/cli/src/x.test.ts', + verdict: 'inconclusive' as const, + reason: reason as 'no-output' | 'no-tests' | 'all-skipped', + }, + ], + ); + expect(detail).toContain(phrase); + expect(detail).toContain(alsoPhrase); + // The runner falling over is not a compile error, and the message that + // says so is the whole point of carrying the reason. + if (reason === 'no-output') { + expect(detail).not.toContain('compile or import error'); + } + }, + ); + it('names the probe and what the passing probes cannot show, per kind', () => { expect( collocatedNotGreenDetail( @@ -1373,6 +1406,58 @@ describe('collocatedNotGreenDetail', () => { }); }); +describe('heldForRedCollocatedTest — the one decision both loops make', () => { + const perFile = [ + { + file: 'packages/cli/src/x.test.ts', + verdict: 'gated' as const, + }, + { + file: 'packages/cli/src/ok.test.ts', + verdict: 'inert' as const, + }, + ]; + const probes = ['packages/cli/src/x.test.ts', 'packages/cli/src/ok.test.ts']; + + it('holds when the collocated test was not green, and explains why', () => { + const detail = heldForRedCollocatedTest( + 'mutant', + 'packages/cli/src/x.ts', + probes, + ['packages/cli/src/ok.test.ts'], + perFile, + ); + expect(detail).toContain('was RED there'); + expect(detail).toContain('packages/cli/src/x.test.ts'); + }); + + it('does not hold when the collocated test was green', () => { + expect( + heldForRedCollocatedTest( + 'hunk', + 'packages/cli/src/ok.ts', + probes, + ['packages/cli/src/ok.test.ts'], + perFile, + ), + ).toBeUndefined(); + }); + + it('does not hold when the file has no collocated probe at all', () => { + // No covering test was measured either way, so this guard has nothing to + // say — the other probes decide. + expect( + heldForRedCollocatedTest( + 'mutant', + 'packages/cli/src/untested.ts', + probes, + [], + perFile, + ), + ).toBeUndefined(); + }); +}); + describe('classifyMutantRun', () => { // Verdicts flow through the SAME per-file classifier the revert probe uses, // so these fixtures are the vitest-JSON shapes classifyProbeRun already diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index 93785eb491c..c3213a3753a 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -76,6 +76,27 @@ import { isWorkspaceMember } from './lib/workspaces.js'; export type ProbeVerdict = 'gated' | 'inert' | 'inconclusive'; +/** + * WHY a probe was `inconclusive`, as the classifier knew it at the time. + * + * Three different things arrive at the same verdict and they are not + * interchangeable: `no-output` is the runner failing to produce parseable JSON, + * where nothing at all is known about collection; `no-tests` is a run that + * produced results without this file, the shape a compile or import error takes; + * `all-skipped` is a file that collected tests and executed none. Anything + * downstream that explains an `inconclusive` has to pick between them, and the + * prose `detail` is written for a human, not for that decision. + */ +export type ProbeReason = 'no-output' | 'no-tests' | 'all-skipped'; + +export interface ProbeResult { + file: string; + verdict: ProbeVerdict; + detail: string; + /** Set only when `verdict` is `inconclusive`. */ + reason?: ProbeReason; +} + export interface FileEntry { path: string; kind: string; @@ -833,15 +854,63 @@ export function collocatedProbe( * the run said nothing about it, which is the same evidentiary hole and never * the claim that its tests failed. */ +/** + * Should this candidate be held `inconclusive` because the test collocated with + * the file it touches was not green in the unmutated baseline — and if so, with + * what explanation? + * + * ONE decision for both loops. The rule and its wording had already been + * duplicated across the mutant and hunk guards, and the duplication had already + * drifted twice: the hunk loop had the rule first and the mutant loop shipped + * eight survivors through the gap before it was copied over, and the shared + * explanation was then corrected in one place and hand-copied to the other. + * A rule stated twice is a rule that will be true in one place. + */ +export function heldForRedCollocatedTest( + kind: 'mutant' | 'hunk', + file: string, + probes: readonly string[], + greenProbes: readonly string[], + perFile: ReadonlyArray>, +): string | undefined { + const own = collocatedProbe(file, [...probes]); + if (!own || greenProbes.includes(own)) return undefined; + return collocatedNotGreenDetail(kind, own, perFile); +} + +const REASON_PHRASE: Record = { + // Nothing is known about collection here — the runner never produced output + // to read. Saying "collected no tests" would be the same invented cause this + // function exists to stop, one layer down: a reader sent after a compile + // error that is not there, while the runner itself is what fell over. + 'no-output': + 'the runner produced no parseable output there, so nothing at all is known about it', + 'no-tests': + 'it collected no tests there — the shape a compile or import error in the probe tree takes', + 'all-skipped': 'it collected tests there but executed none of them', + // An entry that is inconclusive without saying which way. Older artifacts and + // any future branch that forgets to tag itself land here, and the honest + // answer is the disjunction rather than a guess at one of its arms. + unspecified: + 'it did not come back green there (a compile or import error, every test skipped, or no parseable output)', +}; + +/** No entry at all — the baseline never reported this probe. Absent is its own + * state, and is not evidence that its tests failed. */ +const NOT_REPORTED = 'the baseline did not report it'; + export function collocatedNotGreenDetail( kind: 'mutant' | 'hunk', probe: string, - perFile: Array<{ file: string; verdict: ProbeVerdict }>, + perFile: ReadonlyArray>, ): string { + const entry = perFile.find((p) => p.file === probe); const reason = - perFile.find((p) => p.file === probe)?.verdict === 'gated' - ? 'it was RED there' - : 'it collected no tests there (a compile or import error in the probe tree, or every test skipped)'; + entry === undefined + ? NOT_REPORTED + : entry.verdict === 'gated' + ? 'it was RED there' + : REASON_PHRASE[entry.reason ?? 'unspecified']; const what = kind === 'mutant' ? 'the statement' : 'the hunk'; return `this ${kind}'s collocated test ${probe} did not run green in the unmutated baseline — ${reason}, so the remaining probes passing cannot show ${what} is uncovered`; } @@ -935,7 +1004,7 @@ export function classifyProbeRun( stdout: string, probes: string[], stderr = '', -): Array<{ file: string; verdict: ProbeVerdict; detail: string }> { +): ProbeResult[] { let parsed: VitestJson | undefined; const start = stdout.indexOf('{'); if (start >= 0) { @@ -952,6 +1021,7 @@ export function classifyProbeRun( return probes.map((file) => ({ file, verdict: 'inconclusive' as const, + reason: 'no-output' as const, detail: `runner produced no parseable JSON (exit ${exitCode})${why ? `: ${why}` : ''}`, })); } @@ -990,6 +1060,7 @@ export function classifyProbeRun( return { file, verdict: 'inconclusive' as const, + reason: 'no-tests' as const, detail: `collected no tests with the source reverted (run exit ${exitCode}) — likely a compile or import error, which is not evidence either way`, }; } @@ -1008,6 +1079,7 @@ export function classifyProbeRun( return { file, verdict: 'inconclusive' as const, + reason: 'all-skipped' as const, detail: `${assertions.length} test(s) collected but none executed with the source reverted (all skipped) — not evidence either way`, }; } @@ -1263,7 +1335,7 @@ function runProbeSuite( now: () => number = Date.now, dependencyRoot: string = probeTree, ): { - perFile: Array<{ file: string; verdict: ProbeVerdict; detail: string }>; + perFile: ProbeResult[]; ms: number; exposed: { linked: number; failed: number }; } { @@ -1969,8 +2041,6 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { const greenProbes = baseline.perFile .filter((r) => r.verdict === 'inert') .map((r) => r.file); - const notGreen = (kind: 'mutant' | 'hunk', probe: string) => - collocatedNotGreenDetail(kind, probe, baseline.perFile); if (greenProbes.length === 0) { mutantsSkippedForBaseline = candidates.length; // Their own reason, not the budget's: the mutants ran zero suites in @@ -2045,12 +2115,18 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { // of a mutant unguarded against an absent covering test. Checked // before the budget, because a candidate that cannot yield a // verdict should not spend a suite run to say so. - const own = collocatedProbe(c.file, probes); - if (own && !greenProbes.includes(own)) { + const heldDetail = heldForRedCollocatedTest( + 'mutant', + c.file, + probes, + greenProbes, + baseline.perFile, + ); + if (heldDetail) { mutantResults.push({ ...c, verdict: 'inconclusive' as const, - detail: notGreen('mutant', own), + detail: heldDetail, }); continue; } @@ -2089,13 +2165,19 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { // an absent covering test is never `survived` there either. The // two halves are separate guards; having one was read as having // the rule, and eight mutant survivors shipped through the gap. - const own = collocatedProbe(h.file, probes); - if (own && !greenProbes.includes(own)) { + const heldDetail = heldForRedCollocatedTest( + 'hunk', + h.file, + probes, + greenProbes, + baseline.perFile, + ); + if (heldDetail) { const { patch: _patch, ...meta } = h; hunkResults.push({ ...meta, verdict: 'inconclusive' as const, - detail: notGreen('hunk', own), + detail: heldDetail, }); continue; } From ff6b87d788213da284b6c1fb14f6e45eb0b59a04 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 2 Aug 2026 20:42:43 +0800 Subject: [PATCH 3/7] fix(review): make an untagged inconclusive a compile error The tags this PR turns on were the only part of it nothing tested. `reason` was optional and the lookup absorbed a missing one, so deleting a tag left all 116 tests green while every hold of that kind silently degraded to a vague catch-all -- the same defect this PR exists to remove, one layer down. `ProbeResult` is now a discriminated union: `reason` is mandatory on the `inconclusive` arm and absent from the others, so an untagged branch does not compile. Deleting a tag is now caught twice, by tsc and by the new assertions on `classifyProbeRun` itself, which no fixture stood in for before. `no-tests` was also two different observations under one name. `!result` means the run answered and this file was not in the answer -- which a path that failed to match produces as readily as a compile error, and the boundary and case rules right above it are why that is not hypothetical -- so it is now `not-in-results` and says both possibilities instead of picking one. Also from the review: the `collocatedNotGreenDetail` doc block had been left above `heldForRedCollocatedTest`, describing neither, and its last paragraph stated the opposite of what the code returns for an unreported probe. It moves onto the function it documents and now matches it, including that the case is a default rather than one the pipeline can reach. `collocatedProbe` takes a readonly array, dropping a per-candidate copy made only to satisfy the type. --- .../src/commands/review/test-efficacy.test.ts | 84 ++++++++++++++++ .../cli/src/commands/review/test-efficacy.ts | 98 +++++++++++++------ 2 files changed, 154 insertions(+), 28 deletions(-) diff --git a/packages/cli/src/commands/review/test-efficacy.test.ts b/packages/cli/src/commands/review/test-efficacy.test.ts index b5cbfad1f8d..441e35e1ce4 100644 --- a/packages/cli/src/commands/review/test-efficacy.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.test.ts @@ -405,6 +405,90 @@ describe('safeRmWithin', () => { describe('classifyProbeRun', () => { const json = (o: unknown) => JSON.stringify(o); const only = (got: T[]): T => got[0]; + /** The tag, if the verdict is one that carries one. Narrowing is the point: + * `ProbeResult` only offers `reason` on the `inconclusive` arm. */ + const reasonOf = (r: ReturnType[number]) => + r.verdict === 'inconclusive' ? r.reason : undefined; + + // The tags are what `collocatedNotGreenDetail` reads, and an untagged branch + // silently degrades every hold of that kind to a vague catch-all. Measured + // before these existed: deleting one tag left all 116 tests green. + it('tags a run that produced no parseable output as no-output', () => { + const got = classifyProbeRun( + 1, + 'boom', + ['packages/lib/src/a.test.ts'], + 'x', + ); + expect(only(got).verdict).toBe('inconclusive'); + expect(reasonOf(only(got))).toBe('no-output'); + }); + + it('tags a file absent from the results as not-in-results', () => { + // Distinct from collecting zero tests: the run answered and this file was + // not in the answer, which a path miss produces as readily as a compile + // error. + const got = classifyProbeRun( + 1, + json({ + testResults: [ + { + name: '/w/packages/lib/src/other.test.ts', + assertionResults: [{ status: 'passed' }], + }, + ], + }), + ['packages/lib/src/a.test.ts'], + ); + expect(reasonOf(only(got))).toBe('not-in-results'); + expect(only(got).detail).toContain('none for this file'); + }); + + it('tags a file that collected nothing as no-tests', () => { + const got = classifyProbeRun( + 1, + json({ + testResults: [ + { name: '/w/packages/lib/src/a.test.ts', assertionResults: [] }, + ], + }), + ['packages/lib/src/a.test.ts'], + ); + expect(reasonOf(only(got))).toBe('no-tests'); + }); + + it('tags an all-skipped file as all-skipped', () => { + const got = classifyProbeRun( + 0, + json({ + testResults: [ + { + name: '/w/packages/lib/src/a.test.ts', + assertionResults: [{ status: 'skipped' }, { status: 'skipped' }], + }, + ], + }), + ['packages/lib/src/a.test.ts'], + ); + expect(reasonOf(only(got))).toBe('all-skipped'); + }); + + it('leaves a decided verdict untagged', () => { + const got = classifyProbeRun( + 0, + json({ + testResults: [ + { + name: '/w/packages/lib/src/a.test.ts', + assertionResults: [{ status: 'passed' }], + }, + ], + }), + ['packages/lib/src/a.test.ts'], + ); + expect(only(got).verdict).toBe('inert'); + expect('reason' in only(got)).toBe(false); + }); it('calls a test that still passes without the change INERT', () => { // The finding. The source is reverted and the test is green anyway, so it diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index c3213a3753a..6592e6f4d5e 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -87,13 +87,34 @@ export type ProbeVerdict = 'gated' | 'inert' | 'inconclusive'; * downstream that explains an `inconclusive` has to pick between them, and the * prose `detail` is written for a human, not for that decision. */ -export type ProbeReason = 'no-output' | 'no-tests' | 'all-skipped'; +export type ProbeReason = + | 'no-output' + | 'not-in-results' + | 'no-tests' + | 'all-skipped'; -export interface ProbeResult { +/** + * A union, not an optional field: every `inconclusive` MUST say which way, and + * the compiler is what enforces it. Left optional, a branch that forgot to tag + * itself fell through to a vague catch-all wording with nothing failing — + * measured, deleting one tag left all 116 tests green while every hold of that + * kind silently degraded. The one part of this file that has to be right is the + * part a runtime fallback was quietly covering for. + */ +export type ProbeResult = + | { file: string; verdict: 'gated' | 'inert'; detail: string } + | { + file: string; + verdict: 'inconclusive'; + detail: string; + reason: ProbeReason; + }; + +/** What the two explanation helpers read — narrower than `ProbeResult`, whose + * union does not survive a `Pick`. */ +export interface ProbeOutcome { file: string; verdict: ProbeVerdict; - detail: string; - /** Set only when `verdict` is `inconclusive`. */ reason?: ProbeReason; } @@ -829,7 +850,7 @@ export function parseAddedLines(diffText: string): Map { */ export function collocatedProbe( file: string, - testPaths: string[], + testPaths: readonly string[], ): string | undefined { const stem = file.replace(/\.[^./]+$/, ''); return testPaths.find((t) => { @@ -838,22 +859,6 @@ export function collocatedProbe( }); } -/** - * The whole sentence a mutant or hunk is held `inconclusive` with when its own - * collocated test was not green in the unmutated baseline — the ONLY wording - * either guard has, so what this returns is what the report says. - * - * It reads the reason off the baseline rather than asserting one. The two - * reasons are different failures with different fixes, and the guards used to - * name the wrong one flatly: measured on PR #8368, `AuthDialog.test.tsx` - * compiled, collected 26 tests and failed exactly one, and all three mutants in - * its source were held with "likely a compile or import error in the probe - * tree" — sending a reader after an import problem that was never there. - * - * A probe with no baseline entry at all takes the collected-nothing wording: - * the run said nothing about it, which is the same evidentiary hole and never - * the claim that its tests failed. - */ /** * Should this candidate be held `inconclusive` because the test collocated with * the file it touches was not green in the unmutated baseline — and if so, with @@ -871,9 +876,9 @@ export function heldForRedCollocatedTest( file: string, probes: readonly string[], greenProbes: readonly string[], - perFile: ReadonlyArray>, + perFile: readonly ProbeOutcome[], ): string | undefined { - const own = collocatedProbe(file, [...probes]); + const own = collocatedProbe(file, probes); if (!own || greenProbes.includes(own)) return undefined; return collocatedNotGreenDetail(kind, own, perFile); } @@ -885,12 +890,19 @@ const REASON_PHRASE: Record = { // error that is not there, while the runner itself is what fell over. 'no-output': 'the runner produced no parseable output there, so nothing at all is known about it', + // The run answered, and this file was not in the answer. A compile or import + // error looks like this — and so does a path that failed to match, which the + // boundary and case rules in `classifyProbeRun` exist because of. Naming one + // would be picking between two live possibilities. + 'not-in-results': + 'the run produced results there but none for it — which a compile or import error looks like, and so does a path that did not match', 'no-tests': 'it collected no tests there — the shape a compile or import error in the probe tree takes', 'all-skipped': 'it collected tests there but executed none of them', - // An entry that is inconclusive without saying which way. Older artifacts and - // any future branch that forgets to tag itself land here, and the honest - // answer is the disjunction rather than a guess at one of its arms. + // UNREACHABLE through the pipeline: `ProbeResult` makes `reason` mandatory on + // every `inconclusive`, so this arm answers only a hand-built caller. Kept + // because the alternative is a sentence reading "undefined", and named + // honestly rather than counted as covered. unspecified: 'it did not come back green there (a compile or import error, every test skipped, or no parseable output)', }; @@ -899,10 +911,28 @@ const REASON_PHRASE: Record = { * state, and is not evidence that its tests failed. */ const NOT_REPORTED = 'the baseline did not report it'; +/** + * The whole sentence a mutant or hunk is held `inconclusive` with when its own + * collocated test was not green in the unmutated baseline — the ONLY wording + * either guard has, so what this returns is what the report says. + * + * It reads the reason off the baseline rather than asserting one. The ways a + * probe fails to be green are different failures with different fixes, and the + * guards used to name one of them flatly: measured on PR #8368, + * `AuthDialog.test.tsx` compiled, collected 26 tests and failed exactly one, + * and all three mutants in its source were held with "likely a compile or + * import error in the probe tree" — sending a reader after an import problem + * that was never there. + * + * A probe with no baseline entry at all is reported as not measured, which is a + * different claim from measured-and-empty and the one the run actually + * supports. It cannot arise in this pipeline — `classifyProbeRun` maps over + * exactly the probe list `own` is drawn from — so it is a default, not a case. + */ export function collocatedNotGreenDetail( kind: 'mutant' | 'hunk', probe: string, - perFile: ReadonlyArray>, + perFile: readonly ProbeOutcome[], ): string { const entry = perFile.find((p) => p.file === probe); const reason = @@ -1056,7 +1086,19 @@ export function classifyProbeRun( const failed = assertions.filter((a) => a.status === 'failed').length; const passed = assertions.filter((a) => a.status === 'passed').length; - if (!result || assertions.length === 0) { + if (!result) { + // The run produced results and this file is not among them. A compile or + // import error looks like this — and so does a path that did not match + // (the boundary and case rules above are why that is not hypothetical), + // and those are different problems. Say only what was observed. + return { + file, + verdict: 'inconclusive' as const, + reason: 'not-in-results' as const, + detail: `the run produced results but none for this file (run exit ${exitCode}) — a compile or import error looks like this, and so does a path that did not match; not evidence either way`, + }; + } + if (assertions.length === 0) { return { file, verdict: 'inconclusive' as const, From 658c2788be32694c59d255cad691e4cec7c12f25 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 2 Aug 2026 22:37:03 +0800 Subject: [PATCH 4/7] fix(review): let the union reach the array the report serialises The invariant was argued for and then not checked where it counts. `probed` serialises `results`, and `results` was declared as a structural echo of `ProbeResult` rather than `ProbeResult` itself, so two catch paths pushed an untagged `inconclusive` and compiled -- and a third assigned `verdict` in place, which the union cannot see at all. The artifact therefore carried `reason` on classifier-produced entries and not on the others: two conventions for one verdict, described in the PR as one. `results` is now `ProbeResult[]`. That turned all three into compile errors and they needed reasons of their own, neither of which is `no-output`: `not-run` for a probe no suite was attempted for, and `control-failed` for one that read green in a run the positive control had already proved could not go red. The in-place re-class is a replacement now, since assigning `verdict` alone leaves the entry untagged. `ProbeOutcome` was re-opening the optionality one line after the union closed it -- a plain `Omit` collapses the arms -- which let an `inert` entry reach the explanation helper and come back described as "did not come back green", the opposite of what `inert` means. A distributive omit keeps the discrimination, so the helper cannot read `reason` without narrowing first, the `inert` arm has to be written out, and the `unspecified` phrase stops existing rather than being documented as unreachable. Also: `ProbeReason`'s doc described three members of what is now a five-member union and gave `no-tests` the definition of `not-in-results` -- the branch this PR split away from it. And `not-in-results` was the one reason whose report sentence nothing asserted. --- .../src/commands/review/test-efficacy.test.ts | 27 +++-- .../cli/src/commands/review/test-efficacy.ts | 108 ++++++++++++------ 2 files changed, 93 insertions(+), 42 deletions(-) diff --git a/packages/cli/src/commands/review/test-efficacy.test.ts b/packages/cli/src/commands/review/test-efficacy.test.ts index 441e35e1ce4..bd98b867e0a 100644 --- a/packages/cli/src/commands/review/test-efficacy.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.test.ts @@ -1405,6 +1405,9 @@ describe('collocatedNotGreenDetail', () => { { file: 'packages/cli/src/empty.test.ts', verdict: 'inconclusive' as const, + // The union requires it, which is the point: a fixture cannot stand for + // an untagged `inconclusive` because the code cannot produce one. + reason: 'no-tests' as const, }, ]; @@ -1421,16 +1424,17 @@ describe('collocatedNotGreenDetail', () => { expect(detail).not.toContain('compile or import error'); }); - it('gives the disjunction for an inconclusive entry that names no reason', () => { - // An older artifact, or a future branch that forgets to tag itself: say - // what is not known instead of picking one of its arms. + it('refuses to explain a probe the baseline reported GREEN', () => { + // `inert` is what greenProbes is built from, so this sentence does not + // apply to it. The production caller cannot get here; the export can, and + // a fluent claim contradicting the measurement is worse than saying so. const detail = collocatedNotGreenDetail( - 'hunk', - 'packages/cli/src/empty.test.ts', - perFile, + 'mutant', + 'packages/cli/src/green.test.ts', + [{ file: 'packages/cli/src/green.test.ts', verdict: 'inert' as const }], ); - expect(detail).toContain('did not come back green there'); - expect(detail).toContain('no parseable output'); + expect(detail).toContain('reported it GREEN'); + expect(detail).toContain('does not apply'); }); it('says the baseline never reported a probe it has no entry for', () => { @@ -1450,6 +1454,13 @@ describe('collocatedNotGreenDetail', () => { ['no-output', 'produced no parseable output', 'nothing at all is known'], ['no-tests', 'collected no tests there', 'compile or import error'], ['all-skipped', 'executed none of them', 'collected tests there'], + [ + 'not-in-results', + 'produced results there but none for it', + 'a path that did not match', + ], + ['not-run', 'no probe suite ran for it', 'at all there'], + ['control-failed', 'it read green there', 'positive control failed'], ])( 'names %s as the reason rather than guessing one', (reason, phrase, alsoPhrase) => { diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index 6592e6f4d5e..963a93eb0d8 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -77,17 +77,29 @@ import { isWorkspaceMember } from './lib/workspaces.js'; export type ProbeVerdict = 'gated' | 'inert' | 'inconclusive'; /** - * WHY a probe was `inconclusive`, as the classifier knew it at the time. + * WHY a probe was `inconclusive`, as the run knew it at the time. * - * Three different things arrive at the same verdict and they are not - * interchangeable: `no-output` is the runner failing to produce parseable JSON, - * where nothing at all is known about collection; `no-tests` is a run that - * produced results without this file, the shape a compile or import error takes; - * `all-skipped` is a file that collected tests and executed none. Anything - * downstream that explains an `inconclusive` has to pick between them, and the - * prose `detail` is written for a human, not for that decision. + * Five different things arrive at the same verdict and they are not + * interchangeable: + * + * - `control-failed` — the suite ran and read green, but the positive control + * proved the runner could not have gone red, so the reading is not evidence. + * - `not-run` — no probe suite was attempted for it (the worktree could not be + * made, or the spawn itself threw). Nothing was measured at all. + * - `no-output` — the runner ran and produced nothing parseable, so whether it + * collected anything is unknown. + * - `not-in-results` — the run produced results and this file was not among + * them. A compile or import error looks like this, and so does a path that + * failed to match. + * - `no-tests` — the file WAS in the results and collected zero assertions. + * - `all-skipped` — it collected tests and executed none of them. + * + * Anything downstream that explains an `inconclusive` has to pick between + * these, and the prose `detail` is written for a human, not for that decision. */ export type ProbeReason = + | 'control-failed' + | 'not-run' | 'no-output' | 'not-in-results' | 'no-tests' @@ -110,13 +122,20 @@ export type ProbeResult = reason: ProbeReason; }; -/** What the two explanation helpers read — narrower than `ProbeResult`, whose - * union does not survive a `Pick`. */ -export interface ProbeOutcome { - file: string; - verdict: ProbeVerdict; - reason?: ProbeReason; -} +/** + * `Omit` applied across each arm instead of to the union as a whole. A plain + * `Omit`/`Pick` collapses `ProbeResult` into one object type and makes `reason` + * optional again — which is how an `inert` entry could reach the explanation + * helper and come back described as "did not come back green", the opposite of + * what `inert` means. Distributing keeps the discrimination, so the helper + * cannot read `reason` without first narrowing on `verdict`. + */ +type DistributiveOmit = T extends unknown + ? Omit + : never; + +/** What the two explanation helpers read: `ProbeResult` without its prose. */ +export type ProbeOutcome = DistributiveOmit; export interface FileEntry { path: string; @@ -883,7 +902,13 @@ export function heldForRedCollocatedTest( return collocatedNotGreenDetail(kind, own, perFile); } -const REASON_PHRASE: Record = { +const REASON_PHRASE: Record = { + // The suite ran and read green, but the control proved the runner could not + // have reported otherwise. + 'control-failed': + 'it read green there, but the positive control failed, so nothing could have turned that run red', + // Nothing was attempted: no worktree, or the spawn threw before a suite ran. + 'not-run': 'no probe suite ran for it at all there', // Nothing is known about collection here — the runner never produced output // to read. Saying "collected no tests" would be the same invented cause this // function exists to stop, one layer down: a reader sent after a compile @@ -899,12 +924,6 @@ const REASON_PHRASE: Record = { 'no-tests': 'it collected no tests there — the shape a compile or import error in the probe tree takes', 'all-skipped': 'it collected tests there but executed none of them', - // UNREACHABLE through the pipeline: `ProbeResult` makes `reason` mandatory on - // every `inconclusive`, so this arm answers only a hand-built caller. Kept - // because the alternative is a sentence reading "undefined", and named - // honestly rather than counted as covered. - unspecified: - 'it did not come back green there (a compile or import error, every test skipped, or no parseable output)', }; /** No entry at all — the baseline never reported this probe. Absent is its own @@ -938,9 +957,14 @@ export function collocatedNotGreenDetail( const reason = entry === undefined ? NOT_REPORTED - : entry.verdict === 'gated' - ? 'it was RED there' - : REASON_PHRASE[entry.reason ?? 'unspecified']; + : entry.verdict === 'inconclusive' + ? REASON_PHRASE[entry.reason] + : entry.verdict === 'gated' + ? 'it was RED there' + : // `inert` IS green, so the sentence this function builds does not + // apply to it. Say that, rather than produce a fluent claim that + // contradicts the measurement — the caller is what is wrong here. + 'the baseline reported it GREEN, so this explanation does not apply to it'; const what = kind === 'mutant' ? 'the statement' : 'the hunk'; return `this ${kind}'s collocated test ${probe} did not run green in the unmutated baseline — ${reason}, so the remaining probes passing cannot show ${what} is uncovered`; } @@ -1878,11 +1902,12 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { } } - const results: Array<{ - file: string; - verdict: ProbeVerdict; - detail: string; - }> = []; + // `ProbeResult`, not a structural echo of it: this array is what `probed` + // serialises, so an entry pushed here without a reason is an untagged + // `inconclusive` in the artifact — the exact hole the union was introduced to + // close, reopened one level up. Typed this way, the two catch paths below do + // not compile until they say which way they failed. + const results: ProbeResult[] = []; let cleanupFailure: string | undefined; const mutantResults: MutantResult[] = []; let mutantsSkippedForBudget = 0; @@ -2037,7 +2062,12 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { String(sweep?.stderr ?? ''), ); for (const file of probes) { - results.push({ file, verdict: 'inconclusive' as const, detail }); + results.push({ + file, + verdict: 'inconclusive' as const, + reason: 'not-run' as const, + detail, + }); } for (const c of candidates) { mutantResults.push({ ...c, verdict: 'inconclusive' as const, detail }); @@ -2290,6 +2320,7 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { ...probes.map((file) => ({ file, verdict: 'inconclusive' as const, + reason: 'not-run' as const, detail, })), ); @@ -2342,10 +2373,19 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { } // The file-level revert probe's "inert" is the same survivor claim one // level up — a dead runner reports every reverted file green too. - for (const r of results) { + for (let i = 0; i < results.length; i++) { + const r = results[i]; if (r.verdict === 'inert') { - r.verdict = 'inconclusive'; - r.detail = detail; + // Replaced rather than mutated in place: the union makes `reason` + // mandatory on the arm this becomes, and assigning `verdict` alone + // would leave the entry untagged — which is the whole failure the + // union exists to make impossible. + results[i] = { + file: r.file, + verdict: 'inconclusive', + reason: 'control-failed', + detail, + }; } } } From 164017ce7ac035240ee251e9ffe82b8e4559a6e4 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 2 Aug 2026 23:02:27 +0800 Subject: [PATCH 5/7] fix(review): a probe that was killed is not a probe that never ran `not-run` was applied to a catch covering three different failures. The checkout and the tree removal fail before anything runs; the runner throws when it was KILLED, and the per-run timeout fires SIGTERM at a suite that may have executed most of its tests. Tagging that "no probe suite ran" is a cause nothing measured -- the move this file exists to stop, one layer up, in the machine-readable field the description argues a consumer can act on. Split into `runner-died`, derived from the runner's own message prefixes by an exported `probeFailureReason` so both directions are pinned rather than assumed. Three reasons are set only on the run-level results array and can never reach `collocatedNotGreenDetail`, which is passed the baseline. Rendered there, `control-failed` produced "did not run green in the unmutated baseline -- it read green there", a sentence contradicting its own frame. They now report that the baseline did not classify the probe, the same answer the `inert` arm gives, and the parameter is named `baselinePerFile` for the contract it has. The `it.each` cast named three of the six reasons the table feeds, so it compiled regardless and a typo would have surfaced as a confusing `toContain` diff instead of a type error. It is `ProbeReason` now. And the integration hold assertions matched what the old flat wording also satisfied. They now pin the clause that regressed, so the chain the bug shipped on -- baseline classification, reason tag, sentence -- is covered end to end. --- .../review/test-efficacy.integration.test.ts | 6 ++ .../src/commands/review/test-efficacy.test.ts | 48 +++++++++++++++- .../cli/src/commands/review/test-efficacy.ts | 56 ++++++++++++++++--- 3 files changed, 99 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/commands/review/test-efficacy.integration.test.ts b/packages/cli/src/commands/review/test-efficacy.integration.test.ts index a944b4607f1..1be22d8761e 100644 --- a/packages/cli/src/commands/review/test-efficacy.integration.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.integration.test.ts @@ -1049,6 +1049,12 @@ process.stdout.write(JSON.stringify({ expect(m.verdict).toBe('inconclusive'); expect(m.detail).toContain('f.test.ts'); expect(m.detail).toContain('did not run green'); + // The clause that actually regressed. The old flat wording satisfied + // both assertions above, so only this one pins the chain the bug + // shipped on: baseline classification -> reason tag -> sentence. + // `f.test.ts` fails an assertion here, so `gated` is the measured state. + expect(m.detail).toContain('was RED there'); + expect(m.detail).not.toContain('compile or import error'); } // ...and nothing a reader acts on carries a survivor claim for that file. expect( diff --git a/packages/cli/src/commands/review/test-efficacy.test.ts b/packages/cli/src/commands/review/test-efficacy.test.ts index bd98b867e0a..227db4585d8 100644 --- a/packages/cli/src/commands/review/test-efficacy.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.test.ts @@ -20,6 +20,8 @@ import { hasCollocatedNewTest, collocatedProbe, collocatedNotGreenDetail, + probeFailureReason, + type ProbeReason, heldForRedCollocatedTest, fitsAnotherMutantRun, probeCleanupFailureDetail, @@ -1399,6 +1401,26 @@ describe('collocatedProbe', () => { }); }); +describe('probeFailureReason', () => { + it.each([ + // Verbatim from runProbeSuite's own throw sites. + ['runner killed by SIGTERM (probe timed out after 300s)', 'runner-died'], + ['runner killed by SIGKILL', 'runner-died'], + ['runner spawn failed: ENOENT', 'runner-died'], + ])('calls %s a suite that did not survive', (message, expected) => { + expect(probeFailureReason(message)).toBe(expected); + }); + + it.each([ + ['git checkout base failed: fatal: invalid reference', 'not-run'], + ['EACCES: permission denied, rmdir', 'not-run'], + ])('calls %s a suite that never started', (message, expected) => { + // A suite killed at the deadline ran; a checkout that failed did not, and + // the distinction is the whole reason the tag exists. + expect(probeFailureReason(message)).toBe(expected); + }); +}); + describe('collocatedNotGreenDetail', () => { const perFile = [ { file: 'packages/cli/src/red.test.ts', verdict: 'gated' as const }, @@ -1424,6 +1446,28 @@ describe('collocatedNotGreenDetail', () => { expect(detail).not.toContain('compile or import error'); }); + it.each(['not-run', 'runner-died', 'control-failed'] as const)( + 'refuses to explain %s, which a baseline entry cannot carry', + (reason) => { + // These are set on the run-level results array, never by + // classifyProbeRun. Rendered inside this sentence, `control-failed` + // would read "did not run green … it read green there". + const detail = collocatedNotGreenDetail( + 'mutant', + 'packages/cli/src/x.test.ts', + [ + { + file: 'packages/cli/src/x.test.ts', + verdict: 'inconclusive' as const, + reason, + }, + ], + ); + expect(detail).toContain('the baseline did not classify it'); + expect(detail).toContain('does not apply'); + }, + ); + it('refuses to explain a probe the baseline reported GREEN', () => { // `inert` is what greenProbes is built from, so this sentence does not // apply to it. The production caller cannot get here; the export can, and @@ -1459,8 +1503,6 @@ describe('collocatedNotGreenDetail', () => { 'produced results there but none for it', 'a path that did not match', ], - ['not-run', 'no probe suite ran for it', 'at all there'], - ['control-failed', 'it read green there', 'positive control failed'], ])( 'names %s as the reason rather than guessing one', (reason, phrase, alsoPhrase) => { @@ -1471,7 +1513,7 @@ describe('collocatedNotGreenDetail', () => { { file: 'packages/cli/src/x.test.ts', verdict: 'inconclusive' as const, - reason: reason as 'no-output' | 'no-tests' | 'all-skipped', + reason: reason as ProbeReason, }, ], ); diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index 963a93eb0d8..4e85e2b87a7 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -84,8 +84,11 @@ export type ProbeVerdict = 'gated' | 'inert' | 'inconclusive'; * * - `control-failed` — the suite ran and read green, but the positive control * proved the runner could not have gone red, so the reading is not evidence. - * - `not-run` — no probe suite was attempted for it (the worktree could not be - * made, or the spawn itself threw). Nothing was measured at all. + * - `not-run` — no probe suite was attempted for it: the worktree could not be + * made, or the checkout failed. Nothing was measured at all. + * - `runner-died` — a suite WAS started and did not survive: killed at the + * timeout, or the spawn itself failed. It may have executed most of its + * tests, so "nothing ran" is a claim about it that nothing checked. * - `no-output` — the runner ran and produced nothing parseable, so whether it * collected anything is unknown. * - `not-in-results` — the run produced results and this file was not among @@ -100,6 +103,7 @@ export type ProbeVerdict = 'gated' | 'inert' | 'inconclusive'; export type ProbeReason = | 'control-failed' | 'not-run' + | 'runner-died' | 'no-output' | 'not-in-results' | 'no-tests' @@ -902,13 +906,41 @@ export function heldForRedCollocatedTest( return collocatedNotGreenDetail(kind, own, perFile); } +/** + * Which `inconclusive` a throw out of the probe setup-and-run is. + * + * That catch covers three failures and they are not the same fact. The + * checkout and the tree removal fail before anything runs. The runner throws + * when it was KILLED — the per-run timeout fires SIGTERM, and a suite killed at + * the deadline may have executed most of its tests, so "no probe suite ran" is + * a claim about it that nothing checked. Matched on the runner's own message + * prefixes, which are the only thing distinguishing the three at this point. + */ +export function probeFailureReason(message: string): ProbeReason { + return /^runner (killed|spawn failed)/.test(message) + ? 'runner-died' + : 'not-run'; +} + +/** The reasons `classifyProbeRun` can produce — the only ones a baseline entry + * can carry, and so the only ones this explanation is written for. */ +const BASELINE_REASON = new Set([ + 'no-output', + 'not-in-results', + 'no-tests', + 'all-skipped', +]); + const REASON_PHRASE: Record = { // The suite ran and read green, but the control proved the runner could not // have reported otherwise. 'control-failed': 'it read green there, but the positive control failed, so nothing could have turned that run red', - // Nothing was attempted: no worktree, or the spawn threw before a suite ran. + // Nothing was attempted: no worktree, or the checkout failed. 'not-run': 'no probe suite ran for it at all there', + // Started and did not survive. How much of it ran before that is unknown. + 'runner-died': + 'the probe suite was started there and did not survive (a timeout kill, or a spawn that failed)', // Nothing is known about collection here — the runner never produced output // to read. Saying "collected no tests" would be the same invented cause this // function exists to stop, one layer down: a reader sent after a compile @@ -951,14 +983,20 @@ const NOT_REPORTED = 'the baseline did not report it'; export function collocatedNotGreenDetail( kind: 'mutant' | 'hunk', probe: string, - perFile: readonly ProbeOutcome[], + baselinePerFile: readonly ProbeOutcome[], ): string { - const entry = perFile.find((p) => p.file === probe); + const entry = baselinePerFile.find((p) => p.file === probe); const reason = entry === undefined ? NOT_REPORTED : entry.verdict === 'inconclusive' - ? REASON_PHRASE[entry.reason] + ? // Three reasons are set on the run-level `results` array and never by + // `classifyProbeRun`, so a baseline entry cannot carry them. Reaching + // one means the caller passed something other than the baseline — + // say that, rather than emit "did not run green … it read green". + BASELINE_REASON.has(entry.reason) + ? REASON_PHRASE[entry.reason] + : `the baseline did not classify it (${REASON_PHRASE[entry.reason]}), so this explanation does not apply to it` : entry.verdict === 'gated' ? 'it was RED there' : // `inert` IS green, so the sentence this function builds does not @@ -2315,12 +2353,14 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { // The probe could not be set up or run. That is not evidence about any // test — record it and keep going, so the report (and the unreachable // findings, which needed no probe at all) still reaches the caller. - const detail = `probe could not run: ${e instanceof Error ? e.message : String(e)}`; + const message = e instanceof Error ? e.message : String(e); + const detail = `probe could not run: ${message}`; + const reason = probeFailureReason(message); results.push( ...probes.map((file) => ({ file, verdict: 'inconclusive' as const, - reason: 'not-run' as const, + reason, detail, })), ); From 0f4329a6984cc5c29dca4733107e5817ec53e4a1 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 2 Aug 2026 23:17:34 +0800 Subject: [PATCH 6/7] fix(review): read the runner's failure from the result, not from its prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `probeFailureReason` matched `/^runner (killed|spawn failed)/` against the thrown message, and neither real failure produces that text. Measured: `spawnSync` reports a timeout as `error` (`spawnSync … ETIMEDOUT`, with `signal` also set) and a missing binary as `spawnSync … ENOENT`, and `runProbeSuite` throws `r.error` before it ever composes a "runner killed by" sentence. So `runner-died` was never produced -- the tag added last round to stop a cause being invented was itself inoperative -- and the test that covered it asserted `runner spawn failed: ENOENT`, a string that exists nowhere but in that test. The reason is now read off the spawn result's structure by an exported `runnerFailureReason`, and carried on a `ProbeRunFailure` rather than left to be parsed back out. Its cases are driven through real `spawnSync` calls -- a process killed at a 300ms deadline and a binary that does not exist -- so the fixture is measured rather than written. `not-run` widens to "no suite ran: the tree could not be prepared, or the runner could not be started", which is what a spawn that fails actually is. Also from this round: the `ProbeReason` doc said "Five different things" over seven bullets, and `heldForRedCollocatedTest` still called its parameter `perFile` after the helper it forwards to was renamed `baselinePerFile`. --- .../src/commands/review/test-efficacy.test.ts | 42 ++++++----- .../cli/src/commands/review/test-efficacy.ts | 69 +++++++++++++------ 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/packages/cli/src/commands/review/test-efficacy.test.ts b/packages/cli/src/commands/review/test-efficacy.test.ts index 227db4585d8..1d9a645b194 100644 --- a/packages/cli/src/commands/review/test-efficacy.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.test.ts @@ -5,6 +5,7 @@ */ import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'node:child_process'; import { replacementMutantsOf, splitDiffIntoHunks, @@ -20,7 +21,7 @@ import { hasCollocatedNewTest, collocatedProbe, collocatedNotGreenDetail, - probeFailureReason, + runnerFailureReason, type ProbeReason, heldForRedCollocatedTest, fitsAnotherMutantRun, @@ -1401,23 +1402,32 @@ describe('collocatedProbe', () => { }); }); -describe('probeFailureReason', () => { - it.each([ - // Verbatim from runProbeSuite's own throw sites. - ['runner killed by SIGTERM (probe timed out after 300s)', 'runner-died'], - ['runner killed by SIGKILL', 'runner-died'], - ['runner spawn failed: ENOENT', 'runner-died'], - ])('calls %s a suite that did not survive', (message, expected) => { - expect(probeFailureReason(message)).toBe(expected); +describe('runnerFailureReason', () => { + // Driven through the real spawnSync rather than hand-written strings: the + // version this replaced matched a message the code never emits, and a + // fabricated fixture is exactly what let that pass. + it('calls a suite killed at the deadline one that did not survive', () => { + const r = spawnSync( + process.execPath, + ['-e', 'setTimeout(() => {}, 5000)'], + { timeout: 300, encoding: 'utf8' }, + ); + // What node actually reports, and why the old message match failed: + // `error` is set on a timeout, so the throw never reaches a + // "runner killed by" sentence. + expect(r.error?.message).toContain('ETIMEDOUT'); + expect(runnerFailureReason(r)).toBe('runner-died'); }); - it.each([ - ['git checkout base failed: fatal: invalid reference', 'not-run'], - ['EACCES: permission denied, rmdir', 'not-run'], - ])('calls %s a suite that never started', (message, expected) => { - // A suite killed at the deadline ran; a checkout that failed did not, and - // the distinction is the whole reason the tag exists. - expect(probeFailureReason(message)).toBe(expected); + it('calls a runner that could not start one that never ran', () => { + const r = spawnSync('/nonexistent/vitest-bin', [], { encoding: 'utf8' }); + expect(r.error?.message).toContain('ENOENT'); + expect(r.signal).toBeNull(); + expect(runnerFailureReason(r)).toBe('not-run'); + }); + + it('calls a plain failure before any spawn one that never ran', () => { + expect(runnerFailureReason({})).toBe('not-run'); }); }); diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index 4e85e2b87a7..ba89d44e4e0 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -79,16 +79,16 @@ export type ProbeVerdict = 'gated' | 'inert' | 'inconclusive'; /** * WHY a probe was `inconclusive`, as the run knew it at the time. * - * Five different things arrive at the same verdict and they are not + * Seven different things arrive at the same verdict and they are not * interchangeable: * * - `control-failed` — the suite ran and read green, but the positive control * proved the runner could not have gone red, so the reading is not evidence. - * - `not-run` — no probe suite was attempted for it: the worktree could not be - * made, or the checkout failed. Nothing was measured at all. + * - `not-run` — no suite ran for it: the tree could not be prepared, or the + * runner could not be started at all. Nothing was measured. * - `runner-died` — a suite WAS started and did not survive: killed at the - * timeout, or the spawn itself failed. It may have executed most of its - * tests, so "nothing ran" is a claim about it that nothing checked. + * deadline, or by a signal. It may have executed most of its tests, so + * "nothing ran" is a claim about it that nothing checked. * - `no-output` — the runner ran and produced nothing parseable, so whether it * collected anything is unknown. * - `not-in-results` — the run produced results and this file was not among @@ -899,27 +899,45 @@ export function heldForRedCollocatedTest( file: string, probes: readonly string[], greenProbes: readonly string[], - perFile: readonly ProbeOutcome[], + baselinePerFile: readonly ProbeOutcome[], ): string | undefined { const own = collocatedProbe(file, probes); if (!own || greenProbes.includes(own)) return undefined; - return collocatedNotGreenDetail(kind, own, perFile); + return collocatedNotGreenDetail(kind, own, baselinePerFile); } /** - * Which `inconclusive` a throw out of the probe setup-and-run is. + * Did this spawn result start a suite and lose it, or never start one? + * + * Read off the result's STRUCTURE, not its message. The first version of this + * matched `/^runner (killed|spawn failed)/` against the thrown text and was + * inoperative: `spawnSync` reports a timeout as `error` (`ETIMEDOUT`, with + * `signal` also set) and `runProbeSuite` throws `r.error` before it ever + * composes a "runner killed by" sentence, so the real messages are + * `spawnSync … ETIMEDOUT` and `spawnSync … ENOENT` and neither matched. The + * tag it exists to produce was therefore never produced, and the test that + * covered it asserted strings the code does not emit. * - * That catch covers three failures and they are not the same fact. The - * checkout and the tree removal fail before anything runs. The runner throws - * when it was KILLED — the per-run timeout fires SIGTERM, and a suite killed at - * the deadline may have executed most of its tests, so "no probe suite ran" is - * a claim about it that nothing checked. Matched on the runner's own message - * prefixes, which are the only thing distinguishing the three at this point. + * A deadline kill may have executed most of the suite; a runner that could not + * be started ran nothing. That is the distinction a reader acts on. */ -export function probeFailureReason(message: string): ProbeReason { - return /^runner (killed|spawn failed)/.test(message) - ? 'runner-died' - : 'not-run'; +export function runnerFailureReason(r: { + error?: (Error & { code?: string }) | undefined; + signal?: NodeJS.Signals | null; +}): ProbeReason { + return r.signal || r.error?.code === 'ETIMEDOUT' ? 'runner-died' : 'not-run'; +} + +/** A probe run that threw, carrying WHY rather than leaving it to be parsed + * back out of the message. */ +export class ProbeRunFailure extends Error { + constructor( + message: string, + readonly reason: ProbeReason, + ) { + super(message); + this.name = 'ProbeRunFailure'; + } } /** The reasons `classifyProbeRun` can produce — the only ones a baseline entry @@ -1467,10 +1485,18 @@ function runProbeSuite( // fires SIGTERM). Ignoring it reports those as "the runner produced no // parseable JSON", which // blames the runner's output for a run that produced none. - if (r.error) throw r.error; + // `r.error` first, as before: when both are set — ENOBUFS on a run that was + // also killed — the error names the actual failure and the signal only says + // it did not finish. Reversing them buried `ENOBUFS` under "killed by + // SIGTERM", which is a less useful sentence about the same event. The reason + // tag is derived from the whole result either way, so it does not depend on + // which message wins. + if (r.error) + throw new ProbeRunFailure(r.error.message, runnerFailureReason(r)); if (r.signal) { - throw new Error( + throw new ProbeRunFailure( `runner killed by ${r.signal}${r.signal === 'SIGTERM' ? ` (probe timed out after ${Math.round(timeout / 1000)}s)` : ''}`, + runnerFailureReason(r), ); } return { @@ -2355,7 +2381,8 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { // findings, which needed no probe at all) still reaches the caller. const message = e instanceof Error ? e.message : String(e); const detail = `probe could not run: ${message}`; - const reason = probeFailureReason(message); + const reason = + e instanceof ProbeRunFailure ? e.reason : ('not-run' as const); results.push( ...probes.map((file) => ({ file, From 15d1fa78c2629a50e27260410d377d37fef6e2b2 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 2 Aug 2026 23:21:33 +0800 Subject: [PATCH 7/7] docs(review): make the reason phrasings match where each one now routes `runner-died`'s phrase still offered "or a spawn that failed" after a failed spawn was reclassified to `not-run` -- naming a case that no longer arrives there, which is the mismatch this change exists to remove. `not-run`'s comment gained the same case it acquired in the type. And the `results` declaration said "the two catch paths below" when the control-failed re-class is a third site the union now constrains. --- packages/cli/src/commands/review/test-efficacy.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index ba89d44e4e0..4fbe02ce530 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -954,11 +954,12 @@ const REASON_PHRASE: Record = { // have reported otherwise. 'control-failed': 'it read green there, but the positive control failed, so nothing could have turned that run red', - // Nothing was attempted: no worktree, or the checkout failed. + // Nothing ran: no worktree, the checkout failed, or the runner could not be + // started at all — a spawn that fails with ENOENT never gets to a test. 'not-run': 'no probe suite ran for it at all there', // Started and did not survive. How much of it ran before that is unknown. 'runner-died': - 'the probe suite was started there and did not survive (a timeout kill, or a spawn that failed)', + 'the probe suite was started there and did not survive (killed at the deadline, or by a signal)', // Nothing is known about collection here — the runner never produced output // to read. Saying "collected no tests" would be the same invented cause this // function exists to stop, one layer down: a reader sent after a compile @@ -1969,8 +1970,8 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { // `ProbeResult`, not a structural echo of it: this array is what `probed` // serialises, so an entry pushed here without a reason is an untagged // `inconclusive` in the artifact — the exact hole the union was introduced to - // close, reopened one level up. Typed this way, the two catch paths below do - // not compile until they say which way they failed. + // close, reopened one level up. Typed this way, neither catch below nor the + // control-failed re-class compiles until it says which way it failed. const results: ProbeResult[] = []; let cleanupFailure: string | undefined; const mutantResults: MutantResult[] = [];