diff --git a/packages/cli/src/commands/review/test-efficacy.integration.test.ts b/packages/cli/src/commands/review/test-efficacy.integration.test.ts index 52af07c54d3..a944b4607f1 100644 --- a/packages/cli/src/commands/review/test-efficacy.integration.test.ts +++ b/packages/cli/src/commands/review/test-efficacy.integration.test.ts @@ -961,6 +961,112 @@ process.stdout.write(JSON.stringify({ expect(out.findings).toEqual([]); }); + it('holds a mutant at inconclusive when its OWN test was red in the baseline', async () => { + // Measured live on PR #8213: six hunks in `bridge.ts` were correctly held + // at `inconclusive` because `bridge.test.ts` never ran green, while eight + // mutants in the SAME file were scored `survived` and shipped as findings. + // A mutant runs against `greenProbes` only, so the red collocated test is + // excluded from the run, and "every affected test still passed" is then + // computed over a set that omits the one test most likely to catch the + // deletion. Two files here: `f.ts` whose own test is red, and `g.ts` + // whose own test is green — the second is what shows the guard is + // targeted rather than a blanket refusal. + write('package.json', '{"private":true,"workspaces":["packages/*"]}\n'); + write('packages/lib/src/f.ts', 'export const a = new Map();\n'); + write('packages/lib/src/g.ts', 'export const b = new Map();\n'); + const base = commitAll('base'); + write( + 'packages/lib/src/f.ts', + 'export const a = new Map();\nexport function fReset() {\n a.clear();\n}\n', + ); + write( + 'packages/lib/src/g.ts', + 'export const b = new Map();\nexport function gReset() {\n b.clear();\n}\n', + ); + write( + 'packages/lib/src/f.test.ts', + 'import { fReset } from "./f.js"; import { it, expect } from "vitest"; it("t", () => expect(typeof fReset).toBe("function"));\n', + ); + write( + 'packages/lib/src/g.test.ts', + 'import { gReset } from "./g.js"; import { it, expect } from "vitest"; it("t", () => expect(typeof gReset).toBe("function"));\n', + ); + commitAll('pr'); + const wt = join(repo, 'wt'); + git(repo, 'worktree', 'add', '-q', '--detach', wt, 'HEAD'); + writeFileSync( + join(repo, 'report.json'), + JSON.stringify({ + files: [ + { path: 'packages/lib/src/f.ts', kind: 'source' }, + { path: 'packages/lib/src/g.ts', kind: 'source' }, + { path: 'packages/lib/src/f.test.ts', kind: 'test' }, + { path: 'packages/lib/src/g.test.ts', kind: 'test' }, + ], + }), + ); + // `f.test.ts` is red from the start — the baseline shape this is about. + // Everything else is green, and the injected control still turns the run + // red, so the harness is validated and survivors would be licensed. + writeFileSync( + vitestScript(), + `#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +const files = process.argv.slice(2).filter((a) => a.includes('.test.')); +const st = (f) => { + try { + if (fs.readFileSync(f, 'utf8').includes('QWEN-REVIEW-POSITIVE-CONTROL')) return 'failed'; + } catch {} + return path.basename(f) === 'f.test.ts' ? 'failed' : 'passed'; +}; +const results = files.map((f) => ({ + name: path.resolve(f), + assertionResults: [{ status: st(f) }], +})); +const nf = results.filter((r) => r.assertionResults[0].status === 'failed').length; +process.stdout.write(JSON.stringify({ + numPassedTests: results.length - nf, + numFailedTests: nf, + testResults: results, +})); +`, + ); + await runHandler({ + report: join(repo, 'report.json'), + worktree: wt, + base, + out: join(repo, 'out.json'), + }); + + const out = JSON.parse(readFileSync(join(repo, 'out.json'), 'utf8')); + expect(out.harnessValidated).toBe(true); + const forF = out.mutants.probed.filter((m: { file: string }) => + m.file.endsWith('f.ts'), + ); + expect(forF.length).toBeGreaterThan(0); + for (const m of forF) { + expect(m.verdict).toBe('inconclusive'); + expect(m.detail).toContain('f.test.ts'); + expect(m.detail).toContain('did not run green'); + } + // ...and nothing a reader acts on carries a survivor claim for that file. + expect( + (out.findings as Array<{ kind: string; file: string }>).filter( + (f) => f.kind === 'mutant-survived' && f.file.endsWith('f.ts'), + ), + ).toEqual([]); + // The guard is targeted: `g.ts`, whose own test IS green, still gets a + // real verdict rather than being swept up with it. + const forG = out.mutants.probed.filter((m: { file: string }) => + m.file.endsWith('g.ts'), + ); + expect(forG.length).toBeGreaterThan(0); + expect( + forG.every((m: { verdict: string }) => m.verdict !== 'inconclusive'), + ).toBe(true); + }); + it('a control that could not be SET UP leaves the window spendable', async () => { // `null` is not `false`, and this is where the difference is observable. // A control that never ran demonstrated nothing about the runner, so the diff --git a/packages/cli/src/commands/review/test-efficacy.ts b/packages/cli/src/commands/review/test-efficacy.ts index 06205f523af..84f2213e021 100644 --- a/packages/cli/src/commands/review/test-efficacy.ts +++ b/packages/cli/src/commands/review/test-efficacy.ts @@ -1998,6 +1998,31 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { ); } for (const c of harnessValidated === false ? [] : candidates) { + // The hunk loop's rule, which the mutants needed just as much. + // A mutant runs against `greenProbes` only, so a file whose OWN + // test was red in the unmutated baseline has that test excluded + // from the run — and then "every affected test still passed" + // is computed over a set that omits the one test most likely to + // catch the deletion. Measured live on PR #8213: six hunks in + // `bridge.ts` were correctly held at `inconclusive` because + // `bridge.test.ts` was not green, while eight mutants in the SAME + // file were scored `survived` and shipped as findings. + // + // The comment below this loop framed the asymmetry as "mutants + // guard the killed direction, hunks guard the survived one". That + // is true of an inconclusive RUN; it left the survived direction + // of a mutant unguarded against an absent covering test. Checked + // before the budget, because a candidate that cannot yield a + // verdict should not spend a suite run to say so. + const own = collocatedProbe(c.file, probes); + if (own && !greenProbes.includes(own)) { + mutantResults.push({ + ...c, + verdict: 'inconclusive' as const, + detail: `this mutant's collocated test ${own} did not run green in the unmutated baseline (likely a compile or import error in the probe tree), so the remaining probes passing cannot show the statement is uncovered`, + }); + continue; + } const remaining = mutantDeadline - now(); if (!fitsAnotherMutantRun(remaining, estimatedRunMs)) { mutantsSkippedForBudget = @@ -2027,9 +2052,12 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise { // case this exists for: a probe-tree import error that collected // nothing) cannot be scored `survived`: the other probes passing // shows only that THEY do not cover it, not that nothing does, since - // the one test that would catch it never ran. Same asymmetry the - // mutants hold, pointed the other way: there an inconclusive run is - // never `killed`, here an absent covering test is never `survived`. + // the one test that would catch it never ran. The mutants hold + // BOTH halves of this now: an inconclusive run is never `killed`, + // and — since the loop above gained the same collocated check — + // an absent covering test is never `survived` there either. The + // two halves are separate guards; having one was read as having + // the rule, and eight mutant survivors shipped through the gap. const own = collocatedProbe(h.file, probes); if (own && !greenProbes.includes(own)) { const { patch: _patch, ...meta } = h;