Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 106 additions & 0 deletions packages/cli/src/commands/review/test-efficacy.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -961,6 +961,112 @@ process.stdout.write(JSON.stringify({
expect(out.findings).toEqual([]);
});

it('holds a mutant at inconclusive when its OWN test was red in the baseline', async () => {
// Measured live on PR #8213: six hunks in `bridge.ts` were correctly held
// at `inconclusive` because `bridge.test.ts` never ran green, while eight
// mutants in the SAME file were scored `survived` and shipped as findings.
// A mutant runs against `greenProbes` only, so the red collocated test is
// excluded from the run, and "every affected test still passed" is then
// computed over a set that omits the one test most likely to catch the
// deletion. Two files here: `f.ts` whose own test is red, and `g.ts`
// whose own test is green — the second is what shows the guard is
// targeted rather than a blanket refusal.
write('package.json', '{"private":true,"workspaces":["packages/*"]}\n');
write('packages/lib/src/f.ts', 'export const a = new Map();\n');
write('packages/lib/src/g.ts', 'export const b = new Map();\n');
const base = commitAll('base');
write(
'packages/lib/src/f.ts',
'export const a = new Map();\nexport function fReset() {\n a.clear();\n}\n',
);
write(
'packages/lib/src/g.ts',
'export const b = new Map();\nexport function gReset() {\n b.clear();\n}\n',
);
write(
'packages/lib/src/f.test.ts',
'import { fReset } from "./f.js"; import { it, expect } from "vitest"; it("t", () => expect(typeof fReset).toBe("function"));\n',
);
write(
'packages/lib/src/g.test.ts',
'import { gReset } from "./g.js"; import { it, expect } from "vitest"; it("t", () => expect(typeof gReset).toBe("function"));\n',
);
commitAll('pr');
const wt = join(repo, 'wt');
git(repo, 'worktree', 'add', '-q', '--detach', wt, 'HEAD');
writeFileSync(
join(repo, 'report.json'),
JSON.stringify({
files: [
{ path: 'packages/lib/src/f.ts', kind: 'source' },
{ path: 'packages/lib/src/g.ts', kind: 'source' },
{ path: 'packages/lib/src/f.test.ts', kind: 'test' },
{ path: 'packages/lib/src/g.test.ts', kind: 'test' },
],
}),
);
// `f.test.ts` is red from the start — the baseline shape this is about.
// Everything else is green, and the injected control still turns the run
// red, so the harness is validated and survivors would be licensed.
writeFileSync(
vitestScript(),
`#!/usr/bin/env node
import fs from 'node:fs';
import path from 'node:path';
const files = process.argv.slice(2).filter((a) => a.includes('.test.'));
const st = (f) => {
try {
if (fs.readFileSync(f, 'utf8').includes('QWEN-REVIEW-POSITIVE-CONTROL')) return 'failed';
} catch {}
return path.basename(f) === 'f.test.ts' ? 'failed' : 'passed';
};
const results = files.map((f) => ({
name: path.resolve(f),
assertionResults: [{ status: st(f) }],
}));
const nf = results.filter((r) => r.assertionResults[0].status === 'failed').length;
process.stdout.write(JSON.stringify({
numPassedTests: results.length - nf,
numFailedTests: nf,
testResults: results,
}));
`,
);
await runHandler({
report: join(repo, 'report.json'),
worktree: wt,
base,
out: join(repo, 'out.json'),
});

const out = JSON.parse(readFileSync(join(repo, 'out.json'), 'utf8'));
expect(out.harnessValidated).toBe(true);
const forF = out.mutants.probed.filter((m: { file: string }) =>
m.file.endsWith('f.ts'),
);
expect(forF.length).toBeGreaterThan(0);
for (const m of forF) {
expect(m.verdict).toBe('inconclusive');
expect(m.detail).toContain('f.test.ts');
expect(m.detail).toContain('did not run green');
}
// ...and nothing a reader acts on carries a survivor claim for that file.
expect(
(out.findings as Array<{ kind: string; file: string }>).filter(
(f) => f.kind === 'mutant-survived' && f.file.endsWith('f.ts'),
),
).toEqual([]);
// The guard is targeted: `g.ts`, whose own test IS green, still gets a
// real verdict rather than being swept up with it.
const forG = out.mutants.probed.filter((m: { file: string }) =>
m.file.endsWith('g.ts'),
);
expect(forG.length).toBeGreaterThan(0);
expect(
forG.every((m: { verdict: string }) => m.verdict !== 'inconclusive'),
).toBe(true);
});

it('a control that could not be SET UP leaves the window spendable', async () => {
// `null` is not `false`, and this is where the difference is observable.
// A control that never ran demonstrated nothing about the runner, so the
Expand Down
34 changes: 31 additions & 3 deletions packages/cli/src/commands/review/test-efficacy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1998,6 +1998,31 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise<void> {
);
}
for (const c of harnessValidated === false ? [] : candidates) {
// The hunk loop's rule, which the mutants needed just as much.
// A mutant runs against `greenProbes` only, so a file whose OWN
// test was red in the unmutated baseline has that test excluded
// from the run — and then "every affected test still passed"
// is computed over a set that omits the one test most likely to
// catch the deletion. Measured live on PR #8213: six hunks in
// `bridge.ts` were correctly held at `inconclusive` because
// `bridge.test.ts` was not green, while eight mutants in the SAME
// file were scored `survived` and shipped as findings.
//
// The comment below this loop framed the asymmetry as "mutants
// guard the killed direction, hunks guard the survived one". That
// is true of an inconclusive RUN; it left the survived direction
// of a mutant unguarded against an absent covering test. Checked
// before the budget, because a candidate that cannot yield a
// verdict should not spend a suite run to say so.
const own = collocatedProbe(c.file, probes);
if (own && !greenProbes.includes(own)) {
mutantResults.push({
...c,
verdict: 'inconclusive' as const,
detail: `this mutant's collocated test ${own} did not run green in the unmutated baseline (likely a compile or import error in the probe tree), so the remaining probes passing cannot show the statement is uncovered`,
});
continue;
}
const remaining = mutantDeadline - now();
if (!fitsAnotherMutantRun(remaining, estimatedRunMs)) {
mutantsSkippedForBudget =
Expand Down Expand Up @@ -2027,9 +2052,12 @@ async function runTestEfficacy(args: TestEfficacyArgs): Promise<void> {
// case this exists for: a probe-tree import error that collected
// nothing) cannot be scored `survived`: the other probes passing
// shows only that THEY do not cover it, not that nothing does, since
// the one test that would catch it never ran. Same asymmetry the
// mutants hold, pointed the other way: there an inconclusive run is
// never `killed`, here an absent covering test is never `survived`.
// the one test that would catch it never ran. The mutants hold
// BOTH halves of this now: an inconclusive run is never `killed`,
// and — since the loop above gained the same collocated check —
// an absent covering test is never `survived` there either. The
// two halves are separate guards; having one was read as having
// the rule, and eight mutant survivors shipped through the gap.
const own = collocatedProbe(h.file, probes);
if (own && !greenProbes.includes(own)) {
const { patch: _patch, ...meta } = h;
Expand Down
Loading