feat(review): statement-level mutation probes in test-efficacy

packages/cli/src/commands/review/agent-prompt.test.ts

@@ -1864,6 +1864,12 @@ describe('buildRoleBrief — every agent, not just the territory ones', () => {
       '"${QWEN_CODE_CLI:-qwen}" review test-efficacy /tmp/plan.json',
     );
     expect(p).toContain('--base abc123');
+    // All three finding kinds are named, or the agent meets a `mutant-survived`
+    // it was never told how to file — and the skipped/inconclusive mutants must
+    // be fenced off from findings the same way the probes' inconclusive is.
+    expect(p).toContain('`kind: "mutant-survived"`');
+    expect(p).toContain('mutants.skippedForBudget');
+    expect(p).toContain('mutants.skippedForCap');
     // No bare executable `qwen` anywhere in this brief. Agent 7 is the one
     // SUBAGENT that shells out to the review CLI — the one call site neither the
     // SKILL.md sweep nor check-coverage's stderr hints can reach — and its shell

packages/cli/src/commands/review/agent-prompt.ts

@@ -860,7 +860,9 @@ export function buildRoleBrief(
         '',
         '**Then run the test-efficacy probe.** A green suite says the tests pass. It does ' +
           'not say they would have failed had the change been wrong, and those are ' +
-          'different claims:',
+          'different claims. Give this call `timeout: 600000` too — besides the revert ' +
+          'probe it runs up to 8 single-statement deletion mutants, each a suite run, and ' +
+          'it budgets itself to finish inside that ceiling:',
         '',
         '```bash',
         `"\${QWEN_CODE_CLI:-qwen}" review test-efficacy ${resolve(opts.planPath)} \\`,
@@ -872,11 +874,17 @@ export function buildRoleBrief(
         'Read its `findings[]`. `kind: "unreachable"` is a test the project\'s test command ' +
           'never collects — it did not run here and it does not run in CI. `kind: "inert"` is ' +
           'a test that **still passed with the change reverted**: it is green whether or not ' +
-          'the feature exists, so it cannot catch a regression in it. Report each as a ' +
-          '**Suggestion** with `Source: [test]`, saying plainly which behaviour ships ' +
-          'unprotected. **`inconclusive` is not a finding** — reverting the source often ' +
-          "breaks the test's own compile, and that is not the test catching anything. Note it " +
-          'and move on.',
+          'the feature exists, so it cannot catch a regression in it. `kind: "mutant-survived"` ' +
+          'is a single safety statement the diff added (a `.clear()`, an `.abort(…)`, a ' +
+          'reset-to-empty) that was **deleted and every affected test stayed green** — no ' +
+          'test in the diff fails when it is removed, which the whole-file ' +
+          "revert cannot see when the file's other, tested behaviours mask it. Report each as a " +
+          '**Suggestion** with `Source: [test]`, saying plainly which behaviour has no ' +
+          'test in this diff that would catch its removal. **`inconclusive` is not a ' +
+          'finding** — for probes and mutants alike, ' +
+          "reverting or mutating the source often breaks the test's own compile, and that is " +
+          'not the test catching anything. Mutants counted in `mutants.skippedForBudget` ' +
+          'or `mutants.skippedForCap` never ran — not findings either. Note them and move on.',
       );
     }
   }

packages/cli/src/commands/review/test-efficacy.integration.test.ts

@@ -11,7 +11,7 @@
 // verdict logic is unit-tested in `classifyProbeRun`; what these lock down is
 // where the probe runs and what it leaves behind.
 
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import { execFileSync } from 'node:child_process';
 import {
   mkdtempSync,
@@ -97,10 +97,62 @@ function scaffoldModifiedPr(): { wt: string; base: string } {
   return { wt, base };
 }
 
+/**
+ * Swap the fake runner for one that reports every test file as FAILED. Used to
+ * drive the unmutated baseline red, so the mutant phase must skip wholesale.
+ */
+function installFailingVitest(): void {
+  const bin = join(repo, 'node_modules', '.bin', 'vitest');
+  writeFileSync(
+    bin,
+    `#!/usr/bin/env node
+const path = require('path');
+const files = process.argv.slice(2).filter((a) => a.includes('.test.'));
+process.stdout.write(JSON.stringify({
+  numPassedTests: 0,
+  numFailedTests: files.length,
+  testResults: files.map((f) => ({
+    name: path.resolve(f),
+    assertionResults: [{ status: 'failed' }],
+  })),
+}));
+`,
+  );
+  chmodSync(bin, 0o755);
+}
+
+/**
+ * Swap the fake runner for one that reports a file whose path contains "skip"
+ * as all-skipped (collected, but no assertion executed) and every other file as
+ * PASSED. Drives the per-file baseline gate: an unrelated all-skip file is
+ * `inconclusive`, not red, and must not disable the mutant phase.
+ */
+function installMixedVitest(): void {
+  const bin = join(repo, 'node_modules', '.bin', 'vitest');
+  writeFileSync(
+    bin,
+    `#!/usr/bin/env node
+const path = require('path');
+const files = process.argv.slice(2).filter((a) => a.includes('.test.'));
+process.stdout.write(JSON.stringify({
+  testResults: files.map((f) => ({
+    name: path.resolve(f),
+    assertionResults: [{ status: f.includes('skip') ? 'skipped' : 'passed' }],
+  })),
+}));
+`,
+  );
+  chmodSync(bin, 0o755);
+}
+
 beforeEach(() => {
   repo = mkdtempSync(join(tmpdir(), 'efficacy-iso-'));
   outside = mkdtempSync(join(tmpdir(), 'efficacy-outside-'));
   git(repo, 'init', '-q', '-b', 'main', '.');
+  // Keep the fake vitest out of git: `commitAll` runs `git add -A`, and a
+  // committed bin would be checked out into the probe worktree — the stale
+  // passing copy, not the file `installFailingVitest` overwrites.
+  writeFileSync(join(repo, '.gitignore'), 'node_modules\n');
 
   // A fake `vitest` on the up-tree bin path so `npx vitest` in the probe tree
   // resolves locally — fast, deterministic, no network. It echoes each test
@@ -225,6 +277,275 @@ describe('test-efficacy probe isolation (#6832)', () => {
     expect(existsSync(join(repo, 'wt-probe'))).toBe(false);
   });
 
+  it('runs a deletion mutant end-to-end and reports the survivor', async () => {
+    // The dogfood shape at full scale: the PR adds a reset function whose one
+    // safety statement (`state.clear()`) nothing gates. The fake vitest is
+    // green no matter what, so the baseline run passes, the mutant run passes
+    // — a SURVIVOR — and the revert probe still reads the test as inert. Both
+    // trees end clean: the mutation happened only in the disposable worktree.
+    write('package.json', '{"private":true,"workspaces":["packages/*"]}\n');
+    write(
+      'packages/lib/src/f.ts',
+      'export const state = new Map<string, string>();\n' +
+        'export function use(k: string) {\n' +
+        '  return state.get(k);\n' +
+        '}\n',
+    );
+    const base = commitAll('base');
+    const prSource =
+      'export const state = new Map<string, string>();\n' +
+      'export function use(k: string) {\n' +
+      '  return state.get(k);\n' +
+      '}\n' +
+      'export function reset() {\n' +
+      '  state.clear();\n' +
+      '}\n';
+    write('packages/lib/src/f.ts', prSource);
+    write(
+      'packages/lib/src/f.test.ts',
+      'import { reset } from "./f.js"; import { it, expect } from "vitest"; it("t", () => expect(typeof reset).toBe("function"));\n',
+    );
+    commitAll('pr');
+    const wt = join(repo, 'wt');
+    git(repo, 'worktree', 'add', '-q', '--detach', wt, 'HEAD');
+    writeFileSync(
+      join(repo, 'report.json'),
+      JSON.stringify({
+        files: [
+          { path: 'packages/lib/src/f.ts', kind: 'source' },
+          { path: 'packages/lib/src/f.test.ts', kind: 'test' },
+        ],
+      }),
+    );
+
+    const before = treeState(wt);
+    await runHandler({
+      report: join(repo, 'report.json'),
+      worktree: wt,
+      base,
+      out: join(repo, 'out.json'),
+    });
+
+    const out = JSON.parse(readFileSync(join(repo, 'out.json'), 'utf8'));
+    expect(out.mutants.probed).toEqual([
+      {
+        file: 'packages/lib/src/f.ts',
+        line: 6,
+        statement: 'state.clear();',
+        verdict: 'survived',
+        detail: expect.stringContaining('still PASSED'),
+      },
+    ]);
+    expect(out.mutants.survived).toBe(1);
+    expect(out.mutants.skippedForBudget).toBe(0);
+    // The survivor is a finding the orchestrator files; the register matches
+    // the unreachable/inert messages Agent 7's brief already knows how to read.
+    const survivor = (
+      out.findings as Array<{ kind: string; file: string; message: string }>
+    ).find((f) => f.kind === 'mutant-survived');
+    expect(survivor?.file).toBe('packages/lib/src/f.ts');
+    expect(survivor?.message).toContain('state.clear();');
+    // The mutation never touched the shared tree, and the probe tree is gone.
+    expect(treeState(wt)).toBe(before);
+    expect(readFileSync(join(wt, 'packages/lib/src/f.ts'), 'utf8')).toBe(
+      prSource,
+    );
+    expect(existsSync(join(repo, 'wt-probe'))).toBe(false);
+  });
+
+  it('skips the mutants wholesale when the unmutated baseline is not green', async () => {
+    // A mutant is only evidence against a suite that is green WITHOUT it: against
+    // a baseline that already fails, every mutant would be "killed" by failures
+    // it did not cause. So when no probe file is green in the unmutated run, the whole
+    // mutant phase is skipped and the report says so — no probed mutants and no
+    // survivor finding, even though the diff adds an ungated safety statement.
+    write('package.json', '{"private":true,"workspaces":["packages/*"]}\n');
+    write(
+      'packages/lib/src/f.ts',
+      'export const state = new Map<string, string>();\n',
+    );
+    const base = commitAll('base');
+    write(
+      'packages/lib/src/f.ts',
+      'export const state = new Map<string, string>();\n' +
+        'export function reset() {\n' +
+        '  state.clear();\n' +
+        '}\n',
+    );
+    // The test FAILS, so the suite is not cleanly green under a real runner
+    // too — not only under the fake one installed below. Whichever runner the
+    // probe resolves to, the baseline is red and the mutants must be skipped.
+    write(
+      'packages/lib/src/f.test.ts',
+      'import { reset } from "./f.js"; import { it, expect } from "vitest"; it("t", () => { reset(); expect(1).toBe(2); });\n',
+    );
+    commitAll('pr');
+    const wt = join(repo, 'wt');
+    git(repo, 'worktree', 'add', '-q', '--detach', wt, 'HEAD');
+    writeFileSync(
+      join(repo, 'report.json'),
+      JSON.stringify({
+        files: [
+          { path: 'packages/lib/src/f.ts', kind: 'source' },
+          { path: 'packages/lib/src/f.test.ts', kind: 'test' },
+        ],
+      }),
+    );
+    // The unmutated suite is NOT green: the fake runner reports a failure.
+    installFailingVitest();
+
+    await runHandler({
+      report: join(repo, 'report.json'),
+      worktree: wt,
+      base,
+      out: join(repo, 'out.json'),
+    });
+
+    const out = JSON.parse(readFileSync(join(repo, 'out.json'), 'utf8'));
+    expect(out.mutants.probed).toEqual([]);
+    expect(out.mutants.note).toContain('no probe file was green');
+    expect(
+      (out.findings as Array<{ kind: string }>).some(
+        (f) => f.kind === 'mutant-survived',
+      ),
+    ).toBe(false);
+  });
+
+  it('still probes when an UNRELATED probe file is all-skipped (per-file gate)', async () => {
+    // Finding 2's shape: a quarantined suite that is entirely `it.skip`
+    // classifies `inconclusive` — not red, not a failure. The old whole-suite
+    // gate read that as "not cleanly green" and took the ENTIRE mutant phase
+    // down with it, losing the survivor finding below. The gate is per file:
+    // the mutant runs against the probe files that ARE green in the baseline,
+    // so an unrelated all-skip file no longer disables it.
+    write('package.json', '{"private":true,"workspaces":["packages/*"]}\n');
+    write(
+      'packages/lib/src/f.ts',
+      'export const state = new Map<string, string>();\n',
+    );
+    const base = commitAll('base');
+    write(
+      'packages/lib/src/f.ts',
+      'export const state = new Map<string, string>();\n' +
+        'export function reset() {\n' +
+        '  state.clear();\n' +
+        '}\n',
+    );
+    write(
+      'packages/lib/src/f.test.ts',
+      'import { reset } from "./f.js"; import { it, expect } from "vitest"; it("t", () => expect(typeof reset).toBe("function"));\n',
+    );
+    // An unrelated suite that collects but runs nothing (all skipped).
+    write(
+      'packages/lib/src/skipped.test.ts',
+      'import { it } from "vitest"; it.skip("quarantined", () => {});\n',
+    );
+    commitAll('pr');
+    const wt = join(repo, 'wt');
+    git(repo, 'worktree', 'add', '-q', '--detach', wt, 'HEAD');
+    writeFileSync(
+      join(repo, 'report.json'),
+      JSON.stringify({
+        files: [
+          { path: 'packages/lib/src/f.ts', kind: 'source' },
+          { path: 'packages/lib/src/f.test.ts', kind: 'test' },
+          { path: 'packages/lib/src/skipped.test.ts', kind: 'test' },
+        ],
+      }),
+    );
+    // Baseline: f.test.ts passes (inert), skipped.test.ts collects but runs
+    // nothing (inconclusive). The mutant must still run against the green file.
+    installMixedVitest();
+
+    await runHandler({
+      report: join(repo, 'report.json'),
+      worktree: wt,
+      base,
+      out: join(repo, 'out.json'),
+    });
+
+    const out = JSON.parse(readFileSync(join(repo, 'out.json'), 'utf8'));
+    expect(out.mutants.note).toBeUndefined();
+    expect(out.mutants.survived).toBe(1);
+    expect(out.mutants.probed).toEqual([
+      {
+        file: 'packages/lib/src/f.ts',
+        line: 3,
+        statement: 'state.clear();',
+        verdict: 'survived',
+        detail: expect.stringContaining('still PASSED'),
+      },
+    ]);
+  });
+
+  it('reports mutants skipped for budget when time runs out mid-loop', async () => {
+    // Three safety-verb candidates, but the budget expires after two: the
+    // counter, the `skippedForBudget` report field, and the stdout line are
+    // exercised end-to-end. `Date.now()` is mocked to advance 50 s per call
+    // (the real budget is 540 s and a real run cannot reach it in a test):
+    // the baseline measures 50 s, so `estimatedRunMs` is 65 s, and the
+    // remaining budget drops below `2 × 65 s` on the third loop check.
+    write('package.json', '{"private":true,"workspaces":["packages/*"]}\n');
+    write(
+      'packages/lib/src/f.ts',
+      'export let items: string[] = ["a"];\n' +
+        'export const state = new Map<string, string>();\n' +
+        'export const cache = new Set<string>();\n',
+    );
+    const base = commitAll('base');
+    write(
+      'packages/lib/src/f.ts',
+      'export let items: string[] = ["a"];\n' +
+        'export const state = new Map<string, string>();\n' +
+        'export const cache = new Set<string>();\n' +
+        'export function reset() {\n' +
+        '  items = [];\n' +
+        '  state.clear();\n' +
+        '  cache.clear();\n' +
+        '}\n',
+    );
+    write(
+      'packages/lib/src/f.test.ts',
+      'import { reset } from "./f.js"; import { it, expect } from "vitest"; it("t", () => expect(typeof reset).toBe("function"));\n',
+    );
+    commitAll('pr');
+    const wt = join(repo, 'wt');
+    git(repo, 'worktree', 'add', '-q', '--detach', wt, 'HEAD');
+    writeFileSync(
+      join(repo, 'report.json'),
+      JSON.stringify({
+        files: [
+          { path: 'packages/lib/src/f.ts', kind: 'source' },
+          { path: 'packages/lib/src/f.test.ts', kind: 'test' },
+        ],
+      }),
+    );
+
+    let mockNow = 0;
+    const dateSpy = vi.spyOn(Date, 'now').mockImplementation(() => {
+      mockNow += 50_000;
+      return mockNow;
+    });
+    try {
+      await runHandler({
+        report: join(repo, 'report.json'),
+        worktree: wt,
+        base,
+        out: join(repo, 'out.json'),
+      });
+    } finally {
+      dateSpy.mockRestore();
+    }
+
+    const out = JSON.parse(readFileSync(join(repo, 'out.json'), 'utf8'));
+    expect(out.mutants.probed.length).toBe(2);
+    expect(out.mutants.skippedForBudget).toBe(1);
+    expect(out.mutants.probed.length + out.mutants.skippedForBudget).toBe(3);
+    for (const m of out.mutants.probed) {
+      expect(m.verdict).toBe('survived');
+    }
+  });
+
   it('sweeps a stale REGISTERED probe worktree left by a crashed run', async () => {
     const { wt, base } = scaffoldModifiedPr();
     // A prior probe crashed after `worktree add` but before its cleanup, leaving