Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
553 changes: 537 additions & 16 deletions packages/cli/src/commands/review/agent-prompt.test.ts

Large diffs are not rendered by default.

706 changes: 681 additions & 25 deletions packages/cli/src/commands/review/agent-prompt.ts

Large diffs are not rendered by default.

437 changes: 431 additions & 6 deletions packages/cli/src/commands/review/check-coverage.test.ts

Large diffs are not rendered by default.

52 changes: 52 additions & 0 deletions packages/cli/src/commands/review/check-coverage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,58 @@ function runCheckCoverage(args: CheckCoverageArgs): void {
`--chunk <id>\` and pass it verbatim.`,
);
}
// The prompt was built in code and then edited on the way to the agent. Nothing
// else in the run can see this: a paraphrase keeps the diff path, so every other
// check passes.
if (report.rewrittenPrompts.length > 0) {
writeStderrLine(
`ERROR: ${report.rewrittenPrompts.length} agent(s) were not launched with ` +
`the prompt this CLI built for them — ${report.rewrittenPrompts.join('; ')}. ` +
`\`agent-prompt\` prints a prompt to be passed VERBATIM; a summary of it is ` +
`not it. The last run to paraphrase one dropped the rule against reciting a ` +
`stock sentence and replaced the project's review rules with three sentences ` +
`of its own. Re-run \`qwen review agent-prompt\` and pass its output ` +
`unedited — copy it, do not retype it.`,
);
}
// The one failure no other check in this file can see. Every other question is
// asked of an agent that ran; an agent that never ran leaves nothing to ask.
if (report.missingRoles.length > 0) {
writeStderrLine(
`ERROR: ${report.missingRoles.length} required agent(s) never ran — ` +
`${report.missingRoles.join('; ')}. The roster comes from the plan, not ` +
`from anything this run wrote. A dimension nobody reviewed cannot be ` +
`certified clean: build each prompt with the call named above and launch ` +
`an agent with it, verbatim.`,
);
}
if (report.unreadBriefs.length > 0) {
writeStderrLine(
`ERROR: ${report.unreadBriefs.length} agent(s) never opened their brief — ` +
`${report.unreadBriefs.join('; ')}. The launch prompt points at the brief ` +
`instead of containing it, so an agent that did not read it reviewed with ` +
`no dimension, no severity definitions and no project rules. Relaunch each ` +
`once, with the prompt \`agent-prompt\` printed.`,
);
}
if (report.unopenedAgents.length > 0) {
writeStderrLine(
`ERROR: ${report.unopenedAgents.length} agent(s) were pointed at diff lines ` +
`and never opened the diff — ${report.unopenedAgents.join(', ')}. They made ` +
`tool calls, so they are not idle; they simply worked on something else. ` +
`Reviewing the post-change source instead of the diff cannot see a deletion ` +
`at all. Relaunch each once.`,
);
}
if (report.missingChunks.length > 0) {
writeStderrLine(
'NOTE: a chunk counts as read when an agent was pointed at its lines AND ' +
'the harness recorded that agent opening the diff. An agent handed the ' +
'diff with no line ranges covers nothing. Build every whole-diff ' +
"agent's prompt with `qwen review agent-prompt --plan <plan> " +
'--whole-diff` and paste it verbatim ahead of its brief.',
);
}
if (report.idleAgents.length > 0) {
writeStderrLine(
`ERROR: ${report.idleAgents.length} agent(s) made no tool call — ` +
Expand Down
8 changes: 6 additions & 2 deletions packages/cli/src/commands/review/cleanup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

import type { CommandModule } from 'yargs';
import { execFileSync } from 'node:child_process';
import { existsSync, readdirSync, unlinkSync } from 'node:fs';
import { existsSync, readdirSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { writeStdoutLine, writeStderrLine } from '../../utils/stdioHelpers.js';
import { refExists, releaseWorktree } from './lib/git.js';
Expand Down Expand Up @@ -98,7 +98,11 @@ function runCleanup(target: string): void {
if (!file.startsWith(prefix)) continue;
const full = join(REVIEW_TMP_DIR, file);
try {
unlinkSync(full);
// Not every side file is a file. `agent-prompt` records what it handed each
// agent in `<plan>-prompts/`, a directory under this same prefix, and
// `unlinkSync` on a directory is an EISDIR — which this loop would have
// reported as a cleanup failure on every single review.
rmSync(full, { recursive: true, force: true });
writeStdoutLine(`Removed temp file: ${full}`);
removedAny = true;
} catch (err) {
Expand Down
94 changes: 89 additions & 5 deletions packages/cli/src/commands/review/compose-review.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
} from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { promptRecordDir, briefPath } from './lib/prompt-record.js';
import {
composeReview,
composeReviewCommand,
Expand All @@ -24,6 +25,7 @@ import {

vi.mock('../../utils/stdioHelpers.js', () => ({
writeStdoutLine: vi.fn(),
writeStderrLine: vi.fn(),
}));

const MODEL = 'test-model';
Expand All @@ -46,13 +48,23 @@ afterEach(() => {

const DIFF = '/abs/diff.txt';

/** Write a plan with two chunks, and return its path. */
/**
* Write a plan with two chunks, and return its path.
*
* A territory fan-out captured cross-repo, with no deletions: the smallest plan
* whose roster is exactly the chunks plus the test matrix. `coveredPlan()` below
* satisfies that one. A plan that requires nothing is not a plan any capture
* command writes, and coverage now reads the roster out of it.
*/
function plan(): string {
const p = join(dir, 'plan.json');
writeFileSync(
p,
JSON.stringify({
diffPathAbsolute: DIFF,
srcDiffLines: 5000,
diffLines: 5000,
files: [{ path: 'a.ts', kind: 'source', removedLines: 0, heavy: false }],
chunks: [
{ id: 1, startLine: 1, endLine: 100 },
{ id: 2, startLine: 101, endLine: 200 },
Expand All @@ -72,8 +84,13 @@ function plan(): string {
function transcript(
id: string,
launchPrompt: string,
opts: { toolCalls?: number; text?: string } = {},
opts: { toolCalls?: number; text?: string; opens?: string[] } = {},
): void {
const pointedAtBriefs = [
...launchPrompt.matchAll(/read_file\(file_path="([^"]*\.brief\.md)"\)/g),
].map((m) => m[1]);
const working = (opts.toolCalls ?? 0) > 0;
const opens = opts.opens ?? (working ? pointedAtBriefs : []);
const base = { agentId: id, agentName: 'general-purpose', sessionId: 'S1' };
const lines: string[] = [
JSON.stringify({
Expand Down Expand Up @@ -111,6 +128,35 @@ function transcript(
}),
);
}
for (const path of opens) {
lines.push(
JSON.stringify({
...base,
type: 'assistant',
message: {
role: 'model',
parts: [
{ functionCall: { name: 'read_file', args: { file_path: path } } },
],
},
}),
JSON.stringify({
...base,
type: 'tool_result',
message: {
role: 'user',
parts: [
{
functionResponse: {
name: 'read_file',
response: { output: 'brief' },
},
},
],
},
}),
);
}
lines.push(
JSON.stringify({
...base,
Expand All @@ -127,9 +173,43 @@ function transcript(
);
}

/** A prompt the CLI would have built: it names the diff and the read. */
/**
* A prompt the CLI would have built: it names the diff and the read of THIS
* chunk's lines. The offsets are the chunk's own, as `agent-prompt` emits them —
* coverage is attributed from the range delivered, not from the words `chunk N`.
*/
function goodPrompt(chunk: number): string {
return `You are reviewing chunk ${chunk} of 2.\nread_file(file_path="${DIFF}", offset=0, limit=100)`;
const offset = (chunk - 1) * 100;
const brief = briefPath(join(dir, 'plan.json'), `chunk-${chunk}`);
return (
`You are reviewing chunk ${chunk} of 2.\n` +
`read_file(file_path="${brief}")\n` +
`read_file(file_path="${DIFF}", offset=${offset}, limit=100)`
);
}

/** Lay down the CLI's record of the prompt it built for `chunk`. */
function recordBuilt(planPath: string, chunk: number): void {
const d = promptRecordDir(planPath);
mkdirSync(d, { recursive: true });
writeFileSync(join(d, `chunk-${chunk}.txt`), goodPrompt(chunk));
writeFileSync(briefPath(planPath, `chunk-${chunk}`), `chunk-${chunk} brief`);
}

/**
* The one whole-diff agent this plan's roster requires, built and launched.
*
* Its prompt names no line ranges, so it grants no coverage — a review may not
* certify lines on the strength of "somebody had the file open".
*/
function recordMatrix(planPath: string): void {
const d = promptRecordDir(planPath);
mkdirSync(d, { recursive: true });
const brief = briefPath(planPath, 'test-matrix');
writeFileSync(brief, 'The test-matrix brief.');
const launch = `You are the test-coverage matrix agent.\nread_file(file_path="${brief}")\nread_file(file_path="${DIFF}")`;
writeFileSync(join(d, 'test-matrix.txt'), launch);
transcript('tm', launch, { toolCalls: 2, opens: [brief] });
}

/** The prompt the orchestrator actually sent, 23 times: no diff anywhere. */
Expand All @@ -141,7 +221,11 @@ function blindPrompt(chunk: number): string {
function coveredPlan(): string {
transcript('a1', goodPrompt(1), { toolCalls: 3 });
transcript('a2', goodPrompt(2), { toolCalls: 2 });
return plan();
const p = plan();
recordBuilt(p, 1);
recordBuilt(p, 2);
recordMatrix(p);
return p;
}

/** Agents given the diff, that never opened it — and said so at length. */
Expand Down
67 changes: 66 additions & 1 deletion packages/cli/src/commands/review/compose-review.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import type { CommandModule } from 'yargs';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname } from 'node:path';
import { writeStdoutLine } from '../../utils/stdioHelpers.js';
import { writeStdoutLine, writeStderrLine } from '../../utils/stdioHelpers.js';
import {
coverageFromTranscripts,
TranscriptsUnavailableError,
Expand Down Expand Up @@ -226,6 +226,35 @@ export function composeReview(input: ComposeReviewInput): ComposeReviewResult {
'agent-prompt`)',
);
}
// Worked, but not on the diff. Not idle and not blind — it had the path and
// spent its run somewhere else, which on a diff with deletions means it
// reviewed a file the removed lines are simply not in.
for (const label of cov.unopenedAgents) {
unreviewed.push(
`${label} — pointed at diff lines it never opened: it made tool calls, ` +
'but none of them read the diff',
);
}
// The prompt was built in code and edited on the way to the agent. This caps
// for the same reason the others do: what the agent was actually asked is not
// what this skill's guarantees are written against.
for (const label of cov.rewrittenPrompts) {
unreviewed.push(
`${label} — not launched with the prompt the CLI built (pass ` +
'`agent-prompt` output verbatim)',
);
}
// A dimension nobody reviewed. This is exactly what `unreviewedDimensions`
// has always meant, arrived at from the plan instead of from the orchestrator
// noticing — which, on the run that never launched Agent 0, it did not.
for (const label of cov.missingRoles) {
unreviewed.push(`${label} — this agent never ran`);
}
// Launched, but never read the brief it was pointed at: it reviewed with no
// dimension, no severity definitions and no project rules.
for (const label of cov.unreadBriefs) {
unreviewed.push(label);
}
} catch (err) {
// Two different failures, and they must not wear each other's message. A
// malformed plan is the caller's mistake and says so; missing transcripts
Expand Down Expand Up @@ -504,5 +533,41 @@ export const composeReviewCommand: CommandModule = {
writeFileSync(out, json, 'utf8');
}
writeStdoutLine(json);
// The verdict a human reads, next to the JSON a program reads.
//
// Step 6 prints a verdict to the terminal, and until now it *composed* one —
// from the same prose rules this file exists to replace. So a run could skip
// this command entirely and tell the user whatever it had concluded: dogfooded,
// one did, and reported an Approve on a review whose coverage check had refused.
// There is now nothing to compose. This is the sentence; print it.
writeStderrLine(verdictLine(result));
},
};

/** The terminal verdict, in the words Step 6 is told to print. */
export function verdictLine(r: ComposeReviewResult): string {
const label: Record<ReviewEvent, string> = {
APPROVE: 'Approve',
REQUEST_CHANGES: 'Request changes',
COMMENT: 'Comment',
};
const why: Record<string, string> = {
'cannot-tell-existing-critical':
'an existing blocker could not be ruled on',
'chunk-nobody-read': 'part of the diff was never read',
'uncoverable-chunk': 'part of the diff cannot be read at all',
'unreviewed-dimension': 'a dimension nobody reviewed',
'context-unavailable': "the PR's existing discussion could not be read",
};
let line = `Verdict: ${label[r.event]}`;
// A cap only ever takes an Approve away. Naming one on a Request changes — which
// a confirmed blocker earned, and which no cap can soften — would report a
// constraint that did not bind, and the reader would go looking for its effect.
if (r.baseEvent === 'APPROVE' && r.event !== 'APPROVE') {
line +=
` — an Approve was NOT available: ` +
r.cappedBy.map((c) => why[c] ?? c).join('; ');
}
if (r.downgraded) line += ' — downgraded by a presubmit check';
return line;
}
Loading
Loading