Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/pra7-posting-surface.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"review": minor
---

The P1 non-blocking comment budget, plus a collapsed-tail disclosure that names its best finding. Four posting-surface changes, all deterministic (no model behavior changes): at most 3 non-blocking findings post inline per review (the ROUTING `non-blocking-budget` line tunes it; blocking findings are uncapped up to the engine's 20), `nitpick (non-blocking)` never posts inline, documentation-label findings are exempt from the budget (the documentation autofix selects its work by parsing that label off posted threads, so budgeting them would silently shrink a shipped feature's scope), and the collapsed section's summary line now names its top-ranked entry instead of a bare count. The motivating case for the disclosure: three 2026-08-24 approving re-reviews on this repo collapsed correctness findings behind "Non-blocking observations (N)", including Khan/actions#367's report that the acknowledgment feature's own reply guard never fires. Nothing is dropped and the verdict still counts every validated claim; a non-default budget shows in the version footer.
Comment thread
khan-actions-bot marked this conversation as resolved.
Resolved
Comment thread
khan-actions-bot marked this conversation as resolved.
Resolved
10 changes: 10 additions & 0 deletions workflows/review/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,7 @@ rule per line:
# <pattern> [lens=<lens>,…] [tier=trivial|low|medium|high] [direction-dependent]
# enable <reviewer>[,<reviewer>…]
# re-review full|scoped|flip-gated|fast [blocking-only]
# non-blocking-budget <n>
services/**/migrations/** tier=high lens=data-migrations
**/*.graphql lens=api-federation-compat
pkg/auth/** tier=high direction-dependent lens=security-auth
Expand Down Expand Up @@ -396,6 +397,15 @@ re-review scoped
optional `blocking-only` modifier changes the repeat review's posting
surface (see [Re-review modes](#re-review-modes-the-runs-per-pr-cost-lever));
an unknown modifier warns and is ignored, and the mode still applies.
- `non-blocking-budget` sets how many non-blocking findings may post as inline
comments per review (default 3). Blocking findings never count against it;
`nitpick (non-blocking)` findings never post inline at all; documentation
findings are exempt (the documentation reviewer self-caps, and its autofix
selects work by the label on posted threads). Findings over budget collapse
into a `<details>` block in the review body whose summary names the
Comment thread
khan-actions-bot marked this conversation as resolved.
Resolved
top-ranked entry; nothing is dropped and the verdict counts every validated
finding. A malformed value warns and keeps the default; when several lines
Comment thread
khan-actions-bot marked this conversation as resolved.
Resolved
set it, the last one wins.

Glob semantics are a practical subset of gitignore/CODEOWNERS: `**` crosses
directories, `*` and `?` stay within a segment, a trailing `/` matches everything
Expand Down
111 changes: 111 additions & 0 deletions workflows/review/lib/router-non-blocking-budget.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
import {describe, it, expect} from "vitest";

import {runCli} from "./router";
import {parseRoutingConfig, ROUTING_CONFIG_PATH} from "./routing-config";

/**
* The `non-blocking-budget <n>` directive's parse and CLI wiring, split from
* router.test.ts for its max-lines budget (the router-rereview-blocking-only
* precedent). The posting-surface behavior the number drives lives in
* submission-trial-followups.test.ts; here we pin only that the ROUTING line
* parses and that `routing.json` carries the value to the submission CLI.
* The fs fixture is a small local copy of router.test.ts's.
*/

const fakeFs = (inputs: Record<string, string>) => {
const written: Record<string, string> = {};
const fs = {
readFileSync: (p: string, _enc: "utf8"): string => {
const content = inputs[p];
if (content === undefined) {
throw new Error(`unexpected read: ${p}`);
}
return content;
},
writeFileSync: (p: string, data: string): void => {
written[p] = data;
},
existsSync: (p: string): boolean =>
p in inputs ||
Object.keys(inputs).some((key) => key.startsWith(`${p}/`)),
mkdirSync: (_p: string, _opts: {recursive: boolean}): void => {},
readdirSync: (p: string): string[] => {
if (p in inputs) {
throw new Error(`ENOTDIR: not a directory, scandir '${p}'`);
}
const prefix = p.endsWith("/") ? p : `${p}/`;
const names = new Set<string>();
for (const key of Object.keys(inputs)) {
if (key.startsWith(prefix)) {
names.add(key.slice(prefix.length).split("/")[0]);
}
}
return [...names];
},
};
return {fs, written};
};

describe("parseRoutingConfig: non-blocking-budget directive", () => {
it("defaults to 3", () => {
expect(
parseRoutingConfig("docs/** tier=trivial").nonBlockingInlineBudget,
).toBe(3);
});

it("parses a non-negative integer, zero included", () => {
expect(
parseRoutingConfig("non-blocking-budget 5").nonBlockingInlineBudget,
).toBe(5);
expect(
parseRoutingConfig("non-blocking-budget 0").nonBlockingInlineBudget,
).toBe(0);
});

it("warns on a malformed value and keeps the default", () => {
for (const value of ["three", "-1", "2.5"]) {
const config = parseRoutingConfig(`non-blocking-budget ${value}`);
expect(config.nonBlockingInlineBudget).toBe(3);
expect(config.warnings.join("\n")).toContain(
"non-negative integer",
);
}
});

it("skips a line with the wrong arity", () => {
const config = parseRoutingConfig("non-blocking-budget 2 4");
expect(config.nonBlockingInlineBudget).toBe(3);
expect(config.warnings.join("\n")).toContain("exactly one number");
});

it("lets the last of duplicate lines win, with a warning", () => {
const config = parseRoutingConfig(
"non-blocking-budget 5\nnon-blocking-budget 2",
);
expect(config.nonBlockingInlineBudget).toBe(2);
expect(config.warnings.join("\n")).toContain(
"duplicate non-blocking-budget",
);
});
});

describe("runCli: non-blocking budget", () => {
it("surfaces the configured budget in routing.json", () => {
const {fs} = fakeFs({
["/tmp/gh-aw/review/files.json"]: JSON.stringify([
{path: "a.ts", status: "modified"},
]),
[ROUTING_CONFIG_PATH]: "non-blocking-budget 1",
});
expect(runCli(fs).nonBlockingInlineBudget).toBe(1);
});

it("defaults to 3 without a ROUTING config", () => {
const {fs} = fakeFs({
["/tmp/gh-aw/review/files.json"]: JSON.stringify([
{path: "a.ts", status: "modified"},
]),
});
expect(runCli(fs).nonBlockingInlineBudget).toBe(3);
});
});
12 changes: 12 additions & 0 deletions workflows/review/lib/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import {
} from "./lens-payloads";
import {
DEFAULT_DISPATCH_MODE,
DEFAULT_NON_BLOCKING_INLINE_BUDGET,
DEFAULT_RE_REVIEW_MODE,
ENABLEABLE_REVIEWERS,
parseRoutingConfig,
Expand All @@ -65,6 +66,7 @@ import type {
// entry point for routing vocabulary and the ROUTING parser.
export {DEFAULT_MISROUTED_FLOOR_TIER, DEFAULT_TIER_BUDGETS};
export {
DEFAULT_NON_BLOCKING_INLINE_BUDGET,
DEFAULT_RE_REVIEW_MODE,
ENABLEABLE_REVIEWERS,
parseRoutingConfig,
Expand Down Expand Up @@ -752,6 +754,12 @@ export type RoutingJson = {
* collapse into the review body (`submission.ts` reads this).
*/
reReviewBlockingOnly: boolean;
/**
* `non-blocking-budget` line in `ROUTING` (default 3): how many
* non-blocking findings may post inline per review; the overflow
* collapses into the review body (`submission.ts` reads this).
*/
nonBlockingInlineBudget: number;
/**
* The repo's dispatch mode (`dispatch` line in `ROUTING`; `task` when
* absent). `scripted` opts the repo into the deterministic dispatcher
Expand Down Expand Up @@ -782,6 +790,7 @@ export const toRoutingJson = (
reReviewMode: ReReviewMode = DEFAULT_RE_REVIEW_MODE,
dispatchMode: DispatchMode = DEFAULT_DISPATCH_MODE,
reReviewBlockingOnly = false,
nonBlockingInlineBudget: number = DEFAULT_NON_BLOCKING_INLINE_BUDGET,
): RoutingJson => {
const owners: Record<string, string[]> = {};
const generatedFiles: string[] = [];
Expand All @@ -808,6 +817,7 @@ export const toRoutingJson = (
enabledReviewers,
reReviewMode,
reReviewBlockingOnly,
nonBlockingInlineBudget,
dispatchMode,
routingConfig,
};
Expand Down Expand Up @@ -903,6 +913,7 @@ export const runCli = (
enabledReviewers: [],
reReviewMode: DEFAULT_RE_REVIEW_MODE,
reReviewBlockingOnly: false,
nonBlockingInlineBudget: DEFAULT_NON_BLOCKING_INLINE_BUDGET,
dispatchMode: DEFAULT_DISPATCH_MODE,
warnings: [
`routing config missing (${ROUTING_CONFIG_PATH}): no ` +
Expand Down Expand Up @@ -968,6 +979,7 @@ export const runCli = (
routingFileConfig.reReviewMode,
routingFileConfig.dispatchMode,
routingFileConfig.reReviewBlockingOnly,
routingFileConfig.nonBlockingInlineBudget,
);

fs.mkdirSync(REVIEW_DIR, {recursive: true});
Expand Down
62 changes: 60 additions & 2 deletions workflows/review/lib/routing-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,25 @@ export const DEFAULT_RE_REVIEW_MODE: ReReviewMode = "full";
*/
export const RE_REVIEW_MODIFIERS = ["blocking-only"] as const;

/**
* How many non-blocking findings may post as inline comments per review (the
* P1 comment budget). Blocking findings never count against it, and two label
* classes sit outside it: `nitpick (non-blocking)` never posts inline at all,
* and the documentation label is exempt (the documentation reviewer
* self-caps at five per review, and the documentation autofix selects its
* work by parsing that label off posted threads, so collapsing those would
* silently empty the autofix scope). Everything over budget collapses into
* the review body's <details> block; nothing is dropped and the verdict
* still counts every claim.
*
* 3 is set by fiat (the quiet-the-human-surface lane's Q8 decision): at the
* measured 2.91 findings/run it binds rarely and acts as a backstop against
* the wall-of-comments failure mode (webapp#41440: 13 non-blocking inline
* comments in one review). Consumers tune it with a `non-blocking-budget`
* line in ROUTING.
*/
export const DEFAULT_NON_BLOCKING_INLINE_BUDGET = 3;

/**
* How Step 3 runs: the orchestrator invokes the deterministic dispatcher
* (`lib/dispatch.ts`) once, which runs Step 3's phases as code. `scripted`
Expand All @@ -130,6 +149,9 @@ export type RoutingFileConfig = {
/** `re-review <mode> blocking-only`: repeat reviews post only blocking
* findings inline (see {@link RE_REVIEW_MODIFIERS}). */
reReviewBlockingOnly: boolean;
/** `non-blocking-budget <n>`: how many non-blocking findings may post
* inline per review (see {@link DEFAULT_NON_BLOCKING_INLINE_BUDGET}). */
nonBlockingInlineBudget: number;
/** The dispatch mode: always `scripted`. */
dispatchMode: DispatchMode;
/** Fixed-format parse warnings (unknown lens/tier, no-op rule). */
Expand All @@ -145,6 +167,7 @@ const KNOWN_LENS_SET: ReadonlySet<string> = new Set(KNOWN_LENSES);
* <pattern> [lens=<lens>[,<lens>…]] [tier=trivial|low|medium|high] [direction-dependent]
* enable <reviewer>[,<reviewer>…]
* re-review full|scoped|flip-gated|fast [blocking-only]
* non-blocking-budget <n>
*
* `lens=` names specialist lenses to spawn when the pattern is touched (multiple
* matching rules union their lenses). `tier=` assigns a risk tier; when several
Expand All @@ -162,8 +185,12 @@ const KNOWN_LENS_SET: ReadonlySet<string> = new Set(KNOWN_LENSES);
* ({@link RE_REVIEW_MODIFIERS}) makes repeat reviews post only blocking
* findings inline; an unknown modifier warns and is ignored (the mode still
* applies), and `full blocking-only` warns that the modifier never applies
* at full depth. A leftover `dispatch` line from the
* retired dial warns and is ignored (scripted is the only mode).
* at full depth. `non-blocking-budget` sets how many non-blocking findings
* post inline per review ({@link DEFAULT_NON_BLOCKING_INLINE_BUDGET});
* a malformed value warns and keeps the previous value, and when several
* lines set it the last one wins (with a warning). A leftover `dispatch`
* line from the retired dial warns and is ignored (scripted is the only
* mode).
*
* Malformed fields and unknown lens/reviewer names produce a warning and skip
* the lens or line rather than aborting the run: routing degrades to fewer
Expand All @@ -177,6 +204,8 @@ export const parseRoutingConfig = (content: string): RoutingFileConfig => {
let reReviewMode: ReReviewMode = DEFAULT_RE_REVIEW_MODE;
let reReviewBlockingOnly = false;
let reReviewLineSeen = false;
let nonBlockingInlineBudget = DEFAULT_NON_BLOCKING_INLINE_BUDGET;
let budgetLineSeen = false;
let dispatchLineSeen = false;
const warnings: string[] = [];

Expand Down Expand Up @@ -266,6 +295,34 @@ export const parseRoutingConfig = (content: string): RoutingFileConfig => {
continue;
}

if (pattern === "non-blocking-budget") {
if (fields.length !== 1) {
warnings.push(
`ROUTING line ${lineNo}: non-blocking-budget takes ` +
`exactly one number (line skipped)`,
);
continue;
}
const value = Number(fields[0]);
if (!Number.isInteger(value) || value < 0) {
warnings.push(
`ROUTING line ${lineNo}: non-blocking-budget must be a ` +
`non-negative integer, got "${fields[0]}" (kept ` +
`${nonBlockingInlineBudget})`,
);
continue;
}
if (budgetLineSeen) {
warnings.push(
`ROUTING line ${lineNo}: duplicate non-blocking-budget ` +
`line (last one wins)`,
);
}
nonBlockingInlineBudget = value;
budgetLineSeen = true;
continue;
}

if (pattern === "dispatch") {
// The dial is retired: scripted dispatch always runs. A leftover
// line is tolerated (never a crashed run); any value other than
Expand Down Expand Up @@ -361,6 +418,7 @@ export const parseRoutingConfig = (content: string): RoutingFileConfig => {
),
reReviewMode,
reReviewBlockingOnly,
nonBlockingInlineBudget,
dispatchMode: DEFAULT_DISPATCH_MODE,
warnings,
};
Expand Down
20 changes: 13 additions & 7 deletions workflows/review/lib/submission-blocking-only.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ describe("runSubmissionCli: re-review blocking-only", () => {
expect(plan.comments[0].body).not.toContain(
"Non-blocking observations",
);
expect(plan.body).toContain("Non-blocking observations (2)");
expect(plan.body).toContain(
"Non-blocking observations (2; top: a.ts:9 nitpick (non-blocking))",
);
expect(plan.body).toContain(
"- `a.ts:9` nitpick (non-blocking): Rename the helper. " +
"<sub>(correctness-reviewer)</sub>",
Expand Down Expand Up @@ -160,8 +162,8 @@ describe("runSubmissionCli: re-review blocking-only", () => {
depth: "full",
claims: [
claim({
id: "nit",
label: "nitpick (non-blocking)",
id: "sug",
label: "suggestion (non-blocking)",
confidence: 0.9,
}),
],
Expand All @@ -181,8 +183,8 @@ describe("runSubmissionCli: re-review blocking-only", () => {
depth: "scoped",
claims: [
claim({
id: "nit",
label: "nitpick (non-blocking)",
id: "sug",
label: "suggestion (non-blocking)",
confidence: 0.9,
}),
],
Expand Down Expand Up @@ -215,7 +217,9 @@ describe("runSubmissionCli: re-review blocking-only", () => {
const plan = runSubmissionCli(fs);
expect(plan.comments).toHaveLength(20);
expect(plan.body).not.toContain("Non-blocking observations");
expect(plan.body).toContain("Lower-confidence observations (1)");
expect(plan.body).toContain(
"Lower-confidence observations (1; top: a.ts:21 issue (blocking))",
);
expect(plan.body).toContain("issue (blocking)");
expect(plan.notes.join(" ")).toContain(
"1 claim(s) collapsed below the inline bar",
Expand Down Expand Up @@ -243,7 +247,9 @@ describe("runSubmissionCli: re-review blocking-only", () => {
const plan = runSubmissionCli(fs);
expect(plan.event).toBe("APPROVE");
expect(plan.comments).toEqual([]);
expect(plan.body).toContain("Non-blocking observations (1)");
expect(plan.body).toContain(
"Non-blocking observations (1; top: a.ts:2 nitpick (non-blocking))",
);
// A body carrying the collapsed section is never the bare approve
// line, so the redundant-approval skip cannot swallow it.
expect(plan.skipSubmission).toBe(false);
Expand Down
Loading
Loading