diff --git a/tools/hygiene/audit-backlog-status-drift.test.ts b/tools/hygiene/audit-backlog-status-drift.test.ts new file mode 100644 index 000000000..dc7c28725 --- /dev/null +++ b/tools/hygiene/audit-backlog-status-drift.test.ts @@ -0,0 +1,258 @@ +import { test, expect, describe } from "bun:test"; +import { + extractPrimaryArtifacts, + parseFrontmatter, + findDriftCandidates, + type BacklogRow, +} from "./audit-backlog-status-drift"; + +describe("parseFrontmatter", () => { + test("reads status field from YAML frontmatter", () => { + const body = `--- +id: B-0001 +status: open +priority: P3 +--- + +# Body +`; + const fm = parseFrontmatter(body); + expect(fm.status).toBe("open"); + expect(fm.id).toBe("B-0001"); + }); + + test("returns empty object when no frontmatter", () => { + expect(parseFrontmatter("# Just a heading\n")).toEqual({}); + }); + + test("handles colon-in-value correctly", () => { + const body = `--- +title: "foo: bar" +status: open +--- +`; + const fm = parseFrontmatter(body); + expect(fm.title).toBe('"foo: bar"'); + expect(fm.status).toBe("open"); + }); +}); + +describe("extractPrimaryArtifacts", () => { + test("extracts tools/ paths from Acceptance section", () => { + const body = `--- +id: B-0001 +--- + +# Title + +## Source + +Some context. + +## Acceptance + +- New \`tools/hygiene/foo.ts\` +- Tests at \`tools/hygiene/foo.test.ts\` + +## Composes with + +- \`tools/orchestrator-checks/verify-branch.ts\` (sibling) +`; + const paths = extractPrimaryArtifacts(body); + expect(paths).toContain("tools/hygiene/foo.ts"); + expect(paths).toContain("tools/hygiene/foo.test.ts"); + expect(paths).not.toContain("tools/orchestrator-checks/verify-branch.ts"); + }); + + test("skips Composes with section paths (load-bearing false-positive defence)", () => { + const body = `--- +id: B-0002 +--- + +# T + +## Composes with + +- \`tools/foo.ts\` +- \`.claude/rules/bar.md\` +`; + expect(extractPrimaryArtifacts(body)).toEqual([]); + }); + + test("skips Origin, Source, Non-goals, and Resolution sections", () => { + const body = `--- +id: B-0003 +--- + +## Origin + +Per \`tools/old.ts\` — old reference, NOT primary. + +## Non-goals + +Refactoring \`tools/scope-creep.ts\` — explicitly out-of-scope. + +## Resolution + +Closed via \`tools/done.ts\`. + +## Acceptance + +- \`tools/new.ts\` +`; + const paths = extractPrimaryArtifacts(body); + expect(paths).toEqual(["tools/new.ts"]); + expect(paths).not.toContain("tools/old.ts"); + expect(paths).not.toContain("tools/scope-creep.ts"); + expect(paths).not.toContain("tools/done.ts"); + }); + + test("skips backlog cross-refs", () => { + const body = `## Acceptance + +- See \`docs/backlog/P3/B-0001-foo.md\` for context. +- Add \`tools/x.ts\`. +`; + const paths = extractPrimaryArtifacts(body); + expect(paths).toEqual(["tools/x.ts"]); + expect(paths).not.toContain("docs/backlog/P3/B-0001-foo.md"); + }); + + test("extracts paths from Proposed mechanization section", () => { + const body = `## Proposed mechanization + +Add \`tools/hygiene/audit-foo.ts\` that does X. +Also wire \`.claude/rules/foo-rule.md\`. + +## Composes with + +- \`tools/sibling.ts\` +`; + const paths = extractPrimaryArtifacts(body); + expect(paths).toContain("tools/hygiene/audit-foo.ts"); + expect(paths).toContain(".claude/rules/foo-rule.md"); + expect(paths).not.toContain("tools/sibling.ts"); + }); + + test("extracts paths from Scope section", () => { + const body = `## Scope + +Add \`tools/scope-target.ts\`. +`; + expect(extractPrimaryArtifacts(body)).toEqual(["tools/scope-target.ts"]); + }); + + test("INLINE_CROSSREF: 'Composes with X' bullet inside Acceptance section is NOT a deliverable", () => { + // Empirical case from B-0518 (Sharpening 4): an Acceptance sub-section + // contains "Composes with `.claude/rules/encoding-rules-without-mechanizing.md`" + // as a bullet — that's a sibling reference, not a deliverable. + const body = `## Acceptance + +- [ ] New \`tools/foo.ts\` +- [ ] Composes with \`.claude/rules/bar.md\` +`; + const paths = extractPrimaryArtifacts(body); + expect(paths).toEqual(["tools/foo.ts"]); + expect(paths).not.toContain(".claude/rules/bar.md"); + }); + + test("INLINE_CROSSREF: 'sister mechanism' references skip", () => { + const body = `## Proposed mechanization + +- New \`tools/audit.ts\` +- Sister mechanism: \`tools/orchestrator-checks/verify-branch.ts\` +`; + expect(extractPrimaryArtifacts(body)).toEqual(["tools/audit.ts"]); + }); + + test("INLINE_CROSSREF: 'see also' / 'per' / 'references' patterns skip", () => { + const body = `## Acceptance + +- New \`tools/x.ts\` +- See also \`tools/sibling-a.ts\` for shape +- Per \`.claude/rules/some-rule.md\` discipline +- References \`docs/some-doc.md\` for background +`; + const paths = extractPrimaryArtifacts(body); + expect(paths).toEqual(["tools/x.ts"]); + }); + + test("Empirical case from B-0553: composes_with paths NOT in primary sections must be skipped", () => { + const body = `--- +id: B-0116 +status: open +composes_with: + - tools/github/poll-pr-gate.ts +--- + +# B-0116 — tools/gh-jq-safe.sh wrapper + +## Source + +Deepseek peer review 2026-04-30 etc. + +## What + +Add a small wrapper script. + +## Composes with + +- \`tools/github/poll-pr-gate.ts\` — cross-ref + +## Acceptance criteria + +- New \`tools/gh-jq-safe.sh\` wrapper script +`; + const paths = extractPrimaryArtifacts(body); + expect(paths).toEqual(["tools/gh-jq-safe.sh"]); + expect(paths).not.toContain("tools/github/poll-pr-gate.ts"); + }); +}); + +describe("findDriftCandidates", () => { + test("returns rows where all primary artifacts exist on disk", () => { + const rows: readonly BacklogRow[] = [ + { + id: "B-0001", + path: "docs/backlog/P3/fake.md", + status: "open", + primaryArtifacts: ["tools/hygiene/audit-backlog-status-drift.ts"], // exists + }, + { + id: "B-0002", + path: "docs/backlog/P3/fake2.md", + status: "open", + primaryArtifacts: ["tools/hygiene/does-not-exist.ts"], + }, + ]; + const candidates = findDriftCandidates(rows); + expect(candidates.map((r) => r.id)).toEqual(["B-0001"]); + }); + + test("does NOT flag rows with empty primary-artifact lists", () => { + const rows: readonly BacklogRow[] = [ + { + id: "B-9999", + path: "fake", + status: "open", + primaryArtifacts: [], + }, + ]; + expect(findDriftCandidates(rows)).toEqual([]); + }); + + test("requires ALL primary artifacts to exist (mixed → not a candidate)", () => { + const rows: readonly BacklogRow[] = [ + { + id: "B-mixed", + path: "fake", + status: "open", + primaryArtifacts: [ + "tools/hygiene/audit-backlog-status-drift.ts", // exists + "tools/does/not/exist.ts", + ], + }, + ]; + expect(findDriftCandidates(rows)).toEqual([]); + }); +}); diff --git a/tools/hygiene/audit-backlog-status-drift.ts b/tools/hygiene/audit-backlog-status-drift.ts new file mode 100644 index 000000000..27760140b --- /dev/null +++ b/tools/hygiene/audit-backlog-status-drift.ts @@ -0,0 +1,282 @@ +#!/usr/bin/env bun +// audit-backlog-status-drift.ts — detect `status: open` backlog rows whose +// primary artifacts have all shipped, indicating substrate drift. +// +// Per docs/backlog/P3/B-0553-audit-backlog-status-drift-detection-2026-05-16.md +// and memory/feedback_substrate_drift_catch_pattern_claim_acquire_plus_existence_check_otto_cli_2026_05_16.md. +// +// What this does: +// +// - Enumerate docs/backlog/P*/B-*.md rows +// - For each `status: open` row, parse the body to extract primary-artifact +// paths from the **Acceptance / Proposed mechanization / Scope** sections +// ONLY (NOT composes_with: frontmatter, NOT `## Composes with` body section, +// NOT Origin / Source / Why P? / Non-goals / Resolution) +// - Existence-check every primary-artifact path on disk +// - Report rows where ALL primary-artifact paths exist (drift candidates) +// +// Section-aware parsing is the load-bearing detail per B-0553's empirical +// false-positive catalog: a naive `grep -oE 'tools/[a-z0-9_/-]+\.ts'` over the +// whole body had a 4-of-4 false-positive rate because it matched composes_with +// cross-refs as primary artifacts. +// +// Out of scope (next slice): +// +// - `--prune-claims`: release any matching bus claim entries +// - `--open-close-pr`: auto-open close-row PRs +// - Partial-vs-drift verification (the tool flags candidates; human/agent +// verifies each acceptance bullet shipped before close-row PR) +// +// Usage: +// +// bun tools/hygiene/audit-backlog-status-drift.ts # markdown report +// bun tools/hygiene/audit-backlog-status-drift.ts --json # JSON output +// +// Exit codes: +// +// 0 no candidates found (or candidates reported in detect-only mode) +// 64 argument error + +import { readdirSync, readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +type PrimarySectionMatcher = RegExp; + +// Sections where path mentions count as primary-artifact references. +const PRIMARY_SECTIONS: readonly PrimarySectionMatcher[] = [ + /^##\s+Acceptance(\s|$)/i, + /^##\s+Acceptance criteria/i, + /^##\s+Proposed mechanization/i, + /^##\s+Scope(\s|$)/i, +] as const; + +// Sections that mention paths as cross-references, NOT primary artifacts. +const SKIP_SECTIONS: readonly RegExp[] = [ + /^##\s+Composes with/i, + /^##\s+Origin/i, + /^##\s+Source/i, + /^##\s+Why P[0-9]/i, + /^##\s+Non-goals/i, + /^##\s+Substrate-honest framing/i, + /^##\s+Resolution/i, + /^##\s+Background/i, + /^##\s+Problem/i, + /^##\s+Problem statement/i, + /^##\s+Problem class/i, + /^##\s+Empirical anchor/i, + /^##\s+Alternative mitigations/i, + /^##\s+Wire-up/i, +] as const; + +// Path classes we treat as primary artifacts when mentioned in primary sections. +const PATH_REGEX = + /(? { + const match = body.match(/^---\n([\s\S]*?)\n---/); + if (!match) return {}; + const fm: Record = {}; + for (const line of match[1].split("\n")) { + const colonIdx = line.indexOf(":"); + if (colonIdx === -1) continue; + const key = line.slice(0, colonIdx).trim(); + const value = line.slice(colonIdx + 1).trim(); + fm[key] = value; + } + return fm; +} + +/** + * Extract primary-artifact paths from the row body. + * + * A primary-artifact path is a tools/ / .claude/ / docs/ path mentioned in + * an Acceptance / Proposed mechanization / Scope section. Path mentions in + * `Composes with` / Origin / Source / Why P? / Non-goals are treated as + * cross-references and skipped. + * + * Paths under docs/backlog/ are always skipped (they're row cross-refs, not + * deliverables). + */ +export function extractPrimaryArtifacts(body: string): string[] { + const lines = body.split("\n"); + const artifacts = new Set(); + type SectionMode = "primary" | "skip" | "other"; + let sectionMode: SectionMode = "other"; + + for (const line of lines) { + const isH2 = /^## \S/.test(line); + const isH3 = /^### \S/.test(line); + + if (isH2) { + // H2: tri-state classification — PRIMARY_SECTIONS gate extraction, + // SKIP_SECTIONS are explicitly recognised cross-reference sections, + // everything else is "other" (no extraction). Making SKIP_SECTIONS + // load-bearing per github-code-quality finding on PR #3758. + if (PRIMARY_SECTIONS.some((re) => re.test(line))) { + sectionMode = "primary"; + } else if (SKIP_SECTIONS.some((re) => re.test(line))) { + sectionMode = "skip"; + } else { + sectionMode = "other"; + } + continue; + } + if (isH3) { + // H3: per Codex P2 on PR #3758. Two cases: + // (1) `### Acceptance criteria` as top-level heading (no ## parent) + // — must enter primary mode, else systematic false negatives. + // (2) `### Sharpening N` nested inside `## Acceptance criteria` — + // must NOT reset mode (parent already set primary). Inherit. + // Strategy: rewrite ### → ## for pattern matching; if pattern + // matches, switch mode; else KEEP current mode (inherit parent). + const asH2 = line.replace(/^### /, "## "); + if (PRIMARY_SECTIONS.some((re) => re.test(asH2))) { + sectionMode = "primary"; + } else if (SKIP_SECTIONS.some((re) => re.test(asH2))) { + sectionMode = "skip"; + } + // else: keep current mode (nested H3 inherits parent's classification) + continue; + } + const inPrimarySection = sectionMode === "primary"; + if (!inPrimarySection) continue; + + // Skip inline cross-reference lines (e.g. "Composes with `tools/x.ts`" + // bullets inside an Acceptance sub-section — these are siblings, not + // deliverables). + if (INLINE_CROSSREF_PATTERNS.some((re) => re.test(line))) continue; + + for (const match of line.matchAll(PATH_REGEX)) { + const candidate = match[0]; + // Skip backlog-row cross-references (these are siblings, not deliverables). + if (candidate.startsWith("docs/backlog/")) continue; + artifacts.add(candidate); + } + } + + return [...artifacts]; +} + +export function enumerateOpenRows(backlogDir: string = "docs/backlog"): BacklogRow[] { + const rows: BacklogRow[] = []; + for (const priority of ["P0", "P1", "P2", "P3", "P4"]) { + const dir = join(backlogDir, priority); + if (!existsSync(dir)) continue; + for (const file of readdirSync(dir)) { + if (!file.startsWith("B-") || !file.endsWith(".md")) continue; + const path = join(dir, file); + const body = readFileSync(path, "utf-8"); + const fm = parseFrontmatter(body); + if (fm.status !== "open") continue; + const id = fm.id || file.replace(/^(B-\d+(?:\.\d+)?).*/, "$1"); + rows.push({ + id, + path, + status: fm.status, + primaryArtifacts: extractPrimaryArtifacts(body), + }); + } + } + return rows; +} + +export function findDriftCandidates(rows: readonly BacklogRow[]): BacklogRow[] { + return rows.filter((row) => { + if (row.primaryArtifacts.length === 0) return false; + return row.primaryArtifacts.every((p) => existsSync(p)); + }); +} + +function reportMarkdown(candidates: readonly BacklogRow[]): void { + if (candidates.length === 0) { + console.log("No substrate-drift candidates found."); + return; + } + + console.log("# Backlog status-drift candidates\n"); + console.log( + "Rows with `status: open` where ALL primary-artifact paths exist on disk.\n", + ); + console.log("| Row | Path | Primary artifacts |"); + console.log("|---|---|---|"); + for (const row of candidates) { + const artifacts = row.primaryArtifacts.map((a) => `\`${a}\``).join(", "); + console.log(`| ${row.id} | \`${row.path}\` | ${artifacts} |`); + } + console.log(); + console.log( + "**Reminder**: Each candidate requires manual verification before close-row — every Acceptance bullet must have a corresponding merged PR, not just the file present. See `.claude/rules/backlog-item-start-gate.md` step 0 for the partial-vs-drift discriminator.", + ); +} + +function reportJson(candidates: readonly BacklogRow[]): void { + const payload = candidates.map((r) => ({ + id: r.id, + path: r.path, + primaryArtifacts: r.primaryArtifacts, + })); + console.log(JSON.stringify(payload, null, 2)); +} + +function main(): number { + const args = process.argv.slice(2); + + for (const arg of args) { + if (arg !== "--json" && arg !== "--help" && arg !== "-h") { + console.error(`Unknown argument: ${arg}`); + console.error("Usage: bun tools/hygiene/audit-backlog-status-drift.ts [--json]"); + return 64; + } + } + + if (args.includes("--help") || args.includes("-h")) { + console.log( + "Usage: bun tools/hygiene/audit-backlog-status-drift.ts [--json]\n\n" + + "Detects `status: open` backlog rows whose primary-artifact paths all\n" + + "exist on disk (substrate drift candidates).\n\n" + + "Options:\n" + + " --json Emit JSON instead of markdown table\n" + + " --help Show this help", + ); + return 0; + } + + const candidates = findDriftCandidates(enumerateOpenRows()); + + if (args.includes("--json")) { + reportJson(candidates); + } else { + reportMarkdown(candidates); + } + + return 0; +} + +if (import.meta.main) { + process.exit(main()); +}