Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion scripts/check/check-pr-evidence.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,16 @@ if (isMain) {
} else if (result === "pass") {
reportLines.push("Result: PASS", "", reason);
} else {
reportLines.push("Result: FAIL", "", reason);
reportLines.push(
"Result: FAIL",
"",
reason,
"",
"> ℹ️ Editing the PR body to add the evidence does NOT re-run this gate — `ci.yml` " +
"does not listen to the `edited` event. Add the `## Evidence` block, then **push a " +
"commit** (or re-run this job) to re-validate. For releases, put the Evidence block in " +
"the body BEFORE the first push (see the generate-release skill, Phase 0)."
);
}

const report = buildReport(reportLines);
Expand Down
55 changes: 55 additions & 0 deletions tests/unit/check-pr-evidence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import test from "node:test";
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import path from "node:path";
import { fileURLToPath } from "node:url";

const mod = await import("../../scripts/check/check-pr-evidence.mjs");
const { evaluatePrBody } = mod;
const SCRIPT = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"../../scripts/check/check-pr-evidence.mjs"
);

function run(body) {
try {
const out = execFileSync("node", [SCRIPT], {
encoding: "utf8",
env: { ...process.env, PR_BODY: body },
});
return { code: 0, out };
} catch (err) {
return { code: err.status ?? 1, out: `${err.stdout || ""}${err.stderr || ""}` };
}
}

test("evaluatePrBody: no outcome claim → pass (no evidence required)", () => {
const r = evaluatePrBody("Adds a helper module.");
assert.equal(r.result, "pass");
assert.match(r.reason, /no evidence required/i);
});

test("evaluatePrBody: outcome claim + evidence block → pass", () => {
const r = evaluatePrBody("Tests pass.\n\n## Evidence\n```\ntests 20 / pass 20 / fail 0\n```");
assert.equal(r.result, "pass");
});

test("evaluatePrBody: outcome claim without evidence → fail", () => {
const r = evaluatePrBody("All 20 tests pass and 0 errors.");
assert.equal(r.result, "fail");
});

test("the FAIL report explains that editing the body does not re-run the gate (push instead)", () => {
const { code, out } = run("All 20 tests pass and 0 errors."); // claim, no evidence
assert.equal(code, 1, "gate fails on a claim with no evidence");
assert.match(out, /Result: FAIL/);
assert.match(out, /does NOT re-run this gate/);
assert.match(out, /push a commit/i);
});

test("the hint does NOT appear when the gate passes", () => {
const { code, out } = run("Tests pass.\n\n## Evidence\n```\ntests 20 / pass 20 / fail 0\n```");
assert.equal(code, 0);
assert.match(out, /Result: PASS/);
assert.doesNotMatch(out, /does NOT re-run this gate/);
});
Loading