diff --git a/.github/workflows/automated-review-gate.yml b/.github/workflows/automated-review-gate.yml new file mode 100644 index 0000000000..e02b33750d --- /dev/null +++ b/.github/workflows/automated-review-gate.yml @@ -0,0 +1,101 @@ +name: Automated review gate + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + pull_request_review: + types: [submitted, dismissed] + issue_comment: + types: [created, edited] + +permissions: + contents: read + issues: read + pull-requests: read + statuses: write + +concurrency: + group: automated-review-${{ github.event.pull_request.number || github.event.issue.number }} + cancel-in-progress: true + +jobs: + review: + name: publish automated review status + # issue_comment fires for issues too, so keep only pull request comments. + # pull_request_review hands fork pull requests a read-only token that cannot + # write a commit status, so skip it there and let the trusted + # pull_request_target and issue_comment runs publish the decision instead. + if: >- + (github.event_name != 'issue_comment' || github.event.issue.pull_request) && + (github.event_name != 'pull_request_review' || + github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + # This workflow has pull_request_target authority. Check out only the + # trusted default branch and never execute files from the PR head. The + # base commit of an older pull request predates this gate, so reading the + # gate from the default branch is what keeps the status computable. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Require an automated review for the current commit + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 3 + script: | + const { pathToFileURL } = require("node:url"); + const gateUrl = pathToFileURL( + `${process.env.GITHUB_WORKSPACE}/scripts/ci/automated-review-gate.mjs`, + ).href; + let pullRequest = context.payload.pull_request; + if (!pullRequest) { + const response = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number, + }); + pullRequest = response.data; + } + const headSha = pullRequest.head.sha; + let publishAutomatedReviewStatus; + try { + ({ publishAutomatedReviewStatus } = await import(gateUrl)); + } catch (error) { + // A missing or broken gate must show up as a red status, not as + // an absent one. Absent is what a skipped review already looks + // like, and telling those apart is the point of this workflow. + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: headSha, + state: "failure", + context: "Automated review", + description: + "Review gate is unavailable on the default branch", + target_url: pullRequest.html_url, + }); + core.setFailed(`Could not load the review gate: ${error}`); + return; + } + const result = await publishAutomatedReviewStatus({ + github, + owner: context.repo.owner, + repo: context.repo.repo, + pullNumber: pullRequest.number, + headSha, + pullUrl: pullRequest.html_url, + isDraft: pullRequest.draft === true, + }); + if (result.state === "pending") { + core.notice( + "Draft pull request: the automated review status stays pending.", + ); + } else if (result.review) { + core.notice( + `Automated review submitted by ${result.review.reviewer} for ${headSha.slice(0, 12)}.`, + ); + } else { + core.warning(result.failure.message); + } diff --git a/deno.json b/deno.json index da3b7d87ec..1818a55681 100644 --- a/deno.json +++ b/deno.json @@ -559,7 +559,7 @@ "lint:render-mode-defaults": "deno run --allow-read scripts/lint/audit-render-mode-defaults.ts", "lint:dnt-meta-properties": "deno run --config=scripts/test.deno.json --frozen --allow-read scripts/build/dnt-meta-property-safety.ts", "test:layout": "deno run --config=scripts/test.deno.json --no-check --allow-read --allow-run=git --allow-env=TEST_LAYOUT_MIGRATION_BASE_REF scripts/test/test-layout.ts", - "test:scripts": "deno test --config=scripts/test.deno.json --no-check --allow-read --allow-write --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE --allow-run scripts/lint/check-deno-version.test.ts scripts/ci/prepare-rc-build.test.ts scripts/ci/publish-npm-packages.test.ts scripts/ci/setup-deno-workflow.test.ts scripts/test/coverage-ci.test.ts scripts/test/run-test-file.test.ts scripts/test/suites.test.ts scripts/test/test-layout.test.ts scripts/build/compile-binary.test.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.test.ts scripts/build/generate-sbom.test.ts scripts/build/generated-artifact-checks.test.ts scripts/build/npm-dependency-sources.test.ts scripts/build/npm-extension-package-metadata.test.ts scripts/build/npm-package-metadata.test.ts scripts/build/npm-react-shims.test.ts scripts/build/npm-runtime-helper-contract.test.ts scripts/build/prepare-framework-sources.test.ts scripts/build/report-artifact-sizes.test.ts scripts/build/run-generate.test.ts scripts/docs/docs-coverage.test.ts scripts/docs/generate-api-reference.test.ts scripts/docs/guide-validation.test.ts scripts/docs/validate-tracked-docs.test.ts scripts/lint/audit-chat-composability.test.ts scripts/lint/audit-core-deps.test.ts scripts/lint/audit-cwd-relative-test-reads.test.ts scripts/lint/audit-cross-runtime-jsr.test.ts scripts/lint/audit-dependency-boundaries.test.ts scripts/lint/audit-extension-capabilities.test.ts scripts/lint/audit-extension-contracts.test.ts scripts/lint/audit-deps.test.ts scripts/lint/check-module-boundaries.test.ts scripts/lint/lint-config.test.ts scripts/lint/ban-test-only.test.ts scripts/lint/check-sanitizer-baseline.test.ts scripts/lint/check-skipped-tests-baseline.test.ts scripts/lint/audit-anti-slop.test.ts scripts/lint/audit-render-mode-defaults.test.ts scripts/lint/check-test-typecheck-baseline.test.ts scripts/lint/check-coverage.test.ts scripts/security/audit-npm.test.ts scripts/security/secret-scanning-config.test.ts scripts/security/submit-dependency-snapshot.test.ts scripts/test/template-runtime-e2e.test.ts scripts/test/runtime-inference-critical-flow.test.ts && deno task test:tool-search-live", + "test:scripts": "deno test --config=scripts/test.deno.json --no-check --allow-read --allow-write --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE --allow-run scripts/lint/check-deno-version.test.ts scripts/ci/automated-review-gate.test.ts scripts/ci/prepare-rc-build.test.ts scripts/ci/publish-npm-packages.test.ts scripts/ci/setup-deno-workflow.test.ts scripts/test/coverage-ci.test.ts scripts/test/run-test-file.test.ts scripts/test/suites.test.ts scripts/test/test-layout.test.ts scripts/build/compile-binary.test.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.test.ts scripts/build/generate-sbom.test.ts scripts/build/generated-artifact-checks.test.ts scripts/build/npm-dependency-sources.test.ts scripts/build/npm-extension-package-metadata.test.ts scripts/build/npm-package-metadata.test.ts scripts/build/npm-react-shims.test.ts scripts/build/npm-runtime-helper-contract.test.ts scripts/build/prepare-framework-sources.test.ts scripts/build/report-artifact-sizes.test.ts scripts/build/run-generate.test.ts scripts/docs/docs-coverage.test.ts scripts/docs/generate-api-reference.test.ts scripts/docs/guide-validation.test.ts scripts/docs/validate-tracked-docs.test.ts scripts/lint/audit-chat-composability.test.ts scripts/lint/audit-core-deps.test.ts scripts/lint/audit-cwd-relative-test-reads.test.ts scripts/lint/audit-cross-runtime-jsr.test.ts scripts/lint/audit-dependency-boundaries.test.ts scripts/lint/audit-extension-capabilities.test.ts scripts/lint/audit-extension-contracts.test.ts scripts/lint/audit-deps.test.ts scripts/lint/check-module-boundaries.test.ts scripts/lint/lint-config.test.ts scripts/lint/ban-test-only.test.ts scripts/lint/check-sanitizer-baseline.test.ts scripts/lint/check-skipped-tests-baseline.test.ts scripts/lint/audit-anti-slop.test.ts scripts/lint/audit-render-mode-defaults.test.ts scripts/lint/check-test-typecheck-baseline.test.ts scripts/lint/check-coverage.test.ts scripts/security/audit-npm.test.ts scripts/security/secret-scanning-config.test.ts scripts/security/submit-dependency-snapshot.test.ts scripts/test/template-runtime-e2e.test.ts scripts/test/runtime-inference-critical-flow.test.ts && deno task test:tool-search-live", "test:sentry-runtime-packages": "deno test --config=scripts/test.deno.json --no-check --no-lock --allow-read --allow-write --allow-run --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE scripts/build/sentry-runtime-packages.test.ts", "test:tool-search-live": "VF_DISABLE_LRU_INTERVAL=1 deno test --no-check -A tests/agent/verify-tool-search-live.test.ts", "test:cross-runtime": "deno run --allow-all src/platform/compat/cross-runtime.test.ts", diff --git a/scripts/ci/automated-review-gate.mjs b/scripts/ci/automated-review-gate.mjs new file mode 100644 index 0000000000..7d60f5856e --- /dev/null +++ b/scripts/ci/automated-review-gate.mjs @@ -0,0 +1,128 @@ +const AUTOMATED_REVIEW_LOGINS = new Set([ + "coderabbitai[bot]", + "chatgpt-codex-connector[bot]", +]); +const CODERABBIT_LOGIN = "coderabbitai[bot]"; +const CODERABBIT_RECENT_REVIEW_MARKER = ""; +export const AUTOMATED_REVIEW_STATUS_CONTEXT = "Automated review"; +const SUBMITTED_REVIEW_STATES = new Set([ + "APPROVED", + "CHANGES_REQUESTED", + "COMMENTED", +]); + +/** Find an actual automated review submitted against the current PR head. */ +export function findAutomatedReview({ reviews, comments }, headSha) { + for (let index = reviews.length - 1; index >= 0; index--) { + const review = reviews[index]; + const login = review?.user?.login; + const state = review?.state; + if ( + typeof login !== "string" || + !AUTOMATED_REVIEW_LOGINS.has(login.toLowerCase()) || + typeof state !== "string" || + !SUBMITTED_REVIEW_STATES.has(state.toUpperCase()) || + review?.commit_id !== headSha || + typeof review?.submitted_at !== "string" || + review.submitted_at.length === 0 + ) { + continue; + } + return { + reviewer: login, + source: "review", + state: state.toUpperCase(), + url: typeof review.html_url === "string" ? review.html_url : undefined, + }; + } + + for (let index = comments.length - 1; index >= 0; index--) { + const comment = comments[index]; + const login = comment?.user?.login; + const body = comment?.body; + if ( + typeof login !== "string" || + login.toLowerCase() !== CODERABBIT_LOGIN || + typeof body !== "string" || + !body.includes(CODERABBIT_RECENT_REVIEW_MARKER) || + !body.includes(headSha) + ) { + continue; + } + return { + reviewer: login, + source: "summary", + state: "COMMENTED", + url: typeof comment.html_url === "string" ? comment.html_url : undefined, + }; + } + return undefined; +} + +/** Publish the current automated-review decision on the exact PR head SHA. */ +export async function publishAutomatedReviewStatus({ + github, + owner, + repo, + pullNumber, + headSha, + pullUrl, + isDraft = false, +}) { + // Review bots skip drafts, so a draft has no verdict yet. Publish pending so + // "not reviewed yet" never renders as a pass and never as a missing status. + if (isDraft) { + await github.rest.repos.createCommitStatus({ + owner, + repo, + sha: headSha, + state: "pending", + context: AUTOMATED_REVIEW_STATUS_CONTEXT, + description: "Draft pull request waits for ready for review", + target_url: pullUrl, + }); + return { state: "pending", review: undefined, failure: undefined }; + } + + let review; + let failure; + try { + const reviews = await github.paginate(github.rest.pulls.listReviews, { + owner, + repo, + pull_number: pullNumber, + per_page: 100, + }); + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number: pullNumber, + per_page: 100, + }); + review = findAutomatedReview({ reviews, comments }, headSha); + if (!review) { + failure = new Error( + `No automated review was submitted for current commit ${ + headSha.slice(0, 12) + }. ` + + "CodeRabbit and Codex skip or rate-limit comments do not count as reviews.", + ); + } + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + } + + const state = review ? "success" : "failure"; + await github.rest.repos.createCommitStatus({ + owner, + repo, + sha: headSha, + state, + context: AUTOMATED_REVIEW_STATUS_CONTEXT, + description: review + ? `Reviewed by ${review.reviewer}` + : "No CodeRabbit or Codex review for current commit", + target_url: review?.url ?? pullUrl, + }); + return { state, review, failure }; +} diff --git a/scripts/ci/automated-review-gate.test.ts b/scripts/ci/automated-review-gate.test.ts new file mode 100644 index 0000000000..312b6df5e5 --- /dev/null +++ b/scripts/ci/automated-review-gate.test.ts @@ -0,0 +1,316 @@ +import { assert, assertEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { parse } from "#std/yaml/parse"; +import { + findAutomatedReview, + publishAutomatedReviewStatus, +} from "./automated-review-gate.mjs"; + +const HEAD_SHA = "a".repeat(40); +const STALE_SHA = "b".repeat(40); +const WORKFLOW_PATH = new URL( + "../../.github/workflows/automated-review-gate.yml", + import.meta.url, +); + +function review( + overrides: Record = {}, +): Record { + return { + user: { login: "coderabbitai[bot]" }, + state: "COMMENTED", + commit_id: HEAD_SHA, + submitted_at: "2026-08-22T12:00:00Z", + html_url: + "https://github.com/veryfront/veryfront-code/pull/1#pullrequestreview-1", + ...overrides, + }; +} + +function codeRabbitSummary( + overrides: Record = {}, +): Record { + return { + user: { login: "coderabbitai[bot]" }, + body: [ + "", + "No actionable comments were generated in the recent review.", + `Reviewing files between ${STALE_SHA} and ${HEAD_SHA}.`, + ].join("\n"), + html_url: + "https://github.com/veryfront/veryfront-code/pull/1#issuecomment-1", + ...overrides, + }; +} + +function record(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new TypeError(`${label} must be a record`); + } + return value as Record; +} + +describe("automated review gate", () => { + it("accepts submitted CodeRabbit and Codex reviews for the current head", () => { + assertEquals( + findAutomatedReview({ reviews: [review()], comments: [] }, HEAD_SHA) + ?.reviewer, + "coderabbitai[bot]", + ); + assertEquals( + findAutomatedReview({ + reviews: [ + review({ user: { login: "chatgpt-codex-connector[bot]" } }), + ], + comments: [], + }, HEAD_SHA)?.reviewer, + "chatgpt-codex-connector[bot]", + ); + assertEquals( + findAutomatedReview( + { reviews: [], comments: [codeRabbitSummary()] }, + HEAD_SHA, + ) + ?.source, + "summary", + ); + }); + + it("rejects skipped comments, stale reviews, pending reviews, and humans", () => { + const skippedIssueComment = { + user: { login: "coderabbitai[bot]" }, + body: "rate limited, review skipped", + }; + assertEquals( + findAutomatedReview({ + reviews: [ + review({ commit_id: STALE_SHA }), + review({ state: "PENDING" }), + review({ state: "DISMISSED" }), + review({ user: { login: "maintainer" } }), + ], + comments: [ + skippedIssueComment, + codeRabbitSummary({ + body: "\nstale review", + }), + ], + }, HEAD_SHA), + undefined, + ); + }); + + it("publishes the automated review decision on the exact pull request head", async () => { + const statuses: Array> = []; + const listReviews = () => Promise.resolve(); + const listComments = () => Promise.resolve(); + const github = { + paginate: (endpoint: unknown) => + Promise.resolve(endpoint === listReviews ? [review()] : []), + rest: { + issues: { listComments }, + pulls: { listReviews }, + repos: { + createCommitStatus: (status: Record) => { + statuses.push(status); + return Promise.resolve(); + }, + }, + }, + }; + const result = await publishAutomatedReviewStatus({ + github, + owner: "veryfront", + repo: "veryfront-code", + pullNumber: 1, + headSha: HEAD_SHA, + pullUrl: "https://github.com/veryfront/veryfront-code/pull/1", + }); + assertEquals(result.state, "success"); + assertEquals(statuses[0]?.sha, HEAD_SHA); + assertEquals(statuses[0]?.state, "success"); + assertEquals(statuses[0]?.context, "Automated review"); + + github.paginate = () => Promise.resolve([]); + const missing = await publishAutomatedReviewStatus({ + github, + owner: "veryfront", + repo: "veryfront-code", + pullNumber: 1, + headSha: STALE_SHA, + pullUrl: "https://github.com/veryfront/veryfront-code/pull/1", + }); + assertEquals(missing.state, "failure"); + assert( + missing.failure instanceof Error && + missing.failure.message.includes(STALE_SHA.slice(0, 12)), + "the failure must name the unreviewed head commit", + ); + assertEquals(statuses[1]?.sha, STALE_SHA); + assertEquals(statuses[1]?.state, "failure"); + }); + + it("fails closed when the review lookup throws", async () => { + const statuses: Array> = []; + const github = { + paginate: () => Promise.reject(new Error("secondary rate limit")), + rest: { + issues: { listComments: () => Promise.resolve() }, + pulls: { listReviews: () => Promise.resolve() }, + repos: { + createCommitStatus: (status: Record) => { + statuses.push(status); + return Promise.resolve(); + }, + }, + }, + }; + const result = await publishAutomatedReviewStatus({ + github, + owner: "veryfront", + repo: "veryfront-code", + pullNumber: 1, + headSha: HEAD_SHA, + pullUrl: "https://github.com/veryfront/veryfront-code/pull/1", + }); + assertEquals(result.state, "failure"); + assertEquals(result.review, undefined); + assert( + result.failure instanceof Error && + result.failure.message.includes("secondary rate limit"), + "the transport error must be reported, not swallowed", + ); + assertEquals(statuses.length, 1); + assertEquals(statuses[0]?.state, "failure"); + assertEquals(statuses[0]?.sha, HEAD_SHA); + }); + + it("holds a draft pull request at pending instead of success", async () => { + const statuses: Array> = []; + let listed = false; + const github = { + paginate: () => { + listed = true; + return Promise.resolve([review()]); + }, + rest: { + issues: { listComments: () => Promise.resolve() }, + pulls: { listReviews: () => Promise.resolve() }, + repos: { + createCommitStatus: (status: Record) => { + statuses.push(status); + return Promise.resolve(); + }, + }, + }, + }; + const result = await publishAutomatedReviewStatus({ + github, + owner: "veryfront", + repo: "veryfront-code", + pullNumber: 1, + headSha: HEAD_SHA, + pullUrl: "https://github.com/veryfront/veryfront-code/pull/1", + isDraft: true, + }); + assertEquals(result.state, "pending"); + assertEquals(listed, false); + assertEquals(statuses.length, 1); + assertEquals(statuses[0]?.state, "pending"); + assertEquals(statuses[0]?.context, "Automated review"); + assertEquals(statuses[0]?.sha, HEAD_SHA); + }); + + it("uses trusted base code and reruns when a real review is submitted", async () => { + const workflow = record( + parse(await Deno.readTextFile(WORKFLOW_PATH)), + "automated review workflow", + ); + const triggers = record(workflow.on, "automated review triggers"); + assertEquals( + record(triggers.pull_request_target, "pull_request_target trigger").types, + ["opened", "synchronize", "reopened", "ready_for_review"], + ); + assertEquals( + record(triggers.pull_request_review, "pull_request_review trigger").types, + ["submitted", "dismissed"], + ); + assertEquals( + record(triggers.issue_comment, "issue_comment trigger").types, + ["created", "edited"], + ); + + const permissions = record( + workflow.permissions, + "automated review permissions", + ); + assertEquals(permissions, { + contents: "read", + issues: "read", + "pull-requests": "read", + statuses: "write", + }); + + const job = record( + record(workflow.jobs, "automated review jobs").review, + "automated review job", + ); + assertEquals(job.name, "publish automated review status"); + assertEquals(job["timeout-minutes"], 5); + assert( + typeof job.if === "string" && + job.if.includes("github.event.issue.pull_request"), + "issue comments must run the gate only for pull requests", + ); + assert( + typeof job.if === "string" && + job.if.includes( + "github.event.pull_request.head.repo.full_name == github.repository", + ), + "review events must be skipped on forks, where the token cannot write a status", + ); + + const steps = job.steps; + assert(Array.isArray(steps), "automated review job steps must be an array"); + const checkout = record(steps[0], "automated review checkout"); + assertEquals( + record(checkout.with, "automated review checkout inputs").ref, + "${{ github.event.repository.default_branch }}", + ); + assertEquals( + record( + checkout.with, + "automated review checkout inputs", + )["persist-credentials"], + false, + ); + const gate = record(steps[1], "automated review gate step"); + assertEquals( + gate.uses, + "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3", + ); + assert( + typeof record(gate.with, "automated review gate inputs").script === + "string" && + String(record(gate.with, "automated review gate inputs").script) + .includes( + "publishAutomatedReviewStatus", + ), + "the workflow must call the tested review gate", + ); + assert( + String(record(gate.with, "automated review gate inputs").script).includes( + "isDraft: pullRequest.draft === true", + ), + "the workflow must hand draft state to the tested review gate", + ); + const script = String( + record(gate.with, "automated review gate inputs").script, + ); + assert( + script.includes("Review gate is unavailable on the default branch") && + script.includes('state: "failure"'), + "a gate that fails to load must publish a failure status, not no status", + ); + }); +});