ci: publish an honest automated review status

.github/workflows/automated-review-gate.yml

@@ -0,0 +1,101 @@
+name: Automated review gate
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened, ready_for_review]
+  pull_request_review:
+    types: [submitted, dismissed]
+  issue_comment:
+    types: [created, edited]
+
+permissions:
+  contents: read
+  issues: read
+  pull-requests: read
+  statuses: write
+
+concurrency:
+  group: automated-review-${{ github.event.pull_request.number || github.event.issue.number }}
+  cancel-in-progress: true
+
+jobs:
+  review:
+    name: publish automated review status
+    # issue_comment fires for issues too, so keep only pull request comments.
+    # pull_request_review hands fork pull requests a read-only token that cannot
+    # write a commit status, so skip it there and let the trusted
+    # pull_request_target and issue_comment runs publish the decision instead.
+    if: >-
+      (github.event_name != 'issue_comment' || github.event.issue.pull_request) &&
+      (github.event_name != 'pull_request_review' ||
+       github.event.pull_request.head.repo.full_name == github.repository)
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      # This workflow has pull_request_target authority. Check out only the
+      # trusted default branch and never execute files from the PR head. The
+      # base commit of an older pull request predates this gate, so reading the
+      # gate from the default branch is what keeps the status computable.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+      - name: Require an automated review for the current commit
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          retries: 3
+          script: |
+            const { pathToFileURL } = require("node:url");
+            const gateUrl = pathToFileURL(
+              `${process.env.GITHUB_WORKSPACE}/scripts/ci/automated-review-gate.mjs`,
+            ).href;
+            let pullRequest = context.payload.pull_request;
+            if (!pullRequest) {
+              const response = await github.rest.pulls.get({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: context.payload.issue.number,
+              });
+              pullRequest = response.data;
+            }
+            const headSha = pullRequest.head.sha;
+            let publishAutomatedReviewStatus;
+            try {
+              ({ publishAutomatedReviewStatus } = await import(gateUrl));
+            } catch (error) {
+              // A missing or broken gate must show up as a red status, not as
+              // an absent one. Absent is what a skipped review already looks
+              // like, and telling those apart is the point of this workflow.
+              await github.rest.repos.createCommitStatus({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                sha: headSha,
+                state: "failure",
+                context: "Automated review",
+                description:
+                  "Review gate is unavailable on the default branch",
+                target_url: pullRequest.html_url,
+              });
+              core.setFailed(`Could not load the review gate: ${error}`);
+              return;
+            }
+            const result = await publishAutomatedReviewStatus({
+              github,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pullNumber: pullRequest.number,
+              headSha,
+              pullUrl: pullRequest.html_url,
+              isDraft: pullRequest.draft === true,
+            });
+            if (result.state === "pending") {
+              core.notice(
+                "Draft pull request: the automated review status stays pending.",
+              );
+            } else if (result.review) {
+              core.notice(
+                `Automated review submitted by ${result.review.reviewer} for ${headSha.slice(0, 12)}.`,
+              );
+            } else {
+              core.warning(result.failure.message);
+            }

deno.json

@@ -559,7 +559,7 @@
     "lint:render-mode-defaults": "deno run --allow-read scripts/lint/audit-render-mode-defaults.ts",
     "lint:dnt-meta-properties": "deno run --config=scripts/test.deno.json --frozen --allow-read scripts/build/dnt-meta-property-safety.ts",
     "test:layout": "deno run --config=scripts/test.deno.json --no-check --allow-read --allow-run=git --allow-env=TEST_LAYOUT_MIGRATION_BASE_REF scripts/test/test-layout.ts",
-    "test:scripts": "deno test --config=scripts/test.deno.json --no-check --allow-read --allow-write --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE --allow-run scripts/lint/check-deno-version.test.ts scripts/ci/prepare-rc-build.test.ts scripts/ci/publish-npm-packages.test.ts scripts/ci/setup-deno-workflow.test.ts scripts/test/coverage-ci.test.ts scripts/test/run-test-file.test.ts scripts/test/suites.test.ts scripts/test/test-layout.test.ts scripts/build/compile-binary.test.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.test.ts scripts/build/generate-sbom.test.ts scripts/build/generated-artifact-checks.test.ts scripts/build/npm-dependency-sources.test.ts scripts/build/npm-extension-package-metadata.test.ts scripts/build/npm-package-metadata.test.ts scripts/build/npm-react-shims.test.ts scripts/build/npm-runtime-helper-contract.test.ts scripts/build/prepare-framework-sources.test.ts scripts/build/report-artifact-sizes.test.ts scripts/build/run-generate.test.ts scripts/docs/docs-coverage.test.ts scripts/docs/generate-api-reference.test.ts scripts/docs/guide-validation.test.ts scripts/docs/validate-tracked-docs.test.ts scripts/lint/audit-chat-composability.test.ts scripts/lint/audit-core-deps.test.ts scripts/lint/audit-cwd-relative-test-reads.test.ts scripts/lint/audit-cross-runtime-jsr.test.ts scripts/lint/audit-dependency-boundaries.test.ts scripts/lint/audit-extension-capabilities.test.ts scripts/lint/audit-extension-contracts.test.ts scripts/lint/audit-deps.test.ts scripts/lint/check-module-boundaries.test.ts scripts/lint/lint-config.test.ts scripts/lint/ban-test-only.test.ts scripts/lint/check-sanitizer-baseline.test.ts scripts/lint/check-skipped-tests-baseline.test.ts scripts/lint/audit-anti-slop.test.ts scripts/lint/audit-render-mode-defaults.test.ts scripts/lint/check-test-typecheck-baseline.test.ts scripts/lint/check-coverage.test.ts scripts/security/audit-npm.test.ts scripts/security/secret-scanning-config.test.ts scripts/security/submit-dependency-snapshot.test.ts scripts/test/template-runtime-e2e.test.ts scripts/test/runtime-inference-critical-flow.test.ts && deno task test:tool-search-live",
+    "test:scripts": "deno test --config=scripts/test.deno.json --no-check --allow-read --allow-write --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE --allow-run scripts/lint/check-deno-version.test.ts scripts/ci/automated-review-gate.test.ts scripts/ci/prepare-rc-build.test.ts scripts/ci/publish-npm-packages.test.ts scripts/ci/setup-deno-workflow.test.ts scripts/test/coverage-ci.test.ts scripts/test/run-test-file.test.ts scripts/test/suites.test.ts scripts/test/test-layout.test.ts scripts/build/compile-binary.test.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.test.ts scripts/build/generate-sbom.test.ts scripts/build/generated-artifact-checks.test.ts scripts/build/npm-dependency-sources.test.ts scripts/build/npm-extension-package-metadata.test.ts scripts/build/npm-package-metadata.test.ts scripts/build/npm-react-shims.test.ts scripts/build/npm-runtime-helper-contract.test.ts scripts/build/prepare-framework-sources.test.ts scripts/build/report-artifact-sizes.test.ts scripts/build/run-generate.test.ts scripts/docs/docs-coverage.test.ts scripts/docs/generate-api-reference.test.ts scripts/docs/guide-validation.test.ts scripts/docs/validate-tracked-docs.test.ts scripts/lint/audit-chat-composability.test.ts scripts/lint/audit-core-deps.test.ts scripts/lint/audit-cwd-relative-test-reads.test.ts scripts/lint/audit-cross-runtime-jsr.test.ts scripts/lint/audit-dependency-boundaries.test.ts scripts/lint/audit-extension-capabilities.test.ts scripts/lint/audit-extension-contracts.test.ts scripts/lint/audit-deps.test.ts scripts/lint/check-module-boundaries.test.ts scripts/lint/lint-config.test.ts scripts/lint/ban-test-only.test.ts scripts/lint/check-sanitizer-baseline.test.ts scripts/lint/check-skipped-tests-baseline.test.ts scripts/lint/audit-anti-slop.test.ts scripts/lint/audit-render-mode-defaults.test.ts scripts/lint/check-test-typecheck-baseline.test.ts scripts/lint/check-coverage.test.ts scripts/security/audit-npm.test.ts scripts/security/secret-scanning-config.test.ts scripts/security/submit-dependency-snapshot.test.ts scripts/test/template-runtime-e2e.test.ts scripts/test/runtime-inference-critical-flow.test.ts && deno task test:tool-search-live",
     "test:sentry-runtime-packages": "deno test --config=scripts/test.deno.json --no-check --no-lock --allow-read --allow-write --allow-run --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE scripts/build/sentry-runtime-packages.test.ts",
     "test:tool-search-live": "VF_DISABLE_LRU_INTERVAL=1 deno test --no-check -A tests/agent/verify-tool-search-live.test.ts",
     "test:cross-runtime": "deno run --allow-all src/platform/compat/cross-runtime.test.ts",

scripts/ci/automated-review-gate.mjs

@@ -0,0 +1,128 @@
+const AUTOMATED_REVIEW_LOGINS = new Set([
+  "coderabbitai[bot]",
+  "chatgpt-codex-connector[bot]",
+]);
+const CODERABBIT_LOGIN = "coderabbitai[bot]";
+const CODERABBIT_RECENT_REVIEW_MARKER = "<!-- recent_review_start -->";
+export const AUTOMATED_REVIEW_STATUS_CONTEXT = "Automated review";
+const SUBMITTED_REVIEW_STATES = new Set([
+  "APPROVED",
+  "CHANGES_REQUESTED",
+  "COMMENTED",
+]);
+
+/** Find an actual automated review submitted against the current PR head. */
+export function findAutomatedReview({ reviews, comments }, headSha) {
+  for (let index = reviews.length - 1; index >= 0; index--) {
+    const review = reviews[index];
+    const login = review?.user?.login;
+    const state = review?.state;
+    if (
+      typeof login !== "string" ||
+      !AUTOMATED_REVIEW_LOGINS.has(login.toLowerCase()) ||
+      typeof state !== "string" ||
+      !SUBMITTED_REVIEW_STATES.has(state.toUpperCase()) ||
+      review?.commit_id !== headSha ||
+      typeof review?.submitted_at !== "string" ||
+      review.submitted_at.length === 0
+    ) {
+      continue;
+    }
+    return {
+      reviewer: login,
+      source: "review",
+      state: state.toUpperCase(),
+      url: typeof review.html_url === "string" ? review.html_url : undefined,
+    };
+  }
+
+  for (let index = comments.length - 1; index >= 0; index--) {
+    const comment = comments[index];
+    const login = comment?.user?.login;
+    const body = comment?.body;
+    if (
+      typeof login !== "string" ||
+      login.toLowerCase() !== CODERABBIT_LOGIN ||
+      typeof body !== "string" ||
+      !body.includes(CODERABBIT_RECENT_REVIEW_MARKER) ||
+      !body.includes(headSha)
+    ) {
+      continue;
+    }
+    return {
+      reviewer: login,
+      source: "summary",
+      state: "COMMENTED",
+      url: typeof comment.html_url === "string" ? comment.html_url : undefined,
+    };
+  }
+  return undefined;
+}
+
+/** Publish the current automated-review decision on the exact PR head SHA. */
+export async function publishAutomatedReviewStatus({
+  github,
+  owner,
+  repo,
+  pullNumber,
+  headSha,
+  pullUrl,
+  isDraft = false,
+}) {
+  // Review bots skip drafts, so a draft has no verdict yet. Publish pending so
+  // "not reviewed yet" never renders as a pass and never as a missing status.
+  if (isDraft) {
+    await github.rest.repos.createCommitStatus({
+      owner,
+      repo,
+      sha: headSha,
+      state: "pending",
+      context: AUTOMATED_REVIEW_STATUS_CONTEXT,
+      description: "Draft pull request waits for ready for review",
+      target_url: pullUrl,
+    });
+    return { state: "pending", review: undefined, failure: undefined };
+  }
+
+  let review;
+  let failure;
+  try {
+    const reviews = await github.paginate(github.rest.pulls.listReviews, {
+      owner,
+      repo,
+      pull_number: pullNumber,
+      per_page: 100,
+    });
+    const comments = await github.paginate(github.rest.issues.listComments, {
+      owner,
+      repo,
+      issue_number: pullNumber,
+      per_page: 100,
+    });
+    review = findAutomatedReview({ reviews, comments }, headSha);
+    if (!review) {
+      failure = new Error(
+        `No automated review was submitted for current commit ${
+          headSha.slice(0, 12)
+        }. ` +
+          "CodeRabbit and Codex skip or rate-limit comments do not count as reviews.",
+      );
+    }
+  } catch (error) {
+    failure = error instanceof Error ? error : new Error(String(error));
+  }
+
+  const state = review ? "success" : "failure";
+  await github.rest.repos.createCommitStatus({
+    owner,
+    repo,
+    sha: headSha,
+    state,
+    context: AUTOMATED_REVIEW_STATUS_CONTEXT,
+    description: review
+      ? `Reviewed by ${review.reviewer}`
+      : "No CodeRabbit or Codex review for current commit",
+    target_url: review?.url ?? pullUrl,
+  });
+  return { state, review, failure };
+}