diff --git a/.github/scripts/codex-security-review.js b/.github/scripts/codex-security-review.js new file mode 100644 index 00000000000..ac54adb6658 --- /dev/null +++ b/.github/scripts/codex-security-review.js @@ -0,0 +1,746 @@ +"use strict"; + +const MARKER = ""; +const STALE_MARKER = ""; +const REVIEW_COMMAND = "@buzz-security-review"; +const CURRENT_REVIEW_LABEL = "codex-security-review-current"; +const RECONCILIATION_BATCH_SIZE = 32; +const MAX_RECONCILIATION_PASSES = 2; +const GITHUB_RETRY_ATTEMPTS = 3; +const RISKS = ["NONE", "LOW", "MEDIUM", "HIGH", "CRITICAL"]; +const SEVERITIES = new Set(RISKS.slice(1)); +const CATEGORIES = new Set([ + "Isolation", + "Auth", + "Event Integrity", + "Cryptography", + "Injection", + "Agent/Workflow", + "Desktop/Mobile", + "Concurrency", + "Reliability", + "Supply Chain", + "Other", +]); + +const completedMarker = (baseSha, headSha) => + ``; + +const reviewCommand = (headSha) => `${REVIEW_COMMAND} ${headSha}`; + +const isObject = (value) => + value !== null && typeof value === "object" && !Array.isArray(value); + +function requireKeys(value, expected, label) { + if (!isObject(value)) { + throw new Error(`${label} must be an object.`); + } + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) { + throw new Error(`${label} has unexpected or missing properties.`); + } +} + +function requireString(value, maxLength, label) { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > maxLength + ) { + throw new Error( + `${label} must be a non-empty string of at most ${maxLength} characters.`, + ); + } + return value; +} + +function safeCodeText(value, maxLength, label) { + const input = requireString(value, maxLength, label) + .replace( + /[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/g, + " ", + ) + .trim(); + if (!input) { + throw new Error(`${label} is empty after removing control characters.`); + } + + const longestBacktickRun = Math.max( + 0, + ...(input.match(/`+/g) || []).map((run) => run.length), + ); + const fence = "`".repeat(longestBacktickRun + 1); + return `${fence} ${input} ${fence}`; +} + +function validPath(value) { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= 500 && + !value.startsWith("/") && + !value.includes("\\") && + !/[\u0000-\u001f\u007f]/.test(value) && + !value.split("/").includes("..") + ); +} + +const encodeUrlComponent = (value) => + encodeURIComponent(value).replace( + /[!'()*]/g, + (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`, + ); + +const encodePath = (value) => + value.split("/").map(encodeUrlComponent).join("/"); + +const githubErrorStatus = (error) => + Number(error?.status ?? error?.response?.status); + +function isRetryableGithubError(error) { + const status = githubErrorStatus(error); + if (status === 429 || (status >= 500 && status <= 504)) { + return true; + } + if (status !== 403) { + return false; + } + + const headers = error?.response?.headers || {}; + const message = `${error?.message || ""} ${error?.response?.data?.message || ""}`; + return ( + headers["retry-after"] !== undefined || + headers["x-ratelimit-remaining"] === "0" || + message.toLowerCase().includes("rate limit") + ); +} + +function githubRetryDelayMs(error, attempt) { + const headers = error?.response?.headers || {}; + const retryAfterSeconds = Number(headers["retry-after"]); + if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0) { + return Math.min(Math.max(retryAfterSeconds * 1000, 1000), 120000); + } + + const resetSeconds = Number(headers["x-ratelimit-reset"]); + if ( + headers["x-ratelimit-remaining"] === "0" && + Number.isFinite(resetSeconds) + ) { + return Math.min( + Math.max(resetSeconds * 1000 - Date.now() + 1000, 1000), + 120000, + ); + } + + const status = githubErrorStatus(error); + const baseDelay = status === 403 || status === 429 ? 60000 : 2000; + return Math.min(baseDelay * 2 ** (attempt - 1), 120000); +} + +async function withGithubRetry( + operation, + { + core, + sleep = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)), + }, +) { + for (let attempt = 1; attempt <= GITHUB_RETRY_ATTEMPTS; attempt += 1) { + try { + return await operation(); + } catch (error) { + if ( + attempt === GITHUB_RETRY_ATTEMPTS || + !isRetryableGithubError(error) + ) { + throw error; + } + const delay = githubRetryDelayMs(error, attempt); + core.warning( + `GitHub API request failed with status ${githubErrorStatus(error)}; ` + + `retrying in ${Math.ceil(delay / 1000)} seconds ` + + `(attempt ${attempt + 1} of ${GITHUB_RETRY_ATTEMPTS}).`, + ); + await sleep(delay); + } + } + + throw new Error("GitHub API retry loop ended unexpectedly."); +} + +async function findReviewComment({ github, context, prNumber }) { + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + }); + return comments.find( + (comment) => + comment.user?.login === "github-actions[bot]" && + comment.user?.type === "Bot" && + comment.body?.startsWith(`${MARKER}\n`), + ); +} + +async function upsertReviewComment({ github, context, core, prNumber, body }) { + const existing = await findReviewComment({ github, context, prNumber }); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + core.info(`Updated Codex security review comment #${existing.id}.`); + return; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); + core.info(`Posted Codex security review on PR #${prNumber}.`); +} + +async function getPullRequest({ github, context, prNumber }) { + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + if ( + pullRequest.state !== "open" || + pullRequest.base.repo.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + pullRequest.base.ref !== "main" + ) { + return null; + } + return pullRequest; +} + +async function getLiveMainSha({ github, context }) { + const { data: mainRef } = await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: "heads/main", + }); + const sha = mainRef.object?.sha || ""; + if (mainRef.object?.type !== "commit" || !/^[0-9a-f]{40,64}$/.test(sha)) { + throw new Error("refs/heads/main did not resolve to a commit SHA."); + } + return sha; +} + +async function ensureCurrentReviewLabel({ github, context }) { + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: CURRENT_REVIEW_LABEL, + }); + return; + } catch (error) { + if (error?.status !== 404) { + throw error; + } + } + + try { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: CURRENT_REVIEW_LABEL, + color: "1d76db", + description: "The posted Codex security review matches its recorded range.", + }); + } catch (error) { + // Another posting job may create the repository label concurrently. + if (error?.status !== 422) { + throw error; + } + } +} + +async function markReviewCurrent({ github, context, prNumber }) { + await ensureCurrentReviewLabel({ github, context }); + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: [CURRENT_REVIEW_LABEL], + }); +} + +async function clearCurrentReview({ github, context, prNumber }) { + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + name: CURRENT_REVIEW_LABEL, + }); + } catch (error) { + if (error?.status !== 404) { + throw error; + } + } +} + +async function reviewRangeIsCurrent({ + github, + context, + prNumber, + baseSha, + headSha, + headRepo, +}) { + const [pullRequest, liveMainSha] = await Promise.all([ + getPullRequest({ github, context, prNumber }), + getLiveMainSha({ github, context }), + ]); + return ( + pullRequest !== null && + liveMainSha === baseSha && + pullRequest.head.sha === headSha && + pullRequest.head.repo?.full_name === headRepo + ); +} + +async function prepare({ github, context, core }) { + let prNumber; + let requestedHeadSha; + if (context.eventName === "pull_request_target") { + prNumber = Number(context.payload.pull_request?.number); + requestedHeadSha = context.payload.pull_request?.head?.sha || ""; + } else if (context.eventName === "issue_comment") { + prNumber = Number(context.payload.issue?.number); + const command = context.payload.comment?.body || ""; + const match = /^@buzz-security-review ([0-9a-f]{40})$/.exec(command); + if (!match) { + core.setFailed( + `Review commands must be exactly "${REVIEW_COMMAND} ".`, + ); + return; + } + requestedHeadSha = match[1]; + } else { + core.setFailed(`Unsupported review trigger: ${context.eventName}.`); + return; + } + + if (!Number.isSafeInteger(prNumber) || prNumber <= 0) { + core.setFailed("Invalid pull request number for security review."); + return; + } + + const pullRequest = await getPullRequest({ github, context, prNumber }); + if (!pullRequest) { + core.setFailed( + `Pull request #${prNumber} is not an open PR targeting main.`, + ); + return; + } + if (!pullRequest.head.repo?.full_name) { + core.setFailed( + `Pull request #${prNumber} has no available head repository.`, + ); + return; + } + if (pullRequest.head.sha !== requestedHeadSha) { + core.setFailed( + `Pull request #${prNumber} moved after this review was authorized. ` + + `Use "${reviewCommand(pullRequest.head.sha)}" to review the current head.`, + ); + return; + } + + const baseSha = await getLiveMainSha({ github, context }); + const commitRange = `${baseSha}...${pullRequest.head.sha}`; + core.setOutput("pr_number", String(prNumber)); + core.setOutput("trigger_actor", context.actor); + core.setOutput("base_sha", baseSha); + core.setOutput("head_sha", pullRequest.head.sha); + core.setOutput("head_repo", pullRequest.head.repo.full_name); + core.setOutput("commit_range", commitRange); +} + +function setReconciliationOutputs( + core, + { + prNumbers, + mainSha, + shouldContinue = false, + nextAfter = 0, + nextPass = 1, + }, +) { + core.setOutput( + "pr_numbers", + JSON.stringify(prNumbers.length > 0 ? prNumbers : [0]), + ); + core.setOutput("main_sha", mainSha); + core.setOutput("should_continue", String(shouldContinue)); + core.setOutput("next_after", String(nextAfter)); + core.setOutput("next_pass", String(nextPass)); +} + +async function prepareBaseReconciliation({ github, context, core }) { + const reconciliation = context.payload.client_payload || {}; + const afterPrNumber = Number(reconciliation.after_pr || 0); + if (!Number.isSafeInteger(afterPrNumber) || afterPrNumber < 0) { + throw new Error("Invalid reconciliation cursor."); + } + const pass = Number(reconciliation.pass || 1); + if ( + !Number.isSafeInteger(pass) || + pass < 1 || + pass > MAX_RECONCILIATION_PASSES + ) { + throw new Error("Invalid reconciliation pass."); + } + + const requestedMainSha = + context.eventName === "push" + ? context.sha + : reconciliation.main_sha || ""; + if (requestedMainSha && !/^[0-9a-f]{40,64}$/.test(requestedMainSha)) { + throw new Error("Invalid reconciliation main SHA."); + } + const liveMainSha = await getLiveMainSha({ github, context }); + const mainSha = requestedMainSha || liveMainSha; + if (mainSha !== liveMainSha) { + core.info( + `Skipping reconciliation for superseded main commit ${mainSha}.`, + ); + setReconciliationOutputs(core, { prNumbers: [], mainSha, nextPass: pass }); + return; + } + + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: "open", + labels: CURRENT_REVIEW_LABEL, + per_page: 100, + }); + const prNumbers = [ + ...new Set( + issues + .filter((issue) => issue.pull_request) + .map((issue) => issue.number) + .filter((prNumber) => Number.isSafeInteger(prNumber) && prNumber > 0), + ), + ] + .sort((left, right) => left - right) + .filter((prNumber) => prNumber > afterPrNumber); + const batch = prNumbers.slice(0, RECONCILIATION_BATCH_SIZE); + const hasMore = prNumbers.length > RECONCILIATION_BATCH_SIZE; + const startRetryPass = + (batch.length > 0 || afterPrNumber > 0) && + !hasMore && + pass < MAX_RECONCILIATION_PASSES; + setReconciliationOutputs(core, { + prNumbers: batch, + mainSha, + shouldContinue: hasMore || startRetryPass, + nextAfter: hasMore ? batch.at(-1) : 0, + nextPass: startRetryPass ? pass + 1 : pass, + }); +} + +async function invalidatePullRequestUpdate({ github, context, core }) { + const association = context.payload.pull_request?.author_association || ""; + const existingOnly = association === "MEMBER" || association === "OWNER"; + await invalidate({ github, context, core, existingOnly }); +} + +async function invalidate({ + github, + context, + core, + prNumber: requestedPrNumber, + existingOnly = false, +}) { + const prNumber = Number( + requestedPrNumber ?? context.payload.pull_request?.number, + ); + if (!Number.isSafeInteger(prNumber) || prNumber <= 0) { + throw new Error("Invalid pull request number for review invalidation."); + } + + const pullRequest = await getPullRequest({ github, context, prNumber }); + if (!pullRequest) { + await clearCurrentReview({ github, context, prNumber }); + core.notice(`Skipping review invalidation for ineligible PR #${prNumber}.`); + return; + } + + const existing = await findReviewComment({ github, context, prNumber }); + if (!existing && existingOnly) { + await clearCurrentReview({ github, context, prNumber }); + core.info(`PR #${prNumber} has no Codex security review to invalidate.`); + return; + } + + const liveMainSha = await getLiveMainSha({ github, context }); + const currentPrefix = + `${MARKER}\n` + + `${completedMarker(liveMainSha, pullRequest.head.sha)}\n`; + if (existing?.body?.startsWith(currentPrefix)) { + core.info(`PR #${prNumber} already has a review for the current range.`); + return; + } + + const body = `${MARKER} +${STALE_MARKER} +## 🔐 Codex Security Review + +> **Status: review required for the current range.** +> +> The current range is \`${liveMainSha}...${pullRequest.head.sha}\`. +> A new review must complete for this exact range. When manual authorization +> is required, a Block organization member must comment exactly +> \`${reviewCommand(pullRequest.head.sha)}\` to authorize a new review. +> Any previous review applies only to its recorded range. +`; + + if (existing?.body === body) { + core.info(`PR #${prNumber} already has the current stale-review notice.`); + await clearCurrentReview({ github, context, prNumber }); + return; + } + + await upsertReviewComment({ github, context, core, prNumber, body }); + await clearCurrentReview({ github, context, prNumber }); +} + +async function post({ github, context, core }) { + const rawReview = process.env.REVIEW_JSON || ""; + if (rawReview.length === 0 || rawReview.length > 120000) { + throw new Error("Codex output is empty or exceeds the renderer limit."); + } + + const review = JSON.parse(rawReview); + requireKeys(review, ["overall_risk", "summary", "findings", "notes"], "review"); + if (!RISKS.includes(review.overall_risk)) { + throw new Error("Review has an invalid overall risk."); + } + if (!Array.isArray(review.findings) || review.findings.length > 10) { + throw new Error("Review findings must be an array with at most 10 entries."); + } + if (!Array.isArray(review.notes) || review.notes.length > 5) { + throw new Error("Review notes must be an array with at most 5 entries."); + } + + const prNumber = Number(process.env.REVIEW_PR_NUMBER); + if (!Number.isSafeInteger(prNumber) || prNumber <= 0) { + throw new Error("Invalid reviewed pull request number."); + } + const baseSha = process.env.REVIEW_BASE_SHA || ""; + const headSha = process.env.REVIEW_HEAD_SHA || ""; + const headRepo = process.env.REVIEW_HEAD_REPO || ""; + const commitRange = process.env.REVIEW_COMMIT_RANGE || ""; + if (!/^[0-9a-f]{40,64}$/.test(baseSha) || !/^[0-9a-f]{40,64}$/.test(headSha)) { + throw new Error("Invalid reviewed commit SHA."); + } + if (commitRange !== `${baseSha}...${headSha}`) { + throw new Error("Invalid reviewed commit range."); + } + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(headRepo)) { + throw new Error("Invalid reviewed head repository."); + } + + const existingReview = { + github, + context, + core, + prNumber, + existingOnly: true, + }; + const reviewedRange = { + github, + context, + prNumber, + baseSha, + headSha, + headRepo, + }; + + const [pullRequest, liveMainSha] = await Promise.all([ + getPullRequest({ github, context, prNumber }), + getLiveMainSha({ github, context }), + ]); + if ( + !pullRequest || + liveMainSha !== baseSha || + pullRequest.head.sha !== headSha || + pullRequest.head.repo?.full_name !== headRepo + ) { + core.notice(`Skipping stale review for ${commitRange} on PR #${prNumber}.`); + await invalidate(existingReview); + return; + } + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100, + }); + if (files.length !== pullRequest.changed_files) { + throw new Error( + `Expected ${pullRequest.changed_files} changed files, but GitHub returned ${files.length}.`, + ); + } + const changedFiles = new Set(files.map((file) => file.filename)); + + const findingKeys = [ + "severity", + "category", + "title", + "path", + "line", + "description", + "impact", + "recommendation", + ]; + const [headOwner, headName] = headRepo.split("/"); + const renderedFindings = review.findings.map((finding, index) => { + const label = `finding ${index + 1}`; + requireKeys(finding, findingKeys, label); + if (!SEVERITIES.has(finding.severity)) { + throw new Error(`${label} has an invalid severity.`); + } + if (!CATEGORIES.has(finding.category)) { + throw new Error(`${label} has an invalid category.`); + } + if (!validPath(finding.path) || !changedFiles.has(finding.path)) { + throw new Error(`${label} does not reference a changed file.`); + } + if ( + !Number.isSafeInteger(finding.line) || + finding.line < 1 || + finding.line > 10000000 + ) { + throw new Error(`${label} has an invalid line number.`); + } + + const location = + `https://github.com/${encodeUrlComponent(headOwner)}/${encodeUrlComponent(headName)}` + + `/blob/${headSha}/${encodePath(finding.path)}#L${finding.line}`; + const pathLabel = safeCodeText( + `${finding.path}:${finding.line}`, + 520, + `${label} location`, + ); + return [ + `#### [${finding.severity}] ${safeCodeText(finding.title, 200, `${label} title`)}`, + `- **Category**: ${finding.category}`, + `- **Location**: ${pathLabel} ([source](${location}))`, + `- **Description**: ${safeCodeText(finding.description, 1500, `${label} description`)}`, + `- **Impact**: ${safeCodeText(finding.impact, 1500, `${label} impact`)}`, + `- **Recommendation**: ${safeCodeText(finding.recommendation, 1500, `${label} recommendation`)}`, + ].join("\n"); + }); + + const highestFindingRisk = review.findings.reduce( + (highest, finding) => Math.max(highest, RISKS.indexOf(finding.severity)), + 0, + ); + const overallRisk = RISKS[ + Math.max(RISKS.indexOf(review.overall_risk), highestFindingRisk) + ]; + const findingsMarkdown = renderedFindings.length + ? renderedFindings.join("\n\n") + : "No concrete security, correctness, or reliability findings were identified."; + const notesMarkdown = review.notes.length + ? review.notes + .map((note, index) => `- ${safeCodeText(note, 1000, `note ${index + 1}`)}`) + .join("\n") + : "- No additional limitations were reported."; + + const triggerActor = process.env.REVIEW_TRIGGER_ACTOR || ""; + if (!/^[A-Za-z0-9-]{1,39}$/.test(triggerActor)) { + throw new Error("Invalid review trigger actor."); + } + const model = process.env.CODEX_MODEL || ""; + if (!/^[A-Za-z0-9._-]{1,100}$/.test(model)) { + throw new Error("Invalid review model name."); + } + const workflowRun = + `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}` + + `/actions/runs/${process.env.GITHUB_RUN_ID}`; + const body = `${MARKER} +${completedMarker(baseSha, headSha)} +## 🔐 Codex Security Review + +> **Note**: This is an automated, security-focused review generated by Codex. +> Use it as a supplement to human review; false positives are possible. +> +> **Scope** +> - Exact PR diff: \`${commitRange}\` +> - Model: ${model} +> +> 💡 *Click "edited" above to see earlier reviews for this PR.* + +--- + +## Review Summary + +**Overall Risk**: ${overallRisk} + +${safeCodeText(review.summary, 2000, "review summary")} + +### Findings + +${findingsMarkdown} + +### Notes + +${notesMarkdown} + +--- + +Generated by [Codex Security Review](https://github.com/openai/codex-action) | +Requested by: \`@${triggerActor}\` | +[Workflow run](${workflowRun})`; + + if (body.length > 60000) { + throw new Error("Rendered security review exceeds the GitHub comment limit."); + } + + // Register the pending write before the final freshness check. If main moves + // now, either this check clears the label or the main-push reconciler sees it. + await markReviewCurrent({ github, context, prNumber }); + if (!(await reviewRangeIsCurrent(reviewedRange))) { + core.notice( + `Skipping review because ${commitRange} moved while PR #${prNumber} was rendering.`, + ); + await invalidate(existingReview); + return; + } + + await upsertReviewComment({ github, context, core, prNumber, body }); + + if (!(await reviewRangeIsCurrent(reviewedRange))) { + core.notice( + `Review range ${commitRange} moved while posting on PR #${prNumber}; marking it stale.`, + ); + await invalidate(existingReview); + } +} + +module.exports = { + invalidate, + invalidatePullRequestUpdate, + post, + prepare, + prepareBaseReconciliation, + withGithubRetry, +}; diff --git a/.github/scripts/codex-security-review.test.js b/.github/scripts/codex-security-review.test.js new file mode 100644 index 00000000000..0e6d8068ead --- /dev/null +++ b/.github/scripts/codex-security-review.test.js @@ -0,0 +1,568 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const test = require("node:test"); + +const { + invalidate, + invalidatePullRequestUpdate, + post, + prepare, + prepareBaseReconciliation, + withGithubRetry, +} = require("./codex-security-review.js"); + +const BASE_SHA = "a".repeat(40); +const OLD_BASE_SHA = "b".repeat(40); +const HEAD_SHA = "c".repeat(40); +const OTHER_HEAD_SHA = "d".repeat(40); +const NEW_BASE_SHA = "e".repeat(40); +const MARKER = ""; +const CURRENT_REVIEW_LABEL = "codex-security-review-current"; + +function pullRequest({ baseSha = OLD_BASE_SHA, headSha = HEAD_SHA } = {}) { + return { + state: "open", + base: { + ref: "main", + sha: baseSha, + repo: { full_name: "block/buzz" }, + }, + head: { + sha: headSha, + repo: { full_name: "outside/buzz" }, + }, + changed_files: 1, + }; +} + +function harness({ + pull = pullRequest(), + comments = [], + files = [], + labeledIssues = [], + liveMainShas = [BASE_SHA], +} = {}) { + const storedComments = [...comments]; + const created = []; + const updated = []; + const addedLabels = []; + const removedLabels = []; + const outputs = new Map(); + const failures = []; + const notices = []; + const info = []; + const warnings = []; + const listComments = async () => storedComments; + const listFiles = async () => files; + let labelExists = false; + let mainRefRead = 0; + const github = { + paginate: async (method, args) => method(args), + rest: { + git: { + getRef: async () => { + const sha = + liveMainShas[Math.min(mainRefRead, liveMainShas.length - 1)]; + mainRefRead += 1; + return { data: { object: { type: "commit", sha } } }; + }, + }, + issues: { + listComments, + listForRepo: async () => labeledIssues, + getLabel: async () => { + if (!labelExists) { + throw Object.assign(new Error("not found"), { status: 404 }); + } + return { data: { name: CURRENT_REVIEW_LABEL } }; + }, + createLabel: async () => { + labelExists = true; + }, + addLabels: async (input) => { + labelExists = true; + addedLabels.push(input); + }, + removeLabel: async (input) => { + if (!labelExists) { + throw Object.assign(new Error("not found"), { status: 404 }); + } + labelExists = false; + removedLabels.push(input); + }, + createComment: async (input) => { + created.push(input); + storedComments.push( + reviewComment(input.body, { id: 100 + storedComments.length }), + ); + }, + updateComment: async (input) => { + updated.push(input); + const comment = storedComments.find( + (candidate) => candidate.id === input.comment_id, + ); + if (comment) { + comment.body = input.body; + } + }, + }, + pulls: { + get: async () => ({ data: pull }), + listFiles, + }, + }, + }; + const core = { + info: (message) => info.push(message), + notice: (message) => notices.push(message), + warning: (message) => warnings.push(message), + setFailed: (message) => failures.push(message), + setOutput: (name, value) => outputs.set(name, value), + }; + const context = { + actor: "block-member", + eventName: "issue_comment", + payload: { + comment: { body: `@buzz-security-review ${HEAD_SHA}` }, + issue: { number: 6816 }, + }, + repo: { owner: "block", repo: "buzz" }, + }; + return { + context, + core, + addedLabels, + created, + failures, + github, + info, + notices, + outputs, + removedLabels, + storedComments, + updated, + warnings, + }; +} + +function reviewComment(body, { id = 42 } = {}) { + return { + id, + body, + user: { login: "github-actions[bot]", type: "Bot" }, + }; +} + +const NO_FINDINGS_REVIEW = { + overall_risk: "NONE", + summary: "No findings.", + findings: [], + notes: [], +}; + +async function postReview(state, review = NO_FINDINGS_REVIEW) { + const environment = { + CODEX_MODEL: "gpt-5.6-sol", + GITHUB_REPOSITORY: "block/buzz", + GITHUB_RUN_ID: "1234", + GITHUB_SERVER_URL: "https://github.com", + REVIEW_BASE_SHA: BASE_SHA, + REVIEW_COMMIT_RANGE: `${BASE_SHA}...${HEAD_SHA}`, + REVIEW_HEAD_REPO: "outside/buzz", + REVIEW_HEAD_SHA: HEAD_SHA, + REVIEW_JSON: JSON.stringify(review), + REVIEW_PR_NUMBER: "6816", + REVIEW_TRIGGER_ACTOR: "block-member", + }; + const previous = Object.fromEntries( + Object.keys(environment).map((key) => [key, process.env[key]]), + ); + Object.assign(process.env, environment); + + try { + await post(state); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + } +} + +test("prepare binds a member command to the named head SHA", async () => { + const current = harness(); + await prepare(current); + + assert.deepEqual(current.failures, []); + assert.equal(current.outputs.get("head_sha"), HEAD_SHA); + assert.equal(current.outputs.get("base_sha"), BASE_SHA); + assert.notEqual(current.outputs.get("base_sha"), OLD_BASE_SHA); + assert.equal( + current.outputs.get("commit_range"), + `${BASE_SHA}...${HEAD_SHA}`, + ); + + const moved = harness({ pull: pullRequest({ headSha: OTHER_HEAD_SHA }) }); + await prepare(moved); + + assert.equal(moved.outputs.size, 0); + assert.equal(moved.failures.length, 1); + assert.match(moved.failures[0], new RegExp(OTHER_HEAD_SHA)); + assert.match( + moved.failures[0], + new RegExp(`@buzz-security-review ${OTHER_HEAD_SHA}`), + ); +}); + +test("prepare rejects review commands without a full exact SHA", async () => { + const state = harness(); + state.context.payload.comment.body = "@buzz-security-review"; + + await prepare(state); + + assert.equal(state.outputs.size, 0); + assert.equal(state.failures.length, 1); + assert.match(state.failures[0], //); +}); + +test("invalidation compares the complete base and head range", async () => { + const stale = harness({ + comments: [ + reviewComment( + `${MARKER}\n\nold review`, + ), + ], + }); + await stale.github.rest.issues.addLabels({ + issue_number: 6816, + labels: [CURRENT_REVIEW_LABEL], + }); + + await invalidate({ + github: stale.github, + context: stale.context, + core: stale.core, + prNumber: 6816, + existingOnly: true, + }); + + assert.equal(stale.updated.length, 1); + assert.match(stale.updated[0].body, /review required for the current range/); + assert.ok(stale.updated[0].body.includes(`${BASE_SHA}...${HEAD_SHA}`)); + assert.match( + stale.updated[0].body, + new RegExp(`@buzz-security-review ${HEAD_SHA}`), + ); + assert.equal(stale.removedLabels.length, 1); + + await invalidate({ + github: stale.github, + context: stale.context, + core: stale.core, + prNumber: 6816, + existingOnly: true, + }); + + assert.equal(stale.updated.length, 1); + assert.match(stale.info.at(-1), /already has the current stale-review notice/); + + const current = harness({ + comments: [ + reviewComment( + `${MARKER}\n\ncurrent review`, + ), + ], + }); + await invalidate({ + github: current.github, + context: current.context, + core: current.core, + prNumber: 6816, + existingOnly: true, + }); + + assert.equal(current.updated.length, 0); + assert.match(current.info.at(-1), /current range/); +}); + +test("base reconciliation batches every labeled review with a durable cursor", async () => { + const labeledIssues = Array.from({ length: 34 }, (_, index) => ({ + number: index + 1, + pull_request: {}, + })); + const first = harness({ labeledIssues }); + first.context.eventName = "repository_dispatch"; + first.context.payload.client_payload = { + after_pr: "1", + main_sha: BASE_SHA, + }; + + await prepareBaseReconciliation(first); + + const firstBatch = JSON.parse(first.outputs.get("pr_numbers")); + assert.equal(firstBatch.length, 32); + assert.equal(firstBatch[0], 2); + assert.equal(firstBatch.at(-1), 33); + assert.equal(first.outputs.get("main_sha"), BASE_SHA); + assert.equal(first.outputs.get("should_continue"), "true"); + assert.equal(first.outputs.get("next_after"), "33"); + assert.equal(first.outputs.get("next_pass"), "1"); + + const second = harness({ labeledIssues }); + second.context.eventName = "repository_dispatch"; + second.context.payload.client_payload = { + after_pr: "33", + main_sha: BASE_SHA, + }; + + await prepareBaseReconciliation(second); + + assert.deepEqual(JSON.parse(second.outputs.get("pr_numbers")), [34]); + assert.equal(second.outputs.get("should_continue"), "true"); + assert.equal(second.outputs.get("next_after"), "0"); + assert.equal(second.outputs.get("next_pass"), "2"); + + const disappearedTail = harness(); + disappearedTail.context.eventName = "repository_dispatch"; + disappearedTail.context.payload.client_payload = { + after_pr: "33", + main_sha: BASE_SHA, + pass: "1", + }; + + await prepareBaseReconciliation(disappearedTail); + + assert.deepEqual( + JSON.parse(disappearedTail.outputs.get("pr_numbers")), + [0], + ); + assert.equal(disappearedTail.outputs.get("should_continue"), "true"); + assert.equal(disappearedTail.outputs.get("next_after"), "0"); + assert.equal(disappearedTail.outputs.get("next_pass"), "2"); + + const retry = harness({ labeledIssues: [labeledIssues.at(-1)] }); + retry.context.eventName = "repository_dispatch"; + retry.context.payload.client_payload = { + after_pr: "0", + main_sha: BASE_SHA, + pass: "2", + }; + + await prepareBaseReconciliation(retry); + + assert.deepEqual(JSON.parse(retry.outputs.get("pr_numbers")), [34]); + assert.equal(retry.outputs.get("should_continue"), "false"); + assert.equal(retry.outputs.get("next_after"), "0"); + assert.equal(retry.outputs.get("next_pass"), "2"); +}); + +test("base reconciliation stops a continuation from an older main commit", async () => { + const state = harness({ + labeledIssues: [{ number: 6816, pull_request: {} }], + liveMainShas: [NEW_BASE_SHA], + }); + state.context.eventName = "repository_dispatch"; + state.context.payload.client_payload = { + after_pr: "256", + main_sha: BASE_SHA, + }; + + await prepareBaseReconciliation(state); + + assert.deepEqual(JSON.parse(state.outputs.get("pr_numbers")), [0]); + assert.equal(state.outputs.get("main_sha"), BASE_SHA); + assert.equal(state.outputs.get("should_continue"), "false"); + assert.equal(state.outputs.get("next_after"), "0"); + assert.equal(state.outputs.get("next_pass"), "1"); + assert.match(state.info.at(-1), /superseded main commit/); +}); + +test("GitHub rate limits use bounded retry delays", async () => { + const waits = []; + const warnings = []; + let attempts = 0; + const rateLimitError = Object.assign(new Error("secondary rate limit"), { + status: 403, + response: { + status: 403, + headers: { "retry-after": "0" }, + data: { message: "secondary rate limit" }, + }, + }); + + const result = await withGithubRetry( + async () => { + attempts += 1; + if (attempts < 3) { + throw rateLimitError; + } + return "completed"; + }, + { + core: { warning: (message) => warnings.push(message) }, + sleep: async (milliseconds) => waits.push(milliseconds), + }, + ); + + assert.equal(result, "completed"); + assert.equal(attempts, 3); + assert.deepEqual(waits, [1000, 1000]); + assert.equal(warnings.length, 2); +}); + +test("base reconciliation does not create comments on unreviewed PRs", async () => { + const state = harness(); + + await invalidate({ + github: state.github, + context: state.context, + core: state.core, + prNumber: 6816, + existingOnly: true, + }); + + assert.equal(state.created.length, 0); + assert.equal(state.updated.length, 0); +}); + +test("pull request updates invalidate member reviews without adding placeholders", async () => { + const reviewed = harness({ + pull: pullRequest({ headSha: OTHER_HEAD_SHA }), + comments: [ + reviewComment( + `${MARKER}\n\nold review`, + ), + ], + }); + reviewed.context.eventName = "pull_request_target"; + reviewed.context.payload.pull_request = { + number: 6816, + author_association: "MEMBER", + }; + await reviewed.github.rest.issues.addLabels({ + issue_number: 6816, + labels: [CURRENT_REVIEW_LABEL], + }); + + await invalidatePullRequestUpdate(reviewed); + + assert.equal(reviewed.updated.length, 1); + assert.ok( + reviewed.updated[0].body.includes(`${BASE_SHA}...${OTHER_HEAD_SHA}`), + ); + assert.equal(reviewed.removedLabels.length, 1); + + const unreviewed = harness(); + unreviewed.context.eventName = "pull_request_target"; + unreviewed.context.payload.pull_request = { + number: 6816, + author_association: "OWNER", + }; + + await invalidatePullRequestUpdate(unreviewed); + + assert.equal(unreviewed.created.length, 0); + assert.equal(unreviewed.updated.length, 0); + + const external = harness(); + external.context.eventName = "pull_request_target"; + external.context.payload.pull_request = { + number: 6816, + author_association: "CONTRIBUTOR", + }; + + await invalidatePullRequestUpdate(external); + + assert.equal(external.created.length, 1); + assert.match(external.created[0].body, /review required for the current range/); +}); + +test("post preserves finding text while rendering it as inert code", async () => { + const findingPath = "src/x)www.example.com/review.js"; + const state = harness({ files: [{ filename: findingPath }] }); + const summary = + "Keep