From 4fc4bd016efe0a035b28a31586d76e919c9f19ca Mon Sep 17 00:00:00 2001 From: jaylfc Date: Fri, 14 Aug 2026 01:04:05 +0000 Subject: [PATCH] Add merge-gate zero-token scripts and tests Three executables under scripts/merge-gate/: - check_bot_anchoring.sh: verifies at least one substantive bot review is anchored to the PR head oid - check_fake_green.sh: detects rate-limited CodeRabbit status and bare acknowledgement comments with no review object - red_first.sh: mechanizes revert/fail/restore/pass test cycle Tests under tests/test_merge_gate.py exercise the scripts against recorded gh JSON fixtures with no network access. Covers anchored head pass, old sha fail, human-only fail, rate-limited fail, and bare ack fail. --- scripts/merge-gate/check_bot_anchoring.sh | 48 +++++ scripts/merge-gate/check_fake_green.sh | 109 ++++++++++++ scripts/merge-gate/red_first.sh | 165 ++++++++++++++++++ .../fixtures/merge_gate/checkruns_empty.json | 3 + .../fixtures/merge_gate/pr_anchored_head.json | 22 +++ .../merge_gate/pr_anchored_old_sha.json | 14 ++ tests/fixtures/merge_gate/pr_bare_ack.json | 12 ++ tests/fixtures/merge_gate/pr_human_only.json | 14 ++ .../fixtures/merge_gate/pr_rate_limited.json | 6 + tests/fixtures/merge_gate/repo_info.json | 6 + .../merge_gate/status_rate_limited.json | 9 + tests/test_merge_gate.py | 136 +++++++++++++++ 12 files changed, 544 insertions(+) create mode 100755 scripts/merge-gate/check_bot_anchoring.sh create mode 100755 scripts/merge-gate/check_fake_green.sh create mode 100755 scripts/merge-gate/red_first.sh create mode 100644 tests/fixtures/merge_gate/checkruns_empty.json create mode 100644 tests/fixtures/merge_gate/pr_anchored_head.json create mode 100644 tests/fixtures/merge_gate/pr_anchored_old_sha.json create mode 100644 tests/fixtures/merge_gate/pr_bare_ack.json create mode 100644 tests/fixtures/merge_gate/pr_human_only.json create mode 100644 tests/fixtures/merge_gate/pr_rate_limited.json create mode 100644 tests/fixtures/merge_gate/repo_info.json create mode 100644 tests/fixtures/merge_gate/status_rate_limited.json create mode 100644 tests/test_merge_gate.py diff --git a/scripts/merge-gate/check_bot_anchoring.sh b/scripts/merge-gate/check_bot_anchoring.sh new file mode 100755 index 00000000..139d3314 --- /dev/null +++ b/scripts/merge-gate/check_bot_anchoring.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -uo pipefail + +if [[ $# -lt 1 ]]; then + echo "FAILED: usage: check_bot_anchoring.sh " + exit 2 +fi + +PR_NUMBER="$1" + +PR_JSON=$(gh pr view "$PR_NUMBER" --json number,headRefOid,reviews 2>/dev/null) || { + echo "FAILED: gh/network error" + exit 3 +} + +python3 - "$PR_JSON" << 'PYEOF' +import json, sys + +try: + data = json.loads(sys.argv[1]) +except json.JSONDecodeError: + print('FAILED: could not parse PR data') + sys.exit(3) + +head_oid = data.get('headRefOid', '') +reviews = data.get('reviews', []) +bot_authors = {'coderabbitai', 'qodo-code-review', 'kilo-code-bot'} + +for r in reviews: + author = (r.get('author') or {}).get('login', '') + if author not in bot_authors: + continue + state = r.get('state', '') + if state not in ('COMMENTED', 'APPROVED', 'CHANGES_REQUESTED'): + continue + commit_oid = (r.get('commit') or {}).get('oid', '') + if commit_oid != head_oid: + continue + body = r.get('body') or '' + has_inline = r.get('includesCreatedEdit', False) + if body.strip() or has_inline: + print('SUCCESS: anchored substantive bot review found') + sys.exit(0) + +print('FAILED: no anchored substantive bot review found') +sys.exit(10) +PYEOF +exit $? diff --git a/scripts/merge-gate/check_fake_green.sh b/scripts/merge-gate/check_fake_green.sh new file mode 100755 index 00000000..0ab1d304 --- /dev/null +++ b/scripts/merge-gate/check_fake_green.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -uo pipefail + +if [[ $# -lt 1 ]]; then + echo "FAILED: usage: check_fake_green.sh " + exit 2 +fi + +PR_NUMBER="$1" + +PR_JSON=$(gh pr view "$PR_NUMBER" --json number,headRefOid,reviews,comments 2>/dev/null) || { + echo "FAILED: gh/network error" + exit 3 +} + +REPO_JSON=$(gh repo view --json owner,name 2>/dev/null) || { + echo "FAILED: gh/network error" + exit 3 +} + +HEAD_OID=$(echo "$PR_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['headRefOid'])") || { + echo "FAILED: could not parse PR data" + exit 3 +} + +OWNER=$(echo "$REPO_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['owner']['login'])") +REPO_NAME=$(echo "$REPO_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['name'])") + +STATUS_JSON=$(gh api "repos/$OWNER/$REPO_NAME/commits/$HEAD_OID/status" 2>/dev/null) || { + echo "FAILED: gh/network error" + exit 3 +} + +CHECKRUNS_JSON=$(gh api "repos/$OWNER/$REPO_NAME/commits/$HEAD_OID/check-runs" 2>/dev/null) || { + echo "FAILED: gh/network error" + exit 3 +} + +python3 - "$PR_JSON" "$STATUS_JSON" "$CHECKRUNS_JSON" << 'PYEOF' +import json, sys + +try: + pr_data = json.loads(sys.argv[1]) + status_data = json.loads(sys.argv[2]) + checkruns_data = json.loads(sys.argv[3]) +except json.JSONDecodeError: + print('FAILED: could not parse gh response') + sys.exit(3) + +reviews = pr_data.get('reviews', []) +comments = pr_data.get('comments', []) + +# Check (a): success status/check with "Review rate limited" +for item in status_data.get('statuses', []): + if item.get('context') != 'CodeRabbit': + continue + if (item.get('state') or '').upper() != 'SUCCESS': + continue + desc = item.get('description') or '' + if 'Review rate limited' in desc: + print('FAILED: CodeRabbit commit status is SUCCESS but description contains "Review rate limited"') + print('Remediation: the plain "@coderabbitai review" command no-ops on a PR that was reviewed and then pushed to; only "@coderabbitai full review" forces a real pass.') + sys.exit(11) + +for item in checkruns_data.get('check_runs', []): + if 'CodeRabbit' not in item.get('name', ''): + continue + conclusion = (item.get('conclusion') or '').upper() + if conclusion != 'SUCCESS': + continue + output = item.get('output') or {} + output_text = output.get('text') or output.get('title') or output.get('summary') or '' + if 'Review rate limited' in output_text: + print('FAILED: CodeRabbit check run is SUCCESS but output contains "Review rate limited"') + print('Remediation: the plain "@coderabbitai review" command no-ops on a PR that was reviewed and then pushed to; only "@coderabbitai full review" forces a real pass.') + sys.exit(11) + +# Check (b): bare "Review finished" comment with no review object +cr_reviews = [ + r for r in reviews + if (r.get('author') or {}).get('login') == 'coderabbitai' + and r.get('state') in ('COMMENTED', 'APPROVED', 'CHANGES_REQUESTED') +] + +cr_comments = [ + c for c in comments + if (c.get('author') or {}).get('login') == 'coderabbitai' +] + +BARE_ACK_PATTERNS = ['Review finished', 'review finished'] + +def is_bare_ack(body): + body_lower = body.lower() + for pat in BARE_ACK_PATTERNS: + if pat.lower() in body_lower: + return True + return False + +bare_acks = [c for c in cr_comments if is_bare_ack(c.get('body') or '')] + +if bare_acks and not cr_reviews: + print('FAILED: only CodeRabbit artifact is a bare acknowledgement comment with no review object attached') + print('Remediation: the plain "@coderabbitai review" command no-ops on a PR that was reviewed and then pushed to; only "@coderabbitai full review" forces a real pass.') + sys.exit(11) + +print('SUCCESS: no fake-green signals detected') +sys.exit(0) +PYEOF +exit $? \ No newline at end of file diff --git a/scripts/merge-gate/red_first.sh b/scripts/merge-gate/red_first.sh new file mode 100755 index 00000000..591c3ca7 --- /dev/null +++ b/scripts/merge-gate/red_first.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +set -uo pipefail + +if [[ $# -lt 1 ]]; then + echo "FAILED: usage: red_first.sh --paths --tests " + exit 2 +fi + +PR_NUMBER="$1" +shift + +SRC_PATHS=() +TEST_IDS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --paths) + shift + while [[ $# -gt 0 && "$1" != --* ]]; do + SRC_PATHS+=("$1") + shift + done + ;; + --tests) + shift + while [[ $# -gt 0 && "$1" != --* ]]; do + TEST_IDS+=("$1") + shift + done + ;; + *) + echo "FAILED: unknown argument: $1" + exit 2 + ;; + esac +done + +if [[ ${#SRC_PATHS[@]} -eq 0 || ${#TEST_IDS[@]} -eq 0 ]]; then + echo "FAILED: both --paths and --tests are required" + exit 2 +fi + +PR_JSON=$(gh pr view "$PR_NUMBER" --json headRefOid,headRefName,baseRefName 2>/dev/null) || { + echo "FAILED: gh/network error" + exit 3 +} + +python3 - "$PR_JSON" -- "${SRC_PATHS[@]}" -- "${TEST_IDS[@]}" << 'PYEOF' +import json, sys, subprocess, os, tempfile + +args = sys.argv[1:] +sep1 = args.index('--') +sep2 = args.index('--', sep1 + 1) + +try: + data = json.loads(args[0]) +except json.JSONDecodeError: + print('FAILED: could not parse PR data') + sys.exit(3) + +src_paths = args[sep1 + 1:sep2] +test_ids = args[sep2 + 1:] + +head_ref = data.get('headRefName', '') +base_ref = data.get('baseRefName', '') + +def run(cmd, **kwargs): + return subprocess.run(cmd, capture_output=True, text=True, **kwargs) + +def eprint(*a, **kw): + print(*a, file=sys.stderr, **kw) + +repo_root = run(['git', 'rev-parse', '--show-toplevel'], check=True).stdout.strip() +os.chdir(repo_root) + +tmpdir = tempfile.mkdtemp(prefix='red-first-') +try: + result = run(['git', 'worktree', 'add', '-f', tmpdir, head_ref]) + if result.returncode != 0: + eprint('FAILED: could not check out PR branch into worktree') + eprint(result.stderr, end='') + print('FAILED: could not check out PR branch into worktree') + sys.exit(3) +except Exception: + eprint('FAILED: could not check out PR branch into worktree') + print('FAILED: could not check out PR branch into worktree') + sys.exit(3) + +os.chdir(tmpdir) + +# Stage 1: tests must PASS +eprint('Stage 1: running tests on PR branch...') +result = run(['uv', 'run', 'pytest'] + test_ids + ['-q']) +if result.returncode != 0: + eprint(result.stdout, end='') + eprint(result.stderr, end='') + print('FAILED: tests did not pass on PR branch') + os.chdir(repo_root) + run(['git', 'worktree', 'remove', tmpdir], capture_output=True) + sys.exit(13) + +# Get merge base +try: + merge_base = run(['git', 'merge-base', head_ref, 'origin/' + base_ref], check=True).stdout.strip() +except subprocess.CalledProcessError: + try: + merge_base = run(['git', 'merge-base', head_ref, base_ref], check=True).stdout.strip() + except subprocess.CalledProcessError: + eprint('FAILED: could not find merge base') + print('FAILED: could not find merge base') + os.chdir(repo_root) + run(['git', 'worktree', 'remove', tmpdir], capture_output=True) + sys.exit(3) + +# Stage 2: revert source paths to merge-base version +eprint('Stage 2: reverting source paths to merge-base...') +result = run(['git', 'checkout', merge_base, '--'] + src_paths) +if result.returncode != 0: + eprint('FAILED: could not revert source paths') + eprint(result.stderr, end='') + print('FAILED: could not revert source paths') + os.chdir(repo_root) + run(['git', 'worktree', 'remove', tmpdir], capture_output=True) + sys.exit(3) + +# Stage 2 re-run: tests must FAIL +eprint('Stage 2: running tests after revert...') +result = run(['uv', 'run', 'pytest'] + test_ids + ['-q']) +if result.returncode == 0: + eprint(result.stdout, end='') + eprint(result.stderr, end='') + print('FAILED: tests passed without the fix in place') + os.chdir(repo_root) + run(['git', 'worktree', 'remove', tmpdir], capture_output=True) + sys.exit(12) + +# Stage 3: restore +eprint('Stage 3: restoring source paths...') +result = run(['git', 'checkout', 'HEAD', '--'] + src_paths) +if result.returncode != 0: + eprint('FAILED: could not restore source paths') + eprint(result.stderr, end='') + print('FAILED: could not restore source paths') + os.chdir(repo_root) + run(['git', 'worktree', 'remove', tmpdir], capture_output=True) + sys.exit(3) + +# Stage 3 re-run: tests must PASS +eprint('Stage 3: running tests after restore...') +result = run(['uv', 'run', 'pytest'] + test_ids + ['-q']) +if result.returncode != 0: + eprint(result.stdout, end='') + eprint(result.stderr, end='') + print('FAILED: tests did not pass after restore') + os.chdir(repo_root) + run(['git', 'worktree', 'remove', tmpdir], capture_output=True) + sys.exit(13) + +# Cleanup +os.chdir(repo_root) +run(['git', 'worktree', 'remove', tmpdir], capture_output=True) + +print('SUCCESS: red-first cycle completed') +sys.exit(0) +PYEOF +exit $? diff --git a/tests/fixtures/merge_gate/checkruns_empty.json b/tests/fixtures/merge_gate/checkruns_empty.json new file mode 100644 index 00000000..fc992112 --- /dev/null +++ b/tests/fixtures/merge_gate/checkruns_empty.json @@ -0,0 +1,3 @@ +{ + "check_runs": [] +} diff --git a/tests/fixtures/merge_gate/pr_anchored_head.json b/tests/fixtures/merge_gate/pr_anchored_head.json new file mode 100644 index 00000000..d0235a23 --- /dev/null +++ b/tests/fixtures/merge_gate/pr_anchored_head.json @@ -0,0 +1,22 @@ +{ + "number": 218, + "headRefOid": "abc123def456", + "reviews": [ + { + "author": {"login": "coderabbitai"}, + "state": "COMMENTED", + "body": "
Nitpick comments (1)
\n\n**Add coverage for the new import-tracking table.**\n
", + "commit": {"oid": "abc123def456"}, + "includesCreatedEdit": false, + "submittedAt": "2026-07-28T12:21:45Z" + }, + { + "author": {"login": "qodo-code-review"}, + "state": "COMMENTED", + "body": "", + "commit": {"oid": "abc123def456"}, + "includesCreatedEdit": false, + "submittedAt": "2026-07-28T12:22:07Z" + } + ] +} diff --git a/tests/fixtures/merge_gate/pr_anchored_old_sha.json b/tests/fixtures/merge_gate/pr_anchored_old_sha.json new file mode 100644 index 00000000..c31b14c1 --- /dev/null +++ b/tests/fixtures/merge_gate/pr_anchored_old_sha.json @@ -0,0 +1,14 @@ +{ + "number": 218, + "headRefOid": "abc123def456", + "reviews": [ + { + "author": {"login": "coderabbitai"}, + "state": "COMMENTED", + "body": "Review content here", + "commit": {"oid": "oldsha123456"}, + "includesCreatedEdit": false, + "submittedAt": "2026-07-28T12:21:45Z" + } + ] +} diff --git a/tests/fixtures/merge_gate/pr_bare_ack.json b/tests/fixtures/merge_gate/pr_bare_ack.json new file mode 100644 index 00000000..f77142e5 --- /dev/null +++ b/tests/fixtures/merge_gate/pr_bare_ack.json @@ -0,0 +1,12 @@ +{ + "number": 218, + "headRefOid": "abc123def456", + "reviews": [], + "comments": [ + { + "author": {"login": "coderabbitai"}, + "body": "Review finished", + "createdAt": "2026-07-28T12:21:48Z" + } + ] +} diff --git a/tests/fixtures/merge_gate/pr_human_only.json b/tests/fixtures/merge_gate/pr_human_only.json new file mode 100644 index 00000000..651d2370 --- /dev/null +++ b/tests/fixtures/merge_gate/pr_human_only.json @@ -0,0 +1,14 @@ +{ + "number": 218, + "headRefOid": "abc123def456", + "reviews": [ + { + "author": {"login": "jaylfc"}, + "state": "APPROVED", + "body": "LGTM", + "commit": {"oid": "abc123def456"}, + "includesCreatedEdit": false, + "submittedAt": "2026-07-28T12:30:00Z" + } + ] +} diff --git a/tests/fixtures/merge_gate/pr_rate_limited.json b/tests/fixtures/merge_gate/pr_rate_limited.json new file mode 100644 index 00000000..87ba2031 --- /dev/null +++ b/tests/fixtures/merge_gate/pr_rate_limited.json @@ -0,0 +1,6 @@ +{ + "number": 218, + "headRefOid": "abc123def456", + "reviews": [], + "comments": [] +} diff --git a/tests/fixtures/merge_gate/repo_info.json b/tests/fixtures/merge_gate/repo_info.json new file mode 100644 index 00000000..98804758 --- /dev/null +++ b/tests/fixtures/merge_gate/repo_info.json @@ -0,0 +1,6 @@ +{ + "owner": { + "login": "jaylfc" + }, + "name": "taosmd" +} diff --git a/tests/fixtures/merge_gate/status_rate_limited.json b/tests/fixtures/merge_gate/status_rate_limited.json new file mode 100644 index 00000000..2f0f9ac6 --- /dev/null +++ b/tests/fixtures/merge_gate/status_rate_limited.json @@ -0,0 +1,9 @@ +{ + "statuses": [ + { + "context": "CodeRabbit", + "state": "success", + "description": "Review rate limited" + } + ] +} diff --git a/tests/test_merge_gate.py b/tests/test_merge_gate.py new file mode 100644 index 00000000..ecab1289 --- /dev/null +++ b/tests/test_merge_gate.py @@ -0,0 +1,136 @@ +import json +import os +import subprocess +import stat + +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures", "merge_gate") + + +def _read(name): + with open(os.path.join(FIXTURES_DIR, name), "r") as f: + return f.read() + + +def _make_fake_gh(tmpdir, **fixtures): + fake = os.path.join(tmpdir, "gh") + with open(fake, "w") as f: + f.write("#!/usr/bin/env python3\n") + f.write("import sys\n\n") + for key, value in fixtures.items(): + f.write(f"{key.upper()} = {repr(value)}\n") + f.write(""" +def main(): + a = sys.argv[1:] + if not a: + print("{}") + return + if a[0] == 'pr' and len(a) > 1 and a[1] == 'view': + print(PR) + return + if a[0] == 'repo' and len(a) > 1 and a[1] == 'view': + print(REPO) + return + if a[0] == 'api' and len(a) > 1: + p = a[1] + if 'status' in p: + print(STATUS) + return + if 'check-runs' in p: + print(CHECKRUNS) + return + print("{}") + +if __name__ == '__main__': + main() +""") + os.chmod(fake, stat.S_IRWXU) + return fake + + +def _run(script, args, tmpdir, **fixtures): + fake_gh = _make_fake_gh(tmpdir, **fixtures) + env = os.environ.copy() + env["PATH"] = tmpdir + ":" + env.get("PATH", "") + script_path = os.path.abspath(os.path.join("scripts", "merge-gate", script)) + result = subprocess.run( + ["bash", script_path] + args, + capture_output=True, + text=True, + env=env, + ) + return result + + +def test_anchored_head_passes(tmp_path): + result = _run( + "check_bot_anchoring.sh", + ["218"], + str(tmp_path), + pr=_read("pr_anchored_head.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.startswith("SUCCESS:") + + +def test_anchored_old_sha_fails(tmp_path): + result = _run( + "check_bot_anchoring.sh", + ["218"], + str(tmp_path), + pr=_read("pr_anchored_old_sha.json"), + ) + assert result.returncode == 10, result.stdout + result.stderr + assert result.stdout.startswith("FAILED:") + + +def test_human_only_fails(tmp_path): + result = _run( + "check_bot_anchoring.sh", + ["218"], + str(tmp_path), + pr=_read("pr_human_only.json"), + ) + assert result.returncode == 10, result.stdout + result.stderr + assert result.stdout.startswith("FAILED:") + + +def test_rate_limited_fails(tmp_path): + result = _run( + "check_fake_green.sh", + ["218"], + str(tmp_path), + pr=_read("pr_rate_limited.json"), + repo=_read("repo_info.json"), + status=_read("status_rate_limited.json"), + checkruns=_read("checkruns_empty.json"), + ) + assert result.returncode == 11, result.stdout + result.stderr + assert result.stdout.startswith("FAILED:") + assert "Review rate limited" in result.stdout + + +def test_bare_ack_fails(tmp_path): + result = _run( + "check_fake_green.sh", + ["218"], + str(tmp_path), + pr=_read("pr_bare_ack.json"), + repo=_read("repo_info.json"), + status="{}", + checkruns="{}", + ) + assert result.returncode == 11, result.stdout + result.stderr + assert result.stdout.startswith("FAILED:") + + +def test_red_first_usage_error(): + result = subprocess.run( + ["bash", "scripts/merge-gate/red_first.sh"], + capture_output=True, + text=True, + ) + assert result.returncode == 2 + assert result.stdout.startswith("FAILED:")