feat: implement issue #948 — P2 (#860 follow-up): runaway-PR detection in daily health check (commits/comments/cycles/age thresholds)

.github/workflows/daily-pr-review-health.yml

@@ -101,6 +101,50 @@ jobs:
         run: |
           echo "::notice::Health check passed — all runs in the last ${LOOKBACK_DAYS} day(s) are healthy."
 
+      # Runaway-PR detection net (#948, #860 follow-up). Pure gh-API + jq scan of
+      # every open PR against soft, env-overridable thresholds — no LLM, no PR
+      # mutation. #860 ran for four days unnoticed; this makes the next novel
+      # runaway a push signal on day one. Sets HAS_RUNAWAY when any candidate is
+      # found, surfaced through the same health-check / automated-report issue
+      # channel below.
+      - name: Scan for runaway PRs
+        env:
+          GH_TOKEN: ${{ secrets.DON_PETRY_BOT_GH_PAT }}
+          REPO: ${{ github.repository }}
+        run: bash scripts/pr_runaway_scan.sh
+
+      - name: Truncate runaway report if needed
+        run: |
+          if [ -f pr_runaway_report.md ]; then
+            if [ "$(wc -c < pr_runaway_report.md)" -gt 60000 ]; then
+              head -c 60000 pr_runaway_report.md > pr_runaway_report.md.tmp
+              mv pr_runaway_report.md.tmp pr_runaway_report.md
+              printf '\n\n---\n_Report truncated at 60 000 bytes._\n' >> pr_runaway_report.md
+            fi
+          fi
+
+      - name: Create runaway-candidate issue
+        if: env.HAS_RUNAWAY == 'true' && hashFiles('pr_runaway_report.md') != ''
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
+        with:
+          github-token: ${{ github.token }}
+          script: |
+            const fs = require('fs');
+            const report = fs.readFileSync('pr_runaway_report.md', 'utf8');
+            if (!report.trim()) {
+              core.setFailed('Runaway report file is empty');
+              return;
+            }
+            const today = new Date().toISOString().split('T')[0];
+            const issue = await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: `Runaway PR candidate(s) detected ${today}`,
+              body: report,
+              labels: ['health-check', 'automated-report'],
+            });
+            core.notice(`Runaway issue created: ${issue.data.html_url}`);
+
       # Durable, affirmative MCP-connectivity monitor (#903, part of #676).
       # Additive final step: when .github/review-mcp.json is present, assert that
       # the configured server(s) handshake through the same flags engine.sh uses

.github/workflows/lint.yml

@@ -58,6 +58,7 @@ jobs:
             tests/test_review_cycle.bats \
             tests/test_pr_automation_budget.bats \
             tests/test_pr_runaway_regression.bats \
+            tests/test_pr_runaway_detect.bats \
             tests/test_ci_status.bats \
             tests/test_push_protection.bats \
             tests/test_verify_auth_scopes.bats \

scripts/lib/pr-runaway-detect.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# pr-runaway-detect.sh — the runaway-PR DETECTION net (issue #948, #860 follow-up).
+#
+# #860 ran away to 378 commits / 1,582 comments produced entirely by agents
+# looping — and ran for FOUR DAYS with no human noticing, because detection was
+# pull (someone happens to look), not push. The per-PR budget (#926) and the org
+# self-trigger fix *prevent/bound* runaways; this library is the complementary
+# *signal* so the next novel runaway surface is flagged on day one.
+#
+# It is a set of PURE functions: given a PR's already-gathered metrics, decide
+# whether the PR is a runaway CANDIDATE and which soft threshold(s) it crossed.
+# It never mutates a PR — surfacing/routing is the caller's job
+# (scripts/pr_runaway_scan.sh, wired into the daily health check).
+#
+#   pr_runaway_reasons <commits> <comments> <cycles> <age_hours>
+#     Prints one human-readable reason line per crossed threshold; nothing when
+#     the PR is within all thresholds. This is the unit under test.
+#   is_pr_runaway <commits> <comments> <cycles> <age_hours>
+#     Exit 0 when ANY threshold is crossed, 1 otherwise.
+#   pr_age_hours <created_at_iso8601> [now_epoch]
+#     Whole hours between created_at and now (default: current time).
+#   generate_runaway_report <candidates_tsv_file>
+#     Renders the markdown health-report section from a candidates TSV.
+#
+# Thresholds are SOFT and env-overridable with sane defaults (AC#2). All are
+# strict greater-than ("> 50 commits", etc.), matching the issue wording. The
+# age criterion additionally requires ACTIVE agent churn (>= RUNAWAY_CHURN_MIN_
+# CYCLES automated cycles since the last human) so a normal PR that merely sat
+# open past 48h with no agent activity does NOT false-positive (AC#3).
+
+# Soft thresholds — a PR is a candidate when it exceeds ANY of these.
+: "${RUNAWAY_MAX_COMMITS:=50}"       # > this many commits
+: "${RUNAWAY_MAX_COMMENTS:=200}"     # > this many issue comments
+: "${RUNAWAY_MAX_CYCLES:=10}"        # > this many automated cycles (see #926)
+: "${RUNAWAY_MIN_AGE_HOURS:=48}"     # open longer than this, AND churning
+: "${RUNAWAY_CHURN_MIN_CYCLES:=1}"   # min cycles that count as "active churn"
+
+# _runaway_int <value>
+#   Echo the value as a non-negative integer, or 0 for empty/non-numeric input.
+#   Mirrors the defensive degradation in pr-automation-budget.sh so a bad metric
+#   can never break the integer comparisons below ("integer expression expected").
+_runaway_int() {
+  case "${1:-}" in
+    ''|*[!0-9]*) echo 0 ;;
+    *) echo "$1" ;;
+  esac
+}
+
+# _runaway_threshold <value> <default>
+#   Like _runaway_int but falls back to <default> instead of 0 for empty/non-numeric
+#   input. Used for threshold env vars so a bad override can never silently set a
+#   threshold to 0 and spam every open PR as a runaway candidate.
+_runaway_threshold() {
+  local val="${1:-}" default="${2:-0}"
+  case "$val" in
+    ''|*[!0-9]*) echo "$default" ;;
+    *) echo "$val" ;;
+  esac
+}
+
+# pr_runaway_reasons <commits> <comments> <cycles> <age_hours>
+#   Print one reason line per crossed soft threshold (empty when within bounds).
+pr_runaway_reasons() {
+  local commits comments cycles age
+  commits=$(_runaway_int "${1:-0}")
+  comments=$(_runaway_int "${2:-0}")
+  cycles=$(_runaway_int "${3:-0}")
+  age=$(_runaway_int "${4:-0}")
+
+  local max_commits max_comments max_cycles min_age churn_floor
+  max_commits=$(_runaway_threshold "${RUNAWAY_MAX_COMMITS}" 50)
+  max_comments=$(_runaway_threshold "${RUNAWAY_MAX_COMMENTS}" 200)
+  max_cycles=$(_runaway_threshold "${RUNAWAY_MAX_CYCLES}" 10)
+  min_age=$(_runaway_threshold "${RUNAWAY_MIN_AGE_HOURS}" 48)
+  churn_floor=$(_runaway_threshold "${RUNAWAY_CHURN_MIN_CYCLES}" 1)
+
+  [ "$commits" -gt "$max_commits" ] && \
+    printf 'commits: %s (>%s)\n' "$commits" "$max_commits"
+  [ "$comments" -gt "$max_comments" ] && \
+    printf 'comments: %s (>%s)\n' "$comments" "$max_comments"
+  [ "$cycles" -gt "$max_cycles" ] && \
+    printf 'automated cycles: %s (>%s)\n' "$cycles" "$max_cycles"
+  if [ "$age" -gt "$min_age" ] && [ "$cycles" -ge "$churn_floor" ]; then
+    printf 'open %sh with agent churn (%s cycles, >%sh)\n' \
+      "$age" "$cycles" "$min_age"
+  fi
+  return 0
+}
+
+# is_pr_runaway <commits> <comments> <cycles> <age_hours>
+#   Exit 0 when the PR crosses any soft threshold, 1 otherwise.
+is_pr_runaway() {
+  local reasons
+  reasons=$(pr_runaway_reasons "$@")
+  [ -n "$reasons" ]
+}
+
+# pr_age_hours <created_at_iso8601> [now_epoch]
+#   Whole hours since created_at. now_epoch defaults to the current time; it is
+#   injectable so callers/tests are deterministic. Unparseable input -> 0.
+pr_age_hours() {
+  local created="${1:-}" now
+  now=$(_runaway_int "${2:-}")
+  [ "$now" -gt 0 ] || now=$(date -u +%s)
+  local created_epoch
+  created_epoch=$(date -u -d "$created" +%s 2>/dev/null || date -u -v0d -f "%Y-%m-%dT%H:%M:%SZ" "$created" +%s 2>/dev/null || true)
+  if [ -z "$created_epoch" ]; then
+    echo 0
+    return 0
+  fi
+  local diff=$(( now - created_epoch ))
+  [ "$diff" -lt 0 ] && diff=0
+  echo $(( diff / 3600 ))
+}
+
+# generate_runaway_report <candidates_tsv_file>
+#   Render the "Runaway PR Candidates" markdown section for the health report.
+#   Input TSV rows: pr_number <TAB> html_url <TAB> title <TAB> reasons
+#   (reasons already joined with "; "). An empty/missing file prints an
+#   all-clear line and no table, so a clean fleet still gets an explicit signal.
+generate_runaway_report() {
+  local f="${1:-}"
+  local max_commits max_comments max_cycles min_age
+  max_commits=$(_runaway_threshold "${RUNAWAY_MAX_COMMITS}" 50)
+  max_comments=$(_runaway_threshold "${RUNAWAY_MAX_COMMENTS}" 200)
+  max_cycles=$(_runaway_threshold "${RUNAWAY_MAX_CYCLES}" 10)
+  min_age=$(_runaway_threshold "${RUNAWAY_MIN_AGE_HOURS}" 48)
+
+  printf '## Runaway PR Candidates\n\n'
+  printf 'Open PRs exceeding a soft runaway threshold '
+  printf '(commits >%s, comments >%s, automated cycles >%s, or open >%sh with agent churn). ' \
+    "$max_commits" "$max_comments" "$max_cycles" "$min_age"
+  printf 'Detection only — no PR is mutated. See #948 / the #860 post-mortem.\n\n'
+
+  if [ -z "$f" ] || [ ! -s "$f" ]; then
+    printf '✅ No open PR crossed a runaway threshold.\n'
+    return 0
+  fi
+
+  printf '| PR | Title | Triggering metric(s) |\n'
+  printf '|---|---|---|\n'
+  local num url title reasons
+  while IFS=$'\t' read -r num url title reasons; do
+    [ -n "$num" ] || continue
+    printf '| [#%s](%s) | %s | %s |\n' "$num" "$url" "$title" "$reasons"
+  done < "$f"
+}

scripts/pr_runaway_scan.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+# pr_runaway_scan.sh — daily runaway-PR detection net (issue #948, #860 follow-up).
+#
+# Scans every OPEN PR in the target repo and flags any that has crossed a soft
+# runaway threshold (commits / comments / automated cycles / age-with-churn) as a
+# runaway CANDIDATE, with a direct link and the triggering metric. This is the
+# push signal #860 lacked — it ran for four days unnoticed. Detection ONLY: this
+# script never comments on, labels, or otherwise mutates a PR; it just writes a
+# markdown report and sets HAS_RUNAWAY for the health-check workflow to surface
+# through the existing health-check / automated-report issue mechanism.
+#
+# Thresholds are env-overridable (see scripts/lib/pr-runaway-detect.sh).
+#
+# Env vars consumed:
+#   GH_TOKEN        — must have repo read on REPO (PR list/detail, comments)
+#   GH_PAT_FALLBACK — optional secondary token if the primary can't read REPO
+#   REPO / AGENT_REPO — target repo (default: petry-projects/.github-private)
+#   GITHUB_ENV      — written by the Actions runner (HAS_RUNAWAY / RUNAWAY_COUNT)
+#   GITHUB_STEP_SUMMARY — written by the Actions runner
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/lib/pr-automation-budget.sh
+source "${SCRIPT_DIR}/lib/pr-automation-budget.sh"
+# shellcheck source=scripts/lib/pr-runaway-detect.sh
+source "${SCRIPT_DIR}/lib/pr-runaway-detect.sh"
+
+REPO="${REPO:-${AGENT_REPO:-petry-projects/.github-private}}"
+REPORT_FILE="${REPORT_FILE:-pr_runaway_report.md}"
+TODAY=$(date -u +%Y-%m-%d)
+
+echo "=== Runaway-PR Detection — Daily Scan ==="
+echo "  Repo:   $REPO"
+echo "  Date:   $TODAY"
+echo "  Thresholds: commits>${RUNAWAY_MAX_COMMITS} comments>${RUNAWAY_MAX_COMMENTS} cycles>${RUNAWAY_MAX_CYCLES} age>${RUNAWAY_MIN_AGE_HOURS}h(+churn)"
+echo ""
+
+# ---------------------------------------------------------------------------
+# 0. Token selection — fall back to GH_PAT_FALLBACK if REPO is unreachable
+# ---------------------------------------------------------------------------
+if ! gh api "repos/${REPO}" >/dev/null 2>&1; then
+  if [ -n "${GH_PAT_FALLBACK:-}" ]; then
+    echo "::warning::GH_TOKEN cannot access ${REPO} — using GH_PAT_FALLBACK"
+    export GH_TOKEN="$GH_PAT_FALLBACK"
+    if ! gh api "repos/${REPO}" >/dev/null 2>&1; then
+      echo "::error::GH_PAT_FALLBACK also cannot access ${REPO}."
+      exit 1
+    fi
+  else
+    echo "::error::GH_TOKEN cannot access ${REPO} and GH_PAT_FALLBACK is not set."
+    exit 1
+  fi
+fi
+
+# ---------------------------------------------------------------------------
+# 1. Enumerate open PRs
+# ---------------------------------------------------------------------------
+open_prs=$(gh api --paginate "repos/${REPO}/pulls?state=open&per_page=100" \
+  --jq '.[].number' 2>/dev/null) || {
+  echo "::error::Failed to fetch open PRs from GitHub API." >&2
+  exit 1
+}
+
+if [ -z "$open_prs" ]; then
+  echo "No open PRs in ${REPO}."
+fi
+
+# ---------------------------------------------------------------------------
+# 2. Per-PR metrics -> detection (detection only, no mutation)
+# ---------------------------------------------------------------------------
+candidates_file=$(mktemp) || {
+  echo "Failed to create temp file" >&2
+  exit 1
+}
+trap 'rm -f "$candidates_file"' EXIT
+now_epoch=$(date -u +%s)
+scanned=0
+
+while IFS= read -r pr; do
+  [ -n "$pr" ] || continue
+  scanned=$(( scanned + 1 ))
+
+  # Single detail call gives commit/comment counts + created_at + link + title.
+  detail=$(gh api "repos/${REPO}/pulls/${pr}" \
+    --jq '{commits, comments, created_at, html_url, title}' 2>/dev/null || echo '{}')
+  IFS=$'\t' read -r commits comments created_at html_url title <<< "$(jq -r '[.commits // 0, .comments // 0, .created_at // "", .html_url // "", (.title // "" | tostring)] | @tsv' <<< "$detail")"
+
+  # Automated cycles since the last human — reuse the #926 budget computation so
+  # the two guards agree on what "an automated cycle" means.
+  events=$(gather_pr_automation_events "$pr" "$REPO")
+  cycles=$(compute_pr_automation_cycles "$events")
+
+  age_hours=$(pr_age_hours "$created_at" "$now_epoch")
+
+  reasons=$(pr_runaway_reasons "$commits" "$comments" "$cycles" "$age_hours")
+  if [ -n "$reasons" ]; then
+    # Sanitize the title for a single markdown table cell: strip newlines/tabs
+    # (the TSV delimiter) and pipes (the markdown column delimiter).
+    safe_title=$(printf '%s' "$title" | tr '\n\t|' '   ')
+    joined=$(printf '%s' "$reasons" | paste -sd';' - | sed 's/;/; /g')
+    printf '%s\t%s\t%s\t%s\n' "$pr" "$html_url" "$safe_title" "$joined" \
+      >> "$candidates_file"
+    echo "::warning::Runaway candidate PR #${pr} — ${joined}"
+  fi
+done <<< "$open_prs"
+
+# grep -c prints the count to stdout even at zero matches (exiting 1), so `|| true`
+# swallows that exit without appending a second "0".
+runaway_count=$(grep -c . "$candidates_file" 2>/dev/null || true)
+runaway_count=${runaway_count:-0}
+echo "Scanned ${scanned} open PR(s); ${runaway_count} runaway candidate(s)."
+
+# ---------------------------------------------------------------------------
+# 3. Render report + export env flags
+# ---------------------------------------------------------------------------
+{
+  printf '# Runaway PR Detection — %s\n\n' "$TODAY"
+  printf '**Repo:** `%s` | **Open PRs scanned:** %s | **Candidates:** %s\n\n' \
+    "$REPO" "$scanned" "$runaway_count"
+  generate_runaway_report "$candidates_file"
+} > "$REPORT_FILE"
+
+[ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$REPORT_FILE" >> "$GITHUB_STEP_SUMMARY"
+
+if [ -n "${GITHUB_ENV:-}" ]; then
+  echo "RUNAWAY_COUNT=${runaway_count}" >> "$GITHUB_ENV"
+  if [ "$runaway_count" -gt 0 ]; then
+    echo "HAS_RUNAWAY=true" >> "$GITHUB_ENV"
+  else
+    echo "HAS_RUNAWAY=false" >> "$GITHUB_ENV"
+  fi
+fi
+
+echo ""
+echo "Report written to ${REPORT_FILE} ($(wc -c < "$REPORT_FILE") bytes)"
+echo "=== Runaway-PR scan complete ==="