feat: implement issue #737 — [Phase 1] Instrument auto-rebase: agentic-conflict-resolution rate + fan-out CI-run volume report

.github/workflows/auto-rebase-health.yml

@@ -0,0 +1,114 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# SOURCE OF TRUTH: petry-projects/.github-private/.github/workflows/auto-rebase-health.yml
+#
+# AGENTS — READ BEFORE EDITING:
+#   • Daily Auto-rebase Health report (#737, epic #736). Counts conflict-sentinel
+#     fires vs. dev-lead `rebase` responses (the agentic-conflict-resolution rate)
+#     and estimates auto-rebase fan-out CI volume. All logic lives in
+#     scripts/auto_rebase_health.sh (unit-tested in tests/auto_rebase_health.bats).
+#   • No LLM calls — pure gh-API + jq aggregation. Keep it that way; the cost cap
+#     for this report is a single scheduled run/day with no per-PR work.
+#   • Single-repo scan: github.token (with the permissions below) suffices —
+#     actions:read for run telemetry, issues/pull-requests:read for the comment
+#     and PR scans. No cross-org PAT is required (unlike token-report.yml).
+#   • This is a repo-specific workflow with no org template — see AGENTS.md
+#     (Workflow Files exception). Do not let template syncs remove it.
+# ─────────────────────────────────────────────────────────────────────────────
+name: Auto-rebase Health Report
+
+on:
+  schedule:
+    # Daily at 07:00 UTC — between the PR-review health check (06:00) and the
+    # weekly token report (Mon 08:00).
+    - cron: '0 7 * * *'
+  workflow_dispatch:
+    inputs:
+      lookback_days:
+        description: "Days of history to scan for sentinels and run telemetry"
+        required: false
+        default: "7"
+        type: string
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: read
+  actions: read
+
+concurrency:
+  group: auto-rebase-health
+  cancel-in-progress: true
+
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      LOOKBACK_DAYS: ${{ inputs.lookback_days || '7' }}
+
+    steps:
+      - name: Checkout agent repo
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+
+      - name: Generate auto-rebase health report
+        env:
+          GH_TOKEN: ${{ github.token }}
+          AUTO_REBASE_HEALTH_OUT: auto_rebase_health.md
+        run: bash scripts/auto_rebase_health.sh
+
+      - name: Post report to tracking issue
+        if: hashFiles('auto_rebase_health.md') != ''
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
+        with:
+          github-token: ${{ github.token }}
+          script: |
+            const fs = require('fs');
+            const MAX = 65000;
+            let report = fs.readFileSync('auto_rebase_health.md', 'utf8');
+            if (!report.trim()) { core.setFailed('Report file is empty'); return; }
+
+            const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
+            if (report.length > MAX) {
+              report = report.slice(0, MAX) +
+                `\n\n---\n_Report truncated — full output in the [workflow run summary](${runUrl})._`;
+            }
+
+            const LABEL = 'auto-rebase-health';
+            const TITLE = '🔁 Auto-rebase Health — Daily Report';
+
+            // Ensure the tracking label exists.
+            try {
+              await github.rest.issues.getLabel({ owner: context.repo.owner, repo: context.repo.repo, name: LABEL });
+            } catch (e) {
+              await github.rest.issues.createLabel({
+                owner: context.repo.owner, repo: context.repo.repo,
+                name: LABEL, color: '1d76db',
+                description: 'Daily auto-rebase instrumentation report',
+              });
+            }
+
+            // Find-or-create the single pinned tracking issue.
+            const existing = await github.paginate(github.rest.issues.listForRepo, {
+              owner: context.repo.owner, repo: context.repo.repo,
+              labels: LABEL, state: 'open', per_page: 100,
+            });
+
+            let issue = existing.find(i => i.title === TITLE);
+            if (!issue) {
+              const created = await github.rest.issues.create({
+                owner: context.repo.owner, repo: context.repo.repo,
+                title: TITLE, labels: [LABEL],
+                body: 'Tracking issue for the daily Auto-rebase Health report. ' +
+                      'Each run posts a new comment below with the agentic-conflict-resolution ' +
+                      'rate and the auto-rebase fan-out CI-volume estimate.\n\n' +
+                      'Source: `scripts/auto_rebase_health.sh` · `.github/workflows/auto-rebase-health.yml`',
+              });
+              issue = created.data;
+              core.notice(`Created tracking issue #${issue.number}: ${issue.html_url}`);
+            }
+
+            const comment = await github.rest.issues.createComment({
+              owner: context.repo.owner, repo: context.repo.repo,
+              issue_number: issue.number, body: report,
+            });
+            core.notice(`Posted auto-rebase health report: ${comment.data.html_url}`);

.github/workflows/lint.yml

@@ -44,6 +44,7 @@ jobs:
           bats \
             tests/fleet_report.bats \
             tests/token_report.bats \
+            tests/auto_rebase_health.bats \
             tests/model_pricing.bats \
             tests/test_review_cycle.bats \
             tests/test_ci_status.bats \

AGENTS.md

@@ -56,6 +56,14 @@ This is the `.github-private` org infrastructure repo for `petry-projects`. It c
   path, which the thin-caller `auto-rebase.yml` and central reusable cannot cover. It must not be removed by
   template syncs. If the org template gains a retry equivalent, remove this exception and defer to the
   template instead.
+- **Exception:** `auto-rebase-health.yml` (#737, epic #736) is a documented repo-specific workflow with no
+  corresponding org template in `standards/workflows/`. It is a daily (≤1/day) + `workflow_dispatch` report
+  that quantifies the agentic-conflict-resolution rate (conflict-sentinel fires vs. dev-lead `rebase`
+  responses, both HTML-comment markers GitHub search cannot index) and estimates auto-rebase fan-out CI
+  volume from `auto-rebase.yml` run telemetry. It is a no-LLM gh-API + jq report (logic in
+  `scripts/auto_rebase_health.sh`, tests in `tests/auto_rebase_health.bats`) and runs on `github.token`
+  only — no cross-org PAT. It must not be removed by template syncs. If the org template gains an
+  auto-rebase instrumentation equivalent, remove this exception and defer to the template instead.
 
 ### Agent Profiles (`agents/*.md`)
 

scripts/auto_rebase_health.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+# auto_rebase_health.sh — Auto-rebase instrumentation report (#737, epic #736).
+#
+# Measures two things the maintainer needs to make the Merge Queue vs. auto-rebase
+# decision with concrete numbers instead of estimates:
+#
+#   1. Agentic-conflict-resolution rate — how often the auto-rebase conflict
+#      sentinel (`<!-- auto-rebase-conflict:`) fires and how often dev-lead emits
+#      a `rebase` intent in response. Both are HTML-comment markers that GitHub
+#      search cannot index reliably (discussion #735), so we scan the repo's
+#      issue comments directly:
+#        * sentinels  — comments containing `<!-- auto-rebase-conflict:`
+#                       (the marker scripts/dev-lead-intent.sh keys on)
+#        * responses  — dev-lead terminal markers `... intent=rebase status=... -->`
+#                       posted by scripts/dev-lead-fix-reviews.sh (any status)
+#        * applied    — the `status=applied` subset (resolved & force-pushed)
+#
+#   2. Fan-out CI-run volume — the `Auto-rebase non-Dependabot PRs` workflow
+#      (.github/workflows/auto-rebase.yml) runs per push to main and fans
+#      branch-update CI re-runs onto behind PRs. Per-run behind-PR counts are not
+#      logged in this repo (the update-branch calls live in the central reusable),
+#      so the re-run volume is an ESTIMATE: runs × current open non-Dependabot PRs.
+#
+# Layout (mirrors scripts/token_report.sh):
+#   * The count_*/summarize_*/fmt_*/render_report functions are PURE — they take
+#     JSON / scalars and write to stdout. Unit-tested in tests/auto_rebase_health.bats.
+#   * main() does the network I/O: token selection, comment + run telemetry pulls.
+#
+# Env vars consumed:
+#   GH_TOKEN        — primary token (needs actions:read + repo read on this repo)
+#   GH_PAT_FALLBACK — optional fallback PAT if GH_TOKEN lacks run-telemetry access
+#   AGENT_REPO      — repo to scan (default: petry-projects/.github-private)
+#   LOOKBACK_DAYS   — days of history to consider (default: 7)
+#   AUTO_REBASE_HEALTH_OUT — optional path; report is written there in addition to stdout
+#   GITHUB_STEP_SUMMARY — written by the Actions runner when present
+
+set -euo pipefail
+
+WORKFLOW_REPO="${AGENT_REPO:-petry-projects/.github-private}"
+LOOKBACK_DAYS="${LOOKBACK_DAYS:-7}"
+AUTO_REBASE_WORKFLOW="auto-rebase.yml"
+
+# Markers (kept in one place so a rename in the dev-lead scripts is a one-line fix).
+SENTINEL_MARKER='<!-- auto-rebase-conflict:'
+REBASE_RESPONSE_MARKER='intent=rebase status='
+REBASE_APPLIED_MARKER='intent=rebase status=applied'
+
+# ---------------------------------------------------------------------------
+# Pure helpers (unit-tested; no network)
+# ---------------------------------------------------------------------------
+
+# count_marker <comments_json> <substring>
+# Counts comment objects whose `.body` contains <substring>. Absent/empty JSON → 0.
+count_marker() {
+  local json="${1:-}" needle="${2:-}"
+  [ -n "$json" ] || json='[]'
+  printf '%s' "$json" | jq --arg n "$needle" \
+    '[.[] | select((.body // "") | contains($n))] | length'
+}
+
+# summarize_sentinels <comments_json>
+# Emits TSV: sentinels<TAB>responses<TAB>applied
+#   sentinels — conflict sentinels fired
+#   responses — dev-lead rebase intents that ran to a terminal/posted state
+#   applied   — the resolved-and-pushed subset of responses
+summarize_sentinels() {
+  local json="${1:-}"
+  [ -n "$json" ] || json='[]'
+  printf '%s' "$json" | jq -r \
+    --arg sent "$SENTINEL_MARKER" \
+    --arg resp "$REBASE_RESPONSE_MARKER" \
+    --arg appl "$REBASE_APPLIED_MARKER" '
+      [.[] | .body // ""] as $b |
+      [
+        ([$b[] | select(contains($sent))] | length),
+        ([$b[] | select(contains($resp))] | length),
+        ([$b[] | select(contains($appl))] | length)
+      ] | @tsv'
+}
+
+# summarize_runs <runs_json>
+# Emits TSV: total<TAB>success<TAB>failed over the auto-rebase run telemetry.
+summarize_runs() {
+  local json="${1:-}"
+  [ -n "$json" ] || json='[]'
+  printf '%s' "$json" | jq -r '
+    [
+      length,
+      ([.[] | select(.conclusion == "success")] | length),
+      ([.[] | select(.conclusion == "failure")] | length)
+    ] | @tsv'
+}
+
+# estimate_fanout <run_count> <behind_prs>
+# Estimated branch-update CI re-runs = runs × current open non-Dependabot PRs.
+estimate_fanout() {
+  local runs="${1:-0}" behind="${2:-0}"
+  echo $(( runs * behind ))
+}
+
+# fmt_rate <num> <denom>
+# Integer percentage with a zero-denominator guard ("n/a"), so a window with no
+# sentinels never divides by zero.
+fmt_rate() {
+  local num="${1:-0}" denom="${2:-0}"
+  if [ "$denom" -le 0 ]; then
+    echo "n/a"
+    return 0
+  fi
+  echo "$(( num * 100 / denom ))%"
+}
+
+# render_report <comments_json> <runs_json> <lookback_days> <behind_prs> [today]
+# Writes the full Markdown report to stdout. Pure: no network.
+render_report() {
+  local comments_json="${1:-[]}" runs_json="${2:-[]}"
+  local lookback="${3:-7}" behind="${4:-0}" today="${5:-}"
+  [ -n "$today" ] || today="$(date -u +%Y-%m-%d)"
+
+  local sentinels responses applied
+  IFS=$'\t' read -r sentinels responses applied < <(summarize_sentinels "$comments_json")
+
+  local total success failed
+  IFS=$'\t' read -r total success failed < <(summarize_runs "$runs_json")
+
+  local fanout
+  fanout="$(estimate_fanout "$total" "$behind")"
+
+  local runs_per_day rerun_per_day
+  runs_per_day="$(awk -v t="$total" -v d="$lookback" 'BEGIN { printf "%.1f", (d > 0 ? t / d : 0) }')"
+  rerun_per_day="$(awk -v t="$fanout" -v d="$lookback" 'BEGIN { printf "%.1f", (d > 0 ? t / d : 0) }')"
+
+  printf '# 🔁 Auto-rebase Health — %s\n\n' "$today"
+  printf '_Repo `%s` · lookback %s day(s)_\n\n' "$WORKFLOW_REPO" "$lookback"
+
+  printf '## Agentic conflict-resolution rate\n\n'
+  printf -- '- **Sentinels fired** (`%s`): %s\n' "$SENTINEL_MARKER" "$sentinels"
+  printf -- '- **Dev-lead rebase responses** (`intent=rebase`): %s\n' "$responses"
+  printf -- '- **Resolved & pushed** (`status=applied`): %s\n' "$applied"
+  printf -- '- **Resolution rate** (responses ÷ sentinels): %s\n' "$(fmt_rate "$responses" "$sentinels")"
+  printf -- '- **Applied rate** (applied ÷ sentinels): %s\n\n' "$(fmt_rate "$applied" "$sentinels")"
+
+  printf '## Auto-rebase fan-out volume (estimate)\n\n'
+  printf -- '- **Workflow runs** (`%s`): %s total · %s success · %s failed\n' \
+    "$AUTO_REBASE_WORKFLOW" "$total" "$success" "$failed"
+  printf -- '- **Behind-PR multiplier** (open non-Dependabot PRs): %s\n' "$behind"
+  printf -- '- **Estimated branch-update CI re-runs**: ~%s\n' "$fanout"
+  printf -- '- **Per-day baseline**: %s auto-rebase run(s)/day · ~%s CI re-run(s)/day\n\n' \
+    "$runs_per_day" "$rerun_per_day"
+  printf '> Fan-out is an **estimate** — per-run behind-PR counts are not logged in this repo, '
+  printf 'so re-runs = runs × current open non-Dependabot PR count.\n'
+}
+
+# ---------------------------------------------------------------------------
+# Network I/O (main)
+# ---------------------------------------------------------------------------
+
+main() {
+  local today cutoff
+  today="$(date -u +%Y-%m-%d)"
+  # GNU date: -d "N days ago"; BSD/macOS: -v-Nd
+  cutoff="$(date -u -d "${LOOKBACK_DAYS} days ago" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \
+    || date -u -v-"${LOOKBACK_DAYS}"d +%Y-%m-%dT%H:%M:%SZ)"
+
+  echo "=== Auto-rebase Health Report ===" >&2
+  echo "  Repo:     $WORKFLOW_REPO" >&2
+  echo "  Lookback: ${LOOKBACK_DAYS} day(s) (since ${cutoff})" >&2
+  echo "  Date:     $today" >&2
+
+  # Token selection — verify run-telemetry access; fall back to a PAT if needed.
+  if ! gh api "repos/${WORKFLOW_REPO}/actions/workflows/${AUTO_REBASE_WORKFLOW}/runs?per_page=1" \
+       >/dev/null 2>&1; then
+    if [ -n "${GH_PAT_FALLBACK:-}" ]; then
+      echo "::warning::GH_TOKEN cannot read ${AUTO_REBASE_WORKFLOW} runs — using GH_PAT_FALLBACK" >&2
+      export GH_TOKEN="$GH_PAT_FALLBACK"
+    else
+      echo "::error::GH_TOKEN cannot read ${AUTO_REBASE_WORKFLOW} run telemetry and GH_PAT_FALLBACK is unset." >&2
+      echo "::error::Grant actions:read on ${WORKFLOW_REPO} or set the fallback PAT secret." >&2
+      exit 1
+    fi
+  fi
+
+  # 1. Comments — the issues/comments endpoint covers PR comments too (PRs are
+  #    issues), and `since` filters server-side so we only pull the window.
+  local comments_json
+  comments_json="$(gh api \
+    "repos/${WORKFLOW_REPO}/issues/comments?since=${cutoff}&per_page=100" --paginate \
+    --jq '.[] | {body, created_at, html_url}' 2>/dev/null \
+    | jq -s '.' 2>/dev/null || echo '[]')"
+
+  # 2. Auto-rebase run telemetry. --paginate so long manual-dispatch windows are
+  #    not silently truncated at 100 runs (GitHub still caps created>= at 1,000).
+  local runs_json
+  runs_json="$(gh api \
+    "repos/${WORKFLOW_REPO}/actions/workflows/${AUTO_REBASE_WORKFLOW}/runs?per_page=100&created=>=${cutoff}" \
+    --paginate --jq '.workflow_runs | map({conclusion, created_at})' 2>/dev/null \
+    | jq -s 'add // []' 2>/dev/null || echo '[]')"
+
+  # 3. Behind-PR multiplier — open non-Dependabot PRs (proxy for branches the
+  #    fan-out updates). Best-effort; defaults to 0 so the report still renders.
+  local behind
+  behind="$(gh pr list --repo "$WORKFLOW_REPO" --state open --limit 200 --json author \
+    --jq '[.[] | select((.author?.login // "") | test("dependabot"; "i") | not)] | length' \
+    2>/dev/null || echo 0)"
+
+  local report
+  report="$(render_report "$comments_json" "$runs_json" "$LOOKBACK_DAYS" "$behind" "$today")"
+
+  printf '%s\n' "$report"
+  if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+    printf '%s\n' "$report" >> "$GITHUB_STEP_SUMMARY"
+  fi
+  if [ -n "${AUTO_REBASE_HEALTH_OUT:-}" ]; then
+    printf '%s\n' "$report" > "$AUTO_REBASE_HEALTH_OUT"
+  fi
+}
+
+# Only run main when executed directly (not when sourced by tests).
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+  main "$@"
+fi