From 65529d06dbdde0a5fd50e7938397ba0c4d583cf2 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Mon, 3 Aug 2026 01:16:33 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20implement=20issue=20#1440=20?= =?UTF-8?q?=E2=80=94=20auto-rebase:=20extend=20auto=5Frebase=5Fhealth.sh?= =?UTF-8?q?=20with=20fleet=20BEHIND/DIRTY=20observability=20(AC7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/auto_rebase_health.sh | 64 ++++++++++++++++++++++++++++++----- tests/auto_rebase_health.bats | 60 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/scripts/auto_rebase_health.sh b/scripts/auto_rebase_health.sh index 7c60e90f7..dbdc0fa2c 100644 --- a/scripts/auto_rebase_health.sh +++ b/scripts/auto_rebase_health.sh @@ -21,6 +21,13 @@ # logged in this repo (the update-branch calls live in the central reusable), # so the re-run volume is an ESTIMATE: runs × current open non-Dependabot PRs. # +# 3. Fleet merge-state observability (AC7, #1440) — a snapshot of how many open +# non-draft, non-Dependabot PRs are currently `mergeStateStatus: BEHIND` vs +# DIRTY. This makes the effectiveness of the AC1/AC2 auto-rebase fixes +# (petry-projects/.github#926) measurable per run without another manual +# audit. Repo-scoped — fleet-wide would need a cross-repo PAT this report +# does not carry. +# # Layout (mirrors scripts/token_report.sh): # * The count_*/summarize_*/fmt_*/render_report functions are PURE — they take # JSON / scalars and write to stdout. Unit-tested in tests/auto_rebase_health.bats. @@ -78,6 +85,29 @@ summarize_sentinels() { ] | @tsv' } +# summarize_merge_states +# Emits TSV: behinddirty — counts of open PRs whose GitHub `mergeStateStatus` +# is BEHIND (head is behind base; needs an auto-rebase update) vs DIRTY (merge +# conflict; needs human/agent resolution), over the subset of PRs that are +# non-draft AND non-Dependabot-authored. This is the AC7 observability metric +# (#1440): it makes the effectiveness of the AC1/AC2 fixes measurable per run. +# Input JSON is an array of {mergeStateStatus, isDraft, author:{login}} objects +# (as returned by `gh pr list --json mergeStateStatus,isDraft,author`). +# Absent/empty JSON → "0\t0". +summarize_merge_states() { + local json="${1:-}" + [ -n "$json" ] || json='[]' + printf '%s' "$json" | jq -r ' + [ .[] + | select((.isDraft // false) | not) + | select(((.author.login // "") | test("dependabot"; "i")) | not) + ] as $prs | + [ + ([$prs[] | select(.mergeStateStatus == "BEHIND")] | length), + ([$prs[] | select(.mergeStateStatus == "DIRTY")] | length) + ] | @tsv' +} + # summarize_runs # Emits TSV: totalsuccessfailed over the auto-rebase run telemetry. summarize_runs() { @@ -110,11 +140,11 @@ fmt_rate() { echo "$(( num * 100 / denom ))%" } -# render_report [today] +# render_report [today] [prs_json] # Writes the full Markdown report to stdout. Pure: no network. render_report() { local comments_json="${1:-[]}" runs_json="${2:-[]}" - local lookback="${3:-7}" behind="${4:-0}" today="${5:-}" + local lookback="${3:-7}" behind="${4:-0}" today="${5:-}" prs_json="${6:-[]}" [ -n "$today" ] || today="$(date -u +%Y-%m-%d)" local sentinels responses applied @@ -123,6 +153,9 @@ render_report() { local total success failed IFS=$'\t' read -r total success failed < <(summarize_runs "$runs_json") + local ms_behind ms_dirty + IFS=$'\t' read -r ms_behind ms_dirty < <(summarize_merge_states "$prs_json") + local fanout fanout="$(estimate_fanout "$total" "$behind")" @@ -148,7 +181,14 @@ render_report() { printf -- '- **Per-day baseline**: %s auto-rebase run(s)/day · ~%s CI re-run(s)/day\n\n' \ "$runs_per_day" "$rerun_per_day" printf '> Fan-out is an **estimate** — per-run behind-PR counts are not logged in this repo, ' - printf 'so re-runs = runs × current open non-Dependabot PR count.\n' + printf 'so re-runs = runs × current open non-Dependabot PR count.\n\n' + + printf '## Fleet merge-state observability (AC7)\n\n' + printf -- '- **BEHIND** (open non-draft non-Dependabot PRs `mergeStateStatus: BEHIND`): %s\n' "$ms_behind" + printf -- '- **DIRTY** (open non-draft non-Dependabot PRs `mergeStateStatus: DIRTY`): %s\n\n' "$ms_dirty" + printf '> Snapshot of the current open-PR queue, scoped to **`%s`** — fleet-wide counts would ' "$WORKFLOW_REPO" + printf 'require a cross-repo PAT this report does not carry. Tracks the effectiveness of the ' + printf 'AC1/AC2 auto-rebase fixes (petry-projects/.github#926) without a manual audit.\n' } # --------------------------------------------------------------------------- @@ -196,15 +236,23 @@ main() { --paginate --jq '.workflow_runs | map({conclusion, created_at})' 2>/dev/null \ | jq -s 'add // []' 2>/dev/null || echo '[]')" - # 3. Behind-PR multiplier — open non-Dependabot PRs (proxy for branches the - # fan-out updates). Best-effort; defaults to 0 so the report still renders. + # 3. Open-PR snapshot — pulled once with the fields both the behind-PR + # multiplier and the AC7 merge-state (BEHIND/DIRTY) metric need. Best-effort; + # defaults to [] so the report still renders when the query fails. + local prs_json + prs_json="$(gh pr list --repo "$WORKFLOW_REPO" --state open --limit 200 \ + --json author,isDraft,mergeStateStatus 2>/dev/null || echo '[]')" + [ -n "$prs_json" ] || prs_json='[]' + + # Behind-PR multiplier — open non-Dependabot PRs (proxy for branches the + # fan-out updates), derived from the same snapshot. local behind - behind="$(gh pr list --repo "$WORKFLOW_REPO" --state open --limit 200 --json author \ - --jq '[.[] | select((.author?.login // "") | test("dependabot"; "i") | not)] | length' \ + behind="$(printf '%s' "$prs_json" \ + | jq '[.[] | select((.author?.login // "") | test("dependabot"; "i") | not)] | length' \ 2>/dev/null || echo 0)" local report - report="$(render_report "$comments_json" "$runs_json" "$LOOKBACK_DAYS" "$behind" "$today")" + report="$(render_report "$comments_json" "$runs_json" "$LOOKBACK_DAYS" "$behind" "$today" "$prs_json")" printf '%s\n' "$report" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then diff --git a/tests/auto_rebase_health.bats b/tests/auto_rebase_health.bats index 8a12a5a4a..e353d9bad 100644 --- a/tests/auto_rebase_health.bats +++ b/tests/auto_rebase_health.bats @@ -30,6 +30,21 @@ setup() { {"conclusion":"success","created_at":"2026-06-11T00:00:00Z"}, {"conclusion":"success","created_at":"2026-06-11T06:00:00Z"} ]' + + # Open-PR set for merge-state (BEHIND/DIRTY) observability: + # 2 BEHIND (non-draft, human-authored) — counted + # 1 DIRTY (non-draft, human-authored) — counted + # 1 CLEAN (non-draft, human-authored) — ignored (not BEHIND/DIRTY) + # 1 BEHIND but DRAFT — excluded + # 1 DIRTY but Dependabot-authored — excluded + PRS_JSON='[ + {"number":1,"mergeStateStatus":"BEHIND","isDraft":false,"author":{"login":"don-petry"}}, + {"number":2,"mergeStateStatus":"BEHIND","isDraft":false,"author":{"login":"alice"}}, + {"number":3,"mergeStateStatus":"DIRTY","isDraft":false,"author":{"login":"bob"}}, + {"number":4,"mergeStateStatus":"CLEAN","isDraft":false,"author":{"login":"carol"}}, + {"number":5,"mergeStateStatus":"BEHIND","isDraft":true,"author":{"login":"dave"}}, + {"number":6,"mergeStateStatus":"DIRTY","isDraft":false,"author":{"login":"dependabot[bot]"}} + ]' } # --------------------------------------------------------------------------- @@ -80,6 +95,37 @@ setup() { [ "$output" = "$(printf '0\t0\t0')" ] } +# --------------------------------------------------------------------------- +# summarize_merge_states — BEHIND / DIRTY over open non-draft non-Dependabot PRs +# --------------------------------------------------------------------------- + +@test "summarize_merge_states: counts BEHIND and DIRTY over eligible PRs" { + run summarize_merge_states "$PRS_JSON" + [ "$status" -eq 0 ] + # TSV: behinddirty — 2 BEHIND, 1 DIRTY (draft + dependabot excluded) + [ "$output" = "$(printf '2\t1')" ] +} + +@test "summarize_merge_states: excludes draft PRs" { + run summarize_merge_states '[ + {"mergeStateStatus":"BEHIND","isDraft":true,"author":{"login":"alice"}} + ]' + [ "$output" = "$(printf '0\t0')" ] +} + +@test "summarize_merge_states: excludes Dependabot-authored PRs" { + run summarize_merge_states '[ + {"mergeStateStatus":"DIRTY","isDraft":false,"author":{"login":"dependabot[bot]"}} + ]' + [ "$output" = "$(printf '0\t0')" ] +} + +@test "summarize_merge_states: empty/absent JSON returns zeros (no error)" { + run summarize_merge_states "" + [ "$status" -eq 0 ] + [ "$output" = "$(printf '0\t0')" ] +} + # --------------------------------------------------------------------------- # estimate_fanout — run_count × behind_prs # --------------------------------------------------------------------------- @@ -138,3 +184,17 @@ setup() { [ "$status" -eq 0 ] [[ "$output" == *"n/a"* ]] } + +@test "render_report: surfaces BEHIND/DIRTY merge-state observability section" { + run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 3 2026-06-15 "$PRS_JSON" + [ "$status" -eq 0 ] + [[ "$output" == *"merge-state"* ]] + [[ "$output" == *"BEHIND"* ]] + [[ "$output" == *"DIRTY"* ]] +} + +@test "render_report: absent PR JSON renders merge-state counts as zero" { + run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 0 2026-06-15 + [ "$status" -eq 0 ] + [[ "$output" == *"BEHIND"* ]] +} From ea2f2e6b96719308813b758e2895f854bdec5475 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Mon, 3 Aug 2026 01:24:06 +0000 Subject: [PATCH 2/3] fix(reviews): address review comments [skip ci-relay] --- scripts/auto_rebase_health.sh | 6 +++--- tests/auto_rebase_health.bats | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/auto_rebase_health.sh b/scripts/auto_rebase_health.sh index dbdc0fa2c..11426cea8 100644 --- a/scripts/auto_rebase_health.sh +++ b/scripts/auto_rebase_health.sh @@ -97,15 +97,15 @@ summarize_sentinels() { summarize_merge_states() { local json="${1:-}" [ -n "$json" ] || json='[]' - printf '%s' "$json" | jq -r ' + jq -r ' [ .[] | select((.isDraft // false) | not) - | select(((.author.login // "") | test("dependabot"; "i")) | not) + | select(((.author?.login // "" | tostring) | test("dependabot"; "i")) | not) ] as $prs | [ ([$prs[] | select(.mergeStateStatus == "BEHIND")] | length), ([$prs[] | select(.mergeStateStatus == "DIRTY")] | length) - ] | @tsv' + ] | @tsv' <<< "$json" } # summarize_runs diff --git a/tests/auto_rebase_health.bats b/tests/auto_rebase_health.bats index e353d9bad..b725de3f5 100644 --- a/tests/auto_rebase_health.bats +++ b/tests/auto_rebase_health.bats @@ -126,6 +126,15 @@ setup() { [ "$output" = "$(printf '0\t0')" ] } +@test "summarize_merge_states: handles null or missing author safely" { + run summarize_merge_states '[ + {"mergeStateStatus":"BEHIND","isDraft":false,"author":null}, + {"mergeStateStatus":"DIRTY","isDraft":false} + ]' + [ "$status" -eq 0 ] + [ "$output" = "$(printf '1\t1')" ] +} + # --------------------------------------------------------------------------- # estimate_fanout — run_count × behind_prs # --------------------------------------------------------------------------- From c0bbb99936a41a28a67e6f38dfba2fad45acff9e Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Mon, 3 Aug 2026 01:30:38 +0000 Subject: [PATCH 3/3] chore: dev-lead update (review-changes) [skip ci-relay] --- scripts/auto_rebase_health.sh | 25 +++++++++++++++++++++---- tests/auto_rebase_health.bats | 13 +++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/auto_rebase_health.sh b/scripts/auto_rebase_health.sh index 11426cea8..10bd69a98 100644 --- a/scripts/auto_rebase_health.sh +++ b/scripts/auto_rebase_health.sh @@ -38,6 +38,9 @@ # GH_PAT_FALLBACK — optional fallback PAT if GH_TOKEN lacks run-telemetry access # AGENT_REPO — repo to scan (default: petry-projects/.github-private) # LOOKBACK_DAYS — days of history to consider (default: 7) +# PR_LIST_LIMIT — max open PRs fetched for BEHIND/DIRTY/multiplier metrics (default: 1000) +# raise if the repo regularly has >1000 open PRs; a truncation warning +# is rendered in the report whenever the fetched count equals the limit # AUTO_REBASE_HEALTH_OUT — optional path; report is written there in addition to stdout # GITHUB_STEP_SUMMARY — written by the Actions runner when present @@ -45,6 +48,7 @@ set -euo pipefail WORKFLOW_REPO="${AGENT_REPO:-petry-projects/.github-private}" LOOKBACK_DAYS="${LOOKBACK_DAYS:-7}" +PR_LIST_LIMIT="${PR_LIST_LIMIT:-1000}" AUTO_REBASE_WORKFLOW="auto-rebase.yml" # Markers (kept in one place so a rename in the dev-lead scripts is a one-line fix). @@ -140,11 +144,13 @@ fmt_rate() { echo "$(( num * 100 / denom ))%" } -# render_report [today] [prs_json] +# render_report [today] [prs_json] [pr_list_truncated] [pr_list_limit] # Writes the full Markdown report to stdout. Pure: no network. +# pr_list_truncated=true renders a warning that BEHIND/DIRTY counts may undercount. render_report() { local comments_json="${1:-[]}" runs_json="${2:-[]}" local lookback="${3:-7}" behind="${4:-0}" today="${5:-}" prs_json="${6:-[]}" + local pr_list_truncated="${7:-false}" pr_list_limit="${8:-1000}" [ -n "$today" ] || today="$(date -u +%Y-%m-%d)" local sentinels responses applied @@ -186,6 +192,11 @@ render_report() { printf '## Fleet merge-state observability (AC7)\n\n' printf -- '- **BEHIND** (open non-draft non-Dependabot PRs `mergeStateStatus: BEHIND`): %s\n' "$ms_behind" printf -- '- **DIRTY** (open non-draft non-Dependabot PRs `mergeStateStatus: DIRTY`): %s\n\n' "$ms_dirty" + if [ "$pr_list_truncated" = "true" ]; then + printf '> ⚠ **PR list capped at %s** — BEHIND/DIRTY counts cover only the first %s open PRs ' \ + "$pr_list_limit" "$pr_list_limit" + printf 'and may undercount the true backlog. Set `PR_LIST_LIMIT` to raise the cap.\n\n' + fi printf '> Snapshot of the current open-PR queue, scoped to **`%s`** — fleet-wide counts would ' "$WORKFLOW_REPO" printf 'require a cross-repo PAT this report does not carry. Tracks the effectiveness of the ' printf 'AC1/AC2 auto-rebase fixes (petry-projects/.github#926) without a manual audit.\n' @@ -239,10 +250,16 @@ main() { # 3. Open-PR snapshot — pulled once with the fields both the behind-PR # multiplier and the AC7 merge-state (BEHIND/DIRTY) metric need. Best-effort; # defaults to [] so the report still renders when the query fails. - local prs_json - prs_json="$(gh pr list --repo "$WORKFLOW_REPO" --state open --limit 200 \ + local prs_json pr_list_truncated pr_count + prs_json="$(gh pr list --repo "$WORKFLOW_REPO" --state open --limit "$PR_LIST_LIMIT" \ --json author,isDraft,mergeStateStatus 2>/dev/null || echo '[]')" [ -n "$prs_json" ] || prs_json='[]' + pr_count="$(printf '%s' "$prs_json" | jq 'length' 2>/dev/null || echo 0)" + if [ "$pr_count" -ge "$PR_LIST_LIMIT" ]; then + pr_list_truncated=true + else + pr_list_truncated=false + fi # Behind-PR multiplier — open non-Dependabot PRs (proxy for branches the # fan-out updates), derived from the same snapshot. @@ -252,7 +269,7 @@ main() { 2>/dev/null || echo 0)" local report - report="$(render_report "$comments_json" "$runs_json" "$LOOKBACK_DAYS" "$behind" "$today" "$prs_json")" + report="$(render_report "$comments_json" "$runs_json" "$LOOKBACK_DAYS" "$behind" "$today" "$prs_json" "$pr_list_truncated" "$PR_LIST_LIMIT")" printf '%s\n' "$report" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then diff --git a/tests/auto_rebase_health.bats b/tests/auto_rebase_health.bats index b725de3f5..39aa20295 100644 --- a/tests/auto_rebase_health.bats +++ b/tests/auto_rebase_health.bats @@ -207,3 +207,16 @@ setup() { [ "$status" -eq 0 ] [[ "$output" == *"BEHIND"* ]] } + +@test "render_report: truncation flag renders capped-count warning in merge-state section" { + run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 3 2026-06-15 "$PRS_JSON" true 1000 + [ "$status" -eq 0 ] + [[ "$output" == *"PR list capped at 1000"* ]] + [[ "$output" == *"undercount"* ]] +} + +@test "render_report: no truncation flag suppresses capped-count warning" { + run render_report "$COMMENTS_JSON" "$RUNS_JSON" 7 3 2026-06-15 "$PRS_JSON" false 1000 + [ "$status" -eq 0 ] + [[ "$output" != *"PR list capped"* ]] +}