From 647e581698eb6fa8e656e14097c876a72264fdce Mon Sep 17 00:00:00 2001 From: Don Petry Date: Sun, 21 Jun 2026 20:11:07 -0500 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20implement=20issue=20#518=20?= =?UTF-8?q?=E2=80=94=20add-to-project=20backlog=20reconcile=20(workflow=5F?= =?UTF-8?q?dispatch=20+=20cron)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The event-driven add-to-project path misses two structural cases: issues/PRs created by the default GITHUB_TOKEN (GitHub doesn't fire workflows from default-token events) and runs dropped under congestion. Verified live — qualifying post-enablement items (e.g. .github-private#841/842/846) never got added by the event path. Add a scheduled + manual backlog reconcile that closes the gap: - scripts/add-to-project/reconcile-backlog.sh — scans each App-installed repo's OPEN issues/PRs + Ideas discussions and reconciles each via the SAME helpers as the event path (evaluate_noise_gate / reconcile_content_with_project / reconcile_discussion), so missed items land within one cycle. Idempotent; resolves repos from the App installation when RECON_REPOS is unset. - add-to-project-reconcile.yml — runs centrally in .github, daily cron + workflow_dispatch (manual runs default to dry_run); mints the planner App token. Same project_id / noise-gate constants as the stub. - lib.sh add/draft/delete mutators gain DRY_RUN=1 support (log, don't mutate) — used by the reconcile preview; no effect on the event path. - bats: reconcile-backlog.bats (qualify-add, Ideas-draft, repo-count, excluded skip); full add-to-project suite green (47). - docs: initiatives-project.md — describe the reconcile as the durable fix for the GITHUB_TOKEN / congestion gap. Fixes #518. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/add-to-project/lib.sh | 15 ++++ .../add-to-project/reconcile-backlog.sh | 82 +++++++++++++++++++ .../workflows/add-to-project-reconcile.yml | 65 +++++++++++++++ standards/initiatives-project.md | 16 +++- .../add-to-project/reconcile-backlog.bats | 75 +++++++++++++++++ 5 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/add-to-project/reconcile-backlog.sh create mode 100644 .github/workflows/add-to-project-reconcile.yml create mode 100644 test/workflows/add-to-project/reconcile-backlog.bats diff --git a/.github/scripts/add-to-project/lib.sh b/.github/scripts/add-to-project/lib.sh index cb1fd21b0..04d19baea 100644 --- a/.github/scripts/add-to-project/lib.sh +++ b/.github/scripts/add-to-project/lib.sh @@ -185,6 +185,11 @@ add_content_to_project() { fi local content_node_id="$1" + if [ "${DRY_RUN:-}" = "1" ]; then + printf '[dry-run] would add content %s to project\n' "${content_node_id}" + return 0 + fi + # addProjectV2ItemById is idempotent: adding content already on the board # returns the existing item, so the add path needs no find-first dedup. # shellcheck disable=SC2016 # $projectId/$contentId are GraphQL variables @@ -206,6 +211,11 @@ add_draft_item() { local title="$1" local body="$2" + if [ "${DRY_RUN:-}" = "1" ]; then + printf '[dry-run] would add draft: %s\n' "${title}" + return 0 + fi + # shellcheck disable=SC2016 # $projectId/$title/$body are GraphQL variables gh api graphql \ -F projectId="${PROJECT_ID}" \ @@ -225,6 +235,11 @@ delete_project_item() { fi local item_id="$1" + if [ "${DRY_RUN:-}" = "1" ]; then + printf '[dry-run] would delete item %s from project\n' "${item_id}" + return 0 + fi + # Be idempotent on redelivered webhooks / racing runs: a "Could not # resolve" error from a no-longer-present item is the desired final state, # not a real failure. Capture stderr to inspect. diff --git a/.github/scripts/add-to-project/reconcile-backlog.sh b/.github/scripts/add-to-project/reconcile-backlog.sh new file mode 100644 index 000000000..2bc7fa25f --- /dev/null +++ b/.github/scripts/add-to-project/reconcile-backlog.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# reconcile-backlog.sh — periodic/manual backlog reconcile for add-to-project. +# +# Why this exists (petry-projects/.github#518): the event-driven add-to-project +# path misses qualifying items in two structural cases — +# 1. issues/PRs created by the default GITHUB_TOKEN (app/github-actions): +# GitHub does not fire workflows from default-token events, so +# `issues: opened` never triggers the add; and +# 2. runs dropped under runner congestion (no retry). +# This scans each repo's OPEN issues/PRs + Ideas discussions and reconciles each +# with the board using the SAME gate + helpers as the event path +# (evaluate_noise_gate / reconcile_content_with_project / reconcile_discussion), +# so a missed item lands within one reconcile cycle regardless of author/token. +# Idempotent; honours DRY_RUN=1 (logs intended actions, mutates nothing). +# +# Env: +# PROJECT_ID ProjectV2 node ID of the Initiatives project (required) +# PROJECT_URL human-readable URL (logging only) +# REQUIRED_LABEL gate's required label (default: dev-lead) +# EXCLUDED_LABELS gate's excluded labels (default in add-issue-or-pr.sh) +# IDEAS_CATEGORY discussion category tracked as drafts (default: Ideas) +# RECON_REPOS space-separated owner/repo list to scan +# (default: the planner App installation's repositories) +# DRY_RUN "1" → log only +# GH_TOKEN planner App installation token (issues/PRs read + project write) +set -euo pipefail + +_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/add-to-project/add-issue-or-pr.sh +source "${_dir}/add-issue-or-pr.sh" # reconcile_content_with_project (+ lib.sh) +# shellcheck source=.github/scripts/add-to-project/reconcile-discussion.sh +source "${_dir}/reconcile-discussion.sh" # reconcile_discussion + +: "${PROJECT_ID:?PROJECT_ID is required}" +ideas_category="${IDEAS_CATEGORY:-Ideas}" + +# Resolve target repos: explicit RECON_REPOS, else the App installation's repos. +declare -a repos +if [ -n "${RECON_REPOS:-}" ]; then + read -r -a repos <<< "${RECON_REPOS}" +else + mapfile -t repos < <(gh api --paginate /installation/repositories --jq '.repositories[].full_name' 2>/dev/null) +fi +if [ "${#repos[@]}" -eq 0 ]; then + echo "::error::no repos to reconcile (set RECON_REPOS or check the App installation)" >&2 + exit 64 +fi + +scanned=0 +echo "Reconciling ${#repos[@]} repo(s) into ${PROJECT_URL:-the project}${DRY_RUN:+ (DRY RUN)}" +for repo in "${repos[@]}"; do + [ -z "$repo" ] && continue + echo "::group::${repo}" + + # Open issues AND PRs (the issues endpoint returns both; the gate applies to + # both identically). Pass labels as the objects array the gate expects. + while IFS=$'\t' read -r nid url labels; do + [ -z "$nid" ] && continue + scanned=$((scanned + 1)) + reconcile_content_with_project "$nid" "$url" "$labels" || true + done < <(gh api --paginate "repos/${repo}/issues?state=open&per_page=100" \ + --jq '.[] | [.node_id, .html_url, ([.labels[] | {name}] | @json)] | @tsv' 2>/dev/null) + + # Open Ideas-category discussions → draft items. + while IFS=$'\t' read -r number title url; do + [ -z "$number" ] && continue + scanned=$((scanned + 1)) + reconcile_discussion "$number" "$title" "$url" "$ideas_category" || true + done < <(gh api graphql --paginate -f owner="${repo%%/*}" -f name="${repo##*/}" -f query=' + query($owner:String!,$name:String!,$endCursor:String){ + repository(owner:$owner,name:$name){ + discussions(first:100,after:$endCursor,states:OPEN){ + pageInfo{hasNextPage endCursor} + nodes{number title url category{name}} + } + } + }' --jq ".data.repository.discussions.nodes[] | select(.category.name==\"${ideas_category}\") | [(.number|tostring), .title, .url] | @tsv" 2>/dev/null) + + echo "::endgroup::" +done + +echo "Reconcile complete — scanned ${scanned} item(s) across ${#repos[@]} repo(s)." diff --git a/.github/workflows/add-to-project-reconcile.yml b/.github/workflows/add-to-project-reconcile.yml new file mode 100644 index 000000000..bbd5f324e --- /dev/null +++ b/.github/workflows/add-to-project-reconcile.yml @@ -0,0 +1,65 @@ +# Backlog reconcile for the Initiatives project (petry-projects/.github#518). +# +# The event-driven add-to-project automation misses qualifying items created by +# the default GITHUB_TOKEN (GitHub doesn't fire workflows from default-token +# events) and items whose run was dropped under congestion. This scheduled/ +# manual reconcile scans every installed repo's OPEN issues/PRs + Ideas +# discussions and reconciles them onto the board using the same gate + helpers +# as the event path — so missed items land within one cycle. +# +# Runs centrally in petry-projects/.github (the scripts live here); mints the +# petry-projects-planner App token for issues/PRs read + project write. +name: Add-to-Project Reconcile + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Log intended adds/removes without mutating the board" + type: boolean + default: true + repos: + description: "Optional space-separated owner/repo list (default: all App-installed repos)" + type: string + default: "" + schedule: + - cron: "0 7 * * *" # daily 07:00 UTC + +permissions: + contents: read + +concurrency: + group: add-to-project-reconcile + cancel-in-progress: false + +jobs: + reconcile: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Mint App installation token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.INITIATIVES_APP_ID }} + private-key: ${{ secrets.INITIATIVES_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + + - name: Reconcile backlog into the project + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + PROJECT_ID: PVT_kwDOD2inqs4BZq3- + PROJECT_URL: https://github.com/orgs/petry-projects/projects/1 + REQUIRED_LABEL: dev-lead + EXCLUDED_LABELS: compliance-audit,health-check,fleet-tracker,daily-report + IDEAS_CATEGORY: Ideas + RECON_REPOS: ${{ inputs.repos }} + # Manual runs default to dry-run; scheduled runs are live. + DRY_RUN: ${{ (github.event_name == 'workflow_dispatch' && inputs.dry_run) && '1' || '0' }} + run: bash .github/scripts/add-to-project/reconcile-backlog.sh diff --git a/standards/initiatives-project.md b/standards/initiatives-project.md index 6e7e63b7b..80b27d4ea 100644 --- a/standards/initiatives-project.md +++ b/standards/initiatives-project.md @@ -51,8 +51,15 @@ The reconciliation is idempotent — re-delivered webhooks don't double-error. - **Fork PRs labeled by a maintainer when the author is `FIRST_TIMER` / `CONTRIBUTOR`.** The `pull_request_target` gate evaluates the PR author's association; this is documented as a known limit in the workflow header. -- **Historical (already-open) qualifying items.** The workflow only fires on - new events. Backfilled in bulk on 2026-06-07 — see #387 retro. +- **Items the event triggers never fire for.** The event path misses two + structural cases: issues/PRs created by the default `GITHUB_TOKEN` + (`app/github-actions`) — GitHub does not fire workflows from default-token + events — and runs dropped under runner congestion. **A scheduled/manual + backlog reconcile (`add-to-project-reconcile.yml`, #518) closes both:** it + scans every App-installed repo's open issues/PRs + Ideas discussions daily and + reconciles each via the same gate/helpers, so a missed item lands within one + cycle. Run it manually with `dry_run` to preview. Full history (incl. + closed/merged) was bulk-backfilled 2026-06-07 and again 2026-06-21. To add a content-linked item manually: @@ -188,11 +195,14 @@ configured manually in the UI): ```text .github/workflows/add-to-project.yml # Workflow (events → script call) +.github/workflows/add-to-project-reconcile.yml # Scheduled/manual backlog reconcile (#518) .github/scripts/add-to-project/ + lib.sh # find/add/draft/delete helpers (DRY_RUN-aware) add-issue-or-pr.sh # Noise gate + addProjectV2ItemById reconcile-discussion.sh # Paginated find + 4-state reconciler + reconcile-backlog.sh # Scans open issues/PRs + Ideas, reconciles via the above .github/workflows/add-to-project-tests.yml # shellcheck + bats CI gate -test/workflows/add-to-project/ # 35 bats tests, gh stub, fixtures +test/workflows/add-to-project/ # bats tests, gh stub, fixtures ``` **Auth:** A dedicated GitHub App (`petry-projects-planner`, App ID diff --git a/test/workflows/add-to-project/reconcile-backlog.bats b/test/workflows/add-to-project/reconcile-backlog.bats new file mode 100644 index 000000000..20fc31657 --- /dev/null +++ b/test/workflows/add-to-project/reconcile-backlog.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats +# Tests for reconcile-backlog.sh — the periodic/manual backlog reconcile +# (petry-projects/.github#518). Runs the script with a tailored `gh` stub and +# DRY_RUN=1, and asserts it scans a repo's open issues/PRs + Ideas discussions +# and reconciles qualifying ones via the shared event-path helpers (logging the +# intended adds rather than mutating). + +bats_require_minimum_version 1.5.0 + +load 'helpers/setup' + +setup() { + tt_make_tmpdir + # Tailored gh stub: returns one qualifying issue, one Ideas discussion, and + # "not found" for the draft-dedup lookup so reconcile takes the add path. + local bin="${TT_TMP}/bin"; mkdir -p "$bin" + cat > "$bin/gh" <<'STUB' +#!/usr/bin/env bash +args="$*" +case "$args" in + *"repos/"*"/issues"*) + # node_id html_url labels-objects-json (one qualifying issue) + printf 'I_node1\thttps://example.invalid/issues/1\t[{"name":"dev-lead"}]\n' ;; + *graphql*discussions*) + # number title url (one Ideas discussion) + printf '55\tShiny idea\thttps://example.invalid/discussions/55\n' ;; + *graphql*) + # find_project_item lookups (draft-dedup / remove path) → not found + printf '' ;; + *) printf '' ;; +esac +STUB + chmod +x "$bin/gh"; PATH="${bin}:${PATH}"; export PATH + export PROJECT_ID="PVT_test" PROJECT_URL="https://example.invalid/p/1" GH_TOKEN="t" + export RECON_REPOS="petry-projects/demo" # skip installation lookup + export DRY_RUN="1" +} +teardown() { tt_cleanup_tmpdir; } + +run_recon() { run bash "${TT_SCRIPTS_DIR}/reconcile-backlog.sh"; } + +@test "dry-run reconciles a qualifying issue (logs intended add, no mutation)" { + run_recon + [ "$status" -eq 0 ] + echo "$output" | grep -q '\[dry-run\] would add content I_node1 to project' +} + +@test "dry-run reconciles an Ideas discussion as a draft" { + run_recon + [ "$status" -eq 0 ] + echo "$output" | grep -q '\[dry-run\] would add draft: \[Discussion #55\] Shiny idea' +} + +@test "reports the repo count it scanned" { + run_recon + [ "$status" -eq 0 ] + echo "$output" | grep -q 'Reconciling 1 repo(s)' + echo "$output" | grep -q 'Reconcile complete' +} + +@test "an excluded label disqualifies an issue (no add)" { + # override the stub to return a dev-lead + compliance-audit issue + cat > "${TT_TMP}/bin/gh" <<'STUB' +#!/usr/bin/env bash +case "$*" in + *"repos/"*"/issues"*) printf 'I_excl\thttps://example.invalid/issues/2\t[{"name":"dev-lead"},{"name":"compliance-audit"}]\n' ;; + *graphql*discussions*) printf '' ;; + *graphql*) printf '' ;; +esac +STUB + chmod +x "${TT_TMP}/bin/gh" + run_recon + [ "$status" -eq 0 ] + ! echo "$output" | grep -q 'would add content I_excl' +} From 9262241c0caacc694abf64e068c5ab78d12953c8 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Mon, 22 Jun 2026 01:23:54 +0000 Subject: [PATCH 2/2] fix(reviews): address review comments [skip ci-relay] --- .../add-to-project/reconcile-backlog.sh | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/.github/scripts/add-to-project/reconcile-backlog.sh b/.github/scripts/add-to-project/reconcile-backlog.sh index 2bc7fa25f..16c0ed1c6 100644 --- a/.github/scripts/add-to-project/reconcile-backlog.sh +++ b/.github/scripts/add-to-project/reconcile-backlog.sh @@ -33,13 +33,17 @@ source "${_dir}/reconcile-discussion.sh" # reconcile_discussion : "${PROJECT_ID:?PROJECT_ID is required}" ideas_category="${IDEAS_CATEGORY:-Ideas}" +# Exported (with default applied) so the discussions --jq filter can read it via +# `env.RECON_IDEAS_CATEGORY` instead of shell-interpolating it — avoids jq syntax +# breakage / injection if the category name contains quotes or backslashes. +export RECON_IDEAS_CATEGORY="${ideas_category}" # Resolve target repos: explicit RECON_REPOS, else the App installation's repos. declare -a repos if [ -n "${RECON_REPOS:-}" ]; then read -r -a repos <<< "${RECON_REPOS}" else - mapfile -t repos < <(gh api --paginate /installation/repositories --jq '.repositories[].full_name' 2>/dev/null) + mapfile -t repos < <(gh api --paginate /installation/repositories --jq '.repositories[].full_name') fi if [ "${#repos[@]}" -eq 0 ]; then echo "::error::no repos to reconcile (set RECON_REPOS or check the App installation)" >&2 @@ -54,19 +58,25 @@ for repo in "${repos[@]}"; do # Open issues AND PRs (the issues endpoint returns both; the gate applies to # both identically). Pass labels as the objects array the gate expects. + # Fetch into a variable so an API failure is surfaced (not silenced) and + # `set -e`'s check isn't bypassed by a subshell; feed the loop via here-string + # so it stays in the parent shell and `scanned` is updated correctly. + issues_tsv=$(gh api --paginate "repos/${repo}/issues?state=open&per_page=100" \ + --jq '.[] | [.node_id, .html_url, ([.labels[] | {name}] | @json)] | @tsv') || { + echo "::error::failed to fetch issues for ${repo}" >&2 + echo "::endgroup::" + continue + } while IFS=$'\t' read -r nid url labels; do [ -z "$nid" ] && continue scanned=$((scanned + 1)) reconcile_content_with_project "$nid" "$url" "$labels" || true - done < <(gh api --paginate "repos/${repo}/issues?state=open&per_page=100" \ - --jq '.[] | [.node_id, .html_url, ([.labels[] | {name}] | @json)] | @tsv' 2>/dev/null) + done <<< "$issues_tsv" - # Open Ideas-category discussions → draft items. - while IFS=$'\t' read -r number title url; do - [ -z "$number" ] && continue - scanned=$((scanned + 1)) - reconcile_discussion "$number" "$title" "$url" "$ideas_category" || true - done < <(gh api graphql --paginate -f owner="${repo%%/*}" -f name="${repo##*/}" -f query=' + # Open Ideas-category discussions → draft items. Same robustness as above; the + # category is read inside jq via `env.RECON_IDEAS_CATEGORY` (set earlier) rather + # than shell-interpolated, so special characters can't break or inject into it. + discussions_tsv=$(gh api graphql --paginate -f owner="${repo%%/*}" -f name="${repo##*/}" -f query=' query($owner:String!,$name:String!,$endCursor:String){ repository(owner:$owner,name:$name){ discussions(first:100,after:$endCursor,states:OPEN){ @@ -74,7 +84,16 @@ for repo in "${repos[@]}"; do nodes{number title url category{name}} } } - }' --jq ".data.repository.discussions.nodes[] | select(.category.name==\"${ideas_category}\") | [(.number|tostring), .title, .url] | @tsv" 2>/dev/null) + }' --jq '.data.repository.discussions.nodes[] | select(.category.name==env.RECON_IDEAS_CATEGORY) | [(.number|tostring), .title, .url] | @tsv') || { + echo "::error::failed to fetch discussions for ${repo}" >&2 + echo "::endgroup::" + continue + } + while IFS=$'\t' read -r number title url; do + [ -z "$number" ] && continue + scanned=$((scanned + 1)) + reconcile_discussion "$number" "$title" "$url" "$ideas_category" || true + done <<< "$discussions_tsv" echo "::endgroup::" done