diff --git a/.github/scripts/add-to-project/lib.sh b/.github/scripts/add-to-project/lib.sh index cb1fd21b0..04d19baea 100644 --- a/.github/scripts/add-to-project/lib.sh +++ b/.github/scripts/add-to-project/lib.sh @@ -185,6 +185,11 @@ add_content_to_project() { fi local content_node_id="$1" + if [ "${DRY_RUN:-}" = "1" ]; then + printf '[dry-run] would add content %s to project\n' "${content_node_id}" + return 0 + fi + # addProjectV2ItemById is idempotent: adding content already on the board # returns the existing item, so the add path needs no find-first dedup. # shellcheck disable=SC2016 # $projectId/$contentId are GraphQL variables @@ -206,6 +211,11 @@ add_draft_item() { local title="$1" local body="$2" + if [ "${DRY_RUN:-}" = "1" ]; then + printf '[dry-run] would add draft: %s\n' "${title}" + return 0 + fi + # shellcheck disable=SC2016 # $projectId/$title/$body are GraphQL variables gh api graphql \ -F projectId="${PROJECT_ID}" \ @@ -225,6 +235,11 @@ delete_project_item() { fi local item_id="$1" + if [ "${DRY_RUN:-}" = "1" ]; then + printf '[dry-run] would delete item %s from project\n' "${item_id}" + return 0 + fi + # Be idempotent on redelivered webhooks / racing runs: a "Could not # resolve" error from a no-longer-present item is the desired final state, # not a real failure. Capture stderr to inspect. diff --git a/.github/scripts/add-to-project/reconcile-backlog.sh b/.github/scripts/add-to-project/reconcile-backlog.sh new file mode 100644 index 000000000..16c0ed1c6 --- /dev/null +++ b/.github/scripts/add-to-project/reconcile-backlog.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# reconcile-backlog.sh — periodic/manual backlog reconcile for add-to-project. +# +# Why this exists (petry-projects/.github#518): the event-driven add-to-project +# path misses qualifying items in two structural cases — +# 1. issues/PRs created by the default GITHUB_TOKEN (app/github-actions): +# GitHub does not fire workflows from default-token events, so +# `issues: opened` never triggers the add; and +# 2. runs dropped under runner congestion (no retry). +# This scans each repo's OPEN issues/PRs + Ideas discussions and reconciles each +# with the board using the SAME gate + helpers as the event path +# (evaluate_noise_gate / reconcile_content_with_project / reconcile_discussion), +# so a missed item lands within one reconcile cycle regardless of author/token. +# Idempotent; honours DRY_RUN=1 (logs intended actions, mutates nothing). +# +# Env: +# PROJECT_ID ProjectV2 node ID of the Initiatives project (required) +# PROJECT_URL human-readable URL (logging only) +# REQUIRED_LABEL gate's required label (default: dev-lead) +# EXCLUDED_LABELS gate's excluded labels (default in add-issue-or-pr.sh) +# IDEAS_CATEGORY discussion category tracked as drafts (default: Ideas) +# RECON_REPOS space-separated owner/repo list to scan +# (default: the planner App installation's repositories) +# DRY_RUN "1" → log only +# GH_TOKEN planner App installation token (issues/PRs read + project write) +set -euo pipefail + +_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/add-to-project/add-issue-or-pr.sh +source "${_dir}/add-issue-or-pr.sh" # reconcile_content_with_project (+ lib.sh) +# shellcheck source=.github/scripts/add-to-project/reconcile-discussion.sh +source "${_dir}/reconcile-discussion.sh" # reconcile_discussion + +: "${PROJECT_ID:?PROJECT_ID is required}" +ideas_category="${IDEAS_CATEGORY:-Ideas}" +# Exported (with default applied) so the discussions --jq filter can read it via +# `env.RECON_IDEAS_CATEGORY` instead of shell-interpolating it — avoids jq syntax +# breakage / injection if the category name contains quotes or backslashes. +export RECON_IDEAS_CATEGORY="${ideas_category}" + +# Resolve target repos: explicit RECON_REPOS, else the App installation's repos. +declare -a repos +if [ -n "${RECON_REPOS:-}" ]; then + read -r -a repos <<< "${RECON_REPOS}" +else + mapfile -t repos < <(gh api --paginate /installation/repositories --jq '.repositories[].full_name') +fi +if [ "${#repos[@]}" -eq 0 ]; then + echo "::error::no repos to reconcile (set RECON_REPOS or check the App installation)" >&2 + exit 64 +fi + +scanned=0 +echo "Reconciling ${#repos[@]} repo(s) into ${PROJECT_URL:-the project}${DRY_RUN:+ (DRY RUN)}" +for repo in "${repos[@]}"; do + [ -z "$repo" ] && continue + echo "::group::${repo}" + + # Open issues AND PRs (the issues endpoint returns both; the gate applies to + # both identically). Pass labels as the objects array the gate expects. + # Fetch into a variable so an API failure is surfaced (not silenced) and + # `set -e`'s check isn't bypassed by a subshell; feed the loop via here-string + # so it stays in the parent shell and `scanned` is updated correctly. + issues_tsv=$(gh api --paginate "repos/${repo}/issues?state=open&per_page=100" \ + --jq '.[] | [.node_id, .html_url, ([.labels[] | {name}] | @json)] | @tsv') || { + echo "::error::failed to fetch issues for ${repo}" >&2 + echo "::endgroup::" + continue + } + while IFS=$'\t' read -r nid url labels; do + [ -z "$nid" ] && continue + scanned=$((scanned + 1)) + reconcile_content_with_project "$nid" "$url" "$labels" || true + done <<< "$issues_tsv" + + # Open Ideas-category discussions → draft items. Same robustness as above; the + # category is read inside jq via `env.RECON_IDEAS_CATEGORY` (set earlier) rather + # than shell-interpolated, so special characters can't break or inject into it. + discussions_tsv=$(gh api graphql --paginate -f owner="${repo%%/*}" -f name="${repo##*/}" -f query=' + query($owner:String!,$name:String!,$endCursor:String){ + repository(owner:$owner,name:$name){ + discussions(first:100,after:$endCursor,states:OPEN){ + pageInfo{hasNextPage endCursor} + nodes{number title url category{name}} + } + } + }' --jq '.data.repository.discussions.nodes[] | select(.category.name==env.RECON_IDEAS_CATEGORY) | [(.number|tostring), .title, .url] | @tsv') || { + echo "::error::failed to fetch discussions for ${repo}" >&2 + echo "::endgroup::" + continue + } + while IFS=$'\t' read -r number title url; do + [ -z "$number" ] && continue + scanned=$((scanned + 1)) + reconcile_discussion "$number" "$title" "$url" "$ideas_category" || true + done <<< "$discussions_tsv" + + echo "::endgroup::" +done + +echo "Reconcile complete — scanned ${scanned} item(s) across ${#repos[@]} repo(s)." diff --git a/.github/workflows/add-to-project-reconcile.yml b/.github/workflows/add-to-project-reconcile.yml new file mode 100644 index 000000000..bbd5f324e --- /dev/null +++ b/.github/workflows/add-to-project-reconcile.yml @@ -0,0 +1,65 @@ +# Backlog reconcile for the Initiatives project (petry-projects/.github#518). +# +# The event-driven add-to-project automation misses qualifying items created by +# the default GITHUB_TOKEN (GitHub doesn't fire workflows from default-token +# events) and items whose run was dropped under congestion. This scheduled/ +# manual reconcile scans every installed repo's OPEN issues/PRs + Ideas +# discussions and reconciles them onto the board using the same gate + helpers +# as the event path — so missed items land within one cycle. +# +# Runs centrally in petry-projects/.github (the scripts live here); mints the +# petry-projects-planner App token for issues/PRs read + project write. +name: Add-to-Project Reconcile + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Log intended adds/removes without mutating the board" + type: boolean + default: true + repos: + description: "Optional space-separated owner/repo list (default: all App-installed repos)" + type: string + default: "" + schedule: + - cron: "0 7 * * *" # daily 07:00 UTC + +permissions: + contents: read + +concurrency: + group: add-to-project-reconcile + cancel-in-progress: false + +jobs: + reconcile: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Mint App installation token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.INITIATIVES_APP_ID }} + private-key: ${{ secrets.INITIATIVES_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + + - name: Reconcile backlog into the project + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + PROJECT_ID: PVT_kwDOD2inqs4BZq3- + PROJECT_URL: https://github.com/orgs/petry-projects/projects/1 + REQUIRED_LABEL: dev-lead + EXCLUDED_LABELS: compliance-audit,health-check,fleet-tracker,daily-report + IDEAS_CATEGORY: Ideas + RECON_REPOS: ${{ inputs.repos }} + # Manual runs default to dry-run; scheduled runs are live. + DRY_RUN: ${{ (github.event_name == 'workflow_dispatch' && inputs.dry_run) && '1' || '0' }} + run: bash .github/scripts/add-to-project/reconcile-backlog.sh diff --git a/standards/initiatives-project.md b/standards/initiatives-project.md index 6e7e63b7b..80b27d4ea 100644 --- a/standards/initiatives-project.md +++ b/standards/initiatives-project.md @@ -51,8 +51,15 @@ The reconciliation is idempotent — re-delivered webhooks don't double-error. - **Fork PRs labeled by a maintainer when the author is `FIRST_TIMER` / `CONTRIBUTOR`.** The `pull_request_target` gate evaluates the PR author's association; this is documented as a known limit in the workflow header. -- **Historical (already-open) qualifying items.** The workflow only fires on - new events. Backfilled in bulk on 2026-06-07 — see #387 retro. +- **Items the event triggers never fire for.** The event path misses two + structural cases: issues/PRs created by the default `GITHUB_TOKEN` + (`app/github-actions`) — GitHub does not fire workflows from default-token + events — and runs dropped under runner congestion. **A scheduled/manual + backlog reconcile (`add-to-project-reconcile.yml`, #518) closes both:** it + scans every App-installed repo's open issues/PRs + Ideas discussions daily and + reconciles each via the same gate/helpers, so a missed item lands within one + cycle. Run it manually with `dry_run` to preview. Full history (incl. + closed/merged) was bulk-backfilled 2026-06-07 and again 2026-06-21. To add a content-linked item manually: @@ -188,11 +195,14 @@ configured manually in the UI): ```text .github/workflows/add-to-project.yml # Workflow (events → script call) +.github/workflows/add-to-project-reconcile.yml # Scheduled/manual backlog reconcile (#518) .github/scripts/add-to-project/ + lib.sh # find/add/draft/delete helpers (DRY_RUN-aware) add-issue-or-pr.sh # Noise gate + addProjectV2ItemById reconcile-discussion.sh # Paginated find + 4-state reconciler + reconcile-backlog.sh # Scans open issues/PRs + Ideas, reconciles via the above .github/workflows/add-to-project-tests.yml # shellcheck + bats CI gate -test/workflows/add-to-project/ # 35 bats tests, gh stub, fixtures +test/workflows/add-to-project/ # bats tests, gh stub, fixtures ``` **Auth:** A dedicated GitHub App (`petry-projects-planner`, App ID diff --git a/test/workflows/add-to-project/reconcile-backlog.bats b/test/workflows/add-to-project/reconcile-backlog.bats new file mode 100644 index 000000000..20fc31657 --- /dev/null +++ b/test/workflows/add-to-project/reconcile-backlog.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats +# Tests for reconcile-backlog.sh — the periodic/manual backlog reconcile +# (petry-projects/.github#518). Runs the script with a tailored `gh` stub and +# DRY_RUN=1, and asserts it scans a repo's open issues/PRs + Ideas discussions +# and reconciles qualifying ones via the shared event-path helpers (logging the +# intended adds rather than mutating). + +bats_require_minimum_version 1.5.0 + +load 'helpers/setup' + +setup() { + tt_make_tmpdir + # Tailored gh stub: returns one qualifying issue, one Ideas discussion, and + # "not found" for the draft-dedup lookup so reconcile takes the add path. + local bin="${TT_TMP}/bin"; mkdir -p "$bin" + cat > "$bin/gh" <<'STUB' +#!/usr/bin/env bash +args="$*" +case "$args" in + *"repos/"*"/issues"*) + # node_id html_url labels-objects-json (one qualifying issue) + printf 'I_node1\thttps://example.invalid/issues/1\t[{"name":"dev-lead"}]\n' ;; + *graphql*discussions*) + # number title url (one Ideas discussion) + printf '55\tShiny idea\thttps://example.invalid/discussions/55\n' ;; + *graphql*) + # find_project_item lookups (draft-dedup / remove path) → not found + printf '' ;; + *) printf '' ;; +esac +STUB + chmod +x "$bin/gh"; PATH="${bin}:${PATH}"; export PATH + export PROJECT_ID="PVT_test" PROJECT_URL="https://example.invalid/p/1" GH_TOKEN="t" + export RECON_REPOS="petry-projects/demo" # skip installation lookup + export DRY_RUN="1" +} +teardown() { tt_cleanup_tmpdir; } + +run_recon() { run bash "${TT_SCRIPTS_DIR}/reconcile-backlog.sh"; } + +@test "dry-run reconciles a qualifying issue (logs intended add, no mutation)" { + run_recon + [ "$status" -eq 0 ] + echo "$output" | grep -q '\[dry-run\] would add content I_node1 to project' +} + +@test "dry-run reconciles an Ideas discussion as a draft" { + run_recon + [ "$status" -eq 0 ] + echo "$output" | grep -q '\[dry-run\] would add draft: \[Discussion #55\] Shiny idea' +} + +@test "reports the repo count it scanned" { + run_recon + [ "$status" -eq 0 ] + echo "$output" | grep -q 'Reconciling 1 repo(s)' + echo "$output" | grep -q 'Reconcile complete' +} + +@test "an excluded label disqualifies an issue (no add)" { + # override the stub to return a dev-lead + compliance-audit issue + cat > "${TT_TMP}/bin/gh" <<'STUB' +#!/usr/bin/env bash +case "$*" in + *"repos/"*"/issues"*) printf 'I_excl\thttps://example.invalid/issues/2\t[{"name":"dev-lead"},{"name":"compliance-audit"}]\n' ;; + *graphql*discussions*) printf '' ;; + *graphql*) printf '' ;; +esac +STUB + chmod +x "${TT_TMP}/bin/gh" + run_recon + [ "$status" -eq 0 ] + ! echo "$output" | grep -q 'would add content I_excl' +}