feat: implement issue #518 — add-to-project backlog reconcile (workflow_dispatch + cron)

.github/scripts/add-to-project/lib.sh

@@ -185,6 +185,11 @@ add_content_to_project() {
   fi
   local content_node_id="$1"
 
+  if [ "${DRY_RUN:-}" = "1" ]; then
+    printf '[dry-run] would add content %s to project\n' "${content_node_id}"
+    return 0
+  fi
+
   # addProjectV2ItemById is idempotent: adding content already on the board
   # returns the existing item, so the add path needs no find-first dedup.
   # shellcheck disable=SC2016  # $projectId/$contentId are GraphQL variables
@@ -206,6 +211,11 @@ add_draft_item() {
   local title="$1"
   local body="$2"
 
+  if [ "${DRY_RUN:-}" = "1" ]; then
+    printf '[dry-run] would add draft: %s\n' "${title}"
+    return 0
+  fi
+
   # shellcheck disable=SC2016  # $projectId/$title/$body are GraphQL variables
   gh api graphql \
     -F projectId="${PROJECT_ID}" \
@@ -225,6 +235,11 @@ delete_project_item() {
   fi
   local item_id="$1"
 
+  if [ "${DRY_RUN:-}" = "1" ]; then
+    printf '[dry-run] would delete item %s from project\n' "${item_id}"
+    return 0
+  fi
+
   # Be idempotent on redelivered webhooks / racing runs: a "Could not
   # resolve" error from a no-longer-present item is the desired final state,
   # not a real failure. Capture stderr to inspect.

.github/scripts/add-to-project/reconcile-backlog.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# reconcile-backlog.sh — periodic/manual backlog reconcile for add-to-project.
+#
+# Why this exists (petry-projects/.github#518): the event-driven add-to-project
+# path misses qualifying items in two structural cases —
+#   1. issues/PRs created by the default GITHUB_TOKEN (app/github-actions):
+#      GitHub does not fire workflows from default-token events, so
+#      `issues: opened` never triggers the add; and
+#   2. runs dropped under runner congestion (no retry).
+# This scans each repo's OPEN issues/PRs + Ideas discussions and reconciles each
+# with the board using the SAME gate + helpers as the event path
+# (evaluate_noise_gate / reconcile_content_with_project / reconcile_discussion),
+# so a missed item lands within one reconcile cycle regardless of author/token.
+# Idempotent; honours DRY_RUN=1 (logs intended actions, mutates nothing).
+#
+# Env:
+#   PROJECT_ID       ProjectV2 node ID of the Initiatives project   (required)
+#   PROJECT_URL      human-readable URL (logging only)
+#   REQUIRED_LABEL   gate's required label   (default: dev-lead)
+#   EXCLUDED_LABELS  gate's excluded labels  (default in add-issue-or-pr.sh)
+#   IDEAS_CATEGORY   discussion category tracked as drafts (default: Ideas)
+#   RECON_REPOS      space-separated owner/repo list to scan
+#                    (default: the planner App installation's repositories)
+#   DRY_RUN          "1" → log only
+#   GH_TOKEN         planner App installation token (issues/PRs read + project write)
+set -euo pipefail
+
+_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/add-to-project/add-issue-or-pr.sh
+source "${_dir}/add-issue-or-pr.sh"       # reconcile_content_with_project (+ lib.sh)
+# shellcheck source=.github/scripts/add-to-project/reconcile-discussion.sh
+source "${_dir}/reconcile-discussion.sh"  # reconcile_discussion
+
+: "${PROJECT_ID:?PROJECT_ID is required}"
+ideas_category="${IDEAS_CATEGORY:-Ideas}"
+# Exported (with default applied) so the discussions --jq filter can read it via
+# `env.RECON_IDEAS_CATEGORY` instead of shell-interpolating it — avoids jq syntax
+# breakage / injection if the category name contains quotes or backslashes.
+export RECON_IDEAS_CATEGORY="${ideas_category}"
+
+# Resolve target repos: explicit RECON_REPOS, else the App installation's repos.
+declare -a repos
+if [ -n "${RECON_REPOS:-}" ]; then
+  read -r -a repos <<< "${RECON_REPOS}"
+else
+  mapfile -t repos < <(gh api --paginate /installation/repositories --jq '.repositories[].full_name')
+fi
+if [ "${#repos[@]}" -eq 0 ]; then
+  echo "::error::no repos to reconcile (set RECON_REPOS or check the App installation)" >&2
+  exit 64
+fi
+
+scanned=0
+echo "Reconciling ${#repos[@]} repo(s) into ${PROJECT_URL:-the project}${DRY_RUN:+ (DRY RUN)}"
+for repo in "${repos[@]}"; do
+  [ -z "$repo" ] && continue
+  echo "::group::${repo}"
+
+  # Open issues AND PRs (the issues endpoint returns both; the gate applies to
+  # both identically). Pass labels as the objects array the gate expects.
+  # Fetch into a variable so an API failure is surfaced (not silenced) and
+  # `set -e`'s check isn't bypassed by a subshell; feed the loop via here-string
+  # so it stays in the parent shell and `scanned` is updated correctly.
+  issues_tsv=$(gh api --paginate "repos/${repo}/issues?state=open&per_page=100" \
+            --jq '.[] | [.node_id, .html_url, ([.labels[] | {name}] | @json)] | @tsv') || {
+    echo "::error::failed to fetch issues for ${repo}" >&2
+    echo "::endgroup::"
+    continue
+  }
+  while IFS=$'\t' read -r nid url labels; do
+    [ -z "$nid" ] && continue
+    scanned=$((scanned + 1))
+    reconcile_content_with_project "$nid" "$url" "$labels" || true
+  done <<< "$issues_tsv"
+
+  # Open Ideas-category discussions → draft items. Same robustness as above; the
+  # category is read inside jq via `env.RECON_IDEAS_CATEGORY` (set earlier) rather
+  # than shell-interpolated, so special characters can't break or inject into it.
+  discussions_tsv=$(gh api graphql --paginate -f owner="${repo%%/*}" -f name="${repo##*/}" -f query='
+    query($owner:String!,$name:String!,$endCursor:String){
+      repository(owner:$owner,name:$name){
+        discussions(first:100,after:$endCursor,states:OPEN){
+          pageInfo{hasNextPage endCursor}
+          nodes{number title url category{name}}
+        }
+      }
+    }' --jq '.data.repository.discussions.nodes[] | select(.category.name==env.RECON_IDEAS_CATEGORY) | [(.number|tostring), .title, .url] | @tsv') || {
+    echo "::error::failed to fetch discussions for ${repo}" >&2
+    echo "::endgroup::"
+    continue
+  }
+  while IFS=$'\t' read -r number title url; do
+    [ -z "$number" ] && continue
+    scanned=$((scanned + 1))
+    reconcile_discussion "$number" "$title" "$url" "$ideas_category" || true
+  done <<< "$discussions_tsv"
+
+  echo "::endgroup::"
+done
+
+echo "Reconcile complete — scanned ${scanned} item(s) across ${#repos[@]} repo(s)."

.github/workflows/add-to-project-reconcile.yml

@@ -0,0 +1,65 @@
+# Backlog reconcile for the Initiatives project (petry-projects/.github#518).
+#
+# The event-driven add-to-project automation misses qualifying items created by
+# the default GITHUB_TOKEN (GitHub doesn't fire workflows from default-token
+# events) and items whose run was dropped under congestion. This scheduled/
+# manual reconcile scans every installed repo's OPEN issues/PRs + Ideas
+# discussions and reconciles them onto the board using the same gate + helpers
+# as the event path — so missed items land within one cycle.
+#
+# Runs centrally in petry-projects/.github (the scripts live here); mints the
+# petry-projects-planner App token for issues/PRs read + project write.
+name: Add-to-Project Reconcile
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Log intended adds/removes without mutating the board"
+        type: boolean
+        default: true
+      repos:
+        description: "Optional space-separated owner/repo list (default: all App-installed repos)"
+        type: string
+        default: ""
+  schedule:
+    - cron: "0 7 * * *" # daily 07:00 UTC
+
+permissions:
+  contents: read
+
+concurrency:
+  group: add-to-project-reconcile
+  cancel-in-progress: false
+
+jobs:
+  reconcile:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Mint App installation token
+        id: app-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.INITIATIVES_APP_ID }}
+          private-key: ${{ secrets.INITIATIVES_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Reconcile backlog into the project
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          PROJECT_ID: PVT_kwDOD2inqs4BZq3-
+          PROJECT_URL: https://github.com/orgs/petry-projects/projects/1
+          REQUIRED_LABEL: dev-lead
+          EXCLUDED_LABELS: compliance-audit,health-check,fleet-tracker,daily-report
+          IDEAS_CATEGORY: Ideas
+          RECON_REPOS: ${{ inputs.repos }}
+          # Manual runs default to dry-run; scheduled runs are live.
+          DRY_RUN: ${{ (github.event_name == 'workflow_dispatch' && inputs.dry_run) && '1' || '0' }}
+        run: bash .github/scripts/add-to-project/reconcile-backlog.sh

standards/initiatives-project.md

@@ -51,8 +51,15 @@ The reconciliation is idempotent — re-delivered webhooks don't double-error.
 - **Fork PRs labeled by a maintainer when the author is `FIRST_TIMER` /
   `CONTRIBUTOR`.** The `pull_request_target` gate evaluates the PR author's
   association; this is documented as a known limit in the workflow header.
-- **Historical (already-open) qualifying items.** The workflow only fires on
-  new events. Backfilled in bulk on 2026-06-07 — see #387 retro.
+- **Items the event triggers never fire for.** The event path misses two
+  structural cases: issues/PRs created by the default `GITHUB_TOKEN`
+  (`app/github-actions`) — GitHub does not fire workflows from default-token
+  events — and runs dropped under runner congestion. **A scheduled/manual
+  backlog reconcile (`add-to-project-reconcile.yml`, #518) closes both:** it
+  scans every App-installed repo's open issues/PRs + Ideas discussions daily and
+  reconciles each via the same gate/helpers, so a missed item lands within one
+  cycle. Run it manually with `dry_run` to preview. Full history (incl.
+  closed/merged) was bulk-backfilled 2026-06-07 and again 2026-06-21.
 
 To add a content-linked item manually:
 
@@ -188,11 +195,14 @@ configured manually in the UI):
 
 ```text
 .github/workflows/add-to-project.yml             # Workflow (events → script call)
+.github/workflows/add-to-project-reconcile.yml   # Scheduled/manual backlog reconcile (#518)
 .github/scripts/add-to-project/
+    lib.sh                                       # find/add/draft/delete helpers (DRY_RUN-aware)
     add-issue-or-pr.sh                           # Noise gate + addProjectV2ItemById
     reconcile-discussion.sh                      # Paginated find + 4-state reconciler
+    reconcile-backlog.sh                         # Scans open issues/PRs + Ideas, reconciles via the above
 .github/workflows/add-to-project-tests.yml       # shellcheck + bats CI gate
-test/workflows/add-to-project/                   # 35 bats tests, gh stub, fixtures
+test/workflows/add-to-project/                   # bats tests, gh stub, fixtures
 ```
 
 **Auth:** A dedicated GitHub App (`petry-projects-planner`, App ID

test/workflows/add-to-project/reconcile-backlog.bats

@@ -0,0 +1,75 @@
+#!/usr/bin/env bats
+# Tests for reconcile-backlog.sh — the periodic/manual backlog reconcile
+# (petry-projects/.github#518). Runs the script with a tailored `gh` stub and
+# DRY_RUN=1, and asserts it scans a repo's open issues/PRs + Ideas discussions
+# and reconciles qualifying ones via the shared event-path helpers (logging the
+# intended adds rather than mutating).
+
+bats_require_minimum_version 1.5.0
+
+load 'helpers/setup'
+
+setup() {
+  tt_make_tmpdir
+  # Tailored gh stub: returns one qualifying issue, one Ideas discussion, and
+  # "not found" for the draft-dedup lookup so reconcile takes the add path.
+  local bin="${TT_TMP}/bin"; mkdir -p "$bin"
+  cat > "$bin/gh" <<'STUB'
+#!/usr/bin/env bash
+args="$*"
+case "$args" in
+  *"repos/"*"/issues"*)
+    # node_id <TAB> html_url <TAB> labels-objects-json  (one qualifying issue)
+    printf 'I_node1\thttps://example.invalid/issues/1\t[{"name":"dev-lead"}]\n' ;;
+  *graphql*discussions*)
+    # number <TAB> title <TAB> url  (one Ideas discussion)
+    printf '55\tShiny idea\thttps://example.invalid/discussions/55\n' ;;
+  *graphql*)
+    # find_project_item lookups (draft-dedup / remove path) → not found
+    printf '' ;;
+  *) printf '' ;;
+esac
+STUB
+  chmod +x "$bin/gh"; PATH="${bin}:${PATH}"; export PATH
+  export PROJECT_ID="PVT_test" PROJECT_URL="https://example.invalid/p/1" GH_TOKEN="t"
+  export RECON_REPOS="petry-projects/demo"   # skip installation lookup
+  export DRY_RUN="1"
+}
+teardown() { tt_cleanup_tmpdir; }
+
+run_recon() { run bash "${TT_SCRIPTS_DIR}/reconcile-backlog.sh"; }
+
+@test "dry-run reconciles a qualifying issue (logs intended add, no mutation)" {
+  run_recon
+  [ "$status" -eq 0 ]
+  echo "$output" | grep -q '\[dry-run\] would add content I_node1 to project'
+}
+
+@test "dry-run reconciles an Ideas discussion as a draft" {
+  run_recon
+  [ "$status" -eq 0 ]
+  echo "$output" | grep -q '\[dry-run\] would add draft: \[Discussion #55\] Shiny idea'
+}
+
+@test "reports the repo count it scanned" {
+  run_recon
+  [ "$status" -eq 0 ]
+  echo "$output" | grep -q 'Reconciling 1 repo(s)'
+  echo "$output" | grep -q 'Reconcile complete'
+}
+
+@test "an excluded label disqualifies an issue (no add)" {
+  # override the stub to return a dev-lead + compliance-audit issue
+  cat > "${TT_TMP}/bin/gh" <<'STUB'
+#!/usr/bin/env bash
+case "$*" in
+  *"repos/"*"/issues"*) printf 'I_excl\thttps://example.invalid/issues/2\t[{"name":"dev-lead"},{"name":"compliance-audit"}]\n' ;;
+  *graphql*discussions*) printf '' ;;
+  *graphql*) printf '' ;;
+esac
+STUB
+  chmod +x "${TT_TMP}/bin/gh"
+  run_recon
+  [ "$status" -eq 0 ]
+  ! echo "$output" | grep -q 'would add content I_excl'
+}