fix(bin): resolve the CI required-suite roster per repository

CONTRIBUTING.md

@@ -59,6 +59,7 @@ bin/fm-pr-ci-verify.sh <pr-url>   # did repository suites actually run and pass
 
 It reports which suites ran and where, accepts a commit your fork validated while the upstream run is still held, and refuses every outcome that only looks green.
 [`bin/fm-ci-checks-lib.sh`](bin/fm-ci-checks-lib.sh) owns that rule, and everything in this repo that turns checks into a verdict classifies through it.
+The roster it requires is read from the repository the pull request targets, so a change that deliberately adds or removes a CI job is judged against a roster the target branch has not recorded yet; `FM_CI_REQUIRED_SUITES` in that script's header is the override for exactly that case.
 
 ## Repo conventions
 

bin/fm-bearings-snapshot.sh

@@ -210,6 +210,13 @@ gh_bounded() {  # <args...>
     env GH_PROMPT_DISABLED=1 GH_NO_UPDATE_NOTIFIER=1 gh "$@"
 }
 
+# The roster lookup in bin/fm-ci-checks-lib.sh reaches GitHub through this, so
+# its calls carry the same bound every other network read on this path does
+# rather than being the one that can hang the snapshot.
+fm_ci_gh() {  # <args...>
+  gh_bounded "$@" 2>/dev/null
+}
+
 if [ "$INCLUDE_PRS" = 1 ]; then
   if ! command -v gh >/dev/null 2>&1; then
     PR_STATUS='unavailable (gh not found)'
@@ -243,7 +250,15 @@ EOF
         --json number,title,url,headRefName,reviewDecision,mergeable,statusCheckRollup 2>/dev/null) \
         || { nwarn=$((nwarn + 1)); continue; }
       [ -n "$out" ] || out='[]'
-      repo_result=$(printf '%s' "$out" | jq --arg repo "$repo" --argjson limit "$FM_BEARINGS_PR_LIMIT" "$FM_CI_CHECKS_JQ_DEFS"'
+      # Each repository is judged against its own required suite roster
+      # (bin/fm-ci-checks-lib.sh owns where that comes from), resolved once per
+      # repository here rather than once per row. A repository whose roster
+      # cannot be established is classified against an empty one, which the
+      # classifier refuses as incomplete: an unknown standard costs these rows
+      # their green, and can never hand one out.
+      if fm_ci_roster "$repo" 2>/dev/null; then repo_roster=$FM_CI_ROSTER; else repo_roster='[]'; fi
+      repo_result=$(printf '%s' "$out" | jq --arg repo "$repo" --argjson limit "$FM_BEARINGS_PR_LIMIT" \
+        --argjson fm_ci_roster "$repo_roster" "$FM_CI_CHECKS_JQ_DEFS"'
         [ .[] | {
           num:(.number|tostring),
           repo:$repo,

bin/fm-pr-ci-verify.sh

@@ -25,7 +25,19 @@
 # repository and run, so a green verdict always says where its evidence came
 # from and is never confused with the upstream pull request having been checked.
 #
+# The roster of suites a green verdict requires belongs to the repository the
+# pull request targets, and is resolved from it for every run of this script -
+# see fm_ci_roster in bin/fm-ci-checks-lib.sh, which owns where it comes from
+# and why. This script is asked about any repository the fleet works in, so it
+# holds no roster of its own; a repository whose roster cannot be established is
+# refused rather than judged against somebody else's.
+#
 # Usage: fm-pr-ci-verify.sh <pr-url>
+# Env:   FM_CI_REQUIRED_SUITES  JSON array of job names to require instead of
+#          the roster read from the target repository. The escape hatch for a
+#          change that deliberately adds or removes a CI job, whose branch is
+#          therefore judged against a roster the target branch has not
+#          recorded yet.
 # Exit:  0 repository suites ran and passed, upstream or in the head repository
 #        1 they did not: none ran, one is red, one has not finished, or fewer
 #          than the full required roster reported
@@ -65,23 +77,40 @@ URL=$FM_PR_URL
 BASE_REPO="$FM_PR_OWNER/$FM_PR_REPO"
 
 pr=$(gh pr view "$FM_PR_NUMBER" --repo "$BASE_REPO" \
-  --json statusCheckRollup,headRefOid,headRepository,headRepositoryOwner 2>/dev/null) \
+  --json statusCheckRollup,headRefOid,headRepository,headRepositoryOwner,baseRefName 2>/dev/null) \
   || unreadable "the pull request $URL"
 
 rollup=$(printf '%s' "$pr" | jq -c '.statusCheckRollup // []' 2>/dev/null) \
   || unreadable "the checks on $URL"
 head_sha=$(printf '%s' "$pr" | jq -r '.headRefOid // ""' 2>/dev/null) || head_sha=''
+base_ref=$(printf '%s' "$pr" | jq -r '.baseRefName // ""' 2>/dev/null) || base_ref=''
 head_repo=$(printf '%s' "$pr" | jq -r '
   ((.headRepositoryOwner.login // "") | tostring) as $o
   | ((.headRepository.name // "") | tostring) as $n
   | if $o == "" or $n == "" then "" else $o + "/" + $n end' 2>/dev/null) || head_repo=''
 
-state=$(fm_ci_checks_state "$rollup") || unreadable "the checks on $URL"
+# The standard is settled before any verdict is read against it. A roster that
+# cannot be established is a refusal in its own right, because without it no
+# rollup can be told apart from a rollup missing half of itself.
+if ! fm_ci_roster "$BASE_REPO" "$base_ref"; then
+  {
+    printf 'error: refusing to call %s green: could not establish what %s requires of a commit.\n' \
+      "$URL" "$BASE_REPO"
+    echo "See the reason above. Set FM_CI_REQUIRED_SUITES to the JSON array of job names"
+    echo "this change expects if the roster is one the target branch has not recorded yet."
+  } >&2
+  exit 1
+fi
+ROSTER=$FM_CI_ROSTER
+
+state=$(fm_ci_checks_state "$rollup" "$ROSTER") || unreadable "the checks on $URL"
 
 # The roster is printed for every outcome, including the passing one, so the
 # evidence behind a green verdict is on the record rather than only its verdict.
 printf '%s\n' "$URL"
-roster=$(printf '%s' "$rollup" | jq -r "$FM_CI_CHECKS_JQ_DEFS"'
+printf 'required suites: %s, from %s\n' \
+  "$(printf '%s' "$ROSTER" | jq -r 'length')" "$FM_CI_ROSTER_SOURCE"
+roster=$(printf '%s' "$rollup" | jq -r --argjson fm_ci_roster "$ROSTER" "$FM_CI_CHECKS_JQ_DEFS"'
   def row: "  " + (if fm_ci_repo_owned then "suite " else "other " end)
     + ((.conclusion // .state // "pending") | tostring)
     + "\t" + (((.workflowName // "") | tostring) as $w | if $w == "" then "-" else $w end)
@@ -90,15 +119,16 @@ roster=$(printf '%s' "$rollup" | jq -r "$FM_CI_CHECKS_JQ_DEFS"'
   | .[]' 2>/dev/null) || roster=''
 [ -z "$roster" ] || printf '%s\n' "$roster"
 
-own=$(printf '%s' "$rollup" | jq "$FM_CI_CHECKS_JQ_DEFS"'
+own=$(printf '%s' "$rollup" | jq --argjson fm_ci_roster "$ROSTER" "$FM_CI_CHECKS_JQ_DEFS"'
   [.[] | select(fm_ci_repo_owned)] | length' 2>/dev/null) || own='?'
 printf '%s checks: %s (%s repository-owned)\n' "$BASE_REPO" "$state" "$own"
 
 # An incomplete roster names what it is missing, the same way a red or
 # pending check names its own state above, so the refusal is never just "not
 # passing" with no way to tell what would make it so.
 if [ "$state" = incomplete ]; then
-  missing=$(printf '%s' "$rollup" | jq -r "$FM_CI_CHECKS_JQ_DEFS"'fm_ci_missing_suites | .[]' 2>/dev/null) || missing=''
+  missing=$(printf '%s' "$rollup" | jq -r --argjson fm_ci_roster "$ROSTER" \
+    "$FM_CI_CHECKS_JQ_DEFS"'fm_ci_missing_suites | .[]' 2>/dev/null) || missing=''
   [ -z "$missing" ] || printf 'missing required suites:\n%s\n' "$(printf '%s\n' "$missing" | sed 's/^/  /')"
 fi
 
@@ -147,7 +177,7 @@ runs=$(gh api "repos/$head_repo/actions/runs?head_sha=$head_sha&per_page=100" \
 # the jobs of every CI run at this commit are read and judged against the same
 # roster the rollup shape uses. bin/fm-ci-checks-lib.sh owns why a successful
 # run is not that evidence.
-ci_run_ids=$(printf '%s' "$runs" | jq -r "$FM_CI_CHECKS_JQ_DEFS"'
+ci_run_ids=$(printf '%s' "$runs" | jq -r --argjson fm_ci_roster "$ROSTER" "$FM_CI_CHECKS_JQ_DEFS"'
   .[] | select(fm_ci_run_from_ci_workflow) | (.id // empty) | tostring' 2>/dev/null) \
   || unreadable "the workflow runs in $head_repo"
 
@@ -163,7 +193,7 @@ for run_id in $ci_run_ids; do
     || unreadable "the jobs of run $run_id in $head_repo"
 done
 
-fork_state=$(fm_ci_run_jobs_state "$runs" "$ci_jobs") \
+fork_state=$(fm_ci_run_jobs_state "$runs" "$ci_jobs" "$ROSTER") \
   || unreadable "the workflow runs in $head_repo"
 fork_roster=$(printf '%s' "$runs" | jq -r '
   .[] | "  run " + ((.id // "-") | tostring) + " " + ((.conclusion // .status) | tostring)
@@ -179,7 +209,7 @@ job_roster=$(printf '%s' "$ci_jobs" | jq -r '
 printf '%s runs: %s\n' "$head_repo" "$fork_state"
 
 if [ "$fork_state" = incomplete ]; then
-  fork_missing=$(printf '%s' "$ci_jobs" | jq -r "$FM_CI_CHECKS_JQ_DEFS"'
+  fork_missing=$(printf '%s' "$ci_jobs" | jq -r --argjson fm_ci_roster "$ROSTER" "$FM_CI_CHECKS_JQ_DEFS"'
     fm_ci_jobs_missing_suites | .[]' 2>/dev/null) || fork_missing=''
   if [ -n "$fork_missing" ]; then
     printf 'missing required suites:\n%s\n' "$(printf '%s\n' "$fork_missing" | sed 's/^/  /')"

tests/fm-bearings-snapshot.test.sh

@@ -49,6 +49,21 @@ SH
 echo "gh $*" >> "$NET_LOG"
 if [ "${FAKE_GH_FAIL:-0}" = 1 ]; then exit 1; fi
 if [ "${FAKE_GH_SLEEP:-0}" = 1 ]; then sleep 30; fi
+# The per-repository required suite roster (bin/fm-ci-checks-lib.sh): the
+# snapshot asks each repository what its own CI workflow reports before it can
+# call any of that repository's rows green. The fixture answers with the roster
+# the PR fixtures below carry, so a row that is genuinely complete reads as
+# passing rather than as a roster it could not establish.
+if [ "${1:-}" = api ]; then
+  case "${2:-}" in
+    */actions/workflows/*/runs*) printf '{"workflow_runs":[{"id":5150,"name":"CI"}]}\n' ;;
+    */actions/workflows*)        printf '{"total_count":1,"workflows":[{"id":77,"name":"CI"}]}\n' ;;
+    */jobs*)
+      printf '%s\n' '{"total_count":12,"jobs":[{"name":"Lint"},{"name":"Test coverage guard"},{"name":"Behavior portable parallel 1"},{"name":"Behavior portable parallel 2"},{"name":"Behavior portable serial 1"},{"name":"Behavior portable serial 2"},{"name":"Behavior portable serial 3"},{"name":"Behavior portable serial 4"},{"name":"Behavior tests (Herdr)"},{"name":"Behavior timing aggregate"},{"name":"Stock macOS Bash snapshot compatibility"},{"name":"Repo invariants"}]}' ;;
+    *)                           printf '{"default_branch":"main"}\n' ;;
+  esac
+  exit 0
+fi
 if [ "${FAKE_GH_BOT_ONLY:-0}" = 1 ]; then
   # A pull request whose only check is a third-party App: it has no workflow
   # behind it, so it carries no workflow name.
@@ -63,10 +78,10 @@ if [ "${FAKE_GH_MANY:-0}" = 1 ]; then
 JSON
   exit 0
 fi
-# The complete required-suite roster (bin/fm-ci-checks-lib.sh
-# FM_CI_REQUIRED_SUITES), all green - the default fixture stands in for a
-# pull request CI genuinely validated, so it must carry evidence that would
-# actually classify as passing rather than one lone suite.
+# The complete required-suite roster the api fixture above reports for this
+# repository, all green - the default fixture stands in for a pull request CI
+# genuinely validated, so it must carry evidence that would actually classify
+# as passing rather than one lone suite.
 cat <<'JSON'
 [{"number":9,"title":"Ship the thing","url":"https://github.com/kunchenguid/firstmate/pull/9","headRefName":"fm/ship-task","reviewDecision":"APPROVED","mergeable":"MERGEABLE","statusCheckRollup":[{"__typename":"CheckRun","workflowName":"CI","name":"Lint","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Test coverage guard","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior portable parallel 1","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior portable parallel 2","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior portable serial 1","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior portable serial 2","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior portable serial 3","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior portable serial 4","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior tests (Herdr)","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Behavior timing aggregate","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Stock macOS Bash snapshot compatibility","conclusion":"SUCCESS","status":"COMPLETED"},{"__typename":"CheckRun","workflowName":"CI","name":"Repo invariants","conclusion":"SUCCESS","status":"COMPLETED"}]}]
 JSON