ci(nightly-next): read branch refs with git ls-remote so an exhausted API budget never stalls the tick

.github/workflows/auto-resume-nightly-notarization.yml

@@ -37,6 +37,8 @@ jobs:
       ((github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'success') &&
       github.event.workflow_run.head_repository.full_name == github.repository)
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-4vcpu-ubuntu-2404' }}
+    # Room for gh_budget_wait.sh to wait out an exhausted API budget (at most an hour).
+    timeout-minutes: 75
     permissions:
       actions: read
       contents: read
@@ -59,13 +61,36 @@ jobs:
       SOURCE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.source_branch || github.event.workflow_run.head_branch }}
       SOURCE_REPOSITORY: ${{ github.event_name == 'workflow_dispatch' && github.repository || github.event.workflow_run.head_repository.full_name }}
     steps:
+      # Run 41790985234 (2026-11-04) failed here on its first read with "HTTP 403 API rate
+      # limit exceeded for installation". The wrapper waits for the budget's reset and tries
+      # again (scripts/ci/gh_budget_wait.sh), bounded by this job's timeout.
+      - name: Checkout the rate-limit wrapper
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          ref: refs/heads/main
+          path: .preflight-tooling
+          sparse-checkout: |
+            scripts/ci/gh_budget_wait.sh
+          sparse-checkout-cone-mode: false
+
+      - name: Wait through GitHub API rate limits in this job's gh calls
+        run: bash .preflight-tooling/scripts/ci/gh_budget_wait.sh --install
+
       - name: Verify source run and recovery artifacts
         id: check
         run: |
           set -euo pipefail
           [[ "$SOURCE_RUN_ID" =~ ^[0-9]+$ ]] || { echo "invalid source run id" >&2; exit 1; }
           [[ "$SOURCE_RUN_ATTEMPT" =~ ^[0-9]+$ ]] || { echo "invalid source run attempt" >&2; exit 1; }
           [[ "$SOURCE_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]] || { echo "invalid source head SHA" >&2; exit 1; }
+          # nightly-next continues on its own track (below), so its runs, most of this
+          # workflow's triggers, need no API read to be turned away.
+          if [ "$SOURCE_BRANCH" = nightly-next ]; then
+            echo "ready=false" >> "$GITHUB_OUTPUT"
+            echo "reason=nightly-next-continues-on-its-own-track" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID" > "$RUNNER_TEMP/source-run.json"
           python3 - "$RUNNER_TEMP/source-run.json" "$SOURCE_REPOSITORY" "$SOURCE_HEAD_SHA" "$SOURCE_BRANCH" "$SOURCE_RUN_ATTEMPT" <<'PY' > "$RUNNER_TEMP/source-check"
           import json

.github/workflows/nightly-next-tick.yml

@@ -150,8 +150,10 @@ jobs:
 
       # The rolling rule (scripts/ci/nightly_next_rolling.py): promote at once when nothing is
       # building nightly-next, else once the oldest landing the running build does not carry
-      # has waited 10 minutes. A dispatch for one commit skips it. It reads through the API only;
-      # its failure modes are in the script's docstring.
+      # has waited 10 minutes. A dispatch for one commit skips it. It reads nightly-next with
+      # git ls-remote (no API budget; ticks 41791114042 and 41791169626 lost their promotion to
+      # an exhausted budget on the refs API), the rest through the API; its failure modes are in
+      # the script's docstring.
       - name: Apply the rolling rule
         id: rolling
         if: vars.NIGHTLY_NEXT_TRAIN != 'on' && inputs.sha == ''

scripts/ci/nightly_next_feed_watch.py

@@ -41,6 +41,7 @@
 import json
 import os
 import re
+import subprocess
 import sys
 import urllib.parse
 import urllib.request
@@ -229,9 +230,28 @@ def stamp(when: dt.datetime) -> str:
     return when.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
+def ls_remote_sha(repo: str, branch: str, timeout: float = 60) -> str:
+    """A branch's tip from git's smart-HTTP ref advertisement, not the REST API.
+
+    The repository is public, so this needs no token and spends none of the installation's
+    API budget. On 2026-11-04 that budget ran out and the tick's `git/ref/heads/nightly-next`
+    read failed, so ticks 41791114042 and 41791169626 promoted nothing. Raises OSError,
+    subprocess.SubprocessError or ValueError; callers fall back to the API."""
+    server = os.environ.get("GITHUB_SERVER_URL", "https://github.com").rstrip("/")
+    out = subprocess.run(
+        ["git", "ls-remote", "--exit-code", f"{server}/{repo}", f"refs/heads/{branch}"],
+        capture_output=True, text=True, timeout=timeout, check=True, stdin=subprocess.DEVNULL,
+        env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}).stdout
+    sha = out.split()[0].lower() if out.split() else ""
+    if not re.fullmatch(r"[0-9a-f]{40}", sha):
+        raise ValueError(f"git ls-remote {branch}: no sha in {out[:80]!r}")
+    return sha
+
+
 class GitHub:
     def __init__(self, repo: str, token: str, dry_run: bool):
         self.base = f"https://api.github.com/repos/{repo}"
+        self.repo = repo
         self.token = token
         self.dry_run = dry_run
 
@@ -298,6 +318,11 @@ def recent_runs(self, workflow: str, branch: str | None = None,
         return runs
 
     def ref_sha(self, branch: str) -> str:
+        """git ls-remote first (no API budget), the refs API only when that fails."""
+        try:
+            return ls_remote_sha(self.repo, branch)
+        except (OSError, subprocess.SubprocessError, ValueError) as error:
+            print(f"git ls-remote {branch} failed ({error!r}); reading it through the API")
         return self.call("GET", f"/git/ref/heads/{branch}")["object"]["sha"]
 
     def ref_commit(self, branch: str) -> tuple[str, dt.datetime | None]:

scripts/ci/nightly_next_rolling.py

@@ -28,10 +28,13 @@
 A dispatch for one commit (a person, or the feed watch's starvation guard) never
 reaches this rule: the tick runs it only when its `sha` input is empty.
 
-Every read goes through the API with the job's token (no git remote, no
-shallow history). A failed read of nightly-next waits, with a warning: the feed
-watch's dispatch for one commit skips this rule, so that costs at most the
-backstop's delay. Any other failed read, and a gap to the tip that cannot be
+nightly-next's head comes from `git ls-remote` against the public repository
+(no token, no API budget: on 2026-11-04 an exhausted installation budget failed
+the refs API read and ticks 41791114042 and 41791169626 promoted nothing), and
+from the refs API only when that fails. Every other read goes through the API
+with the job's token (no shallow history). A head neither can read waits, with a
+warning: the feed watch's dispatch for one commit skips this rule, so that costs
+at most the backstop's delay. Any other failed read, and a gap to the tip that cannot be
 read, promotes, with a warning: the promotion ignores a request that does not
 move nightly-next forward.
 
@@ -191,15 +194,23 @@ def gh_api(path: str):
 
 
 def evaluate(now: dt.datetime, repo: str, tip: str, api=gh_api,
-             window_minutes: float = WINDOW_MINUTES) -> tuple[bool, str]:
-    """Read everything through the API (the job's token) and decide.
+             window_minutes: float = WINDOW_MINUTES, ls_remote=None) -> tuple[bool, str]:
+    """Read the head with `ls_remote` (main passes git ls-remote) and then the API, the rest
+    through the API (the job's token), and decide.
 
     nightly-next's head unreadable: wait, with a warning. The rule's purpose is to coalesce, and
     the feed watch (every 10 minutes) dispatches a build for one commit, which skips this rule, so
     a failed read here costs at most that backstop's delay and never a burst of builds. Any other
     failed read promotes, with a warning: the promotion ignores a request that moves nothing."""
+    head = None
+    if ls_remote is not None:
+        try:
+            head = ls_remote(repo, "nightly-next")
+        except Exception as error:  # noqa: BLE001 - the refs API below is the fallback
+            print(f"git ls-remote nightly-next failed ({error!r}); reading it through the API")
     try:
-        head = api(f"repos/{repo}/git/ref/heads/nightly-next")["object"]["sha"]
+        if head is None:
+            head = api(f"repos/{repo}/git/ref/heads/nightly-next")["object"]["sha"]
     except Exception as error:  # noqa: BLE001
         print(f"::warning title=nightly-next rolling rule::could not read nightly-next ({error!r}); "
               "waiting for the feed watch's backstop dispatch")
@@ -242,7 +253,8 @@ def main(argv: list[str]) -> int:
     parser.add_argument("--tip", required=True, help="the feat-cmux-next commit the tick checked out")
     parser.add_argument("--window-minutes", type=float, default=WINDOW_MINUTES)
     args = parser.parse_args(argv)
-    go, reason = evaluate(dt.datetime.now(dt.timezone.utc), args.repo, args.tip, window_minutes=args.window_minutes)
+    go, reason = evaluate(dt.datetime.now(dt.timezone.utc), args.repo, args.tip, window_minutes=args.window_minutes,
+                          ls_remote=watch.ls_remote_sha)
     print(f"rolling rule: {'promote' if go else 'wait'}: {reason}")
     output = os.environ.get("GITHUB_OUTPUT")
     if output:

scripts/ci/nightly_next_stuck_run.py

@@ -183,9 +183,20 @@ def gh_cli(args: Sequence[str], write: bool) -> str:
     return result.stdout
 
 
+def ls_remote_sha(repo: str, branch: str) -> str:
+    """A branch's tip from git's ref advertisement of the public repository: no token and none of
+    the installation's API budget, which ran out on 2026-11-04 and failed the refs API read."""
+    server = os.environ.get("GITHUB_SERVER_URL", "https://github.com").rstrip("/")
+    out = subprocess.run(
+        ["git", "ls-remote", "--exit-code", f"{server}/{repo}", f"refs/heads/{branch}"],
+        capture_output=True, text=True, timeout=60, check=True, stdin=subprocess.DEVNULL,
+        env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}).stdout
+    return out.split()[0].lower() if out.split() else ""
+
+
 class Api:
-    def __init__(self, repo: str, gh: Gh = gh_cli):
-        self.repo, self.gh = repo, gh
+    def __init__(self, repo: str, gh: Gh = gh_cli, ls_remote: Callable[[str, str], str] | None = None):
+        self.repo, self.gh, self.ls_remote = repo, gh, ls_remote
 
     def get(self, path: str) -> Any:
         text = self.gh(["api", f"repos/{self.repo}/{path}"], False)
@@ -221,6 +232,15 @@ def listing(self, path: str, key: str) -> list[Any]:
         return items
 
     def tip(self) -> str:
+        """nightly-next's head: git ls-remote when given one (main does), else or on its failure the refs API."""
+        if self.ls_remote is not None:
+            try:
+                sha = self.ls_remote(self.repo, "nightly-next")
+                if SHA.fullmatch(sha):
+                    return sha
+                print(f"git ls-remote nightly-next answered {sha!r}; reading it through the API")
+            except (OSError, subprocess.SubprocessError) as error:
+                print(f"git ls-remote nightly-next failed ({error!r}); reading it through the API")
         sha = field(field(self.get("git/ref/heads/nightly-next"), "object", (dict,)), "sha", (str,))
         if not SHA.fullmatch(sha):
             raise Unreadable(f"nightly-next is {sha!r}")
@@ -442,7 +462,7 @@ def main(argv: list[str]) -> int:
     if len(argv) != 2:
         print("usage: nightly_next_stuck_run.py OWNER/REPO", file=sys.stderr)
         return 2
-    tick(Api(argv[1]))
+    tick(Api(argv[1], ls_remote=ls_remote_sha))
     return 0
 
 

tests/test_nightly_next_rolling.py

@@ -298,16 +298,45 @@ def api_for(compare_answer, ref=None, runs=None, jobs=None) -> Api:
 
 
 class ApiReadTests(unittest.TestCase):
-    """Every read is an API read with the job's token: no git remote (the tick checks out with
-    persist-credentials: false) and no shallow history (it fetches 150 commits)."""
+    """nightly-next's head comes from git ls-remote of the public repository (no token, no API
+    budget), the refs API as its fallback; every other read is an API read with the job's token
+    and no shallow history (it fetches 150 commits)."""
 
     def test_the_head_comes_from_the_refs_api_and_the_gap_from_compare(self):
         api = api_for(compare("ahead", [(WAITING, 4), ("d" * 40, 2)]))
         go, reason = rolling.evaluate(NOW, "o/r", TIP, api=api)
         self.assertFalse(go, reason)
         self.assertIn("repos/o/r/git/ref/heads/nightly-next", api.paths)
         self.assertIn(f"repos/o/r/compare/{HEAD}...{TIP}?per_page=1&page=1", api.paths)
-        self.assertNotIn("ls-remote", SCRIPT.read_text(encoding="utf-8"))
+
+    def test_the_head_comes_from_ls_remote_first_and_spends_no_api_read(self):
+        """2026-11-04: an exhausted installation budget failed the refs API read, and ticks
+        41791114042 and 41791169626 promoted nothing. git ls-remote costs no API budget."""
+        api = api_for(compare("ahead", [(WAITING, 4), ("d" * 40, 2)]))
+        asked = []
+        go, reason = rolling.evaluate(NOW, "o/r", TIP, api=api,
+                                      ls_remote=lambda repo, branch: asked.append((repo, branch)) or HEAD)
+        self.assertFalse(go, reason)
+        self.assertEqual(asked, [("o/r", "nightly-next")])
+        self.assertNotIn("repos/o/r/git/ref/heads/nightly-next", api.paths)
+        self.assertIn(f"repos/o/r/compare/{HEAD}...{TIP}?per_page=1&page=1", api.paths)
+
+    def test_a_rate_limited_refs_api_still_decides_when_ls_remote_answers(self):
+        api = api_for(compare("ahead", [(WAITING, 30)]), ref=OSError("HTTP 403: API rate limit exceeded for installation"))
+        go, reason = rolling.evaluate(NOW, "o/r", TIP, api=api, ls_remote=lambda repo, branch: HEAD)
+        self.assertNotIn("could not read nightly-next", reason)
+        self.assertTrue(go, reason)
+
+    def test_a_failed_ls_remote_falls_back_to_the_refs_api(self):
+        def broken(repo, branch):
+            raise OSError("git: not found")
+        api = api_for(compare("ahead", [(WAITING, 4), ("d" * 40, 2)]))
+        go, reason = rolling.evaluate(NOW, "o/r", TIP, api=api, ls_remote=broken)
+        self.assertFalse(go, reason)
+        self.assertIn("repos/o/r/git/ref/heads/nightly-next", api.paths)
+
+    def test_main_reads_the_head_with_ls_remote(self):
+        self.assertIn("ls_remote=watch.ls_remote_sha", SCRIPT.read_text(encoding="utf-8"))
 
     def test_the_oldest_waiting_commit_is_page_one_of_a_one_per_page_compare(self):
         """Page 1 at per_page=1 is the oldest commit after nightly-next (the API's documented order)."""

tests/test_nightly_next_stuck_run.py

@@ -455,5 +455,35 @@ def test_the_tick_cancels_before_it_re_runs_a_cancelled_tip_in_its_serialized_gr
                         names.index("Re-run a cancelled nightly-next tip"))
         self.assertIn("python3 tests/test_nightly_next_stuck_run.py", CI_GUARDS.read_text())
 
+
+class TipRead(unittest.TestCase):
+    """2026-11-04: the installation's API budget ran out and the refs API read of nightly-next
+    failed. git ls-remote reads the head with no API budget; the refs API is the fallback."""
+
+    def refs_api(self, calls):
+        def gh(args, write):
+            calls.append(args[-1])
+            return json.dumps({"object": {"sha": "c" * 40}})
+        return gh
+
+    def test_ls_remote_answers_without_an_api_read(self):
+        calls = []
+        api = stuck.Api("o/r", self.refs_api(calls), ls_remote=lambda repo, branch: "a" * 40)
+        self.assertEqual(api.tip(), "a" * 40)
+        self.assertEqual(calls, [])
+
+    def test_a_failed_or_odd_ls_remote_falls_back_to_the_refs_api(self):
+        def broken(repo, branch):
+            raise OSError("no git")
+        for reader in (broken, lambda repo, branch: ""):
+            calls = []
+            api = stuck.Api("o/r", self.refs_api(calls), ls_remote=reader)
+            self.assertEqual(api.tip(), "c" * 40)
+            self.assertEqual(calls, ["repos/o/r/git/ref/heads/nightly-next"])
+
+    def test_main_reads_with_ls_remote(self):
+        self.assertIn("Api(argv[1], ls_remote=ls_remote_sha)", Path(stuck.__file__).read_text())
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_nightly_notarization_poll_loop.py

@@ -48,7 +48,7 @@ def test_recovery_is_resolved_once_before_status_polling(self) -> None:
 
     def test_preflight_uses_metadata_only_when_all_variants_exist(self) -> None:
         workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
-        check = workflow["jobs"]["preflight"]["steps"][0]["run"]
+        check = next(s for s in workflow["jobs"]["preflight"]["steps"] if s.get("id") == "check")["run"]
         program = re.search(r'<<\'PY\' > "\$RUNNER_TEMP/artifact-check"\n(.*?)\nPY', check, re.S).group(1)
         variants = ("arm64", "x86_64", "universal")
         recovery = [f"cmux-nightly-notarization-recovery-{v}-aaaaaaa" for v in variants]

tests/test_nightly_resume_tracks.py

@@ -24,7 +24,7 @@
 
 def preflight_check(branch: str) -> tuple[str, str]:
     """Runs the preflight step up to the artifact lookup; returns (exit code, GITHUB_OUTPUT)."""
-    step = yaml.safe_load(WORKFLOW.read_text())["jobs"]["preflight"]["steps"][0]
+    step = next(s for s in yaml.safe_load(WORKFLOW.read_text())["jobs"]["preflight"]["steps"] if s.get("id") == "check")
     script = step["run"].split('gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID/artifacts', 1)[0]
     with tempfile.TemporaryDirectory() as temp:
         temp_path = Path(temp)