ci: read the owned-pool rescue's Actions API through the route App

.github/workflows/ci-owned-pool-rescue.yml

@@ -108,8 +108,25 @@ jobs:
           ref: main
           persist-credentials: false
 
+      # Reads from the org's manaflow-glaeda-route App, on its own 5000 requests
+      # an hour; cancels and re-runs stay on GITHUB_TOKEN (owned_pool_rescue.py,
+      # GitHub). Any failure leaves READ_TOKEN empty and GITHUB_TOKEN reads.
+      - name: Mint the read token
+        id: read-token
+        if: vars.GLAEDA_ROUTE_APP_ID != ''
+        continue-on-error: true
+        timeout-minutes: 1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ vars.GLAEDA_ROUTE_APP_ID }}
+          private-key: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}
+          permission-actions: read
+          permission-contents: read
+          permission-pull-requests: read
+
       - name: Watch the run's persistent pool jobs
         env:
+          READ_TOKEN: ${{ steps.read-token.outputs.token }}
           # A CI run's budget grows by a round per queue round the picker allows.
           QUEUE_ROUNDS: ${{ vars.CI_PR_POOL_QUEUE_ROUNDS }}
           GH_TOKEN: ${{ github.token }}

scripts/ci/owned_pool_rescue.py

@@ -128,8 +128,10 @@
   that job uploaded its marker (it moved jobs onto owned root runners);
 - the run finished, or the watch limit passed.
 
-Request budget: the GITHUB_TOKEN allows about 1000 requests an hour for the
-whole repository. A run on an ephemeral pool costs a jobs listing every
+Request budget: the reads use a manaflow-glaeda-route App token when the
+workflow could mint one (READ_TOKEN; its own 5000 requests an hour), else
+GITHUB_TOKEN, which allows about 1000 requests an hour for the whole
+repository. Cancels and re-runs always use GITHUB_TOKEN (GitHub.__doc__). A run on an ephemeral pool costs a jobs listing every
 POLL_SECONDS until `changes` finishes (usually two or three) plus one artifact
 listing. A run on a persistent pool adds a jobs listing every POLL_SECONDS
 while one of its jobs waits for a runner and every IDLE_POLL_SECONDS otherwise,
@@ -395,20 +397,43 @@ class Aborted(Exception):
     pass
 
 
+def _headers(token: str) -> dict[str, str]:
+    return {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+        "User-Agent": "cmux-ci-owned-pool-rescue",
+    }
+
+
 class GitHub:
-    def __init__(self, token: str, repo: str) -> None:
+    """The Actions API. Reads may use `read_token`, writes always use `token`.
+
+    `read_token` is a manaflow-glaeda-route App installation token, so the
+    watch's polling draws on the App's own 5000 requests an hour instead of
+    the repository's GITHUB_TOKEN budget. Cancels and re-runs keep
+    GITHUB_TOKEN: a re-run's triggering actor must stay github-actions[bot],
+    which ci-macos.yml's attempt-2 routing checks. An installation token
+    lasts an hour and a watch may outlive it, so a 401 on a read drops back to
+    `token` for the rest of the watch.
+    """
+
+    def __init__(self, token: str, repo: str, read_token: str = "") -> None:
         self.repo = repo
-        self.headers = {
-            "Accept": "application/vnd.github+json",
-            "Authorization": f"Bearer {token}",
-            "X-GitHub-Api-Version": "2022-11-28",
-            "User-Agent": "cmux-ci-owned-pool-rescue",
-        }
+        self.headers = _headers(token)
+        self.read_headers = _headers(read_token) if read_token else self.headers
 
     def request(self, method: str, path: str) -> Any:
-        request = urllib.request.Request(f"{API}/repos/{self.repo}{path}", method=method, headers=self.headers)
-        with urllib.request.urlopen(request, timeout=20) as response:
-            body = response.read()
+        headers = self.read_headers if method == "GET" else self.headers
+        request = urllib.request.Request(f"{API}/repos/{self.repo}{path}", method=method, headers=headers)
+        try:
+            with urllib.request.urlopen(request, timeout=20) as response:
+                body = response.read()
+        except urllib.error.HTTPError as error:
+            if error.code != 401 or headers is self.headers:
+                raise
+            self.read_headers = self.headers
+            return self.request(method, path)
         return json.loads(body) if body else None
 
     def run(self, run_id: int) -> Mapping[str, Any]:
@@ -768,7 +793,8 @@ def finish(outcome: str) -> int:
         return finish(f"CI_OWNED_POOL_RESCUE_SECONDS must be {MIN_BUDGET_SECONDS} to {MAX_BUDGET_SECONDS}; "
                       "nothing to watch")
     repository = env.get("GITHUB_REPOSITORY") or ""
-    client = api or GitHub(env.get("GH_TOKEN") or env.get("GITHUB_TOKEN") or "", repository)
+    client = api or GitHub(env.get("GH_TOKEN") or env.get("GITHUB_TOKEN") or "", repository,
+                           read_token=env.get("READ_TOKEN") or "")
     run_id = (env.get("WATCH_RUN_ID") or "").strip()
     if run_id:
         # Dispatched by the picker's job: read the run it names and check it

tests/test_ci_owned_pool_rescue.py

@@ -1085,6 +1085,66 @@ def test_a_refused_main_job_reruns_the_failed_jobs(self):
         self.assertIn("rerun-failed", api.calls)
 
 
+class Tokens(unittest.TestCase):
+    """Reads may use the App's token; writes always use GITHUB_TOKEN."""
+
+    def open_with(self, fail_first_read=False):
+        seen = []
+
+        class Response(io.BytesIO):
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *exc):
+                return False
+
+        def urlopen(request, timeout):
+            seen.append((request.get_method(), request.headers["Authorization"]))
+            if fail_first_read and len(seen) == 1:
+                raise rescue.urllib.error.HTTPError(request.full_url, 401, "expired", {}, None)
+            return Response(b"{}")
+        return seen, unittest.mock.patch.object(rescue.urllib.request, "urlopen", urlopen)
+
+    def test_reads_use_the_app_token_and_writes_keep_github_token(self):
+        seen, patch = self.open_with()
+        with patch:
+            api = rescue.GitHub("repo-token", "o/r", read_token="app-token")
+            api.run(1)
+            api.rerun_failed(1)
+            api.cancel(1)
+        # A re-run started by the App would not be github-actions[bot], which
+        # ci-macos.yml's attempt-2 routing requires.
+        self.assertEqual(seen, [("GET", "Bearer app-token"), ("POST", "Bearer repo-token"),
+                                ("POST", "Bearer repo-token")])
+
+    def test_an_expired_app_token_falls_back_for_the_rest_of_the_watch(self):
+        seen, patch = self.open_with(fail_first_read=True)
+        with patch:
+            api = rescue.GitHub("repo-token", "o/r", read_token="app-token")
+            api.run(1)
+            api.run(1)
+        self.assertEqual(seen, [("GET", "Bearer app-token"), ("GET", "Bearer repo-token"),
+                                ("GET", "Bearer repo-token")])
+
+    def test_without_an_app_token_everything_uses_github_token(self):
+        seen, patch = self.open_with()
+        with patch:
+            rescue.GitHub("repo-token", "o/r").run(1)
+        self.assertEqual(seen, [("GET", "Bearer repo-token")])
+
+    def test_the_workflow_mints_a_read_only_token_and_passes_it(self):
+        steps = yaml.safe_load((ROOT / ".github/workflows/ci-owned-pool-rescue.yml").read_text(
+            encoding="utf-8"))["jobs"]["rescue"]["steps"]
+        mint = next(step for step in steps if step.get("id") == "read-token")
+        self.assertTrue(mint["continue-on-error"])
+        self.assertEqual({key: value for key, value in mint["with"].items() if key.startswith("permission-")},
+                         {"permission-actions": "read", "permission-contents": "read",
+                          "permission-pull-requests": "read"})
+        watch = next(step for step in steps if step.get("name") == "Watch the run's persistent pool jobs")
+        self.assertEqual(watch["env"]["READ_TOKEN"], "${{ steps.read-token.outputs.token }}")
+        self.assertEqual(watch["env"]["GH_TOKEN"], "${{ github.token }}")
+
+
 class Workflow(unittest.TestCase):
     def setUp(self):
         self.text = (ROOT / ".github/workflows/ci-owned-pool-rescue.yml").read_text(encoding="utf-8")