ci: keep the owned-pool snapshot fresh when the janitor cron drifts

.github/workflows/ci-queue-janitor.yml

@@ -26,9 +26,19 @@ name: CI queue janitor
 # Each sweep also publishes the per-pool macOS queue it saw as the
 # `macos-pool-load` artifact, which pull request CI reads to overflow off a
 # backed-up pool.
+#
+# GitHub delays scheduled runs (the */10 cron fired 55 minutes apart on
+# 2026-09-25), and pull request CI trusts a snapshot for at most
+# MAX_SNAPSHOT_MINUTES, so a CI run being requested also starts a sweep. It
+# skips when the newest snapshot is under FRESH_SECONDS old, and the
+# concurrency group keeps one sweep running and one waiting, so a burst of
+# pushes costs at most one extra sweep.
 on:
   schedule:
     - cron: "*/10 * * * *"
+  workflow_run:
+    workflows: [CI]
+    types: [requested]
   workflow_dispatch:
     inputs:
       dry_run:
@@ -54,12 +64,33 @@ jobs:
     runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
     timeout-minutes: 10
     steps:
+      - name: Skip when the pool snapshot is fresh
+        id: fresh
+        if: github.event_name == 'workflow_run'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          FRESH_SECONDS: "240"
+        run: |
+          set -euo pipefail
+          newest="$(gh api "repos/${{ github.repository }}/actions/artifacts?name=macos-pool-load&per_page=20" \
+            --jq '[.artifacts[] | select(.expired | not) | select(.workflow_run.head_branch == "main") | .created_at] | max // ""')" || newest=""
+          age=999999
+          if [ -n "$newest" ]; then
+            age=$(( $(date -u +%s) - $(date -u -d "$newest" +%s) ))
+          fi
+          echo "newest snapshot: ${newest:-none} (${age}s old)"
+          if [ "$age" -lt "$FRESH_SECONDS" ]; then
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Check out trusted janitor
+        if: steps.fresh.outputs.skip != 'true'
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Cancel wasted macOS runs
+        if: steps.fresh.outputs.skip != 'true'
         env:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
@@ -84,7 +115,7 @@ jobs:
       # pull request run lands on, for two API requests instead of one job
       # listing per in-flight run.
       - name: Publish macOS pool load
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && steps.fresh.outputs.skip != 'true' }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: macos-pool-load

scripts/ci/pr_runner_pool.py

@@ -181,7 +181,11 @@
 MAX_RUN_JOBS = SIDE_LANES + APP_HOST_SHARDS + 2
 REPLAYED_RUN_JOBS = SIDE_LANES + 1
 # A snapshot older than this is not trusted to place a run on an owned pool.
-OWNED_MAX_AGE_MINUTES = 20
+# It was 20, but GitHub delays scheduled runs: the janitor's */10 cron fired
+# 55 minutes apart (23:59Z to 00:54Z, 2026-09-25) and every run skipped 40
+# idle minis. ci-queue-janitor.yml now also sweeps when CI is requested, and
+# a mini that turns out busy is caught by the rescue within its budget.
+OWNED_MAX_AGE_MINUTES = 45
 # Pools whose machines are discarded after each job; the only ones a fork run may use.
 EPHEMERAL_PREFIX = "blacksmith-"
 
@@ -1019,7 +1023,9 @@ def count_routed(since: str) -> int:
     )
     problems = slot_problems(env.get("OWNED_SLOTS")) if (env.get("POOL_OWNED") or "").strip() == "1" else []
     for problem in problems:
-        print(f"::warning title={SLOTS_VARIABLE}::{problem}")
+        # An error, not a warning: a malformed entry silently takes the
+        # fleet out of the order (a bare `40` did for 30 minutes on 2026-09-25).
+        print(f"::error title={SLOTS_VARIABLE}::{problem}")
     # A persistent pick names the jobs that take it; every other job of the
     # run takes retry_runner. The marker's jobs are the owned machines held.
     owned_jobs, held = place(plan, choice.owned_budget, gui) if persistent(choice.runner) else ((), plan.peak)

tests/test_ci_pr_runner_pool.py

@@ -629,7 +629,8 @@ def test_route_lookups_stop_at_the_cap(self):
         self.assertEqual(get.call_count, 2 * pool.ROUTE_LOOKUPS)
 
     def test_stale_snapshot_or_no_slots_skips_the_pool(self):
-        self.assertEqual(owned_choice(fleet(age=pool.OWNED_MAX_AGE_MINUTES + 1)).runner, LARGE)
+        self.assertNotEqual(owned_choice(fleet(age=pool.OWNED_MAX_AGE_MINUTES + 1)).runner, MINI)
+        self.assertEqual(owned_choice(fleet(age=40)).runner, MINI)
         self.assertEqual(owned_choice(fleet(), owned_slots="").runner, LARGE)
         self.assertEqual(owned_choice(fleet(), machines=0).runner, LARGE)
 
@@ -655,7 +656,7 @@ def test_main_warns_about_bad_slots_only_while_owned_pools_are_on(self):
                        "GITHUB_STEP_SUMMARY": str(Path(tmp, "summary"))}
                 with unittest.mock.patch("sys.stdout", stdout):
                     pool.main([], env)
-                self.assertEqual("::warning title=CI_OWNED_POOL_SLOTS::" in stdout.getvalue(), warned, owned)
+                self.assertEqual("::error title=CI_OWNED_POOL_SLOTS::" in stdout.getvalue(), warned, owned)
                 self.assertEqual("**Warning:**" in Path(tmp, "summary").read_text(), warned, owned)
 
     def test_slots_ignore_anything_malformed(self):

tests/test_ci_queue_janitor.py

@@ -13,6 +13,8 @@
 from pathlib import Path
 from unittest import mock
 
+import yaml
+
 
 ROOT = Path(__file__).resolve().parents[1]
 SCRIPT = ROOT / "scripts/ci/queue_janitor.py"
@@ -648,6 +650,18 @@ class WorkflowShapeTests(unittest.TestCase):
     def setUp(self):
         self.text = WORKFLOW.read_text(encoding="utf-8")
 
+    def test_a_requested_ci_run_refreshes_a_stale_snapshot(self):
+        # The cron drifts (55 minutes apart on 2026-09-25), so CI being
+        # requested also sweeps, unless the newest snapshot is fresh.
+        workflow = yaml.safe_load(self.text)
+        triggers = workflow[True] if True in workflow else workflow["on"]
+        self.assertEqual(triggers["workflow_run"], {"workflows": ["CI"], "types": ["requested"]})
+        steps = workflow["jobs"]["sweep"]["steps"]
+        self.assertEqual(steps[0]["id"], "fresh")
+        self.assertEqual(steps[0]["if"], "github.event_name == 'workflow_run'")
+        for step in steps[1:]:
+            self.assertIn("steps.fresh.outputs.skip != 'true'", step["if"], step["name"])
+
     def test_triggers_permissions_and_runner(self):
         text = self.text
         self.assertIn('- cron: "*/10 * * * *"', text)