From c65215e65fca54535b4fb1d17085ec26ca46f5b4 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Thu, 24 Sep 2026 21:30:11 -0400 Subject: [PATCH] ci: keep the owned-pool snapshot fresh when the janitor cron drifts Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-queue-janitor.yml | 33 +++++++++++++++++++++++++- scripts/ci/pr_runner_pool.py | 10 ++++++-- tests/test_ci_pr_runner_pool.py | 5 ++-- tests/test_ci_queue_janitor.py | 14 +++++++++++ 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-queue-janitor.yml b/.github/workflows/ci-queue-janitor.yml index 6089a3f627ba..6e54b70467c0 100644 --- a/.github/workflows/ci-queue-janitor.yml +++ b/.github/workflows/ci-queue-janitor.yml @@ -26,9 +26,19 @@ name: CI queue janitor # Each sweep also publishes the per-pool macOS queue it saw as the # `macos-pool-load` artifact, which pull request CI reads to overflow off a # backed-up pool. +# +# GitHub delays scheduled runs (the */10 cron fired 55 minutes apart on +# 2026-09-25), and pull request CI trusts a snapshot for at most +# MAX_SNAPSHOT_MINUTES, so a CI run being requested also starts a sweep. It +# skips when the newest snapshot is under FRESH_SECONDS old, and the +# concurrency group keeps one sweep running and one waiting, so a burst of +# pushes costs at most one extra sweep. on: schedule: - cron: "*/10 * * * *" + workflow_run: + workflows: [CI] + types: [requested] workflow_dispatch: inputs: dry_run: @@ -54,12 +64,33 @@ jobs: runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 10 steps: + - name: Skip when the pool snapshot is fresh + id: fresh + if: github.event_name == 'workflow_run' + env: + GH_TOKEN: ${{ github.token }} + FRESH_SECONDS: "240" + run: | + set -euo pipefail + newest="$(gh api "repos/${{ github.repository }}/actions/artifacts?name=macos-pool-load&per_page=20" \ + --jq '[.artifacts[] | select(.expired | not) | select(.workflow_run.head_branch == "main") | .created_at] | max // ""')" || newest="" + age=999999 + if [ -n "$newest" ]; then + age=$(( $(date -u +%s) - $(date -u -d "$newest" +%s) )) + fi + echo "newest snapshot: ${newest:-none} (${age}s old)" + if [ "$age" -lt "$FRESH_SECONDS" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + - name: Check out trusted janitor + if: steps.fresh.outputs.skip != 'true' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Cancel wasted macOS runs + if: steps.fresh.outputs.skip != 'true' env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} @@ -84,7 +115,7 @@ jobs: # pull request run lands on, for two API requests instead of one job # listing per in-flight run. - name: Publish macOS pool load - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.fresh.outputs.skip != 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: macos-pool-load diff --git a/scripts/ci/pr_runner_pool.py b/scripts/ci/pr_runner_pool.py index 93df6c1cf995..9dfb9ee34554 100644 --- a/scripts/ci/pr_runner_pool.py +++ b/scripts/ci/pr_runner_pool.py @@ -181,7 +181,11 @@ MAX_RUN_JOBS = SIDE_LANES + APP_HOST_SHARDS + 2 REPLAYED_RUN_JOBS = SIDE_LANES + 1 # A snapshot older than this is not trusted to place a run on an owned pool. -OWNED_MAX_AGE_MINUTES = 20 +# It was 20, but GitHub delays scheduled runs: the janitor's */10 cron fired +# 55 minutes apart (23:59Z to 00:54Z, 2026-09-25) and every run skipped 40 +# idle minis. ci-queue-janitor.yml now also sweeps when CI is requested, and +# a mini that turns out busy is caught by the rescue within its budget. +OWNED_MAX_AGE_MINUTES = 45 # Pools whose machines are discarded after each job; the only ones a fork run may use. EPHEMERAL_PREFIX = "blacksmith-" @@ -1019,7 +1023,9 @@ def count_routed(since: str) -> int: ) problems = slot_problems(env.get("OWNED_SLOTS")) if (env.get("POOL_OWNED") or "").strip() == "1" else [] for problem in problems: - print(f"::warning title={SLOTS_VARIABLE}::{problem}") + # An error, not a warning: a malformed entry silently takes the + # fleet out of the order (a bare `40` did for 30 minutes on 2026-09-25). + print(f"::error title={SLOTS_VARIABLE}::{problem}") # A persistent pick names the jobs that take it; every other job of the # run takes retry_runner. The marker's jobs are the owned machines held. owned_jobs, held = place(plan, choice.owned_budget, gui) if persistent(choice.runner) else ((), plan.peak) diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py index d3a458bf8470..337d9a851281 100644 --- a/tests/test_ci_pr_runner_pool.py +++ b/tests/test_ci_pr_runner_pool.py @@ -629,7 +629,8 @@ def test_route_lookups_stop_at_the_cap(self): self.assertEqual(get.call_count, 2 * pool.ROUTE_LOOKUPS) def test_stale_snapshot_or_no_slots_skips_the_pool(self): - self.assertEqual(owned_choice(fleet(age=pool.OWNED_MAX_AGE_MINUTES + 1)).runner, LARGE) + self.assertNotEqual(owned_choice(fleet(age=pool.OWNED_MAX_AGE_MINUTES + 1)).runner, MINI) + self.assertEqual(owned_choice(fleet(age=40)).runner, MINI) self.assertEqual(owned_choice(fleet(), owned_slots="").runner, LARGE) self.assertEqual(owned_choice(fleet(), machines=0).runner, LARGE) @@ -655,7 +656,7 @@ def test_main_warns_about_bad_slots_only_while_owned_pools_are_on(self): "GITHUB_STEP_SUMMARY": str(Path(tmp, "summary"))} with unittest.mock.patch("sys.stdout", stdout): pool.main([], env) - self.assertEqual("::warning title=CI_OWNED_POOL_SLOTS::" in stdout.getvalue(), warned, owned) + self.assertEqual("::error title=CI_OWNED_POOL_SLOTS::" in stdout.getvalue(), warned, owned) self.assertEqual("**Warning:**" in Path(tmp, "summary").read_text(), warned, owned) def test_slots_ignore_anything_malformed(self): diff --git a/tests/test_ci_queue_janitor.py b/tests/test_ci_queue_janitor.py index 5009be63d3b0..7e3e5a33330e 100644 --- a/tests/test_ci_queue_janitor.py +++ b/tests/test_ci_queue_janitor.py @@ -13,6 +13,8 @@ from pathlib import Path from unittest import mock +import yaml + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts/ci/queue_janitor.py" @@ -648,6 +650,18 @@ class WorkflowShapeTests(unittest.TestCase): def setUp(self): self.text = WORKFLOW.read_text(encoding="utf-8") + def test_a_requested_ci_run_refreshes_a_stale_snapshot(self): + # The cron drifts (55 minutes apart on 2026-09-25), so CI being + # requested also sweeps, unless the newest snapshot is fresh. + workflow = yaml.safe_load(self.text) + triggers = workflow[True] if True in workflow else workflow["on"] + self.assertEqual(triggers["workflow_run"], {"workflows": ["CI"], "types": ["requested"]}) + steps = workflow["jobs"]["sweep"]["steps"] + self.assertEqual(steps[0]["id"], "fresh") + self.assertEqual(steps[0]["if"], "github.event_name == 'workflow_run'") + for step in steps[1:]: + self.assertIn("steps.fresh.outputs.skip != 'true'", step["if"], step["name"]) + def test_triggers_permissions_and_runner(self): text = self.text self.assertIn('- cron: "*/10 * * * *"', text)