diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index c927e2a21104..26ebc6e8ad52 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -112,6 +112,7 @@ jobs: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} REQUESTED_RUNNER: ${{ inputs.runner }} + TART_FLEET: ${{ vars.CI_TART_FLEET }} RUNNER_VARIABLE: ${{ vars.MACOS_RUNNER_TESTS }} LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }} POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }} diff --git a/.github/workflows/test-ios.yml b/.github/workflows/test-ios.yml index 29c3a2a70e98..f55df3376573 100644 --- a/.github/workflows/test-ios.yml +++ b/.github/workflows/test-ios.yml @@ -135,6 +135,7 @@ jobs: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} REQUESTED_RUNNER: ${{ inputs.runner }} + TART_FLEET: ${{ vars.CI_TART_FLEET }} RUNNER_VARIABLE: ${{ vars.MACOS_RUNNER_TESTS || vars.MACOS_RUNNER_IOS }} IOS_OWNED: ${{ vars.CI_IOS_OWNED }} POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }} diff --git a/scripts/ci/e2e_runner_pool.py b/scripts/ci/e2e_runner_pool.py index c81b0b45df9d..0fdf9c27a0c7 100644 --- a/scripts/ci/e2e_runner_pool.py +++ b/scripts/ci/e2e_runner_pool.py @@ -102,6 +102,10 @@ SLOTS_VARIABLE = pr_runner_pool.SLOTS_VARIABLE PR_XCODE_VARIABLE = pr_runner_pool.PR_XCODE_VARIABLE OWNED_UI_VARIABLE = "CI_E2E_OWNED_UI" +# The isolated Tart VM runners (tart-canary, tart-dual, tart-small). A request for +# one runs as auto unless this variable is 1 (resolve()). +TART_PREFIX = "tart-" +TART_FLEET_VARIABLE = "CI_TART_FLEET" # The whole API budget of one decision; see the module docstring. MAX_API_CALLS = 4 @@ -304,9 +308,17 @@ def resolve( pr_xcode_app: str | None = None, test_filter: str | None = None, owned_ui: str | None = None, + tart_fleet: str | None = "1", ) -> str: - """The runner label for a workflow run, from its inputs and variables.""" + """The runner label for a workflow run, from its inputs and variables. + + A `tart-*` request runs as `auto` unless `tart_fleet` (vars.CI_TART_FLEET) + is 1: with the Tart VMs offline such a job queues forever (2026-09-25). + """ requested = (requested or "").strip() + if requested.startswith(TART_PREFIX) and (tart_fleet or "").strip() != "1": + log(f"{requested} requested, but {TART_FLEET_VARIABLE} is not 1 (the Tart VMs are offline); running as auto") + requested = "auto" if requested and requested != "auto": return requested if (owned or "").strip() == "1" and not owned_target(test_filter, owned_ui): @@ -359,6 +371,7 @@ def measure() -> PoolLoad | None: overflow=args.overflow, order=args.order, max_queued=args.max_queued, owned=args.owned, owned_slots=args.owned_slots, pr_xcode_app=args.pr_xcode_app, test_filter=args.test_filter, owned_ui=args.owned_ui, + tart_fleet=env.get("TART_FLEET", ""), measure=measure, now=now, log=lambda message: print(message, file=sys.stderr), )) diff --git a/scripts/ci/ios_runner_pool.py b/scripts/ci/ios_runner_pool.py index e14258849d14..15819eb52f90 100644 --- a/scripts/ci/ios_runner_pool.py +++ b/scripts/ci/ios_runner_pool.py @@ -226,10 +226,19 @@ def resolve( measure: Callable[[], IOSLoad], now: dt.datetime, log: Callable[[str], None] = lambda message: None, + tart_fleet: str | None = "1", ) -> Route: - """The route for one run, from its inputs and variables. Raises ValueError on a refused request.""" + """The route for one run, from its inputs and variables. Raises ValueError on a refused request. + + A `tart-*` request (tart-ios) routes as `auto` unless `tart_fleet` + (vars.CI_TART_FLEET) is 1: with the Tart VMs offline it queues forever (2026-09-25). + """ config = LANES[lane] requested = (requested or "").strip() + if requested.startswith(e2e_runner_pool.TART_PREFIX) and (tart_fleet or "").strip() != "1": + log(f"{requested} requested, but {e2e_runner_pool.TART_FLEET_VARIABLE} is not 1 " + "(the Tart VMs are offline); routing as auto") + requested = "auto" default = (variable or "").strip() or SMALL_RUNNER if requested and requested not in ("auto", OWNED_CHOICE): return ephemeral(requested) @@ -343,6 +352,7 @@ def log(message: str) -> None: ios_version=args.ios_version, device_family=args.device_family, swift_package=args.swift_package, upload=args.upload, called=args.called, seed_cache=args.seed_cache, + tart_fleet=env.get("TART_FLEET", ""), measure=measure, now=now, log=log, ) except ValueError as error: diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py index 9466da0dfa6a..129fb58ad67d 100644 --- a/tests/test_ci_pr_runner_pool.py +++ b/tests/test_ci_pr_runner_pool.py @@ -1673,7 +1673,7 @@ def test_package_tests_stay_off_the_pr_lane(self): def ios_route(snap=None, *, lane="test-ios", requested="auto", variable="", ios_owned="1", owned="1", slots=None, ios_version="", device_family="", upload="", called="", ios_since=0, measure=None, - swift_package="", seed_cache=""): + swift_package="", seed_cache="", tart_fleet="1"): calls = [] def measured(): @@ -1687,7 +1687,7 @@ def measured(): owned_slots=json.dumps(IOS_SLOTS if slots is None else slots), pr_xcode_app=PR_XCODE, order="", max_queued="", ios_version=ios_version, device_family=device_family, upload=upload, called=called, - swift_package=swift_package, seed_cache=seed_cache, measure=measured, now=NOW) + swift_package=swift_package, seed_cache=seed_cache, measure=measured, now=NOW, tart_fleet=tart_fleet) return route, len(calls) @@ -1763,6 +1763,15 @@ def test_explicit_runners_and_other_defaults_are_never_rerouted(self): route, calls = ios_route(sim_fleet(), variable="tart-ios") self.assertEqual((route.label, route.persistent, calls), ("tart-ios", False, 0)) + def test_a_tart_request_routes_as_auto_while_the_tart_fleet_is_off(self): + # 2026-09-25: every Tart VM was offline and tart-ios jobs queued for hours. + auto, _ = ios_route(sim_fleet()) + for fleet in ("", "0"): + route, _ = ios_route(sim_fleet(), requested="tart-ios", tart_fleet=fleet) + self.assertEqual((route.label, route.persistent), (auto.label, auto.persistent), fleet) + route, calls = ios_route(sim_fleet(), requested="tart-ios", tart_fleet="1") + self.assertEqual((route.label, calls), ("tart-ios", 0)) + def test_ios_version_upload_release_and_seed_runs_stay_off_the_fleet(self): # seed_cache runs in the ci-cache-writer environment with the R2 write keys. for kwargs in ({"ios_version": "18.5"}, {"upload": "true"}, {"called": "true"}, {"seed_cache": "true"}): diff --git a/tests/test_run_e2e.py b/tests/test_run_e2e.py index 190c6bc77b59..27f9cbf93f7b 100644 --- a/tests/test_run_e2e.py +++ b/tests/test_run_e2e.py @@ -1129,6 +1129,19 @@ def test_an_explicit_choice_or_admin_variable_is_never_rerouted(self): label, calls, _ = self.decide(queue(), variable=OLD) self.assertEqual((label, calls), (OLD, [])) + def test_a_tart_request_runs_as_auto_while_the_tart_fleet_is_off(self): + # 2026-09-25: every Tart VM was offline and tart-small jobs queued for hours. + for fleet in ("", "0"): + with self.subTest(fleet=fleet): + client = FakeActions(queue()) + label = self.pool.resolve( + "tart-small", "", overflow="", order="", max_queued="", + measure=lambda: self.pool.measure_load(client, now=NOW), now=NOW, tart_fleet=fleet) + self.assertEqual(label, self.decide(queue())[0]) + label = self.pool.resolve("tart-small", "", overflow="", order="", max_queued="", + measure=lambda: None, now=NOW, tart_fleet="1") + self.assertEqual(label, "tart-small") + def test_the_commit_does_not_decide(self): for commit in self.COMMITS: with self.subTest(commit=commit): @@ -1222,7 +1235,7 @@ def pool_step(self): return next(step for step in steps if "e2e_runner_pool.py" in step.get("run", "")) def run_pool_step(self, *, requested="auto", variable="", overflow="", order="", - max_queued=""): + max_queued="", tart_fleet=""): """Run the workflow's own step script with the values GitHub would pass. No token reaches it, so a decision that reads the queue fails safe. @@ -1242,6 +1255,7 @@ def run_pool_step(self, *, requested="auto", variable="", overflow="", order="", "${{ vars.CMUX_CI_XCODE_APP_PR }}": "/Applications/Xcode_26.6.app", "${{ inputs.test_filter }}": "cmuxTests/ExampleTests", "${{ vars.CI_E2E_OWNED_UI }}": "", + "${{ vars.CI_TART_FLEET }}": tart_fleet, } for name, expression in step["env"].items(): self.assertIn(expression, values, f"unexpected input {name}: {expression}") @@ -1262,7 +1276,11 @@ def test_the_workflow_step_resolves_through_the_rule(self): self.assertIn("could not read the runner queue", stderr) self.assertEqual(self.run_pool_step(overflow="0")[0], SMALL) self.assertEqual(self.run_pool_step(order=OLD)[0], SMALL) - self.assertEqual(self.run_pool_step(requested="tart-small")[0], "tart-small") + # The Tart VMs are off unless vars.CI_TART_FLEET is 1: a tart pick runs as auto. + label, stderr = self.run_pool_step(requested="tart-small") + self.assertEqual(label, SMALL) + self.assertIn("CI_TART_FLEET is not 1", stderr) + self.assertEqual(self.run_pool_step(requested="tart-small", tart_fleet="1")[0], "tart-small") self.assertEqual(self.run_pool_step(requested=LARGE)[0], LARGE) self.assertEqual(self.run_pool_step(requested=MINI)[0], MINI) self.assertEqual(self.run_pool_step(variable="blacksmith-6vcpu-macos-15")[0],