ci: run a tart-* pick as auto while the Tart VMs are offline

.github/workflows/test-e2e.yml

@@ -112,6 +112,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           REQUESTED_RUNNER: ${{ inputs.runner }}
+          TART_FLEET: ${{ vars.CI_TART_FLEET }}
           RUNNER_VARIABLE: ${{ vars.MACOS_RUNNER_TESTS }}
           LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }}
           POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }}

.github/workflows/test-ios.yml

@@ -135,6 +135,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           REQUESTED_RUNNER: ${{ inputs.runner }}
+          TART_FLEET: ${{ vars.CI_TART_FLEET }}
           RUNNER_VARIABLE: ${{ vars.MACOS_RUNNER_TESTS || vars.MACOS_RUNNER_IOS }}
           IOS_OWNED: ${{ vars.CI_IOS_OWNED }}
           POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }}

scripts/ci/e2e_runner_pool.py

@@ -102,6 +102,10 @@
 SLOTS_VARIABLE = pr_runner_pool.SLOTS_VARIABLE
 PR_XCODE_VARIABLE = pr_runner_pool.PR_XCODE_VARIABLE
 OWNED_UI_VARIABLE = "CI_E2E_OWNED_UI"
+# The isolated Tart VM runners (tart-canary, tart-dual, tart-small). A request for
+# one runs as auto unless this variable is 1 (resolve()).
+TART_PREFIX = "tart-"
+TART_FLEET_VARIABLE = "CI_TART_FLEET"
 
 # The whole API budget of one decision; see the module docstring.
 MAX_API_CALLS = 4
@@ -304,9 +308,17 @@ def resolve(
     pr_xcode_app: str | None = None,
     test_filter: str | None = None,
     owned_ui: str | None = None,
+    tart_fleet: str | None = "1",
 ) -> str:
-    """The runner label for a workflow run, from its inputs and variables."""
+    """The runner label for a workflow run, from its inputs and variables.
+
+    A `tart-*` request runs as `auto` unless `tart_fleet` (vars.CI_TART_FLEET)
+    is 1: with the Tart VMs offline such a job queues forever (2026-09-25).
+    """
     requested = (requested or "").strip()
+    if requested.startswith(TART_PREFIX) and (tart_fleet or "").strip() != "1":
+        log(f"{requested} requested, but {TART_FLEET_VARIABLE} is not 1 (the Tart VMs are offline); running as auto")
+        requested = "auto"
     if requested and requested != "auto":
         return requested
     if (owned or "").strip() == "1" and not owned_target(test_filter, owned_ui):
@@ -359,6 +371,7 @@ def measure() -> PoolLoad | None:
         overflow=args.overflow, order=args.order, max_queued=args.max_queued,
         owned=args.owned, owned_slots=args.owned_slots, pr_xcode_app=args.pr_xcode_app,
         test_filter=args.test_filter, owned_ui=args.owned_ui,
+        tart_fleet=env.get("TART_FLEET", ""),
         measure=measure, now=now,
         log=lambda message: print(message, file=sys.stderr),
     ))

scripts/ci/ios_runner_pool.py

@@ -226,10 +226,19 @@ def resolve(
     measure: Callable[[], IOSLoad],
     now: dt.datetime,
     log: Callable[[str], None] = lambda message: None,
+    tart_fleet: str | None = "1",
 ) -> Route:
-    """The route for one run, from its inputs and variables. Raises ValueError on a refused request."""
+    """The route for one run, from its inputs and variables. Raises ValueError on a refused request.
+
+    A `tart-*` request (tart-ios) routes as `auto` unless `tart_fleet`
+    (vars.CI_TART_FLEET) is 1: with the Tart VMs offline it queues forever (2026-09-25).
+    """
     config = LANES[lane]
     requested = (requested or "").strip()
+    if requested.startswith(e2e_runner_pool.TART_PREFIX) and (tart_fleet or "").strip() != "1":
+        log(f"{requested} requested, but {e2e_runner_pool.TART_FLEET_VARIABLE} is not 1 "
+            "(the Tart VMs are offline); routing as auto")
+        requested = "auto"
     default = (variable or "").strip() or SMALL_RUNNER
     if requested and requested not in ("auto", OWNED_CHOICE):
         return ephemeral(requested)
@@ -343,6 +352,7 @@ def log(message: str) -> None:
             ios_version=args.ios_version, device_family=args.device_family,
             swift_package=args.swift_package, upload=args.upload, called=args.called,
             seed_cache=args.seed_cache,
+            tart_fleet=env.get("TART_FLEET", ""),
             measure=measure, now=now, log=log,
         )
     except ValueError as error:

tests/test_ci_pr_runner_pool.py

@@ -1673,7 +1673,7 @@ def test_package_tests_stay_off_the_pr_lane(self):
 
 def ios_route(snap=None, *, lane="test-ios", requested="auto", variable="", ios_owned="1", owned="1",
               slots=None, ios_version="", device_family="", upload="", called="", ios_since=0, measure=None,
-              swift_package="", seed_cache=""):
+              swift_package="", seed_cache="", tart_fleet="1"):
     calls = []
 
     def measured():
@@ -1687,7 +1687,7 @@ def measured():
         owned_slots=json.dumps(IOS_SLOTS if slots is None else slots),
         pr_xcode_app=PR_XCODE, order="", max_queued="",
         ios_version=ios_version, device_family=device_family, upload=upload, called=called,
-        swift_package=swift_package, seed_cache=seed_cache, measure=measured, now=NOW)
+        swift_package=swift_package, seed_cache=seed_cache, measure=measured, now=NOW, tart_fleet=tart_fleet)
     return route, len(calls)
 
 
@@ -1763,6 +1763,15 @@ def test_explicit_runners_and_other_defaults_are_never_rerouted(self):
         route, calls = ios_route(sim_fleet(), variable="tart-ios")
         self.assertEqual((route.label, route.persistent, calls), ("tart-ios", False, 0))
 
+    def test_a_tart_request_routes_as_auto_while_the_tart_fleet_is_off(self):
+        # 2026-09-25: every Tart VM was offline and tart-ios jobs queued for hours.
+        auto, _ = ios_route(sim_fleet())
+        for fleet in ("", "0"):
+            route, _ = ios_route(sim_fleet(), requested="tart-ios", tart_fleet=fleet)
+            self.assertEqual((route.label, route.persistent), (auto.label, auto.persistent), fleet)
+        route, calls = ios_route(sim_fleet(), requested="tart-ios", tart_fleet="1")
+        self.assertEqual((route.label, calls), ("tart-ios", 0))
+
     def test_ios_version_upload_release_and_seed_runs_stay_off_the_fleet(self):
         # seed_cache runs in the ci-cache-writer environment with the R2 write keys.
         for kwargs in ({"ios_version": "18.5"}, {"upload": "true"}, {"called": "true"}, {"seed_cache": "true"}):

tests/test_run_e2e.py

@@ -1129,6 +1129,19 @@ def test_an_explicit_choice_or_admin_variable_is_never_rerouted(self):
         label, calls, _ = self.decide(queue(), variable=OLD)
         self.assertEqual((label, calls), (OLD, []))
 
+    def test_a_tart_request_runs_as_auto_while_the_tart_fleet_is_off(self):
+        # 2026-09-25: every Tart VM was offline and tart-small jobs queued for hours.
+        for fleet in ("", "0"):
+            with self.subTest(fleet=fleet):
+                client = FakeActions(queue())
+                label = self.pool.resolve(
+                    "tart-small", "", overflow="", order="", max_queued="",
+                    measure=lambda: self.pool.measure_load(client, now=NOW), now=NOW, tart_fleet=fleet)
+                self.assertEqual(label, self.decide(queue())[0])
+        label = self.pool.resolve("tart-small", "", overflow="", order="", max_queued="",
+                                  measure=lambda: None, now=NOW, tart_fleet="1")
+        self.assertEqual(label, "tart-small")
+
     def test_the_commit_does_not_decide(self):
         for commit in self.COMMITS:
             with self.subTest(commit=commit):
@@ -1222,7 +1235,7 @@ def pool_step(self):
         return next(step for step in steps if "e2e_runner_pool.py" in step.get("run", ""))
 
     def run_pool_step(self, *, requested="auto", variable="", overflow="", order="",
-                      max_queued=""):
+                      max_queued="", tart_fleet=""):
         """Run the workflow's own step script with the values GitHub would pass.
 
         No token reaches it, so a decision that reads the queue fails safe.
@@ -1242,6 +1255,7 @@ def run_pool_step(self, *, requested="auto", variable="", overflow="", order="",
             "${{ vars.CMUX_CI_XCODE_APP_PR }}": "/Applications/Xcode_26.6.app",
             "${{ inputs.test_filter }}": "cmuxTests/ExampleTests",
             "${{ vars.CI_E2E_OWNED_UI }}": "",
+            "${{ vars.CI_TART_FLEET }}": tart_fleet,
         }
         for name, expression in step["env"].items():
             self.assertIn(expression, values, f"unexpected input {name}: {expression}")
@@ -1262,7 +1276,11 @@ def test_the_workflow_step_resolves_through_the_rule(self):
         self.assertIn("could not read the runner queue", stderr)
         self.assertEqual(self.run_pool_step(overflow="0")[0], SMALL)
         self.assertEqual(self.run_pool_step(order=OLD)[0], SMALL)
-        self.assertEqual(self.run_pool_step(requested="tart-small")[0], "tart-small")
+        # The Tart VMs are off unless vars.CI_TART_FLEET is 1: a tart pick runs as auto.
+        label, stderr = self.run_pool_step(requested="tart-small")
+        self.assertEqual(label, SMALL)
+        self.assertIn("CI_TART_FLEET is not 1", stderr)
+        self.assertEqual(self.run_pool_step(requested="tart-small", tart_fleet="1")[0], "tart-small")
         self.assertEqual(self.run_pool_step(requested=LARGE)[0], LARGE)
         self.assertEqual(self.run_pool_step(requested=MINI)[0], MINI)
         self.assertEqual(self.run_pool_step(variable="blacksmith-6vcpu-macos-15")[0],