ci: run focused cmuxTests against products CI already compiled

.github/workflows/app-host-test-rerun.yml

@@ -1,6 +1,6 @@
 name: App-host test rerun
 run-name: >-
-  app-host test rerun @ ${{ inputs.ref }} · ${{ inputs.only_testing }}
+  app-host test rerun @ ${{ inputs.ref }} · ${{ inputs.only_testing }}${{ inputs.dispatch_id != '' && format(' [{0}]', inputs.dispatch_id) || '' }}
 
 # Reruns app-host tests for a change that touches only cmuxTests/. Instead of
 # compiling the app, it adopts the app-host products an upstream CI run already
@@ -34,6 +34,10 @@ on:
         description: Repository whose CI products to adopt.
         required: false
         default: manaflow-ai/cmux
+      dispatch_id:
+        description: Optional caller-generated id echoed into the run name so dispatchers can find their run
+        required: false
+        default: ""
       baseline:
         description: Also run the same selectors against the unmodified CI-built bundle, for a before/after.
         required: false

scripts/ci/app_host_test_rerun.py

@@ -69,6 +69,9 @@ def products_artifact(repository: str, run_id: str, api: Callable[[str], dict])
 
 
 MERGE_REF = re.compile(r"refs/pull/\d+/merge")
+E2E_WORKFLOW = ".github/workflows/test-e2e.yml"
+# test-e2e.yml's run title ends "@ <ref> [<dispatch id>]"; run-e2e.sh passes a full SHA.
+DISPATCHED_REVISION = re.compile(r" @ ([0-9a-f]{40})(?: \[[^\]]*\])?$")
 
 
 def commit_parents(revision: str, cwd: str | None = None) -> list[str]:
@@ -97,8 +100,20 @@ def built_revision(run: dict, cwd: str | None = None, fetch: Callable[[str], Non
     loads its reusable workflows from the same ref, and `referenced_workflows`
     names the commit each came from. The merge must name `head_sha` as its
     second parent, which ties it to this run's head rather than any commit.
+    A dispatched test-e2e.yml run's `head_sha` is its workflow's branch, so
+    its title says what it built instead.
     """
     head = run["head_sha"]
+    fetch = fetch or (lambda revision: fetch_commit(revision, cwd=cwd))
+    if run.get("event") == "workflow_dispatch" and run.get("path") == E2E_WORKFLOW:
+        # A dispatched test-e2e.yml run lists under the branch its workflow came
+        # from, usually main, but compiles the `ref` input its title names.
+        # Other dispatches, such as main's ci.yml, build their head.
+        match = DISPATCHED_REVISION.search(str(run.get("display_title", "")))
+        if not match:
+            raise ValueError(f"dispatched run {run.get('id')} names no full revision in its title")
+        fetch(match.group(1))
+        return match.group(1)
     if run.get("event") != "pull_request":
         return head
     merges = {
@@ -112,7 +127,7 @@ def built_revision(run: dict, cwd: str | None = None, fetch: Callable[[str], Non
             "but GitHub recorded no single merge commit for it"
         )
     merge = merges.pop()
-    (fetch or (lambda revision: fetch_commit(revision, cwd=cwd)))(merge)
+    fetch(merge)
     parents = commit_parents(merge, cwd=cwd)
     if len(parents) != 2 or parents[1] != head:
         raise ValueError(f"run {run.get('id')} recorded {merge}, which is not a merge of its head {head}")
@@ -152,14 +167,16 @@ def product_runner(repository: str, run_id: str, api: Callable[[str], dict], pag
 
     The rerun rebuilds cmuxTests with the products' own Xcode, and each macOS
     image carries one pinned Xcode (26.3 on 15, 26.6 on 26). Compile admission
-    follows MACOS_RUNNER_PR, so read the pool it actually ran on. A run
-    without that job keeps the macOS 15 default.
+    follows MACOS_RUNNER_PR, so read the pool it actually ran on; a test-e2e.yml
+    run compiles in its `build` job. A run without either keeps the macOS 15
+    default.
     """
     for page in range(1, pages + 1):
         listing = api(f"repos/{repository}/actions/runs/{run_id}/jobs?filter=latest&per_page=100&page={page}")
         jobs = listing.get("jobs", [])
         for job in jobs:
-            if job.get("name", "").endswith(ADMISSION_JOB):
+            # test-e2e.yml compiles in its `build` job.
+            if job.get("name", "").endswith(ADMISSION_JOB) or job.get("name") == "build":
                 for label in job.get("labels", []):
                     match = re.search(r"macos-(\d+)", label)
                     if match and match.group(1) in PRODUCT_RUNNERS:
@@ -194,8 +211,10 @@ def plan(args: argparse.Namespace, api: Callable[[str], dict] = gh_api) -> dict:
         run = api(f"repos/{args.repository}/actions/runs/{args.source_run_id}")
         try:
             revision = built_revision(run)
+        except ValueError as error:
+            raise SystemExit(str(error))
         except subprocess.CalledProcessError:
-            raise SystemExit(f"run {args.source_run_id} built the merge of {run['head_sha']}, which could not be fetched")
+            raise SystemExit(f"run {args.source_run_id} built a revision other than {run['head_sha']} that could not be fetched")
         try:
             blocking = non_test_changes(revision, head)
         except subprocess.CalledProcessError:

scripts/ci/dispatch-focused-test.py

@@ -19,11 +19,18 @@
 import uuid
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
+import app_host_test_rerun as rerun  # noqa: E402
 import e2e_runner_pool as pool  # noqa: E402
 from e2e_runner_pool import SMALL_RUNNER  # noqa: E402
 
 REPO = "manaflow-ai/cmux"
 WORKFLOW = "test-e2e.yml"
+# Runs cmuxTests against app-host products a CI run already compiled; see
+# reuse_ci_products().
+RERUN_WORKFLOW = "app-host-test-rerun.yml"
+CI_WORKFLOW_PATH = ".github/workflows/ci.yml"
+PRODUCTS_WAIT_SECONDS = 50 * 60
+PRODUCTS_POLL_SECONDS = 60.0
 # The branch `gh workflow run` takes the workflow definition from when no
 # --workflow-ref is given: the repository default branch.
 DEFAULT_WORKFLOW_REF = "main"
@@ -484,29 +491,32 @@ def find_run(
     dispatch_id: str,
     *,
     cancel_event: threading.Event | None = None,
+    workflow: str = WORKFLOW,
 ) -> dict:
     """Correlate this dispatch, never assume the newest run belongs to us."""
     cancel_event = cancel_event or threading.Event()
     suffix = f" @ {commit} [{dispatch_id}]"
+    if workflow == WORKFLOW:
+        def ours(title: str) -> bool:
+            return title.startswith(f"{selector} on ") and title.endswith(suffix)
+    else:
+        def ours(title: str) -> bool:
+            return title.endswith(f" [{dispatch_id}]")
     deadline = time.monotonic() + RUN_DISCOVERY_TIMEOUT_SECONDS
     for attempt in range(RUN_DISCOVERY_ATTEMPTS):
         remaining = deadline - time.monotonic()
         if remaining <= 0:
             break
         runs = json.loads(output(
-            "gh", "run", "list", "--repo", REPO, "--workflow", WORKFLOW,
+            "gh", "run", "list", "--repo", REPO, "--workflow", workflow,
             "--event", "workflow_dispatch", "--limit", "100",
             "--json", "databaseId,displayTitle,url",
             timeout=remaining,
             cancel_event=cancel_event,
         ))
         if cancel_event.is_set():
             raise ValueError("focused-run discovery cancelled")
-        matches = [
-            run for run in runs
-            if run["displayTitle"].startswith(f"{selector} on ")
-            and run["displayTitle"].endswith(suffix)
-        ]
+        matches = [run for run in runs if ours(run["displayTitle"])]
         if len(matches) == 1:
             return matches[0]
         if matches:
@@ -522,11 +532,161 @@ def find_run(
             raise ValueError("focused-run discovery cancelled")
     raise ValueError(
         f"dispatch accepted but its run was not found; request {dispatch_id}. "
-        f"Check https://github.com/{REPO}/actions/workflows/{WORKFLOW} "
+        f"Check https://github.com/{REPO}/actions/workflows/{workflow} "
         "before dispatching again."
     )
 
 
+@contextmanager
+def chdir(path: Path):
+    """contextlib.chdir, which needs Python 3.11; run-e2e.sh may get macOS's 3.9."""
+    previous = os.getcwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(previous)
+
+
+def planned_products(commit: str, only_testing: str, source_run_id: str = "") -> dict | None:
+    """app_host_test_rerun.py's plan for this commit, or None when it finds no products."""
+    args = argparse.Namespace(
+        ref=commit, repository=REPO, only_testing=only_testing,
+        source_run_id=source_run_id, max_commits=200,
+    )
+    try:
+        with chdir(ROOT):
+            return rerun.plan(args)
+    except SystemExit as error:
+        print(f"note: no reusable CI products: {str(error).splitlines()[0]}", file=sys.stderr, flush=True)
+    except (KeyError, ValueError, subprocess.CalledProcessError, json.JSONDecodeError):
+        pass
+    return None
+
+
+def building_producer(commit: str) -> dict | None:
+    """A CI run of this commit still compiling products the commit can use.
+
+    A pull_request run builds the merge with its base, so it qualifies only
+    while that merge differs from the commit under cmuxTests/ alone. Main's
+    ci.yml runs are dispatched by ci-main-full-suite.yml. The
+    guards in main() already refuse a second test-e2e.yml compile of a commit.
+    """
+    try:
+        listing = rerun.gh_api(f"repos/{REPO}/actions/runs?head_sha={commit}&per_page=50")
+        runs = sorted(listing.get("workflow_runs", []), key=lambda run: run.get("created_at", ""), reverse=True)
+    except (subprocess.CalledProcessError, json.JSONDecodeError):
+        runs = []
+    for run in runs:
+        if (run.get("path") != CI_WORKFLOW_PATH or run.get("status") not in UNFINISHED
+                or run.get("event") not in ("push", "pull_request", "workflow_dispatch")):
+            continue
+        try:
+            with chdir(ROOT):
+                if rerun.non_test_changes(rerun.built_revision(run), commit):
+                    continue
+        except (KeyError, ValueError, subprocess.CalledProcessError):
+            continue
+        return {"id": run["id"], "url": run.get("html_url", "")}
+    return None
+
+
+def skips_macos(run_id: int) -> bool:
+    """Whether a CI run decided not to compile for macOS, so it will leave no products."""
+    listing = rerun.gh_api(f"repos/{REPO}/actions/runs/{run_id}/jobs?filter=latest&per_page=100")
+    return any(
+        job.get("name", "").endswith(rerun.ADMISSION_JOB) and job.get("conclusion") == "skipped"
+        for job in listing.get("jobs", [])
+    )
+
+
+def awaited_products(producer: dict, commit: str, only_testing: str) -> dict | None:
+    """Wait for a building run's products, then plan against them."""
+    print(
+        f"{producer['url']} is already compiling {commit}; waiting for its app-host "
+        "products instead of compiling them a second time.",
+        flush=True,
+    )
+    deadline = time.monotonic() + PRODUCTS_WAIT_SECONDS
+    with cancellation_scope() as cancel_event:
+        while True:
+            if rerun.products_artifact(REPO, str(producer["id"]), rerun.gh_api):
+                return planned_products(commit, only_testing, str(producer["id"]))
+            state = rerun.gh_api(f"repos/{REPO}/actions/runs/{producer['id']}")
+            if state.get("status") not in UNFINISHED:
+                print(f"note: {producer['url']} finished without app-host products", file=sys.stderr, flush=True)
+                return None
+            if skips_macos(producer["id"]):
+                print(f"note: {producer['url']} skipped its macOS compile", file=sys.stderr, flush=True)
+                return None
+            if time.monotonic() > deadline:
+                print(f"note: {producer['url']} has not produced app-host products yet", file=sys.stderr, flush=True)
+                return None
+            if wait_for_retry(cancel_event, PRODUCTS_POLL_SECONDS):
+                raise ValueError("waiting for CI products cancelled")
+
+
+def reuse_ci_products(commit: str, entries: list[str], workflow_ref: str | None, wait: bool) -> int | None:
+    """Run cmuxTests selectors against app-host products CI compiled, if any.
+
+    A test-e2e.yml run compiles the whole app, 12 to 27 minutes, to run a few
+    minutes of tests. When CI already compiled this commit's app, or is
+    compiling it now, app-host-test-rerun.yml recompiles only cmuxTests
+    against those products. Returns the exit status, or None to fall back to
+    a full build.
+    """
+    try:
+        only_testing = " ".join(rerun.parse_selectors(" ".join(entries)))
+    except ValueError:
+        return None
+    try:
+        with chdir(ROOT):
+            rerun.fetch_commit(commit)
+    except subprocess.CalledProcessError:
+        return None
+    try:
+        found = planned_products(commit, only_testing)
+        if found is None:
+            producer = building_producer(commit)
+            if producer is None:
+                return None
+            found = awaited_products(producer, commit, only_testing)
+    except (subprocess.CalledProcessError, json.JSONDecodeError, KeyError):
+        return None
+    if found is None:
+        return None
+    dispatch_id = uuid.uuid4().hex
+    command = ["gh", "workflow", "run", RERUN_WORKFLOW, "--repo", REPO]
+    if workflow_ref:
+        command.extend(["--ref", workflow_ref])
+    for key, value in {
+        "ref": commit,
+        "only_testing": only_testing,
+        "source_run_id": found["source_run_id"],
+        "dispatch_id": dispatch_id,
+    }.items():
+        command.extend(["-f", f"{key}={value}"])
+    print(
+        f"Testing {only_testing} at {commit} against the products run {found['source_run_id']} "
+        f"compiled from {found['source_sha']}; only cmuxTests recompiles (request {dispatch_id})",
+        flush=True,
+    )
+    try:
+        subprocess.run(command, cwd=ROOT, check=True)
+    except subprocess.CalledProcessError:
+        # A --workflow-ref whose rerun workflow predates dispatch_id rejects it.
+        print("note: the rerun dispatch was refused; compiling in full instead", file=sys.stderr, flush=True)
+        return None
+    with cancellation_scope() as cancel_event:
+        run = find_run(commit, only_testing, dispatch_id, cancel_event=cancel_event, workflow=RERUN_WORKFLOW)
+    print(f"Run: {run['url']}", flush=True)
+    if wait:
+        return subprocess.run([
+            "gh", "run", "watch", "--repo", REPO, str(run["databaseId"]), "--exit-status",
+        ], cwd=ROOT).returncode
+    return 0
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(
         description="Run one suite or method on an exact pushed commit. "
@@ -549,6 +709,12 @@ def main() -> int:
     parser.add_argument("--job-timeout", type=positive_integer, default=45, help="job timeout in minutes, including compilation (default: 45)")
     parser.add_argument("--workflow-ref", help="workflow-definition branch/tag (default: repository default branch)")
     parser.add_argument("--runner", choices=RUNNERS, help="runner override (default: workflow's configured runner)")
+    parser.add_argument(
+        "--full-build",
+        action="store_true",
+        help="compile the whole app even when CI already compiled this commit's app-host products; "
+        "without it, a cmuxTests run waits (up to 50 min) for a CI run still compiling this commit",
+    )
     parser.add_argument(
         "--force",
         action="store_true",
@@ -693,6 +859,13 @@ def main() -> int:
                     "the new commit. Pass --force to dispatch anyway."
                 )
 
+    # A pinned runner asks about that pool; reused products run on the pool
+    # that compiled them.
+    if test_target == "cmuxTests" and not pinned and not args.full_build:
+        status = reuse_ci_products(commit, args.test_filter, args.workflow_ref, args.wait)
+        if status is not None:
+            return status
+
     runner = args.runner if pinned else routed_runner(default)
     dispatch_id = uuid.uuid4().hex
     video = not args.no_video and test_target != "cmuxTests"

tests/test_app_host_test_rerun.py

@@ -148,6 +148,26 @@ def test_a_pull_request_run_without_a_recorded_merge_is_rejected(self) -> None:
         with self.assertRaisesRegex(ValueError, "no single merge commit"):
             self.built(run)
 
+    def dispatched_run(self, built: str, run_id: int = 6) -> dict:
+        title = f"cmuxTests/ATests on blacksmith-6vcpu-macos-26 @ {built} [abc123]"
+        return {"id": run_id, "event": "workflow_dispatch", "head_sha": self.head, "display_title": title,
+                "path": ".github/workflows/test-e2e.yml"}
+
+    def test_a_dispatched_ci_run_built_its_head(self) -> None:
+        # ci-main-full-suite.yml dispatches main's ci.yml, titled just "CI".
+        run = {"id": 7, "event": "workflow_dispatch", "head_sha": self.head, "display_title": "CI",
+               "path": ".github/workflows/ci.yml"}
+        self.assertEqual(self.built(run), self.head)
+
+    def test_a_dispatched_run_built_the_revision_its_title_names(self) -> None:
+        self.assertEqual(self.built(self.dispatched_run(self.base)), self.base)
+
+    def test_a_dispatched_run_without_a_full_revision_is_rejected(self) -> None:
+        run = self.dispatched_run(self.base)
+        run["display_title"] = "cmuxTests/ATests on blacksmith-6vcpu-macos-26 @ my-branch"
+        with self.assertRaisesRegex(ValueError, "no full revision"):
+            self.built(run)
+
     def plan(self, ref: str, source_run_id: str, api) -> dict:
         self.addCleanup(os.chdir, os.getcwd())
         os.chdir(self.repo.path)
@@ -204,6 +224,21 @@ def api(path: str) -> dict:
         planned = self.plan(self.head, "", api)
         self.assertEqual((planned["source_run_id"], planned["source_sha"]), ("1", self.head))
 
+    def test_automatic_plan_passes_over_a_dispatched_run_of_another_revision(self) -> None:
+        # test-e2e.yml runs list under main's head but compile the branch they test.
+        runs = [
+            {**self.dispatched_run(self.base, run_id=2), "created_at": "2026-01-02"},
+            {"id": 1, "event": "push", "head_sha": self.head, "created_at": "2026-01-01"},
+        ]
+
+        def api(path: str) -> dict:
+            if "head_sha=" in path:
+                return {"workflow_runs": runs if f"head_sha={self.head}" in path else []}
+            return {"artifacts": [self.PRODUCTS]}
+
+        planned = self.plan(self.head, "", api)
+        self.assertEqual((planned["source_run_id"], planned["source_sha"]), ("1", self.head))
+
     def test_lookup_reports_the_built_revision_and_skips_ineligible_merges(self) -> None:
         runs = {"h": [{"id": 1, "created_at": "2026-01-02"}, {"id": 2, "created_at": "2026-01-01"}]}