From b2ba135902732b6007a0fb8cf9ddb5b9323f8b53 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:09:20 -0700 Subject: [PATCH 01/14] test: workflow and guard-only changes must not route to macOS Co-Authored-By: Claude Fable 5.1 --- tests/test_ci_change_areas.py | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 9cca54cc1f70..12bcc72258c0 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -147,6 +147,52 @@ def test_workflow_changes_run_everything() -> None: ) +def test_other_workflow_changes_skip_macos_and_web() -> None: + # ci.yml's macOS and web jobs never read another workflow file. Those edits + # are validated by workflow-guard-tests and by the edited workflow itself. + assert_areas( + [".github/workflows/relay-tls.yml", ".github/actionlint.yaml"], + macos=False, + web=False, + ) + + +def test_guard_only_tests_skip_macos() -> None: + # Referenced in ci.yml only by Linux jobs. + assert_areas(["tests/test_ci_self_hosted_guard.sh"], macos=False, web=False) + assert_areas( + [".github/workflows/ios-testflight.yml", "tests/test_ios_testflight_main_push_filter.py"], + macos=False, + web=False, + ) + + +def test_tests_run_by_macos_jobs_run_macos() -> None: + assert_areas(["tests/test_cli_contract_help.py"], macos=True, web=False) + # A macOS job runs these through a glob. + assert_areas(["tests/test_nushell_integration_hooks.py"], macos=True, web=False) + # Shared by a Linux guard job and release-build. + assert_areas(["tests/test_install_cmux_tui_client.sh"], macos=True, web=False) + + +def test_unreferenced_tests_run_macos() -> None: + # Nothing in ci.yml names it, so a macOS-run test may import it. + assert_areas(["tests/some_new_helper.py"], macos=True, web=False) + + +def test_guard_only_change_with_app_source_runs_macos() -> None: + assert_areas( + [".github/workflows/relay-tls.yml", "Sources/AppDelegate.swift"], + macos=True, + web=False, + ) + + +def test_macos_test_references_fail_open_without_ci_workflow() -> None: + assert module.macos_job_test_references("jobs:\n") is None + assert module.macos_job_test_references("not a workflow") is None + + def test_ci_router_runs_on_every_pr_and_merge_group() -> None: workflow = CI_WORKFLOW.read_text(encoding="utf-8") assert " pull_request:\n merge_group:" in workflow From 982784df5b986caf4b738f35bdda47f33503c341 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:10:18 -0700 Subject: [PATCH 02/14] ci: route other workflow files and Linux-only guard tests away from macOS Any workflow edit forced every CI area, so a change to relay-tls.yml or to a guard script waited for a macOS compile that cannot observe it. ci.yml's jobs read no other workflow file, and those edits are checked by workflow-guard-tests. A tests/ file is now macOS-neutral when ci.yml names it and only Linux jobs name it. ci.yml, scripts/ci/*.py and the router test still force every area, and an unreadable ci.yml or an unnamed tests/ file keeps macOS on. Co-Authored-By: Claude Fable 5.1 --- scripts/ci/detect_ci_change_areas.py | 65 ++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/scripts/ci/detect_ci_change_areas.py b/scripts/ci/detect_ci_change_areas.py index 6ce8bedb937b..64a750e10ab8 100644 --- a/scripts/ci/detect_ci_change_areas.py +++ b/scripts/ci/detect_ci_change_areas.py @@ -5,6 +5,7 @@ import argparse import os +import re import subprocess import sys from dataclasses import dataclass @@ -41,8 +42,15 @@ def normalize_path(path: str) -> str: return normalized -def is_workflow(path: str) -> bool: - return path.startswith(".github/workflows/") +CI_WORKFLOW_PATH = ".github/workflows/ci.yml" + + +def is_other_workflow_config(path: str) -> bool: + # ci.yml's macOS and web jobs read no other workflow file. An edit to one is + # checked by workflow-guard-tests and by that workflow's own triggers. + if path == CI_WORKFLOW_PATH: + return False + return path.startswith(".github/workflows/") or path == ".github/actionlint.yaml" def forces_all_areas(path: str) -> bool: @@ -50,7 +58,55 @@ def forces_all_areas(path: str) -> bool: is_direct_ci_python = path.startswith(ci_script_prefix) and path.endswith(".py") if is_direct_ci_python: is_direct_ci_python = "/" not in path[len(ci_script_prefix) :] - return is_workflow(path) or is_direct_ci_python or path == "tests/test_ci_change_areas.py" + return path == CI_WORKFLOW_PATH or is_direct_ci_python or path == "tests/test_ci_change_areas.py" + + +_TEST_REFERENCE_RE = re.compile(r"tests/[A-Za-z0-9_./-]*") + + +def macos_job_test_references(workflow: str) -> Optional[tuple[frozenset[str], frozenset[str]]]: + """Return the tests/ paths ci.yml names in macOS jobs and in all jobs. + + A macOS job that runs tests through a glob yields the glob's literal prefix. + Returns None when the jobs cannot be read, so the caller fails open. + """ + _, found, body = workflow.partition("\njobs:\n") + if not found: + return None + macos: set[str] = set() + everywhere: set[str] = set() + jobs = 0 + for block in re.split(r"(?m)^ (?=[A-Za-z0-9_-]+:\s*$)", body): + runs_on = re.search(r"(?m)^ runs-on:\s*(.+)$", block) + if not runs_on: + continue + jobs += 1 + references = set(_TEST_REFERENCE_RE.findall(block)) + everywhere |= references + if re.search(r"macos", runs_on.group(1), re.IGNORECASE): + macos |= references + if jobs == 0: + return None + return frozenset(macos), frozenset(everywhere) + + +def load_macos_job_test_references() -> Optional[tuple[frozenset[str], frozenset[str]]]: + try: + return macos_job_test_references(Path(CI_WORKFLOW_PATH).read_text(encoding="utf-8")) + except OSError: + return None + + +def is_guard_only_test(path: str, references: Optional[tuple[frozenset[str], frozenset[str]]]) -> bool: + # A tests/ file is macOS-neutral only when ci.yml names it and every job + # that names it runs on Linux. An unnamed file may be imported by a test a + # macOS job runs, so it stays macOS-relevant. + if references is None or not path.startswith("tests/"): + return False + macos, everywhere = references + if path not in everywhere: + return False + return not any(path.startswith(reference) for reference in macos) def is_web_change(path: str) -> bool: @@ -130,6 +186,7 @@ def classify_files(paths: Iterable[str]) -> ChangeAreas: macos = False web = False agent_session_web = False + test_references = load_macos_job_test_references() for raw_path in paths: path = normalize_path(raw_path) @@ -140,6 +197,8 @@ def classify_files(paths: Iterable[str]) -> ChangeAreas: web = True agent_session_web = True continue + if is_other_workflow_config(path) or is_guard_only_test(path, test_references): + continue if is_web_change(path): web = True if is_agent_session_web_change(path): From fcbeaccf9e2fc26a900987dc4d5944042ee4cece Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:15:39 -0700 Subject: [PATCH 03/14] test: only a plainly Linux runs-on may make a test guard-only Co-Authored-By: Claude Fable 5.1 --- tests/test_ci_change_areas.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 12bcc72258c0..111c00659ab5 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -188,6 +188,32 @@ def test_guard_only_change_with_app_source_runs_macos() -> None: ) +def test_only_a_plainly_linux_job_makes_a_test_guard_only() -> None: + def workflow(runs_on: str) -> str: + return ( + "name: CI\njobs:\n guard:\n" + " runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}\n" + " steps:\n - run: python3 tests/test_guard.py\n" + f" other:\n runs-on:{runs_on}\n" + " steps:\n - run: python3 tests/test_other.py\n" + ) + + for runs_on in ( + " ${{ matrix.runner }}", + " ${{ needs.pick.outputs.runner }}", + "\n - self-hosted\n - arm64", + "\n group: big-macs", + " ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }}", + " ${{ vars.LINUX_RUNNER || vars.MACOS_RUNNER_15 }}", + ): + references = module.macos_job_test_references(workflow(runs_on)) + assert module.is_guard_only_test("tests/test_guard.py", references), runs_on + assert not module.is_guard_only_test("tests/test_other.py", references), runs_on + + references = module.macos_job_test_references(workflow(" ubuntu-24.04")) + assert module.is_guard_only_test("tests/test_other.py", references) + + def test_macos_test_references_fail_open_without_ci_workflow() -> None: assert module.macos_job_test_references("jobs:\n") is None assert module.macos_job_test_references("not a workflow") is None From a2e41a3007b9479fc9e5b38437c15252e8f4134d Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:16:23 -0700 Subject: [PATCH 04/14] ci: treat every runs-on that is not plainly Linux as macOS A job whose runner comes from a matrix, a needs output, or a list on the following lines was read as Linux, so a test only it runs could be classified guard-only. Co-Authored-By: Claude Fable 5.1 --- scripts/ci/detect_ci_change_areas.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/ci/detect_ci_change_areas.py b/scripts/ci/detect_ci_change_areas.py index 64a750e10ab8..e932cb7f9f78 100644 --- a/scripts/ci/detect_ci_change_areas.py +++ b/scripts/ci/detect_ci_change_areas.py @@ -64,8 +64,17 @@ def forces_all_areas(path: str) -> bool: _TEST_REFERENCE_RE = re.compile(r"tests/[A-Za-z0-9_./-]*") +def is_plainly_linux_runner(runs_on: str) -> bool: + # Anything else counts as macOS: a matrix or needs expression, a list or + # group on the following lines, or a label this does not recognize. + value = runs_on.strip() + if not value or re.search(r"macos|matrix\.|needs\.|inputs\.", value, re.IGNORECASE): + return False + return bool(re.search(r"LINUX_RUNNER|LINUX_ARM64_RUNNER|ubuntu", value)) + + def macos_job_test_references(workflow: str) -> Optional[tuple[frozenset[str], frozenset[str]]]: - """Return the tests/ paths ci.yml names in macOS jobs and in all jobs. + """Return the tests/ paths ci.yml names in non-Linux jobs and in all jobs. A macOS job that runs tests through a glob yields the glob's literal prefix. Returns None when the jobs cannot be read, so the caller fails open. @@ -77,13 +86,13 @@ def macos_job_test_references(workflow: str) -> Optional[tuple[frozenset[str], f everywhere: set[str] = set() jobs = 0 for block in re.split(r"(?m)^ (?=[A-Za-z0-9_-]+:\s*$)", body): - runs_on = re.search(r"(?m)^ runs-on:\s*(.+)$", block) + runs_on = re.search(r"(?m)^ runs-on:[ \t]*(.*)$", block) if not runs_on: continue jobs += 1 references = set(_TEST_REFERENCE_RE.findall(block)) everywhere |= references - if re.search(r"macos", runs_on.group(1), re.IGNORECASE): + if not is_plainly_linux_runner(runs_on.group(1)): macos |= references if jobs == 0: return None From fdf9b6cc2fc6322a1bff9a33a470a41df23839ee Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:16:51 -0700 Subject: [PATCH 05/14] test: a ci.yml edit confined to Linux jobs must not route to macOS Co-Authored-By: Claude Fable 5.1 --- tests/test_ci_change_areas.py | 111 +++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 111c00659ab5..eda57122e867 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -214,6 +214,114 @@ def workflow(runs_on: str) -> str: assert module.is_guard_only_test("tests/test_other.py", references) +CI_DIFF_BASE = """name: CI +on: + pull_request: +env: + FOO: "1" +jobs: + changes: + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + steps: + - run: route + workflow-guard-tests: + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + steps: + - run: guard + macos-compile-admission: + runs-on: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} + steps: + - run: compile + ci-status: + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + steps: + - run: gate +""" + + +def test_ci_workflow_change_is_linux_only_for_linux_job_edits() -> None: + linux_only = module.ci_workflow_change_is_linux_only + assert linux_only(CI_DIFF_BASE, CI_DIFF_BASE.replace("- run: guard", "- run: guard\n - run: more")) + added_linux_job = CI_DIFF_BASE.replace( + " ci-status:", + " new-linux:\n runs-on: ubuntu-24.04\n steps:\n - run: x\n ci-status:", + ) + assert linux_only(CI_DIFF_BASE, added_linux_job) + + +def test_ci_workflow_change_runs_macos_when_it_could_matter() -> None: + linux_only = module.ci_workflow_change_is_linux_only + for head in ( + CI_DIFF_BASE.replace("- run: compile", "- run: compile --faster"), + CI_DIFF_BASE.replace("blacksmith-6vcpu-macos-15", "blacksmith-6vcpu-macos-26"), + CI_DIFF_BASE.replace('FOO: "1"', 'FOO: "2"'), + CI_DIFF_BASE.replace("- run: route", "- run: route --differently"), + CI_DIFF_BASE.replace("- run: gate", "- run: gate || true"), + # A Linux job that becomes a macOS job, and a removed macOS job. + CI_DIFF_BASE.replace( + " workflow-guard-tests:\n runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}", + " workflow-guard-tests:\n runs-on: ${{ matrix.runner }}", + ), + CI_DIFF_BASE.replace( + " macos-compile-admission:\n runs-on: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }}\n steps:\n - run: compile\n", + "", + ), + "not a workflow", + ): + assert not linux_only(CI_DIFF_BASE, head), head + assert not linux_only("not a workflow", CI_DIFF_BASE) + assert not linux_only(CI_DIFF_BASE, CI_DIFF_BASE) + + +def run_detect_step_for_ci_workflow_edit(base: str, head: str) -> tuple[subprocess.CompletedProcess[str], list[str]]: + script = detect_step_script() + with tempfile.TemporaryDirectory() as temp_dir: + repo = Path(temp_dir) + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "ci@example.test"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "CI Test"], cwd=repo, check=True) + helper_copy = repo / "scripts" / "ci" / "detect_ci_change_areas.py" + helper_copy.parent.mkdir(parents=True, exist_ok=True) + helper_copy.write_text(HELPER.read_text(encoding="utf-8"), encoding="utf-8") + workflow = repo / ".github" / "workflows" / "ci.yml" + workflow.parent.mkdir(parents=True, exist_ok=True) + workflow.write_text(base, encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, check=True) + base_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + workflow.write_text(head, encoding="utf-8") + subprocess.run(["git", "commit", "-q", "-am", "head"], cwd=repo, check=True) + head_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + output_path = repo / "github-output.txt" + env = { + **os.environ, + "EVENT_NAME": "pull_request", + "BASE_SHA": base_sha, + "HEAD_SHA": head_sha, + "MERGE_SHA": head_sha, + "GITHUB_OUTPUT": str(output_path), + } + result = subprocess.run( + ["bash", "-c", script], cwd=repo, env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, + ) + return result, output_path.read_text(encoding="utf-8").splitlines() + + +def test_workflow_routes_linux_only_ci_workflow_edit_away_from_macos() -> None: + _, outputs = run_detect_step_for_ci_workflow_edit( + CI_DIFF_BASE, CI_DIFF_BASE.replace("- run: guard", "- run: guard\n - run: more") + ) + assert outputs == ["macos=false", "web=false", "agent_session_web=false"] + + +def test_workflow_routes_macos_job_edit_to_every_area() -> None: + _, outputs = run_detect_step_for_ci_workflow_edit( + CI_DIFF_BASE, CI_DIFF_BASE.replace("- run: compile", "- run: compile --faster") + ) + assert outputs == ["macos=true", "web=true", "agent_session_web=true"] + + def test_macos_test_references_fail_open_without_ci_workflow() -> None: assert module.macos_job_test_references("jobs:\n") is None assert module.macos_job_test_references("not a workflow") is None @@ -660,9 +768,10 @@ def test_ghosttykit_guard_wiring_pr_stays_on_release_guard() -> None: def test_workflow_only_pr_keeps_fail_open_routing() -> None: + # The base has no ci.yml to compare against. result, outputs = run_detect_step_for_paths([".github/workflows/ci.yml"]) - assert "CI router changed; running all CI areas." in result.stdout + assert "running all CI areas" in result.stdout + result.stderr assert outputs == [ "macos=true", "web=true", From 799d2190f947044223404350c30c192e359705a4 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:17:52 -0700 Subject: [PATCH 06/14] ci: route a ci.yml edit confined to Linux jobs away from macOS Any edit to ci.yml ran every area, so adding a step to workflow-guard-tests waited for a macOS compile. The routing step now hands the detector the base ci.yml, and the detector compares the two job by job. macOS is skipped only when the text before jobs: is unchanged, every changed, added or removed job plainly runs on Linux, and neither changes nor ci-status is among them. The shell pre-check still runs every area when the detector or its test changed, and an unreadable base or head does too. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 19 ++++++- scripts/ci/detect_ci_change_areas.py | 81 +++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b5877b49254..8d7160571a97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,7 @@ jobs: ghosttykit_guard_only=true has_ghosttykit_guard_file=false ci_router_changed=false + ci_workflow_changed=false while IFS= read -r changed_file; do case "$changed_file" in ghostty|scripts/download-prebuilt-ghosttykit.sh|scripts/validate-xcframework-archive.py|scripts/ghosttykit-checksums.txt|tests/test_ci_ghosttykit_release_check.sh) @@ -81,7 +82,7 @@ jobs: ci_router_changed=true ;; .github/workflows/ci.yml) - ci_router_changed=true + ci_workflow_changed=true ;; scripts/ci/*) ci_router_changed=true @@ -111,9 +112,23 @@ jobs: exit 0 fi + # The detector is unchanged here, so it may judge a ci.yml edit: only + # an edit confined to Linux jobs skips macOS, and anything it cannot + # read runs every area. + workflow_base_args=() + if [ "$ci_workflow_changed" = true ]; then + if ! git show "$BASE_SHA:.github/workflows/ci.yml" > /tmp/cmux-ci-base-workflow.yml; then + echo "Could not read the base ci.yml; running all CI areas." >&2 + emit_all_areas + exit 0 + fi + workflow_base_args=(--ci-workflow-base /tmp/cmux-ci-base-workflow.yml) + fi + python3 scripts/ci/detect_ci_change_areas.py \ --event-name "$EVENT_NAME" \ - --files-from /tmp/cmux-ci-changed-files.txt + --files-from /tmp/cmux-ci-changed-files.txt \ + ${workflow_base_args[@]+"${workflow_base_args[@]}"} exit 0 fi diff --git a/scripts/ci/detect_ci_change_areas.py b/scripts/ci/detect_ci_change_areas.py index e932cb7f9f78..b99c3154d88e 100644 --- a/scripts/ci/detect_ci_change_areas.py +++ b/scripts/ci/detect_ci_change_areas.py @@ -73,6 +73,63 @@ def is_plainly_linux_runner(runs_on: str) -> bool: return bool(re.search(r"LINUX_RUNNER|LINUX_ARM64_RUNNER|ubuntu", value)) +_JOB_SPLIT_RE = re.compile(r"(?m)^ (?=[A-Za-z0-9_-]+:\s*$)") + +# `changes` routes every other job and `ci-status` is the required gate, so an +# edit to either always runs every area. +_ROUTING_JOBS = frozenset({"changes", "ci-status"}) + + +def split_workflow_jobs(workflow: str) -> Optional[tuple[str, dict[str, str]]]: + """Return the text before `jobs:` and each job's block, or None if unreadable.""" + preamble, found, body = workflow.partition("\njobs:\n") + if not found: + return None + jobs: dict[str, str] = {} + for block in _JOB_SPLIT_RE.split(body): + name, _, _ = block.partition(":") + if not block.strip(): + continue + if not re.fullmatch(r"[A-Za-z0-9_-]+", name) or name in jobs: + return None + jobs[name] = block + return (preamble, jobs) if jobs else None + + +def job_is_plainly_linux(block: str) -> bool: + runs_on = re.search(r"(?m)^ runs-on:[ \t]*(.*)$", block) + return bool(runs_on) and is_plainly_linux_runner(runs_on.group(1)) + + +def ci_workflow_change_is_linux_only(base: str, head: str) -> bool: + """True when base and head ci.yml differ only in jobs that run on Linux. + + Triggers, env, permissions and concurrency live before `jobs:` and reach + every job, so any change there is not Linux-only. Unreadable input and an + unchanged file are not Linux-only either, so the caller fails open. + """ + base_parts = split_workflow_jobs(base) + head_parts = split_workflow_jobs(head) + if base_parts is None or head_parts is None: + return False + (base_preamble, base_jobs), (head_preamble, head_jobs) = base_parts, head_parts + if base_preamble != head_preamble: + return False + changed = { + name + for name in base_jobs.keys() | head_jobs.keys() + if base_jobs.get(name) != head_jobs.get(name) + } + if not changed or changed & _ROUTING_JOBS: + return False + return all( + job_is_plainly_linux(jobs[name]) + for name in changed + for jobs in (base_jobs, head_jobs) + if name in jobs + ) + + def macos_job_test_references(workflow: str) -> Optional[tuple[frozenset[str], frozenset[str]]]: """Return the tests/ paths ci.yml names in non-Linux jobs and in all jobs. @@ -191,7 +248,7 @@ def is_macos_change(path: str) -> bool: return not is_macos_neutral(path) -def classify_files(paths: Iterable[str]) -> ChangeAreas: +def classify_files(paths: Iterable[str], *, ci_workflow_linux_only: bool = False) -> ChangeAreas: macos = False web = False agent_session_web = False @@ -201,6 +258,8 @@ def classify_files(paths: Iterable[str]) -> ChangeAreas: path = normalize_path(raw_path) if not path: continue + if path == CI_WORKFLOW_PATH and ci_workflow_linux_only: + continue if forces_all_areas(path): macos = True web = True @@ -222,6 +281,19 @@ def classify_files(paths: Iterable[str]) -> ChangeAreas: ) +def ci_workflow_linux_only(base_path: Optional[Path]) -> bool: + if base_path is None: + return False + try: + base = base_path.read_text(encoding="utf-8") + head = Path(CI_WORKFLOW_PATH).read_text(encoding="utf-8") + except OSError: + return False + linux_only = ci_workflow_change_is_linux_only(base, head) + print(f"ci.yml changed; only Linux jobs differ: {bool_output(linux_only)}") + return linux_only + + def run_git(args: list[str]) -> str: return subprocess.check_output(["git", *args], text=True, stderr=subprocess.STDOUT).strip() @@ -250,6 +322,11 @@ def parse_args(argv: list[str]) -> argparse.Namespace: default=os.environ.get("GITHUB_OUTPUT"), help="Path to append GitHub Actions step outputs to.", ) + parser.add_argument( + "--ci-workflow-base", + type=Path, + help="The base revision of ci.yml, to compare its jobs with the checked-out one.", + ) parser.add_argument( "--files-from", type=Path, @@ -277,7 +354,7 @@ def main(argv: list[str]) -> int: raise RuntimeError("pull_request event is missing base/head SHA") files = changed_files(args.base_sha, args.head_sha) if files: - areas = classify_files(files) + areas = classify_files(files, ci_workflow_linux_only=ci_workflow_linux_only(args.ci_workflow_base)) else: areas = ChangeAreas.all() print("PR diff is empty; running all CI areas.") From ab7f94ca94e0bec9afa95aa9c48c92f21a676250 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:23:33 -0700 Subject: [PATCH 07/14] ci: keep web validation and the activation benchmark routed to their own workflows Both relied on any workflow edit forcing every area. The web gate now names web-validation.yml itself, so an edit to it still runs web validation. The activation benchmark's pre-check fails open for perf-activation.yml and the detector, no longer for every workflow file, so an unrelated workflow edit stops taking a macOS runner for it. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/perf-activation.yml | 6 +++--- scripts/ci/web_validation.py | 12 +++++++++++- tests/test_ci_change_areas.py | 10 ++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/perf-activation.yml b/.github/workflows/perf-activation.yml index 595a0c80465a..35d89edd7050 100644 --- a/.github/workflows/perf-activation.yml +++ b/.github/workflows/perf-activation.yml @@ -85,9 +85,9 @@ jobs: exit 0 fi - # This guard runs before the PR-editable Python detector. Workflow - # and detector edits must fail open to the benchmark. - if grep -Eq '^(\.github/workflows/[^/]+\.ya?ml|scripts/ci/[^/]+\.py|tests/test_ci_change_areas\.py)$' /tmp/cmux-activation-changed-files.txt; then + # This guard runs before the PR-editable Python detector. Edits to + # this workflow and to the detector must fail open to the benchmark. + if grep -Eq '^(\.github/workflows/perf-activation\.ya?ml|scripts/ci/[^/]+\.py|tests/test_ci_change_areas\.py)$' /tmp/cmux-activation-changed-files.txt; then echo "CI router changed; running activation benchmark." emit_all_areas exit 0 diff --git a/scripts/ci/web_validation.py b/scripts/ci/web_validation.py index 9679db16d522..30669dda509a 100644 --- a/scripts/ci/web_validation.py +++ b/scripts/ci/web_validation.py @@ -14,7 +14,17 @@ def requires_web(paths: list[str]) -> bool: return classify_files(paths).web or any( - path in {".vercelignore", "vercel.json", "bunfig.toml", ".npmrc", "tests/test_web_validation.py"} + path + in { + ".vercelignore", + "vercel.json", + "bunfig.toml", + ".npmrc", + "tests/test_web_validation.py", + # The CI router treats other workflow files as neutral, so this + # gate names its own. + ".github/workflows/web-validation.yml", + } or path.startswith(("config/", "workers/")) for path in paths ) diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index eda57122e867..8185336c7514 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -1237,6 +1237,16 @@ def test_agent_session_web_resources_runs_only_for_agent_session_web_area() -> N assert "if: ${{ needs.changes.outputs.agent_session_web == 'true' }}" in block +def test_perf_activation_runs_for_its_own_workflow_and_not_for_others() -> None: + _, outputs = run_detect_step_for_paths([".github/workflows/relay-tls.yml"], PERF_ACTIVATION_WORKFLOW) + assert outputs == ["macos=false", "web=false", "agent_session_web=false"] + + for path in (".github/workflows/perf-activation.yml", "scripts/ci/subprocess.py"): + result, outputs = run_detect_step_for_paths([path], PERF_ACTIVATION_WORKFLOW) + assert "CI router changed; running activation benchmark." in result.stdout, path + assert outputs[0] == "macos=true", (path, outputs) + + def test_perf_activation_workflow_keeps_required_status_while_gating_benchmark() -> None: result, outputs = run_detect_step_for_paths(["docs/ci-runners.md"], PERF_ACTIVATION_WORKFLOW) From 14995fe6e499c14abe3d9b261b0010248ad831c7 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:37:06 -0700 Subject: [PATCH 08/14] ci: retry native tsgo aborts once --- .github/workflows/ci.yml | 23 +++++++++++++++++++++-- tests/test_ci_change_areas.py | 10 +++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d7160571a97..d2bb31341e59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -500,7 +500,26 @@ jobs: run: bunx playwright install --with-deps chromium - name: Instant navigation tests - run: bun run test:instant + run: | + set -o pipefail + log="$RUNNER_TEMP/web-test-instant.log" + set +e + bun run test:instant 2>&1 | tee "$log" + status=${PIPESTATUS[0]} + set -e + + # The native TypeScript preview compiler can abort while Playwright + # starts its web server. Retry that transient compiler crash once, + # while preserving immediate failures for real type or test errors. + if [ "$status" -ne 0 ] \ + && grep -Fq '[WebServer] $ tsgo --noEmit' "$log" \ + && grep -Fq 'Aborted (core dumped)' "$log"; then + echo "::warning::native tsgo aborted during instant navigation tests; retrying once" + bun run test:instant + exit $? + fi + + exit "$status" # Checks for in-app React webviews (currently the diff viewer; more cmux React # surfaces will live alongside it). @@ -1685,7 +1704,7 @@ jobs: - app-host-unit-tests - swift-package-tests - agent-session-web-resources - if: ${{ always() }} + if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' }} runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 5 steps: diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 8185336c7514..503701574edc 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -893,12 +893,20 @@ def test_required_tests_status_waits_for_app_host_matrix() -> None: assert " - linux-preflight" in block assert " - macos-compile-admission" in block assert " - app-host-unit-tests" in block - assert "if: ${{ always() }}" in block + assert "if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' }}" in block assert 'preflight["result"] != "success"' in block assert 'macos == "true" and tests["result"] != "success"' in block assert 'tests["result"] not in {"success", "skipped"}' in block +def test_web_instant_navigation_retries_native_tsgo_abort() -> None: + block = workflow_job_block("web-typecheck") + + assert "grep -Fq '[WebServer] $ tsgo --noEmit' \"$log\"" in block + assert "grep -Fq 'Aborted (core dumped)' \"$log\"" in block + assert "retrying once" in block + + def test_macos_jobs_wait_for_linux_preflight() -> None: # The staged macOS jobs must gate on their direct needs explicitly. # A bare `if: needs.changes.outputs.macos == 'true'` keeps the implicit From 26f35d6e4a3153049a190170f43f63af2e263091 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:43:18 -0700 Subject: [PATCH 09/14] ci: keep the tests gate running after a prerequisite fails The aggregate job turns a failed changes or linux-preflight result into a failing check. With a condition on those results it is skipped instead, and a skipped check does not block a merge. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 2 +- tests/test_ci_change_areas.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2bb31341e59..1c904886251c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1704,7 +1704,7 @@ jobs: - app-host-unit-tests - swift-package-tests - agent-session-web-resources - if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' }} + if: ${{ always() }} runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 5 steps: diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 503701574edc..b6f14a013d45 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -893,7 +893,7 @@ def test_required_tests_status_waits_for_app_host_matrix() -> None: assert " - linux-preflight" in block assert " - macos-compile-admission" in block assert " - app-host-unit-tests" in block - assert "if: ${{ !cancelled() && needs.changes.result == 'success' && needs.linux-preflight.result == 'success' }}" in block + assert "if: ${{ always() }}" in block assert 'preflight["result"] != "success"' in block assert 'macos == "true" and tests["result"] != "success"' in block assert 'tests["result"] not in {"success", "skipped"}' in block From a93251813ced4bbe8114794236f95e090b532ae2 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:54:06 -0700 Subject: [PATCH 10/14] test: route a pull request whose event base is behind main Co-Authored-By: Claude Fable 5.1 --- tests/test_ci_change_areas.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index b6f14a013d45..4d7908de358a 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -617,7 +617,7 @@ def test_workflow_diff_failure_runs_all_areas() -> None: ] -def test_workflow_routes_from_shallow_synthetic_merge() -> None: +def run_detect_step_on_shallow_synthetic_merge(*, stale_event_base: bool) -> tuple[subprocess.CompletedProcess[str], list[str]]: script = detect_step_script() with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -634,6 +634,7 @@ def test_workflow_routes_from_shallow_synthetic_merge() -> None: (source / "common.txt").write_text("common\n", encoding="utf-8") subprocess.run(["git", "add", "."], cwd=source, check=True) subprocess.run(["git", "commit", "-q", "-m", "common"], cwd=source, check=True) + common_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() subprocess.run(["git", "branch", "feature"], cwd=source, check=True) (source / "base-only.txt").write_text("base\n", encoding="utf-8") @@ -681,7 +682,10 @@ def test_workflow_routes_from_shallow_synthetic_merge() -> None: env={ **os.environ, "EVENT_NAME": "pull_request", - "BASE_SHA": base_sha, + # The event payload keeps the base the pull request was last + # synced against. Once main moves on, that commit is outside + # the depth-2 checkout of the synthetic merge. + "BASE_SHA": common_sha if stale_event_base else base_sha, "HEAD_SHA": head_sha, "MERGE_SHA": merge_sha, "GITHUB_OUTPUT": str(output_path), @@ -691,13 +695,23 @@ def test_workflow_routes_from_shallow_synthetic_merge() -> None: stderr=subprocess.PIPE, check=True, ) + return result, output_path.read_text(encoding="utf-8").splitlines() - assert "Could not compute PR diff" not in result.stderr - assert output_path.read_text(encoding="utf-8").splitlines() == [ - "macos=false", - "web=true", - "agent_session_web=false", - ] + +def test_workflow_routes_from_shallow_synthetic_merge() -> None: + result, outputs = run_detect_step_on_shallow_synthetic_merge(stale_event_base=False) + + assert "Could not compute PR diff" not in result.stderr + assert outputs == ["macos=false", "web=true", "agent_session_web=false"] + + +def test_workflow_routes_when_main_moved_past_the_event_base() -> None: + result, outputs = run_detect_step_on_shallow_synthetic_merge(stale_event_base=True) + + assert "Could not compute PR diff" not in result.stderr + # base-only.txt landed on main after the event base. It is not part of the + # pull request and must not route macOS. + assert outputs == ["macos=false", "web=true", "agent_session_web=false"] def test_workflow_empty_diff_runs_all_areas() -> None: From fbff73ee689959183fceb08e9f464842255b2075 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:55:03 -0700 Subject: [PATCH 11/14] test: web validation must route from the merge parent when the event base is gone Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 7 +++++++ tests/test_web_validation.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c904886251c..b9b30009b90a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,13 @@ jobs: if [ "$EVENT_NAME" = "pull_request" ] || [ "$EVENT_NAME" = "merge_group" ]; then # Checkout depth 2 contains the synthetic merge commit and its # base parent, even when base/head histories are shallow boundaries. + # The event's base SHA is where the pull request last synced. Once + # main moves on it is outside this checkout, and a diff from it + # would also count everything main gained since. The merge commit's + # first parent is the base it was actually built on. + if git rev-parse -q --verify "$MERGE_SHA^2" > /dev/null; then + BASE_SHA="$(git rev-parse "$MERGE_SHA^1")" + fi if ! git diff --name-only "$BASE_SHA" "$MERGE_SHA" > /tmp/cmux-ci-changed-files.txt; then echo "Could not compute PR diff; running all CI areas." >&2 emit_all_areas diff --git a/tests/test_web_validation.py b/tests/test_web_validation.py index 0706f53190d0..bdf8459cdd45 100644 --- a/tests/test_web_validation.py +++ b/tests/test_web_validation.py @@ -28,6 +28,39 @@ def test_web_inputs_and_mixed_changes_are_selected(self): self.assertTrue(gate.requires_web([path, "README.md"])) self.assertFalse(gate.requires_web(["README.md", "docs/cli.md", "Sources/AppDelegate.swift"])) + def test_pull_request_routes_from_the_merge_parent_when_the_event_base_is_gone(self): + with tempfile.TemporaryDirectory() as directory: + repo = Path(directory) + def git(*args): + return subprocess.check_output([ + "git", "-c", "user.name=CI", "-c", "user.email=ci@example.test", + "-c", "core.hooksPath=/dev/null", *args, + ], cwd=repo, text=True, stderr=subprocess.DEVNULL).strip() + git("init", "-q", "-b", "main") + (repo / "README.md").write_text("base\n") + git("add", ".") + git("commit", "-qm", "base") + git("branch", "feature") + (repo / "web").mkdir() + (repo / "web/main-only.ts").write_text("export const value = 1;\n") + git("add", ".") + git("commit", "-qm", "main gains a web file") + git("checkout", "-q", "feature") + (repo / "docs").mkdir() + (repo / "docs/note.md").write_text("docs only\n") + git("add", ".") + git("commit", "-qm", "docs") + git("checkout", "-q", "main") + git("merge", "-q", "--no-ff", "feature", "-m", "synthetic merge") + output = repo / "outputs" + output.write_text("") + # The event base is a commit this checkout does not have. + subprocess.run([sys.executable, str(ROOT / "scripts/ci/web_validation.py"), "route"], + cwd=repo, env={**os.environ, "EVENT_NAME": "pull_request", "BASE_SHA": "1" * 40, + "HEAD_SHA": git("rev-parse", "HEAD"), "GITHUB_OUTPUT": str(output)}, check=True, + capture_output=True) + self.assertEqual(output.read_text().strip(), "required=false") + def test_real_pr_and_push_diffs_and_missing_history(self): with tempfile.TemporaryDirectory() as directory: repo = Path(directory) From f2b4f4629261d2f55a28fd551186915e881d32bd Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:55:37 -0700 Subject: [PATCH 12/14] ci: diff a pull request against the base its merge commit was built on The routing steps diffed from the event's base SHA, which is where the pull request last synced. Once main moves on, that commit is outside the depth-2 checkout, the diff fails and every area runs. In a sample of 40 recent runs that happened in 11. A diff from the old base would also count what main gained since. CI and web validation now diff from the synthetic merge commit's first parent. Co-Authored-By: Claude Fable 5.1 --- scripts/ci/web_validation.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/ci/web_validation.py b/scripts/ci/web_validation.py index 30669dda509a..9c6674486778 100644 --- a/scripts/ci/web_validation.py +++ b/scripts/ci/web_validation.py @@ -30,9 +30,28 @@ def requires_web(paths: list[str]) -> bool: ) +def merge_parent(head: str) -> str: + """Return the base a pull request's synthetic merge commit was built on. + + The event's base SHA is where the pull request last synced. Once main + moves on it is outside the depth-2 checkout, and a diff from it would also + count what main gained since. + """ + def resolve(revision: str) -> str: + result = subprocess.run( + ["git", "rev-parse", "-q", "--verify", revision], text=True, capture_output=True + ) + return result.stdout.strip() if result.returncode == 0 else "" + + # Only a merge commit has a second parent. + return resolve(f"{head}^1") if resolve(f"{head}^2") else "" + + def required_for_event(event: str, base: str, head: str) -> bool: if event not in {"pull_request", "push"} or not base or not head: return True + if event == "pull_request": + base = merge_parent(head) or base try: paths = subprocess.check_output( ["git", "diff", "--no-renames", "--name-only", "-z", base, head, "--"], text=True, From 813db13d8cb6654d87b1e404267a1b022cf80462 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:59:38 -0700 Subject: [PATCH 13/14] test: agent instructions and skill docs must not route to macOS Co-Authored-By: Claude Fable 5.1 --- tests/test_ci_change_areas.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 4d7908de358a..3d7b1f395413 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -44,6 +44,27 @@ def test_docs_only_skips_expensive_areas() -> None: assert_areas(["docs/ci.md", "README.md"], macos=False, web=False) +def test_agent_instructions_and_skill_docs_skip_expensive_areas() -> None: + assert_areas( + [ + "CLAUDE.md", + "AGENTS.md", + "Packages/iOS/AGENTS.md", + "skills/cmux-testing/references/local-vs-ci-validation.md", + "skills/cmux/SKILL.md", + ], + macos=False, + web=False, + ) + + +def test_bundled_and_executable_skill_files_run_macos() -> None: + # The app bundles skills/cmux-cua as a folder resource. + assert_areas(["skills/cmux-cua/SKILL.md"], macos=True, web=False) + assert_areas(["skills/cmux-settings/scripts/cmux-settings"], macos=True, web=False) + assert_areas(["skills/cmux-browser/agents/openai.yaml"], macos=True, web=False) + + def test_cli_contract_doc_runs_macos_contract_tests() -> None: assert_areas(["docs/cli-contract.md"], macos=True, web=False) From 78a5712b1cbb8af8ac28aa5730bccf6e24c0cca0 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 17:00:07 -0700 Subject: [PATCH 14/14] ci: treat agent instructions and skill docs as documentation CLAUDE.md, AGENTS.md at any depth, and Markdown under skills/ routed to the macOS suite. No macOS job reads them. skills/cmux-cua stays macOS-relevant because the app bundles it as a folder resource, and so do skill scripts and manifests. Co-Authored-By: Claude Fable 5.1 --- scripts/ci/detect_ci_change_areas.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/ci/detect_ci_change_areas.py b/scripts/ci/detect_ci_change_areas.py index b99c3154d88e..464250219708 100644 --- a/scripts/ci/detect_ci_change_areas.py +++ b/scripts/ci/detect_ci_change_areas.py @@ -233,7 +233,14 @@ def is_macos_neutral(path: str) -> bool: ) ): return True - return path == "README.md" or (path.startswith("README.") and path.endswith(".md")) + if path == "README.md" or (path.startswith("README.") and path.endswith(".md")): + return True + # Agent instructions at any depth, and skill documentation. The app bundles + # skills/cmux-cua as a folder resource, and skill scripts and manifests are + # executable inputs, so only Markdown outside that folder is neutral. + if path.rsplit("/", 1)[-1] in {"CLAUDE.md", "AGENTS.md"}: + return True + return path.startswith("skills/") and path.endswith(".md") and not path.startswith("skills/cmux-cua/") def is_macos_change(path: str) -> bool: