diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c2039ae9f47..7557d3ee6497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,16 @@ jobs: exit 0 fi + # Registry edits that only register Linux guards do not change the + # native test lanes. Missing history keeps the detector fail-open. + registry_base_args=() + if grep -Fxq 'tests/test-execution.toml' /tmp/cmux-ci-changed-files.txt; then + registry_base="$(mktemp "${RUNNER_TEMP:-/tmp}/cmux-test-registry-base.XXXXXX")" + if git show "$BASE_SHA:tests/test-execution.toml" > "$registry_base"; then + registry_base_args=(--test-registry-base "$registry_base") + fi + fi + # A provenance-only PR gets the cheap Linux guard and avoids # queuing the unrelated macOS/web/Go suites. Keep this exception # narrow: an unrelated workflow edit or source/configuration file @@ -308,6 +318,11 @@ jobs: fi echo "CI routing policy changed; classifying product inputs with the trusted base router." + if [ "${#registry_base_args[@]}" -gt 0 ] && ! python3 "$trusted_root/scripts/ci/detect_ci_change_areas.py" --help | grep -q -- '--test-registry-base'; then + echo "Trusted router predates registry comparison; running all CI areas." + emit_all_areas + exit 0 + fi trusted_areas=/tmp/cmux-ci-trusted-areas.txt : > "$trusted_areas" ( @@ -316,6 +331,7 @@ jobs: python3 scripts/ci/detect_ci_change_areas.py \ --event-name "$EVENT_NAME" \ --files-from "$product_files" \ + ${registry_base_args[@]+"${registry_base_args[@]}"} \ --github-output "$trusted_areas" ) if [ "$cli_call_site_changed" = true ]; then @@ -342,6 +358,7 @@ jobs: python3 scripts/ci/detect_ci_change_areas.py \ --event-name "$EVENT_NAME" \ --files-from /tmp/cmux-ci-changed-files.txt \ + ${registry_base_args[@]+"${registry_base_args[@]}"} \ ${workflow_base_args[@]+"${workflow_base_args[@]}"} exit 0 fi diff --git a/scripts/ci/detect_ci_change_areas.py b/scripts/ci/detect_ci_change_areas.py index 0231f0f4d689..959066d809fe 100644 --- a/scripts/ci/detect_ci_change_areas.py +++ b/scripts/ci/detect_ci_change_areas.py @@ -1033,7 +1033,53 @@ def is_release_build_neutral(path: str) -> bool: return is_test_only_source(path) or path in RELEASE_BUILD_NEUTRAL_INPUTS -def classify_files(paths: Iterable[str], *, ci_workflow_linux_only: bool = False) -> ChangeAreas: +def test_registry_change_is_linux_only(base: str, head: str) -> bool: + """Ignore only Linux registrations; native execution entries must match.""" + try: + import tomllib + + def native_entries(text: str) -> list[dict]: + registry = tomllib.loads(text) + if set(registry) != {"version", "test"} or registry["version"] != 1: + raise ValueError("unknown registry schema") + entries = registry["test"] + if not isinstance(entries, list) or not entries: + raise ValueError("missing test entries") + seen = set() + native = [] + for entry in entries: + if not isinstance(entry, dict) or set(entry) - {"path", "lane", "requirements", "reason"}: + raise ValueError("unknown test entry") + path, lane = entry.get("path"), entry.get("lane") + if not isinstance(path, str) or not isinstance(lane, str) or path in seen: + raise ValueError("missing or duplicate test identity") + seen.add(path) + if lane != "linux-guard": + native.append(entry) + elif entry.get("requirements"): + raise ValueError("guard entry with runtime requirements") + return native + + return native_entries(base) == native_entries(head) + except (ImportError, ValueError, TypeError, KeyError): + return False + + +def test_registry_linux_only(base_path: Optional[Path]) -> bool: + if base_path is None: + return False + root = Path(os.environ.get("CMUX_CI_HEAD_TEST_REFERENCE_ROOT") or Path.cwd()) + try: + return test_registry_change_is_linux_only( + base_path.read_text(encoding="utf-8"), + (root / "tests/test-execution.toml").read_text(encoding="utf-8"), + ) + except OSError: + return False + + +def classify_files(paths: Iterable[str], *, ci_workflow_linux_only: bool = False, + test_registry_linux_only: bool = False) -> ChangeAreas: macos = False web = False agent_session_web = False @@ -1050,6 +1096,14 @@ def classify_files(paths: Iterable[str], *, ci_workflow_linux_only: bool = False path = normalize_path(raw_path) if not path: continue + if path == "tests/test-execution.toml": + # The guard workflow references this registry too, but native + # Python lanes consume it indirectly through their lane runner. + # A Linux reference alone cannot prove native execution unchanged. + if not test_registry_linux_only: + macos = True + release_build = True + continue if is_cli_change(path, cli_inputs, macos_ios_packages): cli = True if path == CI_WORKFLOW_PATH and ci_workflow_linux_only: @@ -1162,6 +1216,11 @@ def parse_args(argv: list[str]) -> argparse.Namespace: type=Path, help="The base revision of ci.yml, to compare its jobs with the checked-out one.", ) + parser.add_argument( + "--test-registry-base", + type=Path, + help="Base test registry; Linux-only entry changes do not select native CI.", + ) parser.add_argument( "--files-from", type=Path, @@ -1189,7 +1248,11 @@ def main(argv: list[str]) -> int: raise RuntimeError("pull_request event is missing base/head SHA") files = changed_files(args.base_sha, args.head_sha) if files: - areas = classify_files(files, ci_workflow_linux_only=ci_workflow_linux_only(args.ci_workflow_base)) + areas = classify_files( + files, + ci_workflow_linux_only=ci_workflow_linux_only(args.ci_workflow_base), + test_registry_linux_only=test_registry_linux_only(args.test_registry_base), + ) else: areas = ChangeAreas.all() print("PR diff is empty; running all CI areas.") diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index aa63bfedbfab..40a117474c97 100755 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -14,6 +14,7 @@ import tempfile import textwrap from pathlib import Path +from unittest.mock import patch import yaml @@ -1084,6 +1085,50 @@ def test_other_workflow_changes_skip_macos_and_web() -> None: ) +def test_linux_registry_changes_skip_native_but_preserve_native_changes() -> None: + base = 'version = 1\n[[test]]\npath = "tests/native.py"\nlane = "macos-shell"\n' + guard = '\n[[test]]\npath = "tests/guard.py"\nlane = "linux-guard"\n' + assert module.test_registry_change_is_linux_only(base, base + guard) + assert module.test_registry_change_is_linux_only(base + guard, base) + assert module.test_registry_change_is_linux_only(base, base + '\n# comment\n') + for candidate in ( + base.replace('macos-shell', 'linux-guard'), + base.replace('native.py', 'other.py'), + base + 'requirements = ["fish"]\n', + base.replace('version = 1', 'version = 2'), + 'invalid TOML', + base + guard + guard, + ): + assert not module.test_registry_change_is_linux_only(base, candidate), candidate + assert not module.test_registry_change_is_linux_only('invalid TOML', base) + + +def test_registry_cli_uses_base_and_keeps_mixed_product_changes() -> None: + base = 'version = 1\n[[test]]\npath = "tests/native.py"\nlane = "macos-shell"\n' + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / 'tests').mkdir() + head = root / 'tests/test-execution.toml' + head.write_text(base + '\n[[test]]\npath = "tests/guard.py"\nlane = "linux-guard"\n') + before = root / 'base.toml' + before.write_text(base) + files = root / 'files.txt' + files.write_text('tests/test-execution.toml\n') + env = {**os.environ, 'CMUX_CI_HEAD_TEST_REFERENCE_ROOT': str(root)} + command = [sys.executable, str(HELPER), '--event-name', 'pull_request', + '--files-from', str(files), '--test-registry-base', str(before)] + result = subprocess.run(command, env=env, capture_output=True, text=True, check=True) + assert 'macos=false' in result.stdout, result.stdout + assert 'release_build=false' in result.stdout, result.stdout + files.write_text('tests/test-execution.toml\nSources/AppDelegate.swift\n') + result = subprocess.run(command, env=env, capture_output=True, text=True, check=True) + assert 'macos=true' in result.stdout, result.stdout + before.unlink() + files.write_text('tests/test-execution.toml\n') + result = subprocess.run(command, env=env, capture_output=True, text=True, check=True) + assert 'macos=true' in result.stdout, result.stdout + + def test_guard_only_tests_skip_macos() -> None: # Referenced only by Linux jobs in the CI caller or reusable guard workflow. assert_areas(["tests/test_ci_self_hosted_guard.sh"], macos=False, web=False) @@ -1682,14 +1727,21 @@ def run_macos_status( def run_detect_step_for_paths( paths: list[str], workflow_path: Path = CI_WORKFLOW, + *, + base_files: dict[str, str] | None = None, + head_files: dict[str, str] | None = None, ) -> tuple[subprocess.CompletedProcess[str], list[str]]: script = detect_step_script(workflow_path) with tempfile.TemporaryDirectory() as temp_dir: repo = Path(temp_dir) - runner_temp = Path(temp_dir) / "runner-temp" - subprocess.run(["git", "init", "-q"], cwd=repo, check=True) - subprocess.run(["git", "config", "user.email", "ci@example.test"], cwd=repo, check=True) - subprocess.run(["git", "config", "user.name", "CI Test"], cwd=repo, check=True) + git_env = os.environ.copy() + for name in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"): + git_env.pop(name, None) + # Parallel local checkouts must not share the workflow's fixed /tmp files. + script = script.replace("/tmp/cmux-ci-", str(repo / "cmux-ci-")) + subprocess.run(["git", "init", "-q"], cwd=repo, env=git_env, check=True) + subprocess.run(["git", "config", "user.email", "ci@example.test"], cwd=repo, env=git_env, check=True) + subprocess.run(["git", "config", "user.name", "CI Test"], cwd=repo, env=git_env, check=True) helper_copy = repo / "scripts" / "ci" / "detect_ci_change_areas.py" helper_copy.parent.mkdir(parents=True, exist_ok=True) helper_copy.write_text(HELPER.read_text(encoding="utf-8"), encoding="utf-8") @@ -1710,34 +1762,36 @@ def run_detect_step_for_paths( target = repo / relative target.parent.mkdir(parents=True, exist_ok=True) target.write_text(support.read_text(encoding="utf-8"), encoding="utf-8") + for path, content in (base_files or {}).items(): + target = repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") (repo / "base.txt").write_text("base\n", encoding="utf-8") - subprocess.run(["git", "add", "."], cwd=repo, check=True) - subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, check=True) - base_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + subprocess.run(["git", "add", "."], cwd=repo, env=git_env, check=True) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=repo, env=git_env, check=True) + base_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, env=git_env, text=True).strip() if paths: for path in paths: target = repo / path target.parent.mkdir(parents=True, exist_ok=True) - target.write_text("changed\n", encoding="utf-8") - subprocess.run(["git", "add", "."], cwd=repo, check=True) - subprocess.run(["git", "commit", "-q", "-m", "head"], cwd=repo, check=True) - head_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + target.write_text((head_files or {}).get(path, "changed\n"), encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=repo, env=git_env, check=True) + subprocess.run(["git", "commit", "-q", "-m", "head"], cwd=repo, env=git_env, check=True) + head_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, env=git_env, text=True).strip() else: head_sha = base_sha output_path = repo / "github-output.txt" env = { - **os.environ, + **git_env, "EVENT_NAME": "pull_request", "BASE_SHA": base_sha, "HEAD_SHA": head_sha, "MERGE_SHA": head_sha, "GITHUB_OUTPUT": str(output_path), - # The trusted base router lays its checkout out under $RUNNER_TEMP, - # and the step runs under `set -u`. GitHub sets it; a local run - # does not, so without this the suite only passes inside CI. - "RUNNER_TEMP": os.environ.get("RUNNER_TEMP") or str(runner_temp), + "GITHUB_WORKSPACE": str(repo), + "RUNNER_TEMP": str(repo), } result = subprocess.run( ["bash", "-c", script], @@ -1751,6 +1805,38 @@ def run_detect_step_for_paths( return result, output_path.read_text(encoding="utf-8").splitlines() +def test_detect_step_ignores_inherited_git_location() -> None: + # Each Git location override must be ignored, including the custom index + # that otherwise silently redirects writes outside the fixture repository. + with tempfile.TemporaryDirectory() as foreign_dir: + foreign = Path(foreign_dir) + for variable, value in { + "GIT_DIR": str(foreign / "not-a-repository"), + "GIT_WORK_TREE": str(foreign / "missing-worktree"), + "GIT_INDEX_FILE": str(foreign / "foreign-index"), + }.items(): + with patch.dict(os.environ, {variable: value}): + result, outputs = run_detect_step_for_paths(["Sources/AppDelegate.swift"]) + assert result.returncode == 0, result.stderr + assert "macos=true" in outputs, outputs + assert not Path(value).exists(), f"fixture wrote through {variable}" + + +def test_workflow_registry_diff_reaches_normal_and_trusted_router() -> None: + registry = "tests/test-execution.toml" + base = 'version = 1\n[[test]]\npath = "tests/native.py"\nlane = "macos-shell"\n' + guard = '\n[[test]]\npath = "tests/guard.py"\nlane = "linux-guard"\n' + for policy_change in ([], ["scripts/ci/detect_ci_change_areas.py"]): + for candidate, expected in ((base + guard, "false"), + (base.replace("native.py", "other.py"), "true")): + result, outputs = run_detect_step_for_paths( + [registry, *policy_change], + base_files={registry: base}, head_files={registry: candidate}, + ) + assert f"macos={expected}" in outputs, (result.stdout, result.stderr) + assert f"release_build={expected}" in outputs, outputs + + def test_workflow_self_change_guard_runs_before_detector_imports() -> None: result, outputs = run_detect_step_for_paths(["scripts/ci/subprocess.py"])