diff --git a/.github/workflows/cmux-next.yml b/.github/workflows/cmux-next.yml index 7e19ce339894..e6395545277a 100644 --- a/.github/workflows/cmux-next.yml +++ b/.github/workflows/cmux-next.yml @@ -1276,6 +1276,8 @@ jobs: python3 tests/test_job_input_fingerprint.py # A hand merge of a Swift PR waits for its Release compile (scripts/ci/merge_compile_guard.py). python3 tests/test_merge_compile_guard.py + # The advisory test-compile check: its plan, the lane phase and red only for a compile failure. + python3 tests/test_ci_compile_test_plan.py # The merge guards run from the feat-cmux-next tip, never a pull request's checkout. python3 tests/test_guard_hooks_from_base.py # ci-guards.yml's `ci` group runs for feat-cmux-next too (cmux-next-guards.yml). diff --git a/.github/workflows/fleet-test-compile.yml b/.github/workflows/fleet-test-compile.yml new file mode 100644 index 000000000000..ac0c3e37e102 --- /dev/null +++ b/.github/workflows/fleet-test-compile.yml @@ -0,0 +1,131 @@ +name: fleet test-compile + +# Advisory: compile the Swift test targets a pull request can break, on a +# fleet Mac, and fail this check when one does not compile. +# +# fleet/compile-check (fleet-compile-check.yml) compiles only the app +# executable, which links no test target, and merge_compile_guard.py accepts +# it in place of a queued Release compile. #23360 (a throwing call inside +# #expect without try) and #23163 (a fake that missed a protocol requirement) +# passed it and broke feat-cmux-next's test build. This check covers that gap +# until the Linux pool compiles test targets (its SDK gains Xcode's Testing +# search paths with the next pool image, cmuxterm-hq #2750). +# +# It gates nothing: fleet/compile-check, merge_compile_guard.py and the +# Release compile neither wait for it nor read it. A red here is a test +# target that does not compile at the PR's merge commit. +# +# The plan (scripts/ci/compile_test_plan.py, over the merge commit's diff from +# its base) is the CmuxNext test targets that depend on a changed target in +# the SwiftPM target graph, and the macOS package lane's packages whose +# sources or path dependencies changed. An empty plan passes on a Linux runner +# in seconds and queues no Mac; a plan runs `package-test-lane.sh +# compile-tests BASE` as a light fleet step. + +on: + pull_request: + branches: [feat-cmux-next] + paths: + - "Packages/**" + - "vendor/**" + - "scripts/ci/compile_test_plan.py" + - "scripts/ci/package-test-lane.sh" + - ".github/workflows/fleet-test-compile.yml" + +permissions: {} + +concurrency: + group: fleet-test-compile-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + plan: + name: fleet test-compile plan + # The fleet takes same-repository pull requests only (as the Swift canary's fleet job). + if: ${{ github.repository_owner == 'manaflow-ai' && github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 5 + outputs: + needed: ${{ steps.plan.outputs.needed }} + base: ${{ steps.plan.outputs.base }} + steps: + - name: Checkout the merge commit and its base + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + submodules: false + fetch-depth: 2 + + - name: Plan the test targets this change reaches + id: plan + run: | + set -euo pipefail + base="$(git rev-parse HEAD^1)" + git diff --no-renames --name-only "$base" HEAD > "$RUNNER_TEMP/changed.txt" + python3 scripts/ci/compile_test_plan.py --changed-files "$RUNNER_TEMP/changed.txt" > "$RUNNER_TEMP/plan.txt" + { + echo "### fleet test-compile plan" + echo + if [ -s "$RUNNER_TEMP/plan.txt" ]; then + echo 'Package, then its test targets (none listed: every one):' + echo + echo '```' + cat "$RUNNER_TEMP/plan.txt" + echo '```' + else + echo "The change reaches no test target: nothing to compile, no Mac queued." + fi + } >> "$GITHUB_STEP_SUMMARY" + cat "$RUNNER_TEMP/plan.txt" + echo "base=$base" >> "$GITHUB_OUTPUT" + if [ -s "$RUNNER_TEMP/plan.txt" ]; then echo "needed=true"; else echo "needed=false"; fi >> "$GITHUB_OUTPUT" + + compile: + name: fleet/test-compile (advisory) + needs: plan + # An empty plan reports on a Linux runner at once; only a plan takes the gateway and a fleet Mac. + runs-on: ${{ needs.plan.outputs.needed == 'true' && vars.CI_COMPILE_CHECK_GATEWAY || 'ubuntu-24.04' }} + if: ${{ needs.plan.outputs.needed == 'false' || (needs.plan.outputs.needed == 'true' && vars.CI_COMPILE_CHECK_GATEWAY != '') }} + permissions: {} + # Longer than the client's queue (1200 s) plus run (1500 s) timeouts and its + # report: a job timeout concludes as a failure, which would read as red. + timeout-minutes: 55 + steps: + # The gateway checks nothing out: the fleet fetches the merge commit itself. + # Exit 0 is a pass. The lane's "compile-tests: a test target did not + # compile" marker is a compile failure (this check goes red) whatever the + # exit: the lane prints it as soon as a target fails, and a later + # package's setup may end the run with another status. Anything else + # (queue or run timeout, setup, the client) is no verdict: a warning, + # and this check stays green. + - name: Compile the planned test targets on a fleet Mac + env: + NEEDED: ${{ needs.plan.outputs.needed }} + REF: ${{ github.sha }} + BASE: ${{ needs.plan.outputs.base }} + run: | + set -uo pipefail + if [[ "$NEEDED" != true ]]; then + echo "The change reaches no test target: nothing to compile." + exit 0 + fi + client="${CMUX_CI_CLIENT:-$HOME/.local/bin/cmux-ci}" + if ! [[ -x "$client" ]]; then + echo "::warning title=fleet test-compile::$client is not installed on runner $RUNNER_NAME; no test-compile verdict." + exit 0 + fi + log="$RUNNER_TEMP/test-compile.log" + status=0 + "$client" run --class light --script scripts/ci/package-test-lane.sh --ref "$REF" \ + --arg=compile-tests --arg="$BASE" \ + --timeout 1500 --queue-timeout 1200 2>&1 | tee "$log" || status=$? + if [[ "$status" -eq 0 ]]; then + echo "Every planned test target compiled." + elif grep -q 'compile-tests: a test target did not compile' "$log"; then + echo "::error title=fleet test-compile::A test target this change reaches does not compile (advisory; see the errors above)." + exit 1 + else + echo "::warning title=fleet test-compile::No test-compile verdict (exit $status)." + fi diff --git a/scripts/ci/compile_test_plan.py b/scripts/ci/compile_test_plan.py new file mode 100644 index 000000000000..1c915b9ccb8a --- /dev/null +++ b/scripts/ci/compile_test_plan.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Pick the Swift test targets a change can break, for the fleet test compile. + +fleet/compile-check (.github/workflows/fleet-compile-check.yml) compiles the +app executable, which links no test target. A change can still break a test +target's compile (#23360: a throwing call inside `#expect` without `try`; +#23163: a fake that missed a protocol requirement). This picks what the +advisory fleet-test-compile.yml check builds (`package-test-lane.sh +compile-tests` on a fleet Mac), from the files the change touches; an empty +plan queues no Mac: + + CmuxNext the test targets that depend on a changed target, from the + SwiftPM target graph (Packages/macOS/CmuxNext/ci-target-graph.json, + the same routing scripts/ci/cmux_next_route.py gives the swift + test tier). A manifest or graph change is every test target. + packages each package of the macOS package lane (PACKAGES in + scripts/ci/package-test-lane.sh) whose directory or local path + dependencies changed: all its test targets. + +Only files inside a package or one of its path dependencies count; a Markdown +file never does. Other local packages (iOS, Shared libraries without a macOS +lane) build in their own jobs, not with a macOS `swift build`. + +Prints one line per package: its directory, then the test targets to build, +or the directory alone for every test target of the package. + +Usage: compile_test_plan.py --changed-files FILE [--root DIR] +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from cmux_next_route import PACKAGE, closure, load_graph, route # noqa: E402,F401 +from select_package_tests import input_prefixes, package_dirs, under # noqa: E402 + +ROOT = Path(__file__).resolve().parents[2] +CMUXNEXT = PACKAGE.rstrip("/") +LANE = "scripts/ci/package-test-lane.sh" +LANE_PACKAGES = re.compile(r"^\s*PACKAGES=\(\n(.*?)\n\s*\)", re.S | re.M) + + +def lane_packages(root: Path) -> list[str]: + """The packages the macOS package lane tests (its PACKAGES list).""" + found = LANE_PACKAGES.search((root / LANE).read_text(encoding="utf-8")) + if not found: + raise SystemExit(f"no PACKAGES=( ... ) list in {LANE}") + return [line.strip() for line in found.group(1).splitlines() if line.strip() and not line.strip().startswith("#")] + + +def cmuxnext_targets(root: Path, changed: list[str]) -> list[str] | None: + """The CmuxNext test targets `changed` can break; None for every one.""" + result = route(root, "pull_request", changed, set()) + graph = load_graph(root) + every = sorted(name for name, target in graph["targets"].items() if target["kind"] == "test") + targets = every if result.full else sorted(result.tests) + return None if targets == every else targets + + +def plan(root: Path, changed: list[str]) -> list[tuple[str, list[str] | None]]: + """[(package directory, test targets or None for all)] for the changed files.""" + changed = [path for path in changed if path and not path.endswith(".md")] + dirs = package_dirs(root) + result: list[tuple[str, list[str] | None]] = [] + for name in [Path(CMUXNEXT).name, *lane_packages(root)]: + if name not in dirs or any(directory == dirs[name] for directory, _ in result): + continue + prefixes = input_prefixes(root, name, dirs) + hits = [path for path in changed if any(under(path, prefix) for prefix in prefixes)] + if not hits: + continue + if dirs[name] == CMUXNEXT: + targets = cmuxnext_targets(root, hits) + if targets == []: + continue + result.append((dirs[name], targets)) + else: + result.append((dirs[name], None)) + return result + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--changed-files", type=Path, required=True, help="one path per line") + args = parser.parse_args(argv) + changed = args.changed_files.read_text(encoding="utf-8").splitlines() + for directory, targets in plan(args.root, changed): + print(" ".join([directory, *(targets or [])])) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/ci/package-test-lane.sh b/scripts/ci/package-test-lane.sh index 4f5a890f692b..51c3f2681e7c 100755 --- a/scripts/ci/package-test-lane.sh +++ b/scripts/ci/package-test-lane.sh @@ -32,6 +32,19 @@ # scripts/ci/package-test-lane.sh --ref SHA --arg=suite # --arg=Packages/macOS/CmuxNext --arg=SuiteA,SuiteB,SuiteC # --env CMUX_CI_XCODE_APP=/Applications/Xcode_26.6.app +# compile-tests MERGE_BASE_SHA +# compile, without running, the test targets this commit's change +# can break: scripts/ci/compile_test_plan.py over `git diff +# MERGE_BASE_SHA HEAD` (the CmuxNext test targets that depend on a +# changed target, and the lane's packages whose sources or path +# dependencies changed). fleet-test-compile.yml runs it on pull +# requests as an advisory check (fleet/compile-check compiles only +# the app, which links no test target: #23360, #23163). Exit 0 +# when every planned target compiled (or none was +# planned), 1 after "compile-tests: a test target did not +# compile", anything else when there is no verdict. The marker is +# printed as soon as a target fails, so a later package's setup +# failure (any exit) never hides an earlier compile failure. # # --event and --full-suite default to EVENT_NAME and FULL_SUITE. Run from the # repository root; every helper path is relative to it. @@ -41,6 +54,14 @@ phase=run case "${1:-}" in run|select|packages|ghostty-sha) phase="$1"; shift ;; prebuild-one) phase="$1"; prebuild_package="$2"; prebuild_log="$3"; shift 3 ;; + compile-tests) + phase="$1" + if [ "$#" -ne 2 ] || ! [[ "$2" =~ ^[0-9a-f]{40}$ ]]; then + echo "usage: package-test-lane.sh compile-tests MERGE_BASE_SHA (the full sha of the change's merge base)" >&2; exit 2 + fi + compile_base="$2" + shift 2 + ;; suite) phase="$1"; suite_package="${2:-}" if [ "$#" -lt 3 ]; then @@ -746,10 +767,130 @@ run_suite() { echo "All ${#suite_filters[@]} suites passed." } +# compile_one LOG SWIFT_BUILD_ARGS...: one `swift build` into LOG (and the step +# log), built again from clean outputs when it read a stale explicit module +# (the backstop run_suite has). Returns the build's status. +compile_one() { + local log="$1" status=0 + shift + swift build "$@" < /dev/null 2>&1 | tee "$log" || status=$? + if [ "$status" -ne 0 ] && swiftpm_stale_module_failure "$log"; then + echo "::warning title=Stale precompiled module::the warm build read a stale explicit module; building again from clean outputs" + swiftpm_purge_build_outputs "$compile_scratch" + swiftpm_binary_module_guard "$compile_pkgdir" "$compile_scratch" + status=0 + swift build "$@" < /dev/null 2>&1 | tee "$log" || status=$? + fi + return "$status" +} + +run_compile_tests() { + # The merge base may be missing from a shallow step checkout: fetch that one commit. + if ! git cat-file -e "$compile_base^{commit}" 2>/dev/null; then + local depth=() + [ "$(git rev-parse --is-shallow-repository)" = true ] && depth=(--depth=1) + git fetch --no-tags --no-write-fetch-head ${depth[@]+"${depth[@]}"} \ + "$(git remote get-url origin 2>/dev/null || echo https://github.com/manaflow-ai/cmux.git)" "$compile_base" \ + || { echo "compile-tests: could not fetch the merge base $compile_base" >&2; exit 3; } + fi + local changed="$work/compile-tests-changed.txt" plan="$work/compile-tests-plan.txt" + git diff --no-renames --name-only "$compile_base" HEAD > "$changed" \ + || { echo "compile-tests: no diff from $compile_base" >&2; exit 3; } + python3 scripts/ci/compile_test_plan.py --changed-files "$changed" > "$plan" \ + || { echo "compile-tests: compile_test_plan.py failed" >&2; exit 3; } + echo "compile-tests: $(wc -l < "$changed" | tr -d ' ') files changed since ${compile_base:0:12}" + if [ ! -s "$plan" ]; then + echo "compile-tests: the change reaches no test target; nothing to compile." + return 0 + fi + echo "compile-tests plan (package, then its test targets; none listed is every one):" + sed 's/^/ /' "$plan" + select_xcode + echo "Xcode: $DEVELOPER_DIR" + local words target log status started failed=0 rows=() builds submodules submodule refused + local debug_args=(${swift_test_debug_info_args[@]+"${swift_test_debug_info_args[@]}"}) + log="$(mktemp -t swift-compile-tests.XXXXXX)" + # The plan on fd 9: nothing a build step reads from stdin can eat its lines. + while read -r -a words <&9; do + [ "${#words[@]}" -gt 0 ] || continue + compile_pkgdir="${words[0]}" + compile_scratch="$compile_pkgdir/.build" + if grep -q 'GhosttyKit\.xcframework' "$compile_pkgdir/Package.swift"; then + ensure_ghosttykit + fi + if [ "$compile_pkgdir" = Packages/macOS/CmuxNext ]; then + ensure_web_bundles + fi + # A fleet step checks the commit out with every submodule deinitialized: + # populate the ones this package reaches (run-swift-testing-suites.sh does too). + submodules=() + while IFS= read -r submodule; do + [ -n "$(ls -A "$submodule" 2>/dev/null)" ] || submodules+=("$submodule") + done < <(python3 scripts/ci/swift_package_submodules.py "$PWD" "$compile_pkgdir") + if [ "${#submodules[@]}" -gt 0 ]; then + git submodule update --init --depth 1 -- "${submodules[@]}" + fi + swiftpm_binary_module_guard "$compile_pkgdir" "$compile_scratch" + if [ "${#words[@]}" -gt 1 ]; then + builds=("${words[@]:1}") + else + builds=(--build-tests) + fi + for target in "${builds[@]}"; do + started=$SECONDS + status=0 + refused=0 + if [ "$target" = --build-tests ]; then + echo "::group::swift build --build-tests $compile_pkgdir" + compile_one "$log" --build-tests ${debug_args[@]+"${debug_args[@]}"} --package-path "$compile_pkgdir" || status=$? + else + echo "::group::swift build --target $target ($compile_pkgdir)" + compile_one "$log" ${debug_args[@]+"${debug_args[@]}"} --package-path "$compile_pkgdir" --target "$target" || status=$? + # A SwiftPM that will not build one test target on its own: build them + # all, once, in place of this package's remaining targets. + if [ "$status" -ne 0 ] && grep -qiE "no (target|module) named|unknown target" "$log"; then + echo "::warning title=compile-tests::swift build --target $target was refused; building every test target of $compile_pkgdir instead" + refused=1 + target=--build-tests + status=0 + compile_one "$log" --build-tests ${debug_args[@]+"${debug_args[@]}"} --package-path "$compile_pkgdir" || status=$? + fi + fi + echo "::endgroup::" + if [ "$status" -eq 0 ]; then + rows+=("$(printf '%-34s %-30s %-10s %6ss' "${compile_pkgdir##*/}" "${target#--}" compiled "$((SECONDS - started))")") + else + failed=$((failed + 1)) + rows+=("$(printf '%-34s %-30s %-10s %6ss' "${compile_pkgdir##*/}" "${target#--}" FAILED "$((SECONDS - started))")") + echo "::error title=Test target did not compile::$compile_pkgdir: ${target#--} (exit $status)" + # The verdict now, before any later setup can end the run. + echo "compile-tests: a test target did not compile: ${compile_pkgdir##*/} ${target#--}" + fi + [ "$refused" -eq 0 ] || break + done + [ "$failed" -ne 0 ] || record_built_ref "$compile_pkgdir" + done 9< "$plan" + rm -f "$log" + local table + table="$(printf '%-34s %-30s %-10s %7s\n' package target result time; printf '%s\n' "${rows[@]}")" + printf 'compile-tests results:\n%s\n' "$table" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '### Test target compile\n\n```\n%s\n```\n' "$table" >> "$GITHUB_STEP_SUMMARY" + fi + if [ "$failed" -ne 0 ]; then + echo "compile-tests: a test target did not compile ($failed of ${#rows[@]})" + exit 1 + fi + echo "compile-tests: every planned test target compiled (${#rows[@]})" +} + case "$phase" in suite) run_suite ;; + compile-tests) + run_compile_tests + ;; prebuild-one) prebuild_one "$prebuild_package" "$prebuild_log" ;; diff --git a/scripts/ci/select_package_tests.py b/scripts/ci/select_package_tests.py index 423da63d2149..a74117ff260b 100755 --- a/scripts/ci/select_package_tests.py +++ b/scripts/ci/select_package_tests.py @@ -27,6 +27,7 @@ "scripts/ci/git-seed.sh", "scripts/ci/release-build-archs.sh", "scripts/ci/ci_process_tree.py", + "scripts/ci/compile_test_plan.py", "scripts/ci/ensure-web-bundles.sh", "scripts/ci/hung_test_watchdog.py", "scripts/ci/owned_spm_scratch.py", diff --git a/tests/test-execution.toml b/tests/test-execution.toml index bbd8dcaff382..288ace41eeb6 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -1605,6 +1605,10 @@ lane = "linux-guard" path = "tests/test_ci_target_graph.py" lane = "linux-guard" +[[test]] +path = "tests/test_ci_compile_test_plan.py" +lane = "linux-guard" + [[test]] path = "tests/test_ios_internal_builder.py" lane = "linux-guard" diff --git a/tests/test_ci_compile_test_plan.py b/tests/test_ci_compile_test_plan.py new file mode 100644 index 000000000000..1742591d1e89 --- /dev/null +++ b/tests/test_ci_compile_test_plan.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""The fleet compile check compiles the test targets a change can break. + +fleet/compile-check compiled only the app executable. On 2026-10-21 two pull +requests passed it and then broke feat-cmux-next's test build: #23360 put a +throwing call inside `#expect` without `try` in PaletteMenuVocabularyTests +(CmuxNextActionsTests), and #23163 added a FakeClient to CmuxNextAppTests that +missed a requirement #21258 had added. Every Mac job that builds tests was +still queued when they merged. + +scripts/ci/compile_test_plan.py picks what the check compiles from the +changed files: the CmuxNext test targets that depend on a changed target (the +SwiftPM target graph cmux_next_route.py already reads), and the macOS-tested +local packages whose sources or path dependencies changed. package-test-lane.sh +`compile-tests MERGE_BASE` builds that plan on a fleet Mac, as the advisory +fleet-test-compile.yml check; fleet/compile-check stays the app compile. +""" + +from __future__ import annotations + +import importlib.util +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "ci" / "compile_test_plan.py" +LANE = ROOT / "scripts" / "ci" / "package-test-lane.sh" +CMUXNEXT = "Packages/macOS/CmuxNext" + +# #23360's diff against its base (ef40118cea25..4f867f5e57c2). +PR_23360 = [ + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/ActionCatalog.xcstrings", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/AppStoreActions.xcstrings", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/Catalog/AppStoreActionCatalog.swift", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/Catalog/BrowserActionCatalog.swift", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/Catalog/CloudActionCatalog.swift", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/Catalog/HistoryActionCatalog.swift", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/Catalog/RemoteActionCatalog.swift", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/Catalog/WorkspaceActionCatalog.swift", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/HistoryActions.xcstrings", + "Packages/macOS/CmuxNext/Sources/CmuxNextActions/RemoteActions.xcstrings", + "Packages/macOS/CmuxNext/Tests/CmuxNextActionsTests/PaletteMenuVocabularyTests.swift", + "plans/cmux-next/palette-audit.md", +] + + +def load(): + spec = importlib.util.spec_from_file_location("compile_test_plan", SCRIPT) + module = importlib.util.module_from_spec(spec) + sys.modules["compile_test_plan"] = module + spec.loader.exec_module(module) + return module + + +class CompileTestPlanTests(unittest.TestCase): + def setUp(self) -> None: + self.module = load() + + def plan(self, changed: list[str]) -> dict[str, list[str] | None]: + return dict(self.module.plan(ROOT, changed)) + + def test_pr_23360_compiles_the_test_target_it_broke(self) -> None: + plan = self.plan(PR_23360) + self.assertIn(CMUXNEXT, plan) + targets = plan[CMUXNEXT] + self.assertIsNotNone(targets, "a CmuxNextActions change is not every test target") + self.assertIn("CmuxNextActionsTests", targets) + # Test targets that depend on CmuxNextActions are reverse dependents: they compile too. + self.assertIn("CmuxNextAppTests", targets) + # One that does not reach CmuxNextActions stays out. + graph = self.module.load_graph(ROOT) + unrelated = [ + name for name, target in graph["targets"].items() + if target["kind"] == "test" and "CmuxNextActions" not in self.module.closure(graph, {name}) + ] + self.assertTrue(unrelated, "every test target depends on CmuxNextActions; pick another fixture") + self.assertFalse(set(unrelated) & set(targets), unrelated) + self.assertEqual(targets, sorted(targets)) + + def test_a_new_test_file_compiles_its_own_target(self) -> None: + # #23163: the FakeClient in a new CmuxNextAppTests file. + plan = self.plan([f"{CMUXNEXT}/Tests/CmuxNextAppTests/RemoteChildSessionsTests.swift"]) + self.assertEqual(plan, {CMUXNEXT: ["CmuxNextAppTests"]}) + + def test_manifest_change_compiles_every_test_target(self) -> None: + self.assertEqual(self.plan([f"{CMUXNEXT}/Package.swift"]), {CMUXNEXT: None}) + + def test_local_package_change_compiles_it_and_its_dependents(self) -> None: + plan = self.plan(["Packages/Shared/CMUXAuthCore/Sources/CMUXAuthCore/AuthTokens.swift"]) + # CMUXAuthCore and CmuxAuthRuntime are in the macOS package lane; CmuxNext uses both. + self.assertIsNone(plan.get("Packages/Shared/CMUXAuthCore", "missing")) + self.assertIsNone(plan.get("Packages/Shared/CmuxAuthRuntime", "missing")) + self.assertIn(CMUXNEXT, plan) + self.assertTrue(plan[CMUXNEXT], "the CmuxNext test targets that use CMUXAuthCore") + + def test_packages_outside_the_macos_lanes_are_not_compiled(self) -> None: + # iOS packages build in the iOS jobs, not with a macOS `swift build`. + plan = self.plan(["Packages/iOS/CmuxTabsKit/Sources/CmuxTabsKit/Tabs.swift"]) + self.assertNotIn("Packages/iOS/CmuxTabsKit", plan) + + def test_changes_that_compile_no_test_are_an_empty_plan(self) -> None: + self.assertEqual(self.plan(["docs/ci/merge-freeze.md", "web/app/page.tsx", "README.md"]), {}) + self.assertEqual(self.plan([f"{CMUXNEXT}/README.md"]), {}) + # CI and app-host changes compile no package test: the advisory check queues no Mac. + self.assertEqual(self.plan(["scripts/ci/cmux_next_route.py", "scripts/cmux-next/prepush-gate.py", + "App/CmuxNextApp.swift", "webviews/src/app.tsx"]), {}) + + def test_cli_prints_one_line_per_package(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + changed = Path(scratch) / "changed.txt" + changed.write_text("\n".join(PR_23360) + "\n", encoding="utf-8") + done = subprocess.run( + [sys.executable, str(SCRIPT), "--root", str(ROOT), "--changed-files", str(changed)], + capture_output=True, text=True, check=True, + ) + lines = done.stdout.splitlines() + self.assertEqual(len(lines), 1, done.stdout) + words = lines[0].split() + self.assertEqual(words[0], CMUXNEXT) + self.assertIn("CmuxNextActionsTests", words[1:]) + + +class CompileTestsPhaseArgsTests(unittest.TestCase): + """package-test-lane.sh compile-tests refuses a bad merge base before it builds anything.""" + + def run_lane(self, *args: str) -> subprocess.CompletedProcess: + env = {key: value for key, value in os.environ.items() if not key.startswith("GITHUB_")} + return subprocess.run(["bash", str(LANE), "compile-tests", *args], cwd=ROOT, env=env, + capture_output=True, text=True, timeout=60) + + def test_needs_one_full_merge_base_sha(self) -> None: + for args in ((), ("abc123",), ("A" * 40,), ("0" * 40, "extra")): + done = self.run_lane(*args) + self.assertEqual(done.returncode, 2, (args, done.stdout, done.stderr)) + self.assertIn("compile-tests MERGE_BASE_SHA", done.stderr) + + +MARKER = "compile-tests: a test target did not compile" + +FAKE_SWIFT = r"""#!/usr/bin/env bash +echo "swift $*" >> "$FAKE_SEEN" +[ "$1" = build ] || exit 0 +target="" +prev="" +for a in "$@"; do + [ "$prev" = --target ] && target="$a" + prev="$a" +done +case " $FAKE_BAD " in *" $target "*) + [ -n "$target" ] && { echo "Tests/$target/T.swift:74:106: error: call can throw, but it is not marked with 'try'"; exit 1; } ;; +esac +case " $FAKE_REFUSED " in *" $target "*) + [ -n "$target" ] && { echo "error: no target named '$target'"; exit 1; } ;; +esac +echo "Build complete!" +""" + +FAKE_XCRUN = r"""#!/usr/bin/env bash +for a in "$@"; do + [ "$a" = --show-sdk-path ] && { echo "$DEVELOPER_DIR/SDKs/MacOSX.sdk"; exit 0; } +done +exit 1 +""" + +# The plan comes from FAKE_PLAN (";" between packages), and one package's +# submodule setup fails when it is FAKE_BROKEN_SETUP. +FAKE_PLAN = r"""import os +for line in os.environ.get("FAKE_PLAN", "").split(";"): + if line.strip(): + print(line.strip()) +""" + +FAKE_SUBMODULES = r"""import os, sys +if sys.argv[2] == os.environ.get("FAKE_BROKEN_SETUP"): + print("vendor/not-a-submodule") +""" + + +class CompileTestsPhaseTests(unittest.TestCase): + """package-test-lane.sh compile-tests with a fake `swift`: what reaches red.""" + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(subprocess.run, ["rm", "-rf", str(self.tmp)]) + repo = self.repo = self.tmp / "repo" + bin_dir = self.bin = self.tmp / "bin" + (repo / "scripts" / "ci").mkdir(parents=True) + (repo / "scripts" / "cmux-next").mkdir(parents=True) + (repo / "vendor" / "not-a-submodule").mkdir(parents=True) + bin_dir.mkdir() + for name in ("package-test-lane.sh", "swift-test-debug-info.sh", "swiftpm-binary-module-guard.sh"): + (repo / "scripts" / "ci" / name).write_bytes((ROOT / "scripts" / "ci" / name).read_bytes()) + stale = ROOT / "scripts" / "cmux-next" / "stale-pcm-retry-needed.sh" + (repo / "scripts" / "cmux-next" / stale.name).write_bytes(stale.read_bytes()) + (repo / "scripts" / "ci" / "compile_test_plan.py").write_text(FAKE_PLAN) + (repo / "scripts" / "ci" / "swift_package_submodules.py").write_text(FAKE_SUBMODULES) + for package in ("Alpha", "Beta"): + pkg = repo / "Packages" / "macOS" / package + pkg.mkdir(parents=True) + (pkg / "Package.swift").write_text( + f'// swift-tools-version:5.9\nimport PackageDescription\nlet package = Package(name: "{package}")\n') + for name, text in (("swift", FAKE_SWIFT), ("xcrun", FAKE_XCRUN)): + (bin_dir / name).write_text(text) + (bin_dir / name).chmod(0o755) + (bin_dir / "python3").symlink_to(sys.executable) + self.dev = self.tmp / "Xcode.app" / "Contents" / "Developer" + (self.dev / "SDKs" / "MacOSX.sdk").mkdir(parents=True) + git = ["git", "-c", "user.email=t@t", "-c", "user.name=t", "-C", str(repo)] + subprocess.run(["git", "init", "-q", str(repo)], check=True) + subprocess.run([*git, "add", "-A"], check=True) + subprocess.run([*git, "commit", "-qm", "base"], check=True) + self.base = subprocess.run([*git, "rev-parse", "HEAD"], capture_output=True, text=True, + check=True).stdout.strip() + (repo / "Packages" / "macOS" / "Alpha" / "A.swift").write_text("let a = 1\n") + subprocess.run([*git, "add", "-A"], check=True) + subprocess.run([*git, "commit", "-qm", "change"], check=True) + self.seen = self.tmp / "seen" + + def lane(self, plan: str, bad: str = "", refused: str = "", broken: str = "") -> subprocess.CompletedProcess: + self.seen.write_text("") + env = {key: value for key, value in os.environ.items() if not key.startswith("GITHUB_")} + env.update(PATH=f"{self.bin}:{env['PATH']}", DEVELOPER_DIR=str(self.dev), RUNNER_TEMP=str(self.tmp), + FAKE_SEEN=str(self.seen), FAKE_PLAN=plan, FAKE_BAD=bad, FAKE_REFUSED=refused, + FAKE_BROKEN_SETUP=broken) + return subprocess.run(["bash", "scripts/ci/package-test-lane.sh", "compile-tests", self.base], + cwd=self.repo, env=env, capture_output=True, text=True, timeout=120) + + def builds(self) -> list[str]: + return [line for line in self.seen.read_text().splitlines() if line.startswith("swift build")] + + def test_every_target_compiles(self) -> None: + done = self.lane("Packages/macOS/Alpha AlphaTests OtherTests;Packages/macOS/Beta") + self.assertEqual(done.returncode, 0, done.stdout + done.stderr) + self.assertIn("every planned test target compiled (3)", done.stdout) + self.assertNotIn(MARKER, done.stdout) + self.assertEqual(len(self.builds()), 3, self.builds()) + self.assertTrue(any("--build-tests" in b and "Beta" in b for b in self.builds())) + + def test_an_empty_plan_builds_nothing(self) -> None: + done = self.lane("") + self.assertEqual(done.returncode, 0, done.stdout + done.stderr) + self.assertIn("nothing to compile", done.stdout) + self.assertEqual(self.builds(), []) + + def test_a_target_that_does_not_compile_is_red(self) -> None: + done = self.lane("Packages/macOS/Alpha AlphaTests BadTests;Packages/macOS/Beta", bad="BadTests") + self.assertEqual(done.returncode, 1, done.stdout + done.stderr) + self.assertIn(MARKER, done.stdout) + self.assertIn("::error title=Test target did not compile::Packages/macOS/Alpha: BadTests", done.stdout) + # A failing target does not stop the ones after it. + self.assertTrue(any("Beta" in b for b in self.builds()), self.builds()) + + def test_a_later_setup_failure_keeps_an_earlier_target_failure_red(self) -> None: + # Beta's submodule setup fails after Alpha's BadTests did not compile: + # the run ends early, but the failure marker must already be out. + done = self.lane("Packages/macOS/Alpha BadTests;Packages/macOS/Beta", bad="BadTests", + broken="Packages/macOS/Beta") + self.assertNotEqual(done.returncode, 0, done.stdout + done.stderr) + self.assertIn(MARKER, done.stdout, done.stdout + done.stderr) + + def test_a_setup_failure_alone_is_no_verdict(self) -> None: + done = self.lane("Packages/macOS/Alpha AlphaTests;Packages/macOS/Beta", broken="Packages/macOS/Beta") + self.assertNotEqual(done.returncode, 0, done.stdout + done.stderr) + self.assertNotIn(MARKER, done.stdout) + + def test_a_refused_target_falls_back_to_build_tests_once_per_package(self) -> None: + done = self.lane("Packages/macOS/Alpha OneTests TwoTests ThreeTests", refused="OneTests TwoTests ThreeTests") + self.assertEqual(done.returncode, 0, done.stdout + done.stderr) + whole = [b for b in self.builds() if "--build-tests" in b] + self.assertEqual(len(whole), 1, self.builds()) + self.assertEqual(len([b for b in self.builds() if "--target" in b]), 1, self.builds()) + + +WORKFLOW = ROOT / ".github" / "workflows" / "fleet-test-compile.yml" +STEP = "Compile the planned test targets on a fleet Mac" + + +def step_script() -> str: + lines = WORKFLOW.read_text().splitlines() + start = next(i for i, line in enumerate(lines) if line.strip() == f"- name: {STEP}") + run = next(i for i in range(start, len(lines)) if lines[i].strip() == "run: |") + indent = len(lines[run]) - len(lines[run].lstrip()) + 2 + body = [] + for line in lines[run + 1:]: + if line.strip() and len(line) - len(line.lstrip()) < indent: + break + body.append(line[indent:]) + return "\n".join(body) + "\n" + + +class AdvisoryVerdictTests(unittest.TestCase): + """fleet-test-compile.yml: red only for a test target that does not compile; anything else is no verdict.""" + + def verdict(self, status: int, output: str, needed: str = "true", client: bool = True) -> subprocess.CompletedProcess: + with tempfile.TemporaryDirectory() as scratch: + fake = Path(scratch) / "cmux-ci" + if client: + fake.write_text(f"#!/usr/bin/env bash\ncat <<'OUT'\n{output}\nOUT\nexit {status}\n") + fake.chmod(0o755) + env = {key: value for key, value in os.environ.items() if not key.startswith("GITHUB_")} + env.update(NEEDED=needed, REF="a" * 40, BASE="b" * 40, RUNNER_TEMP=scratch, RUNNER_NAME="gateway", + CMUX_CI_CLIENT=str(fake)) + return subprocess.run(["bash", "-c", step_script()], env=env, capture_output=True, text=True, timeout=60) + + def assert_green(self, done, warned: bool) -> None: + self.assertEqual(done.returncode, 0, done.stdout + done.stderr) + self.assertNotIn("::error", done.stdout) + self.assertEqual("::warning" in done.stdout, warned, done.stdout) + + def test_exit_codes_map_to_red_warning_or_green(self) -> None: + self.assert_green(self.verdict(0, "compile-tests: every planned test target compiled (3)"), warned=False) + red = self.verdict(1, f"{MARKER} (1 of 3)") + self.assertEqual(red.returncode, 1, red.stdout) + self.assertIn("::error", red.stdout) + # A later setup failure ends the lane with another status after the marker: still red. + red = self.verdict(128, f"{MARKER}: Alpha BadTests\nfatal: not a submodule") + self.assertEqual(red.returncode, 1, red.stdout) + # No marker: the queue, a timeout, a setup failure or the client itself. A warning, never red. + for status in (1, 2, 3, 124, 137): + with self.subTest(status=status): + self.assert_green(self.verdict(status, "error: queue timeout"), warned=True) + + def test_no_plan_or_no_client_is_green(self) -> None: + self.assert_green(self.verdict(1, MARKER, needed="false"), warned=False) + self.assert_green(self.verdict(0, "", client=False), warned=True) + + def test_the_job_outlasts_the_client_so_a_slow_fleet_never_reads_red(self) -> None: + # A GitHub job timeout concludes as a failure: the client's queue and run + # timeouts must end first, with room for the client to report. + text = WORKFLOW.read_text() + job = text.split("\n compile:\n", 1)[1] + minutes = int(re.search(r"^ timeout-minutes: (\d+)$", job, re.M).group(1)) + queue = int(re.search(r"--queue-timeout (\d+)", job).group(1)) + run = int(re.search(r"--timeout (\d+)", job).group(1)) + self.assertGreaterEqual(minutes * 60, queue + run + 300, (minutes, queue, run)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_merge_compile_guard.py b/tests/test_merge_compile_guard.py index 600594daf929..5510d9ba9fc8 100644 --- a/tests/test_merge_compile_guard.py +++ b/tests/test_merge_compile_guard.py @@ -131,6 +131,38 @@ def test_the_workflow_posts_the_status_the_guard_reads_on_the_sha_it_was_given(s self.assertNotIn("pull_request", text) +class AdvisoryTestCompile(unittest.TestCase): + """fleet-test-compile.yml's check is advisory: the guard neither waits for it nor needs it. + + It compiles the test targets a PR reaches on a fleet Mac (#23360, #23163 broke the base's test build + after the app-only fleet/compile-check passed). A red or pending one never holds a merge here, and it + never posts the fleet/compile-check status the guard trusts. + """ + + ADVISORY = "fleet/test-compile (advisory)" + WORKFLOW = ROOT / ".github/workflows/fleet-test-compile.yml" + + def test_a_red_or_pending_advisory_check_never_holds_a_merge(self): + for state, conclusion in (("completed", "failure"), ("queued", None), ("in_progress", None)): + with self.subTest(state=state): + advisory = check(self.ADVISORY, state, conclusion or "success") + self.assertEqual(guard.verdict(pr(), SWIFT, [check(COMPILE), advisory]), "") + self.assertEqual(guard.verdict(pr(), SWIFT, [check(COMPILE, "queued"), advisory], fleet_passed=True), "") + # And it never stands in for the Release compile. + self.assertIn("still running", guard.verdict(pr(), SWIFT, [check(COMPILE, "queued"), advisory])) + + def test_it_is_not_the_fleet_check_the_guard_trusts(self): + self.assertNotEqual(guard.FLEET_WORKFLOW, self.WORKFLOW.name) + text = self.WORKFLOW.read_text() + self.assertIn(f"name: {self.ADVISORY}\n", text) + self.assertNotIn(guard.FLEET_CONTEXT + "\n", text) + self.assertNotIn("statuses", text) + self.assertNotIn(guard.FLEET_TITLE, text) + fleet = (ROOT / ".github/workflows" / guard.FLEET_WORKFLOW).read_text() + self.assertNotIn("compile-tests", fleet) + self.assertNotIn("test-compile", fleet) + + def status(state="success", creator="github-actions[bot]", run=99): return {"state": state, "creator": creator, "url": f"https://github.com/manaflow-ai/cmux/actions/runs/{run}"}