diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2da6cba62abd..6b4de42da1ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,6 +263,9 @@ jobs: - name: Validate xcodebuild failure diagnostics run: ./tests/test_ci_xcodebuild_with_diagnostics.sh + - name: Validate focused test launcher + run: python3 tests/test_run_e2e.py + - name: Validate app-host xcodebuild retry guard run: ./tests/test_ci_app_host_xcodebuild_retry.sh diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 2990f5fd3562..cfbae0f0217b 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,5 +1,5 @@ name: E2E test with video recording -run-name: ${{ inputs.test_filter }} on ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner }} @ ${{ inputs.ref || github.ref_name }} +run-name: ${{ inputs.test_filter }} on ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner }} @ ${{ inputs.ref || github.ref_name }}${{ inputs.dispatch_id != '' && format(' [{0}]', inputs.dispatch_id) || '' }} on: workflow_dispatch: @@ -11,6 +11,10 @@ on: test_filter: description: "Test class or class/method; optionally target-qualified as cmuxUITests/Class or cmuxTests/Class" required: true + dispatch_id: + description: "Optional caller-generated id echoed into the run name so dispatchers can resolve their exact run" + required: false + default: "" test_timeout: description: "Per-test timeout in seconds" required: false diff --git a/scripts/ci/dispatch-focused-test.py b/scripts/ci/dispatch-focused-test.py new file mode 100644 index 000000000000..5ffa4292efce --- /dev/null +++ b/scripts/ci/dispatch-focused-test.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Dispatch the existing E2E workflow for an exact revision and selected test.""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import json +from pathlib import Path +import re +import signal +import subprocess +import sys +import threading +import time +from urllib.parse import quote +import uuid + +REPO = "manaflow-ai/cmux" +WORKFLOW = "test-e2e.yml" +ROOT = Path(__file__).resolve().parents[2] +RUN_DISCOVERY_ATTEMPTS = 12 +RUN_DISCOVERY_TIMEOUT_SECONDS = 60.0 +SELECTOR = re.compile( + r"(?:(?:cmuxTests|cmuxUITests)/)?" + r"[A-Za-z_][A-Za-z0-9_]*(?:/[A-Za-z_][A-Za-z0-9_]*(?:\(\))?)?" +) + + +def positive_integer(value: str) -> int: + if not re.fullmatch(r"[1-9][0-9]*", value): + raise argparse.ArgumentTypeError("must be a positive integer") + return int(value) + + +def output( + *command: str, + timeout: float | None = None, + cancel_event: threading.Event | None = None, +) -> str: + if cancel_event is None: + try: + return subprocess.check_output( + command, cwd=ROOT, text=True, timeout=timeout + ).strip() + except subprocess.TimeoutExpired as error: + raise ValueError("GitHub command timed out during focused-run discovery") from error + + process = subprocess.Popen( + command, + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + ) + try: + while True: + if cancel_event.is_set(): + process.terminate() + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise ValueError("focused-run discovery cancelled") + try: + stdout, _ = process.communicate( + timeout=min(0.25, timeout) if timeout is not None else 0.25 + ) + except subprocess.TimeoutExpired: + if timeout is not None: + timeout -= 0.25 + if timeout <= 0: + process.kill() + process.wait() + raise ValueError( + "GitHub command timed out during focused-run discovery" + ) + continue + if process.returncode: + raise subprocess.CalledProcessError( + process.returncode, command, output=stdout + ) + return stdout.strip() + finally: + if process.poll() is None: + process.kill() + process.wait() + if process.stdout is not None: + process.stdout.close() + + +def wait_for_retry(cancel_event: threading.Event, delay_seconds: float) -> bool: + """Wait for the next discovery attempt, allowing cancellation to interrupt it.""" + return cancel_event.wait(delay_seconds) + + +@contextmanager +def cancellation_scope(): + """Turn termination signals into a cancellable run-discovery wait.""" + cancel_event = threading.Event() + previous = {} + + def cancel(_signum, _frame): + cancel_event.set() + + try: + for signum in (signal.SIGINT, signal.SIGTERM): + previous[signum] = signal.signal(signum, cancel) + yield cancel_event + finally: + for signum, handler in previous.items(): + signal.signal(signum, handler) + + +def find_run( + commit: str, + selector: str, + dispatch_id: str, + *, + cancel_event: threading.Event | None = None, +) -> dict: + """Correlate this dispatch, never assume the newest run belongs to us.""" + cancel_event = cancel_event or threading.Event() + suffix = f" @ {commit} [{dispatch_id}]" + deadline = time.monotonic() + RUN_DISCOVERY_TIMEOUT_SECONDS + for attempt in range(RUN_DISCOVERY_ATTEMPTS): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + runs = json.loads(output( + "gh", "run", "list", "--repo", REPO, "--workflow", WORKFLOW, + "--event", "workflow_dispatch", "--limit", "100", + "--json", "databaseId,displayTitle,url", + timeout=remaining, + cancel_event=cancel_event, + )) + if cancel_event.is_set(): + raise ValueError("focused-run discovery cancelled") + matches = [ + run for run in runs + if run["displayTitle"].startswith(f"{selector} on ") + and run["displayTitle"].endswith(suffix) + ] + if len(matches) == 1: + return matches[0] + if matches: + raise ValueError("multiple runs matched this dispatch; refusing to guess") + remaining = deadline - time.monotonic() + if attempt + 1 >= RUN_DISCOVERY_ATTEMPTS or remaining <= 0: + break + # Back off while the Actions API registers the run. The monotonic + # deadline bounds the total wait, and Event.wait lets cancellation + # interrupt the delay instead of trapping the caller in a fixed sleep. + delay = min(2 ** min(attempt, 3), 8, remaining) + if wait_for_retry(cancel_event, delay): + raise ValueError("focused-run discovery cancelled") + raise ValueError( + f"dispatch accepted but its run was not found; request {dispatch_id}. " + f"Check https://github.com/{REPO}/actions/workflows/{WORKFLOW} " + "before dispatching again." + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run one suite or method on an exact pushed commit. " + "This focused result does not replace the full CI merge checks.", + epilog="Examples: scripts/run-e2e.sh cmuxTests/RemoteTmuxMirrorPaneInputMappingTests --wait; " + "scripts/run-e2e.sh UpdatePillUITests/testFoo --ref my-branch --no-video", + ) + parser.add_argument("test_filter", help="cmuxTests/Suite[/method] or cmuxUITests/Class[/method]; bare names target UI tests") + parser.add_argument("--ref", help="remote branch, tag, or SHA; default: clean local HEAD, already pushed") + parser.add_argument("--wait", action="store_true", help="wait and return a nonzero status if the run fails") + parser.add_argument("--no-video", action="store_true") + parser.add_argument("--timeout", type=positive_integer, default=120, help="per-test timeout in seconds (default: 120)") + parser.add_argument("--job-timeout", type=positive_integer, default=45, help="job timeout in minutes, including compilation (default: 45)") + parser.add_argument("--workflow-ref", help="workflow-definition branch/tag (default: repository default branch)") + args = parser.parse_args() + if not SELECTOR.fullmatch(args.test_filter): + parser.error("test_filter must name one suite or method, optionally prefixed with cmuxTests/ or cmuxUITests/") + if args.ref is not None and not args.ref.strip(): + parser.error("--ref must not be empty") + if args.workflow_ref is not None and not args.workflow_ref.strip(): + parser.error("--workflow-ref must not be empty") + + requested_ref = args.ref + if requested_ref is None: + if output("git", "status", "--porcelain", "--untracked-files=normal"): + raise ValueError("commit and push local changes first, or use --ref to explicitly test a remote revision") + requested_ref = output("git", "rev-parse", "HEAD") + # Resolve once before spending a runner. A subsequent branch push cannot + # change which source revision checkout receives. + commit = json.loads(output( + "gh", "api", f"repos/{REPO}/commits/{quote(requested_ref, safe='')}", + ))["sha"] + if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("GitHub did not resolve the requested revision to a full commit SHA") + if args.ref is None and commit != requested_ref: + raise ValueError("GitHub revision differs from local HEAD; push the intended commit first") + + dispatch_id = uuid.uuid4().hex + video = not args.no_video and not args.test_filter.startswith("cmuxTests/") + fields = { + "ref": commit, + "test_filter": args.test_filter, + "record_video": str(video).lower(), + "test_timeout": str(args.timeout), + "job_timeout": str(args.job_timeout), + "dispatch_id": dispatch_id, + } + command = ["gh", "workflow", "run", WORKFLOW, "--repo", REPO] + if args.workflow_ref: + command.extend(["--ref", args.workflow_ref]) + for key, value in fields.items(): + command.extend(["-f", f"{key}={value}"]) + print(f"Testing {args.test_filter} at {commit} (request {dispatch_id})", flush=True) + subprocess.run(command, cwd=ROOT, check=True) + with cancellation_scope() as cancel_event: + run = find_run( + commit, args.test_filter, dispatch_id, cancel_event=cancel_event + ) + print(f"Run: {run['url']}", flush=True) + if args.wait: + return subprocess.run([ + "gh", "run", "watch", "--repo", REPO, str(run["databaseId"]), + "--exit-status", + ], cwd=ROOT).returncode + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, KeyError, subprocess.CalledProcessError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index 06baab768ca3..5e8f7e52eb46 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -1,97 +1,5 @@ #!/usr/bin/env bash -# Trigger the test-e2e.yml workflow and optionally wait for results. -# -# Usage: -# ./scripts/run-e2e.sh UpdatePillUITests -# ./scripts/run-e2e.sh UpdatePillUITests --wait -# ./scripts/run-e2e.sh UpdatePillUITests/testFoo --ref my-branch -# ./scripts/run-e2e.sh cmuxTests/ForkParentFallbackGeneralizationTests -# ./scripts/run-e2e.sh UpdatePillUITests --no-video --timeout 300 +# Launch a focused suite or method on an exact pushed commit. set -euo pipefail - -REPO="manaflow-ai/cmux" -WORKFLOW="test-e2e.yml" - -# Defaults -REF="" -WAIT=false -RECORD_VIDEO=true -TIMEOUT=120 - -usage() { - cat < [options] - -Arguments: - test_filter Test class or class/method. Bare filters target cmuxUITests; - use cmuxUITests/Class or cmuxTests/Class for explicit targets. - -Options: - --ref Branch or SHA to test (default: current branch) - --wait Wait for the run to complete and print result - --no-video Disable video recording - --timeout Per-test timeout in seconds (default: 120) - -h, --help Show this help -EOF - exit 0 -} - -if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - usage -fi - -TEST_FILTER="$1" -shift - -while [ $# -gt 0 ]; do - case "$1" in - --ref) - REF="$2" - shift 2 - ;; - --wait) - WAIT=true - shift - ;; - --no-video) - RECORD_VIDEO=false - shift - ;; - --timeout) - TIMEOUT="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - usage - ;; - esac -done - -# Build workflow dispatch fields -FIELDS=(-f "test_filter=$TEST_FILTER" -f "record_video=$RECORD_VIDEO" -f "test_timeout=$TIMEOUT") -if [ -n "$REF" ]; then - FIELDS+=(-f "ref=$REF") -fi - -echo "Triggering $WORKFLOW with test_filter=$TEST_FILTER ref=${REF:-} video=$RECORD_VIDEO timeout=$TIMEOUT" -gh workflow run "$WORKFLOW" --repo "$REPO" "${FIELDS[@]}" - -# Wait a moment for the run to register -sleep 3 - -# Get the latest run ID -RUN_ID=$(gh run list --repo "$REPO" --workflow "$WORKFLOW" --limit 1 --json databaseId --jq '.[0].databaseId') -RUN_URL="https://github.com/$REPO/actions/runs/$RUN_ID" - -echo "Run: $RUN_URL" - -if [ "$WAIT" = true ]; then - echo "Waiting for run to complete..." - gh run watch --repo "$REPO" "$RUN_ID" --exit-status || true - - STATUS=$(gh run view --repo "$REPO" "$RUN_ID" --json conclusion --jq '.conclusion') - echo "" - echo "Result: $STATUS" - echo "Run: $RUN_URL" -fi +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec python3 "$script_dir/ci/dispatch-focused-test.py" "$@" diff --git a/tests/test_run_e2e.py b/tests/test_run_e2e.py new file mode 100644 index 000000000000..0f68e56f10ad --- /dev/null +++ b/tests/test_run_e2e.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Exercise the focused-run launcher against a fake GitHub CLI.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +HEAD = "a" * 40 +REMOTE_HEAD = "b" * 40 +FAKE_GH = r'''#!/usr/bin/env python3 +import json, os, pathlib, sys +args = sys.argv[1:] +root = pathlib.Path(os.environ["LAUNCHER_TEST_DIR"]) +with (root / "calls.jsonl").open("a") as f: + f.write(json.dumps(args) + "\n") +if args[0] == "api": + if os.environ.get("LAUNCHER_MISSING_COMMIT"): + sys.exit(1) + print(json.dumps({"sha": "b" * 40 if "topic%2Ffix" in args[1] else "a" * 40})) +elif args[:2] == ["workflow", "run"]: + fields = dict(arg.split("=", 1) for arg in args if "=" in arg) + (root / "dispatch.json").write_text(json.dumps(fields)) +elif args[:2] == ["run", "list"]: + fields = json.loads((root / "dispatch.json").read_text()) + print(json.dumps([ + {"databaseId": 999, "displayTitle": "someone else's newer run", "url": "https://github.com/manaflow-ai/cmux/actions/runs/999"}, + {"databaseId": 123, "displayTitle": fields["test_filter"] + " on mac @ " + fields.get("ref", "main") + " [" + fields.get("dispatch_id", "") + "]", "url": "https://github.com/manaflow-ai/cmux/actions/runs/123"} + ])) +elif args[:2] == ["run", "watch"]: + sys.exit(int(os.environ.get("LAUNCHER_WATCH_STATUS", "0"))) +elif args[:2] == ["run", "view"]: + print("failure") +else: + sys.exit(2) +''' + + +class FocusedLauncherTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.bin = self.root / "bin" + self.bin.mkdir() + for name, source in { + "gh": FAKE_GH, + "git": '#!/bin/sh\ncase "$*" in\n*status*) printf "%s" "${LAUNCHER_DIRTY:-}";;\n*) printf "%s\\n" "' + HEAD + '";;\nesac\n', + "sleep": "#!/bin/sh\nexit 0\n", + }.items(): + path = self.bin / name + path.write_text(source) + path.chmod(0o755) + self.env = { + **os.environ, + "PATH": str(self.bin) + os.pathsep + os.environ["PATH"], + "LAUNCHER_TEST_DIR": str(self.root), + } + + def launch(self, *args, **env): + return subprocess.run( + ["bash", str(ROOT / "scripts/run-e2e.sh"), *args], + env={**self.env, **env}, text=True, capture_output=True, + ) + + def calls(self): + path = self.root / "calls.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + def dispatch(self): + return json.loads((self.root / "dispatch.json").read_text()) + + def test_default_dispatches_exact_local_commit_and_finds_its_own_run(self): + result = self.launch("cmuxTests/ExampleTests") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.dispatch()["ref"], HEAD) + self.assertTrue(self.dispatch()["dispatch_id"]) + self.assertEqual(self.dispatch()["record_video"], "false") + self.assertIn("/actions/runs/123", result.stdout) + self.assertNotIn("/actions/runs/999", result.stdout) + + def test_explicit_remote_ref_is_resolved_before_dispatch(self): + result = self.launch("cmuxTests/ExampleTests/testOne", "--ref", "topic/fix") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.dispatch()["ref"], REMOTE_HEAD) + + def test_dirty_default_checkout_does_not_dispatch(self): + result = self.launch("cmuxTests/ExampleTests", LAUNCHER_DIRTY=" M Sources/App.swift") + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.root / "dispatch.json").exists()) + + def test_explicit_remote_ref_does_not_claim_to_test_dirty_local_files(self): + result = self.launch("ExampleUITests", "--ref", "topic/fix", LAUNCHER_DIRTY=" M local.txt") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.dispatch()["ref"], REMOTE_HEAD) + self.assertEqual(self.dispatch()["record_video"], "true") + + def test_unpushed_commit_does_not_dispatch(self): + result = self.launch("cmuxTests/ExampleTests", LAUNCHER_MISSING_COMMIT="1") + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.root / "dispatch.json").exists()) + + def test_wait_preserves_failure_and_watches_matching_run(self): + result = self.launch("cmuxTests/ExampleTests", "--wait", LAUNCHER_WATCH_STATUS="1") + self.assertEqual(result.returncode, 1, result.stderr) + watch = next(call for call in self.calls() if call[:2] == ["run", "watch"]) + self.assertIn("123", watch) + self.assertNotIn("999", watch) + + def test_rejects_invalid_selectors_before_dispatch(self): + for selector in ("", "cmuxTests/", "cmuxTests/Example/extra/method", "cmuxTests/A\ndispatch_id=bad", "cmuxTests/A;echo bad"): + with self.subTest(selector=selector): + self.assertNotEqual(self.launch(selector).returncode, 0) + self.assertFalse((self.root / "dispatch.json").exists()) + + def test_rejects_invalid_or_missing_options(self): + for args in (("--timeout", "0"), ("--timeout", "bad"), ("--ref",), ("--unknown",)): + with self.subTest(args=args): + self.assertNotEqual(self.launch("ExampleTests", *args).returncode, 0) + self.assertFalse((self.root / "dispatch.json").exists()) + + +class RunDiscoveryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + spec = importlib.util.spec_from_file_location( + "focused_dispatch", ROOT / "scripts/ci/dispatch-focused-test.py" + ) + cls.dispatch = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.dispatch) + + def test_waits_for_matching_dispatch_without_choosing_another_run(self): + other = {"databaseId": 999, "displayTitle": "Other on mac @ " + HEAD + " [other]"} + own = {"databaseId": 123, "displayTitle": "cmuxTests/Example on mac @ " + HEAD + " [mine]"} + with mock.patch.object(self.dispatch, "output", side_effect=[json.dumps([other]), json.dumps([other, own])]), mock.patch.object(self.dispatch, "wait_for_retry", return_value=False) as wait: + result = self.dispatch.find_run(HEAD, "cmuxTests/Example", "mine") + self.assertEqual(result["databaseId"], 123) + wait.assert_called_once_with(mock.ANY, 1) + + def test_missing_dispatch_fails_without_redispatching(self): + with mock.patch.object(self.dispatch, "output", return_value="[]") as output, mock.patch.object(self.dispatch, "wait_for_retry", return_value=False) as wait: + with self.assertRaisesRegex(ValueError, "before dispatching again"): + self.dispatch.find_run(HEAD, "cmuxTests/Example", "mine") + self.assertEqual(output.call_count, 12) + self.assertEqual(wait.call_count, 11) + self.assertTrue(all(call.args[1:3] == ("run", "list") for call in output.call_args_list)) + + def test_cancellation_interrupts_discovery(self): + cancelled = threading.Event() + cancelled.set() + with mock.patch.object(self.dispatch, "output", return_value="[]"): + with self.assertRaisesRegex(ValueError, "cancelled"): + self.dispatch.find_run( + HEAD, "cmuxTests/Example", "mine", cancel_event=cancelled + ) + + def test_cancellation_terminates_inflight_command(self): + cancelled = threading.Event() + timer = threading.Timer(0.1, cancelled.set) + timer.start() + try: + with self.assertRaisesRegex(ValueError, "cancelled"): + self.dispatch.output( + self.dispatch.sys.executable, + "-c", + "import time; time.sleep(30)", + timeout=60, + cancel_event=cancelled, + ) + finally: + timer.cancel() + + def test_cancellation_scope_handles_sigint_and_restores_handlers(self): + original = self.dispatch.signal.getsignal(self.dispatch.signal.SIGINT) + with self.dispatch.cancellation_scope() as cancelled: + self.dispatch.signal.raise_signal(self.dispatch.signal.SIGINT) + self.assertTrue(cancelled.is_set()) + self.assertIs(self.dispatch.signal.getsignal(self.dispatch.signal.SIGINT), original) + + def test_ambiguous_dispatch_fails(self): + run = {"databaseId": 123, "displayTitle": "cmuxTests/Example on mac @ " + HEAD + " [mine]"} + with mock.patch.object(self.dispatch, "output", return_value=json.dumps([run, run])): + with self.assertRaisesRegex(ValueError, "refusing to guess"): + self.dispatch.find_run(HEAD, "cmuxTests/Example", "mine") + + +if __name__ == "__main__": + unittest.main()