diff --git a/.github/workflows/pr-media.yml b/.github/workflows/pr-media.yml index 8387c71111de..e9af09bba690 100644 --- a/.github/workflows/pr-media.yml +++ b/.github/workflows/pr-media.yml @@ -92,6 +92,7 @@ jobs: sparse-checkout: | scripts/ci/pr_media.py scripts/ci/find_admitted_build.py + scripts/ci/product_input_identity.py dogfood/scenarios sparse-checkout-cone-mode: false diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index a22fcae878b0..0d36f31d23c5 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -593,8 +593,32 @@ jobs: # PR media tours (pr-media.yml) ask for CI's product and never pay for # a second compile of the app. - - name: Refuse to compile for a dispatch that requires an adopted product + # A reuse error is no evidence the product is unloadable, so it fails + # under its own name (pr_media.py compiles only after a refusal). The + # lookup records API and transfer failures as miss reasons, so those + # count as errors too. + - name: Fail a dispatch that requires an adopted product when reuse errored + id: reuse_error if: ${{ inputs.require_adopted_product && steps.reuse.outputs.hit != 'true' }} + env: + OUTCOME: ${{ steps.reuse.outcome }} + REASON: ${{ steps.reuse.outputs.reason }} + MISSES: ${{ steps.reuse.outputs.miss_reasons }} + run: | + errors=",artifact_listing_unavailable,consumer_provenance_unavailable,artifact_download_error,fingerprint_unavailable,reuse_api_or_validation_error," + errored="" + if [ "$OUTCOME" != success ] || [ "$REASON" != miss ]; then errored=1; fi + IFS=, read -ra reasons <<<"$MISSES" + for reason in "${reasons[@]}"; do + case "$errors" in *",$reason,"*) errored=1 ;; esac + done + if [ -n "$errored" ]; then + echo "::error::Compiled-product reuse did not complete (${MISSES:-no reason recorded}), so this dispatch cannot tell whether a product loads." + exit 1 + fi + + - name: Refuse to compile for a dispatch that requires an adopted product + if: ${{ inputs.require_adopted_product && steps.reuse.outputs.hit != 'true' && steps.reuse_error.outcome == 'success' }} env: MISSES: ${{ steps.reuse.outputs.miss_reasons }} run: | diff --git a/scripts/ci/pr_media.py b/scripts/ci/pr_media.py index db9a99c8a6a1..36cb533cdc84 100644 --- a/scripts/ci/pr_media.py +++ b/scripts/ci/pr_media.py @@ -63,12 +63,9 @@ # its runner (an app pull request), or straight away (a manual dispatch). COMPILE_NEVER, COMPILE_FALLBACK, COMPILE_NOW = "never", "fallback", "now" COMPILE_MODES = (COMPILE_NEVER, COMPILE_FALLBACK, COMPILE_NOW) -# Changes to these build the app; a pull request touching none (a CLI-only -# one, say) never compiles an app just for its tours. -APP_PATH_PREFIXES = ("Sources/", "Packages/macOS/", "Packages/Shared/", "TunnelExtension/", "Resources/", - "Assets.xcassets/", "AppIcon.icon/", "vendor/", "ghostty", "cmux.xcodeproj/", - "cmux-Bridging-Header.h", "cmux.entitlements", "cmux-helper.entitlements", "webviews/", - "cmuxUITests/", f"{SCENARIOS_DIR}/") +# Product inputs no tour shows (the CLI lane, the app-host unit tests): a +# pull request that only changes these never compiles an app for its tours. +NON_TOUR_PRODUCT_PREFIXES = ("CLI/", "cmuxCLITests/", "cmuxCLITestSupport/", "cmuxTests/") GATE_WAIT_SECONDS = 25 * 60 # Reads share the repository's token budget with the dispatcher, so waits poll slowly. GATE_POLL_SECONDS = 60 @@ -77,6 +74,8 @@ ADMISSION_JOB_SUFFIX = "macOS compile admission" # test-e2e.yml's step that fails a require_adopted_product run whose reuse missed. REFUSE_STEP = "Refuse to compile for a dispatch that requires an adopted product" +# ... and the one that fails it when reuse errored (no evidence either way). +REUSE_ERROR_STEP = "Fail a dispatch that requires an adopted product when reuse errored" MERGE_REF = re.compile(r"refs/pull/\d+/merge") TESTED_LINE = re.compile(r"^Testing \S+ at ([0-9a-f]{40}) \(request ") MAX_KEY_SHOTS = 4 @@ -182,6 +181,22 @@ def select_tours(scenarios: dict[str, object], changed: Iterable[str], body: str return [], "no tour matched and the default tour is missing" +def reaches_app(path: str) -> bool: + """Whether a changed path can change the app a tour shows: an input of the + app-host product (product_input_identity.reaches_product, what CI keys its + build on) outside NON_TOUR_PRODUCT_PREFIXES.""" + import importlib.util + spec = importlib.util.spec_from_file_location("product_input_identity", + ROOT / "scripts/ci/product_input_identity.py") + assert spec and spec.loader + identity = sys.modules.get(spec.name) + if identity is None: + identity = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = identity + spec.loader.exec_module(identity) + return identity.reaches_product(path) and not path.startswith(NON_TOUR_PRODUCT_PREFIXES) + + def head_scenarios(head_sha: str) -> dict[str, object]: """Tours at the head commit, read from git objects (never checked out or run).""" listing = subprocess.run(["git", "ls-tree", "--name-only", head_sha, f"{SCENARIOS_DIR}/"], @@ -396,7 +411,7 @@ def plan(repository: str) -> int: except json.JSONDecodeError: pass tours, reason = select_tours(scenarios, changed, pull.get("body")) - app_change = any(path.startswith(APP_PATH_PREFIXES) for path in changed) + app_change = any(reaches_app(path) for path in changed) force = os.environ.get("FORCE", "").lower() == "true" pending = [tour for tour in tours if force or published(repository, pr, head_sha, tour) is None] print(f"#{pr} at {head_sha}: tours {tours or 'none'} ({reason}); to run: {pending or 'none'}", flush=True) @@ -478,19 +493,28 @@ def wait(self, sleep: Callable[[float], None] = time.sleep) -> dict: def refused_after(dispatch: "Dispatch", repository: str) -> bool: """Wait for an adopt-only tour run; whether it stopped because it could - not load CI's build. The finished run is kept on `dispatch.completed`.""" + not load CI's build. The finished run is kept on `dispatch.completed`; + a run whose reuse errored is kept with conclusion "reuse_error".""" run = dispatch.wait() - if run.get("conclusion") == "failure" and refused_to_compile(repository, str(dispatch.run_id)): + if run.get("conclusion") != "failure": + dispatch.completed = run + return False + if refused_to_compile(repository, str(dispatch.run_id)): return True - dispatch.completed = run + errored = REUSE_ERROR_STEP in failed_steps(repository, str(dispatch.run_id)) + dispatch.completed = {**run, "conclusion": "reuse_error"} if errored else run return False +def failed_steps(repository: str, run_id: str) -> set[str]: + jobs = (gh_json([f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=100"]) or {}).get("jobs", []) + return {str(step.get("name")) for job in jobs for step in job.get("steps") or [] + if step.get("conclusion") == "failure"} + + def refused_to_compile(repository: str, run_id: str) -> bool: """Whether the tour run stopped because it could not load CI's build.""" - jobs = (gh_json([f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=100"]) or {}).get("jobs", []) - return any(step.get("name") == REFUSE_STEP and step.get("conclusion") == "failure" - for job in jobs for step in job.get("steps") or []) + return REFUSE_STEP in failed_steps(repository, run_id) def frames_of(run_id: str, out: Path, repository: str) -> list[dict]: @@ -674,6 +698,9 @@ def tour(repository: str, name: str, scenario: Path, head_sha: str, out: Path, c if conclusion in ("success", "failure"): manifest["result"] = "passed" if conclusion == "success" else "failure" manifest.update(tour_media(dispatch.run_id, name, head_sha, out, repository)) + elif conclusion == "reuse_error": + manifest["note"] = ("skipped: the tour run could not check CI's build (a reuse error); " + "the next CI attempt tries again") else: manifest["note"] = (f"skipped: the tour run ended {conclusion or 'unfinished'}; " "the next CI attempt tries again") diff --git a/tests/test_ci_pr_media.py b/tests/test_ci_pr_media.py index 74cc24ba0ac1..ee5284f81191 100644 --- a/tests/test_ci_pr_media.py +++ b/tests/test_ci_pr_media.py @@ -331,6 +331,14 @@ def test_a_pull_request_on_mains_build_compiles_its_head(self) -> None: self.assertEqual(outputs["compile_tour"], "sidebar-and-chrome-tour") self.assertEqual(json.loads(outputs["run"]), ["sidebar-and-chrome-tour"]) + def test_app_changes_follow_cis_build_inputs(self) -> None: + self.assertTrue(media.reaches_app("config/IrohRelayPolicyProduction.xcconfig")) + self.assertTrue(media.reaches_app("Sources/ContentView.swift")) + self.assertTrue(media.reaches_app("Packages/Shared/CmuxAuthRuntime/Sources/A.swift")) + for path in ("CLI/cmux.swift", "cmuxTests/AppTests.swift", "docs/a.md", "web/app/page.tsx", "tests/test_x.py", "scripts/ci/pr_media.py"): + with self.subTest(path=path): + self.assertFalse(media.reaches_app(path)) + def test_only_app_changes_may_compile(self) -> None: outputs = self.plan({"repos/o/r/actions/workflows/ci.yml/runs": {"workflow_runs": []}, "repos/o/r/pulls/42/files": [[{"filename": "CLI/cmux.swift"}]]}, {}) @@ -495,6 +503,37 @@ def test_only_one_tour_per_head_compiles(self) -> None: self.assertEqual(len(self.commands), 1) self.assertIn("only other compiles", manifest["note"]) + def test_a_reuse_error_is_not_a_refusal(self) -> None: + original = media.failed_steps + media.failed_steps = lambda *_: {media.REUSE_ERROR_STEP} + self.addCleanup(setattr, media, "failed_steps", original) + manifest = self.run_tour("failure", {"gif": "tour.gif", "shots": []}, "fallback") + self.assertEqual(len(self.commands), 1) + self.assertEqual(manifest["result"], "not run") + self.assertIn("reuse error", manifest["note"]) + + def test_the_reuse_error_step_tells_errors_from_misses(self) -> None: + import subprocess + workflow = yaml.safe_load((ROOT / ".github/workflows/test-e2e.yml").read_text()) + step = next(step for job in workflow["jobs"].values() for step in job.get("steps", []) + if step.get("name") == media.REUSE_ERROR_STEP) + cases = [("success", "miss", "no_matching_contract_artifact,artifact_expired", 0), + ("success", "miss", "", 0), + ("success", "miss", "artifact_expired,artifact_listing_unavailable", 1), + ("success", "miss", "fingerprint_unavailable", 1), + ("success", "fallback", "reuse_api_or_validation_error", 1), + ("failure", "", "", 1)] + for outcome, reason, misses, expected in cases: + with self.subTest(outcome=outcome, reason=reason, misses=misses): + done = subprocess.run(["bash", "-eo", "pipefail", "-c", step["run"]], capture_output=True, text=True, + env={"PATH": os.environ["PATH"], "OUTCOME": outcome, "REASON": reason, + "MISSES": misses}) + self.assertEqual(done.returncode, expected, done.stdout + done.stderr) + + def test_the_reuse_error_step_is_named_as_in_test_e2e(self) -> None: + workflow = (ROOT / ".github/workflows/test-e2e.yml").read_text() + self.assertIn(f"- name: {media.REUSE_ERROR_STEP}", workflow) + def test_a_moved_head_does_not_compile(self) -> None: original = media.head_moved media.head_moved = lambda *_: True