PR media: classify app changes by CI's build inputs; a reuse error is no refusal

.github/workflows/pr-media.yml

@@ -92,6 +92,7 @@ jobs:
           sparse-checkout: |
             scripts/ci/pr_media.py
             scripts/ci/find_admitted_build.py
+            scripts/ci/product_input_identity.py
             dogfood/scenarios
           sparse-checkout-cone-mode: false
 

.github/workflows/test-e2e.yml

@@ -593,8 +593,32 @@ jobs:
 
       # PR media tours (pr-media.yml) ask for CI's product and never pay for
       # a second compile of the app.
-      - name: Refuse to compile for a dispatch that requires an adopted product
+      # A reuse error is no evidence the product is unloadable, so it fails
+      # under its own name (pr_media.py compiles only after a refusal). The
+      # lookup records API and transfer failures as miss reasons, so those
+      # count as errors too.
+      - name: Fail a dispatch that requires an adopted product when reuse errored
+        id: reuse_error
         if: ${{ inputs.require_adopted_product && steps.reuse.outputs.hit != 'true' }}
+        env:
+          OUTCOME: ${{ steps.reuse.outcome }}
+          REASON: ${{ steps.reuse.outputs.reason }}
+          MISSES: ${{ steps.reuse.outputs.miss_reasons }}
+        run: |
+          errors=",artifact_listing_unavailable,consumer_provenance_unavailable,artifact_download_error,fingerprint_unavailable,reuse_api_or_validation_error,"
+          errored=""
+          if [ "$OUTCOME" != success ] || [ "$REASON" != miss ]; then errored=1; fi
+          IFS=, read -ra reasons <<<"$MISSES"
+          for reason in "${reasons[@]}"; do
+            case "$errors" in *",$reason,"*) errored=1 ;; esac
+          done
+          if [ -n "$errored" ]; then
+            echo "::error::Compiled-product reuse did not complete (${MISSES:-no reason recorded}), so this dispatch cannot tell whether a product loads."
+            exit 1
+          fi
+
+      - name: Refuse to compile for a dispatch that requires an adopted product
+        if: ${{ inputs.require_adopted_product && steps.reuse.outputs.hit != 'true' && steps.reuse_error.outcome == 'success' }}
         env:
           MISSES: ${{ steps.reuse.outputs.miss_reasons }}
         run: |

scripts/ci/pr_media.py

@@ -63,12 +63,9 @@
 # its runner (an app pull request), or straight away (a manual dispatch).
 COMPILE_NEVER, COMPILE_FALLBACK, COMPILE_NOW = "never", "fallback", "now"
 COMPILE_MODES = (COMPILE_NEVER, COMPILE_FALLBACK, COMPILE_NOW)
-# Changes to these build the app; a pull request touching none (a CLI-only
-# one, say) never compiles an app just for its tours.
-APP_PATH_PREFIXES = ("Sources/", "Packages/macOS/", "Packages/Shared/", "TunnelExtension/", "Resources/",
-                     "Assets.xcassets/", "AppIcon.icon/", "vendor/", "ghostty", "cmux.xcodeproj/",
-                     "cmux-Bridging-Header.h", "cmux.entitlements", "cmux-helper.entitlements", "webviews/",
-                     "cmuxUITests/", f"{SCENARIOS_DIR}/")
+# Product inputs no tour shows (the CLI lane, the app-host unit tests): a
+# pull request that only changes these never compiles an app for its tours.
+NON_TOUR_PRODUCT_PREFIXES = ("CLI/", "cmuxCLITests/", "cmuxCLITestSupport/", "cmuxTests/")
 GATE_WAIT_SECONDS = 25 * 60
 # Reads share the repository's token budget with the dispatcher, so waits poll slowly.
 GATE_POLL_SECONDS = 60
@@ -77,6 +74,8 @@
 ADMISSION_JOB_SUFFIX = "macOS compile admission"
 # test-e2e.yml's step that fails a require_adopted_product run whose reuse missed.
 REFUSE_STEP = "Refuse to compile for a dispatch that requires an adopted product"
+# ... and the one that fails it when reuse errored (no evidence either way).
+REUSE_ERROR_STEP = "Fail a dispatch that requires an adopted product when reuse errored"
 MERGE_REF = re.compile(r"refs/pull/\d+/merge")
 TESTED_LINE = re.compile(r"^Testing \S+ at ([0-9a-f]{40}) \(request ")
 MAX_KEY_SHOTS = 4
@@ -182,6 +181,22 @@ def select_tours(scenarios: dict[str, object], changed: Iterable[str], body: str
     return [], "no tour matched and the default tour is missing"
 
 
+def reaches_app(path: str) -> bool:
+    """Whether a changed path can change the app a tour shows: an input of the
+    app-host product (product_input_identity.reaches_product, what CI keys its
+    build on) outside NON_TOUR_PRODUCT_PREFIXES."""
+    import importlib.util
+    spec = importlib.util.spec_from_file_location("product_input_identity",
+                                                  ROOT / "scripts/ci/product_input_identity.py")
+    assert spec and spec.loader
+    identity = sys.modules.get(spec.name)
+    if identity is None:
+        identity = importlib.util.module_from_spec(spec)
+        sys.modules[spec.name] = identity
+        spec.loader.exec_module(identity)
+    return identity.reaches_product(path) and not path.startswith(NON_TOUR_PRODUCT_PREFIXES)
+
+
 def head_scenarios(head_sha: str) -> dict[str, object]:
     """Tours at the head commit, read from git objects (never checked out or run)."""
     listing = subprocess.run(["git", "ls-tree", "--name-only", head_sha, f"{SCENARIOS_DIR}/"],
@@ -396,7 +411,7 @@ def plan(repository: str) -> int:
             except json.JSONDecodeError:
                 pass
     tours, reason = select_tours(scenarios, changed, pull.get("body"))
-    app_change = any(path.startswith(APP_PATH_PREFIXES) for path in changed)
+    app_change = any(reaches_app(path) for path in changed)
     force = os.environ.get("FORCE", "").lower() == "true"
     pending = [tour for tour in tours if force or published(repository, pr, head_sha, tour) is None]
     print(f"#{pr} at {head_sha}: tours {tours or 'none'} ({reason}); to run: {pending or 'none'}", flush=True)
@@ -478,19 +493,28 @@ def wait(self, sleep: Callable[[float], None] = time.sleep) -> dict:
 
 def refused_after(dispatch: "Dispatch", repository: str) -> bool:
     """Wait for an adopt-only tour run; whether it stopped because it could
-    not load CI's build. The finished run is kept on `dispatch.completed`."""
+    not load CI's build. The finished run is kept on `dispatch.completed`;
+    a run whose reuse errored is kept with conclusion "reuse_error"."""
     run = dispatch.wait()
-    if run.get("conclusion") == "failure" and refused_to_compile(repository, str(dispatch.run_id)):
+    if run.get("conclusion") != "failure":
+        dispatch.completed = run
+        return False
+    if refused_to_compile(repository, str(dispatch.run_id)):
         return True
-    dispatch.completed = run
+    errored = REUSE_ERROR_STEP in failed_steps(repository, str(dispatch.run_id))
+    dispatch.completed = {**run, "conclusion": "reuse_error"} if errored else run
     return False
 
 
+def failed_steps(repository: str, run_id: str) -> set[str]:
+    jobs = (gh_json([f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=100"]) or {}).get("jobs", [])
+    return {str(step.get("name")) for job in jobs for step in job.get("steps") or []
+            if step.get("conclusion") == "failure"}
+
+
 def refused_to_compile(repository: str, run_id: str) -> bool:
     """Whether the tour run stopped because it could not load CI's build."""
-    jobs = (gh_json([f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=100"]) or {}).get("jobs", [])
-    return any(step.get("name") == REFUSE_STEP and step.get("conclusion") == "failure"
-               for job in jobs for step in job.get("steps") or [])
+    return REFUSE_STEP in failed_steps(repository, run_id)
 
 
 def frames_of(run_id: str, out: Path, repository: str) -> list[dict]:
@@ -674,6 +698,9 @@ def tour(repository: str, name: str, scenario: Path, head_sha: str, out: Path, c
             if conclusion in ("success", "failure"):
                 manifest["result"] = "passed" if conclusion == "success" else "failure"
                 manifest.update(tour_media(dispatch.run_id, name, head_sha, out, repository))
+            elif conclusion == "reuse_error":
+                manifest["note"] = ("skipped: the tour run could not check CI's build (a reuse error); "
+                                    "the next CI attempt tries again")
             else:
                 manifest["note"] = (f"skipped: the tour run ended {conclusion or 'unfinished'}; "
                                     "the next CI attempt tries again")

tests/test_ci_pr_media.py

@@ -331,6 +331,14 @@ def test_a_pull_request_on_mains_build_compiles_its_head(self) -> None:
         self.assertEqual(outputs["compile_tour"], "sidebar-and-chrome-tour")
         self.assertEqual(json.loads(outputs["run"]), ["sidebar-and-chrome-tour"])
 
+    def test_app_changes_follow_cis_build_inputs(self) -> None:
+        self.assertTrue(media.reaches_app("config/IrohRelayPolicyProduction.xcconfig"))
+        self.assertTrue(media.reaches_app("Sources/ContentView.swift"))
+        self.assertTrue(media.reaches_app("Packages/Shared/CmuxAuthRuntime/Sources/A.swift"))
+        for path in ("CLI/cmux.swift", "cmuxTests/AppTests.swift", "docs/a.md", "web/app/page.tsx", "tests/test_x.py", "scripts/ci/pr_media.py"):
+            with self.subTest(path=path):
+                self.assertFalse(media.reaches_app(path))
+
     def test_only_app_changes_may_compile(self) -> None:
         outputs = self.plan({"repos/o/r/actions/workflows/ci.yml/runs": {"workflow_runs": []},
                              "repos/o/r/pulls/42/files": [[{"filename": "CLI/cmux.swift"}]]}, {})
@@ -495,6 +503,37 @@ def test_only_one_tour_per_head_compiles(self) -> None:
         self.assertEqual(len(self.commands), 1)
         self.assertIn("only other compiles", manifest["note"])
 
+    def test_a_reuse_error_is_not_a_refusal(self) -> None:
+        original = media.failed_steps
+        media.failed_steps = lambda *_: {media.REUSE_ERROR_STEP}
+        self.addCleanup(setattr, media, "failed_steps", original)
+        manifest = self.run_tour("failure", {"gif": "tour.gif", "shots": []}, "fallback")
+        self.assertEqual(len(self.commands), 1)
+        self.assertEqual(manifest["result"], "not run")
+        self.assertIn("reuse error", manifest["note"])
+
+    def test_the_reuse_error_step_tells_errors_from_misses(self) -> None:
+        import subprocess
+        workflow = yaml.safe_load((ROOT / ".github/workflows/test-e2e.yml").read_text())
+        step = next(step for job in workflow["jobs"].values() for step in job.get("steps", [])
+                    if step.get("name") == media.REUSE_ERROR_STEP)
+        cases = [("success", "miss", "no_matching_contract_artifact,artifact_expired", 0),
+                 ("success", "miss", "", 0),
+                 ("success", "miss", "artifact_expired,artifact_listing_unavailable", 1),
+                 ("success", "miss", "fingerprint_unavailable", 1),
+                 ("success", "fallback", "reuse_api_or_validation_error", 1),
+                 ("failure", "", "", 1)]
+        for outcome, reason, misses, expected in cases:
+            with self.subTest(outcome=outcome, reason=reason, misses=misses):
+                done = subprocess.run(["bash", "-eo", "pipefail", "-c", step["run"]], capture_output=True, text=True,
+                                      env={"PATH": os.environ["PATH"], "OUTCOME": outcome, "REASON": reason,
+                                           "MISSES": misses})
+                self.assertEqual(done.returncode, expected, done.stdout + done.stderr)
+
+    def test_the_reuse_error_step_is_named_as_in_test_e2e(self) -> None:
+        workflow = (ROOT / ".github/workflows/test-e2e.yml").read_text()
+        self.assertIn(f"- name: {media.REUSE_ERROR_STEP}", workflow)
+
     def test_a_moved_head_does_not_compile(self) -> None:
         original = media.head_moved
         media.head_moved = lambda *_: True