From 2fdefe5369d74d4c3d5c5fa6e9739832aeb08562 Mon Sep 17 00:00:00 2001 From: teamleaderleo Date: Wed, 30 Sep 2026 04:06:43 -0400 Subject: [PATCH] feat: add a static UI iteration mirror --- .github/actions/e2e-run-tests/action.yml | 6 + .github/workflows/test-e2e.yml | 2 + scripts/ci/e2e-frames.py | 18 ++ scripts/ci/ui-mirror.py | 238 ++++++++++++++++++ .../cmux-testing/references/ui-test-frames.md | 10 + tests/test_ci_e2e_compilation_cache.py | 2 +- tests/test_ui_mirror.py | 70 ++++++ 7 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 scripts/ci/ui-mirror.py create mode 100644 tests/test_ui_mirror.py diff --git a/.github/actions/e2e-run-tests/action.yml b/.github/actions/e2e-run-tests/action.yml index 288d7f56cf68..7175259ba959 100644 --- a/.github/actions/e2e-run-tests/action.yml +++ b/.github/actions/e2e-run-tests/action.yml @@ -829,6 +829,7 @@ runs: - name: Upload UI test steps # No hashFiles gate: it ignores paths outside the workspace, and # runner.temp is outside it. if-no-files-found covers an empty build. + id: ui-frames if: ${{ always() && inputs.target == 'cmuxUITests' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -850,6 +851,7 @@ runs: RUN_ID: ${{ github.run_id }} RECORD_VIDEO: ${{ inputs.record-video }} RECORDING_URL: ${{ steps.recording.outputs.artifact-url }} + UI_FRAMES_URL: ${{ steps.ui-frames.outputs.artifact-url }} shell: bash run: | set -euo pipefail @@ -885,6 +887,10 @@ runs: fi fi + if [ -n "$UI_FRAMES_URL" ]; then + echo "**UI mirror:** [Download the interactive mirror artifact]($UI_FRAMES_URL) (open \`index.html\` locally)" + fi + if [ -n "$TEST_OUTPUT" ]; then echo echo "
Test output (last 200 lines)" diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 4bdd474d6727..35eb32f2cfe3 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1145,6 +1145,7 @@ jobs: sparse-checkout: | .github/actions/e2e-run-tests scripts/ci/e2e-frames.py + scripts/ci/ui-mirror.py sparse-checkout-cone-mode: false persist-credentials: false @@ -1364,6 +1365,7 @@ jobs: sparse-checkout: | .github/actions/e2e-run-tests scripts/ci/e2e-frames.py + scripts/ci/ui-mirror.py sparse-checkout-cone-mode: false persist-credentials: false diff --git a/scripts/ci/e2e-frames.py b/scripts/ci/e2e-frames.py index 95ac6b45c4d1..c9a2fef36e25 100644 --- a/scripts/ci/e2e-frames.py +++ b/scripts/ci/e2e-frames.py @@ -9,6 +9,8 @@ frames/NN-.jpg the screen right after that action sheet-N.jpg 3x4 contact sheets, each tile captioned with its action attachments/ text a test attached (a dogfood tour's trees, socket replies, step log) + index.html an interactive local-first UI mirror of the run + mirror.json the mirror's machine-readable manifest A frame is the last screenshot XCUITest saved under a top-level action. On hosts where XCTest keeps a screen recording of a failing test instead, the recording @@ -385,6 +387,21 @@ def markdown(summary: list[dict]) -> str: return "\n".join(lines) + "\n" +def write_mirror(out: Path, quiet: bool = False) -> None: + """Build the interactive mirror next to the extracted frames.""" + mirror = Path(__file__).with_name("ui-mirror.py") + result = subprocess.run( + [sys.executable, str(mirror), str(out)], + check=False, + text=True, + capture_output=True, + ) + if result.returncode == 0 and not quiet: + print(result.stdout.strip()) + elif result.returncode != 0: + print(f"warning: could not build UI mirror: {result.stderr.strip()[:240]}", file=sys.stderr) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("source", help="run id, run URL, an .xcresult, or a directory of them") @@ -417,6 +434,7 @@ def main() -> int: with args.summary.open("a") as handle: handle.write(markdown(summary)) + write_mirror(out, quiet=args.json) if args.json: print(json.dumps(summary, indent=2)) return 0 diff --git a/scripts/ci/ui-mirror.py b/scripts/ci/ui-mirror.py new file mode 100644 index 000000000000..8de609c9318c --- /dev/null +++ b/scripts/ci/ui-mirror.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Build a small, local-first frontend for cmux UI-test artifacts. + +The E2E lane already exports screenshots, accessibility trees, socket replies, +and step logs. This turns that directory into an interactive mirror that can +be opened with a browser without a server or a JavaScript dependency. + + python3 scripts/ci/ui-mirror.py "$TMPDIR/cmux-ui-frames/123" + +The generated ``index.html`` is deliberately static: it can be uploaded as a +CI artifact, copied to another machine, or opened with ``open``. It references +the images and text attachments next to it instead of embedding or uploading +their contents anywhere. +""" + +from __future__ import annotations + +import argparse +import html +import json +import re +from pathlib import Path +from typing import Any + + +STEP = re.compile( + r"^(?P\d+)\. (?P.*?) \(frames/(?P<frame>[^)]+)\)" + r"(?P<failed> <- failed here)?$" +) + + +def relative(path: Path, root: Path) -> str: + """Return a browser-safe path for a file beneath ``root``.""" + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError as error: + raise ValueError(f"artifact path escapes mirror root: {path}") from error + + +def read_test(test_dir: Path, root: Path) -> dict[str, Any]: + steps_file = test_dir / "steps.md" + lines = steps_file.read_text(encoding="utf-8", errors="replace").splitlines() + heading = lines[0] if lines else test_dir.name + identifier, separator, result = heading[2:].rpartition(": ") + if not separator: + identifier, result = heading, "Unknown" + + failures = [line.removeprefix("Failure: ") for line in lines if line.startswith("Failure: ")] + steps: list[dict[str, Any]] = [] + for line in lines: + match = STEP.match(line) + if not match: + continue + frame = test_dir / "frames" / match.group("frame") + if not frame.is_file(): + continue + steps.append( + { + "number": int(match.group("number")), + "title": match.group("title"), + "frame": relative(frame, root), + "failed": bool(match.group("failed")), + } + ) + + attachments: list[dict[str, str]] = [] + attachment_dir = test_dir / "attachments" + if attachment_dir.is_dir(): + for path in sorted(p for p in attachment_dir.rglob("*") if p.is_file()): + attachment: dict[str, str] = { + "name": path.relative_to(attachment_dir).as_posix(), + "path": relative(path, root), + } + if path.suffix.lower() in {".txt", ".log", ".json", ".md"}: + # Keep the mirror usable from file://, where fetch() is often + # blocked by the browser's local-file origin policy. + attachment["text"] = path.read_text( + encoding="utf-8", errors="replace" + )[:200_000] + attachments.append(attachment) + + sheets = [ + relative(path, root) + for path in sorted(test_dir.glob("sheet-*.jpg")) + if path.is_file() + ] + return { + "id": identifier, + "result": result, + "failures": failures, + "steps": steps, + "attachments": attachments, + "sheets": sheets, + } + + +def collect(root: Path) -> list[dict[str, Any]]: + tests = [] + for steps_file in sorted(root.rglob("steps.md")): + tests.append(read_test(steps_file.parent, root)) + return tests + + +def write_mirror(root: Path, tests: list[dict[str, Any]]) -> Path: + payload = json.dumps( + {"version": 1, "root": root.name, "tests": tests}, + ensure_ascii=False, + separators=(",", ":"), + ).replace("<", "\\u003c") + title = html.escape(root.name) + document = f"""<!doctype html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>cmux UI mirror · {title} + + + + +
+

Select a test

+
Choose a test and step to inspect the UI.
+
+
+
+
Failures
Attachments

+

+
+
+
+
+"""
+    output = root / "index.html"
+    output.write_text(document, encoding="utf-8")
+    (root / "mirror.json").write_text(
+        json.dumps({"version": 1, "root": root.name, "tests": tests}, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    return output
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("artifact", type=Path, help="a ui-frames artifact or extracted e2e-frames directory")
+    args = parser.parse_args()
+    root = args.artifact.resolve()
+    if not root.is_dir():
+        parser.error(f"artifact directory does not exist: {root}")
+    tests = collect(root)
+    output = write_mirror(root, tests)
+    print(f"cmux UI mirror: {output}")
+    print(f"tests: {len(tests)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/skills/cmux-testing/references/ui-test-frames.md b/skills/cmux-testing/references/ui-test-frames.md
index 2bbe7c122f80..7803ec094a8c 100644
--- a/skills/cmux-testing/references/ui-test-frames.md
+++ b/skills/cmux-testing/references/ui-test-frames.md
@@ -10,6 +10,11 @@ It prints each test's result, its failure, the action it failed at, and paths to
 - `steps.md`: the test as numbered actions ("Click "SidebarHelpMenuOptionSettings" MenuItem"), with the failing one marked.
 - `frames/NN-.jpg`: the screen right after each action. Open the failing one first.
 - `sheet-N.jpg`: 3x4 contact sheets of the frames, tile N being step N.
+- `index.html`: a local-first cmux UI mirror with a searchable test list, step
+  timeline, large screenshot viewer, contact sheets, failures, and text
+  attachments. Open it directly in a browser; it needs no server or build.
+- `mirror.json`: the same run manifest in a machine-readable form for other
+  review tools.
 
 CI builds these for every UI run: the action lists are in the run's job summary, and the frames in its `ui-frames` artifact, which `scripts/ui-test` downloads.
 
@@ -19,3 +24,8 @@ Keep in mind:
 - **Expected Failure** is not a pass. It usually means the harness absorbed a launch or activation failure and the test ended before the behavior under test.
 - Hosts that record a failing test's screen instead of screenshots give one frame a second, captioned with the running action.
 - If a frame shows a system dialog over the app, that explains focus and activation failures. The E2E action closes leftover dialogs before tests run.
+
+The mirror is generated automatically whenever `e2e-frames.py` extracts a run,
+including the `ui-frames` artifact that CI uploads. That makes a built app and a
+JSON dogfood tour enough to produce a reviewable frontend artifact: iterate on
+the tour or the UI branch, dispatch again, and compare the resulting mirrors.
diff --git a/tests/test_ci_e2e_compilation_cache.py b/tests/test_ci_e2e_compilation_cache.py
index 49895fe472bb..aa4e330478b8 100644
--- a/tests/test_ci_e2e_compilation_cache.py
+++ b/tests/test_ci_e2e_compilation_cache.py
@@ -231,7 +231,7 @@ def test_the_build_runner_runs_the_tests_so_a_run_queues_once(self):
             self.assertEqual(checkout['with']['path'], '.e2e-workflow')
             self.assertEqual(
                 checkout['with']['sparse-checkout'].split(),
-                ['.github/actions/e2e-run-tests', 'scripts/ci/e2e-frames.py'],
+                ['.github/actions/e2e-run-tests', 'scripts/ci/e2e-frames.py', 'scripts/ci/ui-mirror.py'],
             )
         # The build job's budget covers compiling and testing.
         self.assertEqual(WORKFLOW['jobs']['build']['timeout-minutes'],
diff --git a/tests/test_ui_mirror.py b/tests/test_ui_mirror.py
new file mode 100644
index 000000000000..9e5416217731
--- /dev/null
+++ b/tests/test_ui_mirror.py
@@ -0,0 +1,70 @@
+import importlib.util
+import json
+import pathlib
+import tempfile
+import unittest
+
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+SPEC = importlib.util.spec_from_file_location("cmux_ui_mirror", ROOT / "scripts/ci/ui-mirror.py")
+assert SPEC and SPEC.loader
+MIRROR = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MIRROR)
+
+
+class UIMirrorTests(unittest.TestCase):
+    def make_artifact(self, root: pathlib.Path) -> pathlib.Path:
+        test = root / "DogfoodScenarioUITests" / "testRunScenario"
+        (test / "frames").mkdir(parents=True)
+        (test / "attachments").mkdir()
+        (test / "steps.md").write_text(
+            "# DogfoodScenarioUITests/testRunScenario: Passed\n\n"
+            "1. Click Preview  (frames/0001-click-preview.jpg)\n"
+            "2. Final  (frames/0002-final.jpg)  <- failed here\n"
+            "Failure: expected preview\n",
+            encoding="utf-8",
+        )
+        for name in ("0001-click-preview.jpg", "0002-final.jpg"):
+            (test / "frames" / name).write_bytes(b"jpg")
+        (test / "attachments" / "01-reply.json").write_text(
+            '{"ok":true}\n', encoding="utf-8"
+        )
+        (test / "attachments" / "02-clip.mp4").write_bytes(b"video")
+        return test
+
+    def test_collects_steps_failures_and_attachments(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="cmux-ui-mirror-") as directory:
+            root = pathlib.Path(directory)
+            self.make_artifact(root)
+            tests = MIRROR.collect(root)
+
+            self.assertEqual(len(tests), 1)
+            self.assertEqual(len(tests[0]["steps"]), 2)
+            self.assertTrue(tests[0]["steps"][1]["failed"])
+            self.assertEqual(tests[0]["attachments"][0]["name"], "01-reply.json")
+            self.assertEqual(tests[0]["attachments"][1]["name"], "02-clip.mp4")
+
+    def test_manifest_and_html_are_written_without_script_breakout(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="cmux-ui-mirror-") as directory:
+            root = pathlib.Path(directory)
+            test = self.make_artifact(root)
+            steps = test / "steps.md"
+            steps.write_text(
+                steps.read_text(encoding="utf-8").replace(
+                    "Click Preview", "Click