From bfa7a7d55cd6c9184063868b6439762379031017 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 27 Sep 2026 06:45:55 -0400 Subject: [PATCH 1/2] UI test frames: sample XCTest screen recordings; SIGKILL stuck prompts - On cmux12s, XCTest attached a screen recording of each failing test and no step screenshots, so e2e-frames.py found 0 frames. It now samples .mp4/.mov attachments at 2 fps into the same frames and sheets (17 frames for run 36312258398's Settings test). - The dialog step sends SIGKILL. A SecurityAgent showing a prompt ignores SIGTERM: on cmux12s the step's pkill reported success and the 7 h old keychain prompt stayed; SIGKILL closed it, and with glaeda#1309 unlocking the keychain in the GUI session no new prompt appeared. Co-Authored-By: Claude Opus 5.5 --- .github/actions/e2e-run-tests/action.yml | 4 +- scripts/ci/e2e-frames.py | 50 +++++++++++++++++-- .../cmux-testing/references/ui-test-frames.md | 1 + 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.github/actions/e2e-run-tests/action.yml b/.github/actions/e2e-run-tests/action.yml index 434664c69b7a..9d6c9fd50520 100644 --- a/.github/actions/e2e-run-tests/action.yml +++ b/.github/actions/e2e-run-tests/action.yml @@ -316,8 +316,10 @@ runs: run: | defaults write com.apple.CrashReporter DialogType none 2>/dev/null || true uid="$(id -u)" + # SIGKILL: a SecurityAgent showing a prompt ignores SIGTERM (cmux12s, + # 2026-09-27: pkill "succeeded" and the 7 h old prompt stayed). for process in UserNotificationCenter SecurityAgent; do - if pkill -x -u "$uid" "$process" 2>/dev/null; then + if pkill -KILL -x -u "$uid" "$process" 2>/dev/null; then echo "Closed $process (its dialogs are cancelled; launchd restarts it on demand)" fi done diff --git a/scripts/ci/e2e-frames.py b/scripts/ci/e2e-frames.py index bf1ddaa2eeac..9c48b160961a 100644 --- a/scripts/ci/e2e-frames.py +++ b/scripts/ci/e2e-frames.py @@ -37,6 +37,10 @@ REPO = "manaflow-ai/cmux" ARTIFACT = "test-results" IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".heic"} +# Some hosts' XCTest attaches a screen recording of each failing test instead +# of per-step screenshots; frames are sampled from it. +VIDEO_SUFFIXES = {".mp4", ".mov"} +VIDEO_FPS = 2 FRAME_WIDTH = 960 SHEET_COLUMNS, SHEET_ROWS = 3, 4 SHEET_TILE_WIDTH = 640 @@ -108,6 +112,23 @@ def to_png(source: Path, destination: Path) -> None: run(["sips", "-s", "format", "png", "--resampleWidth", str(FRAME_WIDTH), str(source), "--out", str(destination)]) +def video_frames(source: Path, frames: Path, start: int) -> int: + """Sample a recording at VIDEO_FPS into frames numbered after `start`; returns how many.""" + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + return 0 + with tempfile.TemporaryDirectory() as tmp: + subprocess.run( + [ffmpeg, "-loglevel", "error", "-y", "-i", str(source), + "-vf", f"fps={VIDEO_FPS},scale={FRAME_WIDTH}:-2", str(Path(tmp) / "%04d.png")], + check=False, + ) + written = sorted(Path(tmp).glob("*.png")) + for offset, frame in enumerate(written, start=1): + shutil.move(frame, frames / f"{start + offset:03d}-video.png") + return len(written) + + def build_sheets(frames: Path, test_dir: Path) -> list[Path]: ffmpeg = shutil.which("ffmpeg") if not ffmpeg or not any(frames.iterdir()): @@ -195,20 +216,36 @@ def main() -> int: shutil.rmtree(test_dir) frames.mkdir(parents=True) - images = sorted( + media = sorted( (a for a in entry.get("attachments", []) - if Path(a.get("exportedFileName", "")).suffix.lower() in IMAGE_SUFFIXES), + if Path(a.get("exportedFileName", "")).suffix.lower() in IMAGE_SUFFIXES | VIDEO_SUFFIXES), key=lambda a: a.get("timestamp", 0), ) - captures, failure_frame = [], None - for index, attachment in enumerate(images, start=1): + captures, failure_frame, recordings, index = [], None, [], 0 + images = [] + for attachment in media: name = attachment.get("suggestedHumanReadableName", "") + source = exported / attachment["exportedFileName"] + if source.suffix.lower() in VIDEO_SUFFIXES: + recordings.append(str(source)) + written = video_frames(source, frames, index) + if not written: + print(f"skipped recording {source.name} ({identifier}): needs ffmpeg", file=sys.stderr) + images.extend([attachment] * written) + index += written + if written and failure_frame is None and attachment.get("isAssociatedWithFailure"): + # A recording ends at the failure. + failure_frame = str(frames / f"{index:03d}-video.png") + continue + index += 1 frame = frames / f"{index:03d}-{label_for(name)}.png" try: - to_png(exported / attachment["exportedFileName"], frame) + to_png(source, frame) except subprocess.CalledProcessError: print(f"skipped unreadable image {attachment['exportedFileName']} ({identifier})", file=sys.stderr) + index -= 1 continue + images.append(attachment) if not name.startswith("Screenshot "): captures.append(str(frame)) if attachment.get("isAssociatedWithFailure") and failure_frame is None: @@ -222,6 +259,7 @@ def main() -> int: "frames": len(images), "captures": captures, "failure_frame": failure_frame, + "recordings": recordings, "sheets": [str(p) for p in build_sheets(frames, test_dir)], "slideshow": str(test_dir / "steps.mp4") if (test_dir / "steps.mp4").exists() else None, "dir": str(test_dir), @@ -241,6 +279,8 @@ def main() -> int: print(f" failure: {failure.splitlines()[0] if failure else ''}") if item["failure_frame"]: print(f" at failure: {item['failure_frame']}") + for recording in item.get("recordings", []): + print(f" recording: {recording}") for capture in item["captures"]: print(f" capture: {capture}") for sheet in item["sheets"]: diff --git a/skills/cmux-testing/references/ui-test-frames.md b/skills/cmux-testing/references/ui-test-frames.md index 2e865797b57b..5aa5dd3521d8 100644 --- a/skills/cmux-testing/references/ui-test-frames.md +++ b/skills/cmux-testing/references/ui-test-frames.md @@ -29,6 +29,7 @@ Under `$TMPDIR/cmux-e2e-frames////`: | `sheet-N.png` | 3x4 grid of the steps in time order, 1920 px wide. Open these first. | | `frames/NNN-step.png` | XCUITest's screenshot for one step, 960 px wide. | | `frames/NNN-.png` | A named capture the test attached. | +| `frames/NNN-video.png` | Frames sampled at 2 fps from XCTest's screen recording, which some hosts attach to a failing test instead of step screenshots. | | `steps.mp4` | The frames at 2 fps, for a person to scrub. | `--json` prints the same summary for scripts. From a64682c7e9a0da0d1e9243e10528463e78e8d295 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 27 Sep 2026 06:52:19 -0400 Subject: [PATCH 2/2] UI test frames: find the failure inside a recording; review fixes - XCTest flags the failure's snapshot and logs, not the recording, and stamps a recording with its start; the failure frame is now taken at the first failure-flagged timestamp inside the clip (Settings test of run 36312258398: frame 17, the failure at 8.3 s). - Attachments with a null timestamp no longer crash the summary. - Frame numbers are 4 digits, so long recordings keep their order. - A failed ffmpeg run no longer reports "needs ffmpeg", and tests with no attachments carry the recordings key in --json. Co-Authored-By: Claude Opus 5.5 --- scripts/ci/e2e-frames.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/scripts/ci/e2e-frames.py b/scripts/ci/e2e-frames.py index 9c48b160961a..fa9f45793937 100644 --- a/scripts/ci/e2e-frames.py +++ b/scripts/ci/e2e-frames.py @@ -125,7 +125,7 @@ def video_frames(source: Path, frames: Path, start: int) -> int: ) written = sorted(Path(tmp).glob("*.png")) for offset, frame in enumerate(written, start=1): - shutil.move(frame, frames / f"{start + offset:03d}-video.png") + shutil.move(frame, frames / f"{start + offset:04d}-video.png") return len(written) @@ -202,7 +202,7 @@ def main() -> int: # captures kept with `.keepAlways` survive. summary.append({ "test": identifier, "result": outcome["result"], "failures": outcome["failures"], - "frames": 0, "captures": [], "failure_frame": None, "sheets": [], "slideshow": None, "dir": None, + "frames": 0, "captures": [], "failure_frame": None, "recordings": [], "sheets": [], "slideshow": None, "dir": None, }) for entry in manifest: @@ -219,9 +219,10 @@ def main() -> int: media = sorted( (a for a in entry.get("attachments", []) if Path(a.get("exportedFileName", "")).suffix.lower() in IMAGE_SUFFIXES | VIDEO_SUFFIXES), - key=lambda a: a.get("timestamp", 0), + key=lambda a: a.get("timestamp") or 0, ) captures, failure_frame, recordings, index = [], None, [], 0 + clips = [] # (start timestamp, frames before it, frames written) per recording images = [] for attachment in media: name = attachment.get("suggestedHumanReadableName", "") @@ -230,15 +231,15 @@ def main() -> int: recordings.append(str(source)) written = video_frames(source, frames, index) if not written: - print(f"skipped recording {source.name} ({identifier}): needs ffmpeg", file=sys.stderr) + why = "ffmpeg failed" if shutil.which("ffmpeg") else "needs ffmpeg" + print(f"skipped recording {source.name} ({identifier}): {why}", file=sys.stderr) + else: + clips.append((attachment.get("timestamp") or 0, index, written)) images.extend([attachment] * written) index += written - if written and failure_frame is None and attachment.get("isAssociatedWithFailure"): - # A recording ends at the failure. - failure_frame = str(frames / f"{index:03d}-video.png") continue index += 1 - frame = frames / f"{index:03d}-{label_for(name)}.png" + frame = frames / f"{index:04d}-{label_for(name)}.png" try: to_png(source, frame) except subprocess.CalledProcessError: @@ -251,6 +252,17 @@ def main() -> int: if attachment.get("isAssociatedWithFailure") and failure_frame is None: failure_frame = str(frame) + # XCTest flags the failure's snapshot and logs, not the recording, and + # stamps a recording with its start; find the failure inside the clip. + failed_at = min((a["timestamp"] for a in entry.get("attachments", []) + if a.get("isAssociatedWithFailure") and a.get("timestamp") is not None), + default=None) + if failure_frame is None and failed_at is not None: + for start_ts, before, written in clips: + offset = int((failed_at - start_ts) * VIDEO_FPS) + 1 + if 1 <= offset: + failure_frame = str(frames / f"{before + min(offset, written):04d}-video.png") + outcome = results.get(identifier, {"result": "?", "failures": []}) summary.append({ "test": identifier,