diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 3dd23c357083..dc28f1f6799f 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -257,6 +257,10 @@ jobs: if: ${{ matrix.group == 'preflight' }} run: python3 tests/test_pr_media.py + - name: Test the cmux-next tour media + if: ${{ matrix.group == 'preflight' }} + run: python3 tests/test_cmux_next_media.py + - name: Test XCStrings catalog lint if: ${{ matrix.group == 'preflight' }} run: python3 tests/test_lint_xcstrings.py diff --git a/.github/workflows/cmux-next-media.yml b/.github/workflows/cmux-next-media.yml new file mode 100644 index 000000000000..6ac6ef98c6d9 --- /dev/null +++ b/.github/workflows/cmux-next-media.yml @@ -0,0 +1,227 @@ +name: cmux-next media + +# Screenshots and a short video of each cmux-next UI pull request, posted in +# one sticky comment (scripts/cmux-next/media/publish.py). The tour builds the +# head as a tagged dev app on an owned Mac's gui runner, launches it, drives it +# through the bundled cmux CLI (scripts/cmux-next/media/tour.py) and captures +# every step. Tours are data in scripts/cmux-next/media/tours. +# +# Not a required check. It never runs on Blacksmith, whose macOS sessions sit +# at a locked screen and cannot capture: with no owned gui runner the tour +# does not run and the comment says why (scripts/ci/cmux_next_media_runner.py). +# Same-repository heads only: owned Macs run trusted code, and the publish job +# holds the pr-media write token. That job never checks out or runs the pull +# request's code: its scripts come from the base revision, and it reads the +# tour artifact as data (publish.py validates names and paths before upload). +# A pull_request run still takes this file from the pull request, so the +# boundary that holds is same-repository write access: anyone who can push a +# branch here could edit these jobs, and forks never reach them. + +on: + pull_request: + branches: [feat-cmux-next, feat-cmux-next-acpmux] + types: [opened, synchronize, reopened] + paths: + - Packages/macOS/CmuxNext/Sources/** + - webviews/src/** + - scripts/cmux-next/media/** + - scripts/ci/cmux_next_media_runner.py + - .github/workflows/cmux-next-media.yml + +permissions: {} + +concurrency: + group: cmux-next-media-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + route: + name: Pick the tour runner + if: ${{ github.repository_owner == 'manaflow-ai' && github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 5 + permissions: + contents: read + outputs: + runs_on: ${{ steps.pick.outputs.runs_on }} + reason: ${{ steps.pick.outputs.reason }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + scripts/ci/cmux_next_media_runner.py + scripts/ci/pr_runner_pool.py + sparse-checkout-cone-mode: false + + - name: Pick an owned gui runner + id: pick + env: + POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }} + OWNED_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }} + PR_XCODE_APP: ${{ github.event.pull_request.head.repo.full_name == github.repository && vars.CMUX_CI_XCODE_APP_PR || '' }} + SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + run: python3 scripts/ci/cmux_next_media_runner.py --same-repo "$SAME_REPO" + + tour: + name: Tour the app + needs: route + if: ${{ needs.route.outputs.runs_on != '' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || needs.route.outputs.runs_on }} + timeout-minutes: 60 + permissions: + contents: read + env: + CMUX_CI_XCODE_APP: ${{ github.event.pull_request.head.repo.full_name == github.repository && vars.CMUX_CI_XCODE_APP_PR || '' }} + # GhosttyKit comes prebuilt; the ghostty CLI helper zig build is skipped. + CMUX_SKIP_ZIG_BUILD: "1" + CMUX_GHOSTTYKIT_PREPROVISIONED: "1" + # One tag per runner keeps its DerivedData warm across pull requests; + # a mini has one gui runner, so two tours never share it at once. + TOUR_TAG: ci-media + steps: + - name: Clear stale git locks (self-hosted reused workspace) + run: rm -f "${GITHUB_WORKSPACE:-$PWD}/.git/index.lock" 2>/dev/null || true + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + submodules: recursive + + - name: Select Xcode + run: ./scripts/select-ci-xcode.sh + + - name: Check the tours + run: python3 scripts/cmux-next/media/tour.py --check scripts/cmux-next/media/tours + + - name: Check screen capture + run: | + set -euo pipefail + echo "runner user: $(id -un), console user: $(stat -f %Su /dev/console)" + system_profiler SPDisplaysDataType 2>/dev/null | sed -n '/Displays:/,$p' | head -20 || true + if [ -f scripts/ci/preflight-e2e-screen-capture.py ]; then + python3 scripts/ci/preflight-e2e-screen-capture.py + fi + + # The same grant main's E2E recorder makes on these Macs + # (.github/actions/e2e-run-tests on main): screencapture from a job + # otherwise fails with "could not create image". Best effort; tour.py + # probes which capture route works and reports it. + - name: Grant screen capture to screencapture + continue-on-error: true + run: | + tcc="/Library/Application Support/com.apple.TCC/TCC.db" + sudo -n sqlite3 "$tcc" "INSERT OR REPLACE INTO access (service, client, client_type, auth_value, auth_reason, auth_version, csreq, policy_id, indirect_object_identifier_type, indirect_object_identifier, indirect_object_code_identity, flags, last_modified) VALUES ('kTCCServiceScreenCapture', '/usr/sbin/screencapture', 1, 2, 4, 1, NULL, NULL, 0, 'UNUSED', NULL, 0, $(date +%s));" \ + || echo "::warning::could not grant screen capture in the system TCC database" + + - name: Download pre-built GhosttyKit.xcframework + run: | + set -euo pipefail + GHOSTTY_SHA="$(git rev-parse HEAD:ghostty)" ./scripts/download-prebuilt-ghosttykit.sh + + - name: Build the tagged app + id: build + run: | + set -euo pipefail + log="$RUNNER_TEMP/reload.out" + ./scripts/reload.sh --tag "$TOUR_TAG" --no-global-cli-links 2>&1 | tee "$log" + app="$(awk '/^App path:/ { getline; sub(/^ +/, ""); print; exit }' "$log")" + if [ -z "$app" ] || [ ! -d "$app" ]; then + app="$(find "$HOME/Library/Developer/Xcode/DerivedData/cmux-$TOUR_TAG/Build/Products/Debug" -maxdepth 1 -name '*.app' | head -1)" + fi + [ -d "$app" ] || { echo "::error::reload.sh built no app"; exit 1; } + echo "app=$app" >> "$GITHUB_OUTPUT" + echo "Built $app" + + - name: Run the tours + env: + APP: ${{ steps.build.outputs.app }} + # --fresh-state: the ci-media tag's saved workspaces would otherwise + # carry one tour's windows into the next run on the same Mac. + # exec: the runner signals the step's shell on cancel, and bash would + # not pass it on, so tour.py could not stop the app and its daemon. + run: exec python3 scripts/cmux-next/media/tour.py --app "$APP" --out "$RUNNER_TEMP/media" --fresh-state scripts/cmux-next/media/tours + + - name: Keep the media for the publish job + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-next-media + path: ${{ runner.temp }}/media + if-no-files-found: ignore + retention-days: 3 + + publish: + name: Post the tour media + needs: [route, tour] + if: ${{ !cancelled() && needs.route.result == 'success' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 15 + # contents: write reaches the pr-media branch; main and the feature + # branches are protected by their rulesets. + permissions: + contents: write + pull-requests: write + steps: + # The publisher comes from the base revision, never the head: this job + # holds a write token and runs no pull request code. + - name: Checkout the publisher from the base revision + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + sparse-checkout: | + scripts/pr-media.py + scripts/cmux-next/media/publish.py + sparse-checkout-cone-mode: false + + - name: Download the tour media + if: ${{ needs.tour.result != 'skipped' }} + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: cmux-next-media + path: ${{ runner.temp }}/media + + - name: Check for a publisher in the base revision + id: publisher + run: | + if [ -f scripts/cmux-next/media/publish.py ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "::notice::The base revision has no cmux-next media publisher yet; the media stays in the run artifact." + fi + + # Without ffmpeg the comment still carries the screenshots, only no + # video, so a slow or stuck apt never holds the comment back. + - name: Install ffmpeg + if: ${{ steps.publisher.outputs.present == 'true' }} + timeout-minutes: 4 + continue-on-error: true + env: + DEBIAN_FRONTEND: noninteractive + run: command -v ffmpeg >/dev/null || (sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg >/dev/null) + + - name: Upload the media and update the comment + if: ${{ steps.publisher.outputs.present == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + ROUTE_REASON: ${{ needs.route.outputs.reason }} + TOUR_RESULT: ${{ needs.tour.result }} + run: | + set -euo pipefail + note="" + case "$TOUR_RESULT" in + skipped) note="The tour did not run: $ROUTE_REASON." ;; + failure) + if compgen -G "$RUNNER_TEMP/media/*/manifest.json" >/dev/null; then + note="The tour failed; the failed steps are below." + else + note="The tour job failed before the tour ran (build or setup); the run log has the output." + fi ;; + esac + python3 scripts/cmux-next/media/publish.py --repo "$GITHUB_REPOSITORY" \ + --media "$RUNNER_TEMP/media" --pr "$PR" --sha "$HEAD_SHA" --run-url "$RUN_URL" --note "$note" diff --git a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Usage/ResourceControl.swift b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Usage/ResourceControl.swift index bcf4f8cde929..9db0bb55a1e0 100644 --- a/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Usage/ResourceControl.swift +++ b/Packages/macOS/CmuxNext/Sources/CmuxNextApp/Usage/ResourceControl.swift @@ -18,7 +18,7 @@ enum ResourceControl { static func run(_ params: [String: JSONValue], services: AppServices?) async throws -> JSONValue { guard let services else { throw ControlError(code: "unavailable", message: RefusalStrings.noWindowShowsWorkspace) } - let target = try await MainActor.run { try target(params, services: services) } + let target = try await MainActor.run { try Self.target(params, services: services) } let milliseconds = min(max(params["interval_ms"]?.intValue ?? 1000, 100), 1500) let first = await services.resources.sample(target) await waitOnce(.milliseconds(milliseconds)) diff --git a/scripts/ci/cmux_next_media_runner.py b/scripts/ci/cmux_next_media_runner.py new file mode 100755 index 000000000000..3e05080d9f86 --- /dev/null +++ b/scripts/ci/cmux_next_media_runner.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Pick the runner for cmux-next-media.yml's tour: an owned Mac's gui runner, or none. + +The tour launches cmux-next and screenshots it, so it needs a logged-in +console session that can come to the front and be captured. Blacksmith's +macOS sessions sit at a locked screen (e2e_runner_pool.py on main has the +runs), so the tour never falls back to Blacksmith: with no owned gui runner it +does not run, and the comment says why. + +The label is the gui runner of the owned pool for the pull request lane's +Xcode pin (pr_runner_pool.gui_label(), `glaeda-gui--xcode-`), +one per mini, the runner that holds the mini's console session. Workflows +never name an owned label (tests/test_ci_self_hosted_guard.sh), so it reaches +runs-on only as this script's output. + +Rules, in order: a fork head gets nothing (owned Macs run trusted code only); +CI_PR_POOL_OWNED must be 1; the first owned pool class whose gui label has +machines in CI_OWNED_POOL_SLOTS wins. + +Writes `runs_on` (a label, or "") and `reason` to $GITHUB_OUTPUT, and prints them. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pr_runner_pool # noqa: E402 + + +def pick(*, same_repo: bool, owned: str | None, owned_slots: str | None, + pr_xcode_app: str | None) -> tuple[str, str]: + """(label, reason); the label is "" when the tour should not run.""" + if not same_repo: + return "", "a fork head never runs on an owned Mac" + if (owned or "").strip() != "1": + return "", "owned Macs are off (CI_PR_POOL_OWNED is not 1)" + pools = pr_runner_pool.owned_pools(pr_xcode_app) + if not pools: + return "", f"no owned pool for the lane's Xcode pin ({pr_xcode_app or 'unset'})" + slots = pr_runner_pool.slots(owned_slots, pr_xcode_app) + for pool in pools: + gui = pr_runner_pool.gui_label(pool) + if gui and slots.get(gui, 0) > 0: + return gui, f"{slots[gui]} gui runner(s) on {pool}" + return "", "no owned pool has gui runners in CI_OWNED_POOL_SLOTS" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--same-repo", required=True, choices=("true", "false")) + args = parser.parse_args(argv) + label, reason = pick( + same_repo=args.same_repo == "true", + owned=os.environ.get("POOL_OWNED"), + owned_slots=os.environ.get("OWNED_SLOTS"), + pr_xcode_app=os.environ.get("PR_XCODE_APP"), + ) + lines = [f"runs_on={label}", f"reason={reason}"] + print("\n".join(lines)) + output = os.environ.get("GITHUB_OUTPUT") + if output: + with open(output, "a", encoding="utf-8") as handle: + handle.write("\n".join(lines) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/cmux-next/media/publish.py b/scripts/cmux-next/media/publish.py new file mode 100755 index 000000000000..7e538a4895f0 --- /dev/null +++ b/scripts/cmux-next/media/publish.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Post cmux-next tour media on its pull request: one sticky comment per PR. + + publish.py --media DIR --pr N --sha SHA [--run-url URL] [--note TEXT] [--dry-run] + +DIR holds tour.py's output (/manifest.json, shots/, frames/). For each +tour this turns the frames into an mp4 and a GIF, uploads them and the +screenshots to the `pr-media` branch at //next-/ (the layout +scripts/ci/prune_pr_media.py bounds), and writes the comment marked +COMMENT_MARKER: the GIF, a step table, failed steps' screenshots inline and +the rest folded. With no tour output, `--note` says why in the same comment. + +Uploads go through scripts/pr-media.py's contents-API writer. Nothing is +posted when the pull request's head has moved past SHA: the newer push's run +posts its own. +""" + +from __future__ import annotations + +import argparse +import html +import importlib.util +import json +import re +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from typing import Any, Callable + +ROOT = Path(__file__).resolve().parents[3] +COMMENT_MARKER = "" +TOURS_DIR = "scripts/cmux-next/media/tours" +BOT = "github-actions[bot]" +# Frames are taken about every 0.5 s; played at 4 per second the video runs at 2x. +VIDEO_FPS = 4 +GIF_FPS = 4 +VIDEO_SIZE = (1280, 800) +# tour.py's MAX_FRAMES: a larger artifact does not make a longer video. +MAX_FRAMES = 600 +GIF_WIDTHS = (720, 560, 440) +TOUR_NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,39}") +SHOT_PATH = re.compile(r"shots/[0-9]{2}-[a-z0-9-]{1,40}(?:-failed)?\.png") +STATUS_TEXT = {"ok": "ok", "failed": "**failed**", "unavailable": "unavailable"} + + +def load_pr_media() -> Any: + spec = importlib.util.spec_from_file_location("pr_media_tool", ROOT / "scripts/pr-media.py") + module = importlib.util.module_from_spec(spec) + assert spec.loader + # Its dataclasses look their module up in sys.modules while it loads. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def manifests(media: Path) -> list[tuple[Path, dict[str, Any]]]: + """Each tour's folder and manifest. The artifact came from a job that ran + the pull request's code, so it is data: a folder whose name is not a tour + slug, or a manifest that is not an object, is skipped.""" + found = [] + for path in sorted(media.glob("*/manifest.json")): + if not TOUR_NAME.fullmatch(path.parent.name) or path.is_symlink(): + print(f"warning: skipping {path}: not a tour folder", file=sys.stderr) + continue + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + print(f"warning: skipping {path}: {error}", file=sys.stderr) + continue + if isinstance(manifest, dict) and isinstance(manifest.get("steps", []), list): + manifest["steps"] = [clean_step(step) for step in manifest.get("steps", []) if isinstance(step, dict)] + manifest["frames"] = manifest.get("frames") if isinstance(manifest.get("frames"), list) else [] + for key in ("title", "error", "capture_mode"): + if key in manifest: + manifest[key] = str(manifest[key]) + found.append((path.parent, manifest)) + return found + + +def clean_step(step: dict[str, Any]) -> dict[str, Any]: + """A step with only the fields the comment shows, each a plain string or int.""" + status = step.get("status") + index = step.get("index") + return { + "index": index if isinstance(index, int) and not isinstance(index, bool) else "", + "title": str(step.get("title", ""))[:200], + "status": status if isinstance(status, str) and status in STATUS_TEXT else "failed", + "detail": str(step.get("detail") or "")[:500], + "command": str(step.get("command") or "")[:500], + "shot": step["shot"] if isinstance(step.get("shot"), str) else "", + } + + +def shot_file(directory: Path, shot: Any) -> Path | None: + """The screenshot a step names, when it is one tour.py could have written: + shots/NN-slug.png inside the tour folder, a regular file, not a link.""" + if not isinstance(shot, str) or not SHOT_PATH.fullmatch(shot): + return None + path = directory / shot + if path.is_symlink() or not path.is_file(): + return None + return path + + +def make_video(directory: Path, manifest: dict[str, Any], into: Path, pr_media: Any, + run: Callable[..., Any] = subprocess.run) -> tuple[Path | None, Path | None]: + """(mp4, gif) from the tour's frames; (None, None) with fewer than two frames.""" + if len(manifest.get("frames") or []) < 2 or not shutil.which("ffmpeg"): + return None, None + mp4 = into / "tour.mp4" + done = run(["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-framerate", str(VIDEO_FPS), + "-i", str(directory / "frames/frame-%05d.jpg"), + "-frames:v", str(MAX_FRAMES), + # A fixed output size: a frame of another size cannot break the stream. + "-vf", f"scale={VIDEO_SIZE[0]}:{VIDEO_SIZE[1]}:force_original_aspect_ratio=decrease," + f"pad={VIDEO_SIZE[0]}:{VIDEO_SIZE[1]}:(ow-iw)/2:(oh-ih)/2:color=white", + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(mp4)], + capture_output=True, text=True) + if done.returncode != 0 or not mp4.is_file(): + print(f"warning: no video for {directory.name}: {done.stderr.strip()}", file=sys.stderr) + return None, None + gif = into / "tour.gif" + for width in GIF_WIDTHS: + done = run(pr_media.gif_argv(mp4, gif, GIF_FPS, width), capture_output=True, text=True) + if done.returncode == 0 and gif.is_file() and gif.stat().st_size <= pr_media.INLINE_MAX_BYTES - 512 * 1024: + return mp4, gif + return mp4, None + + +def step_row(step: dict[str, Any]) -> str: + status = STATUS_TEXT.get(step.get("status", ""), "") + detail = f": {cell(step['detail'])}" if step.get("detail") else "" + # A code span cannot hold its own backticks, and inside one nothing else is Markdown. + one_line = re.sub(r"[\r\n]+", " ", str(step.get("command") or "")).replace("`", "'") + command = f"`{one_line}`" if one_line else "" + return f"| {cell(step.get('index', ''))} | {cell(step.get('title', ''))} | {status}{detail} | {command.replace('|', chr(92) + '|')} |" + + +MARKDOWN_SPECIAL = re.compile(r"([\\`*_{}\[\]()#+!<>~|-])") + + +def cell(text: Any) -> str: + """Manifest text for the comment, inert: it comes from a job that ran the + pull request's code. HTML and Markdown are escaped, and @ is broken so no + mention or team ping is sent.""" + escaped = MARKDOWN_SPECIAL.sub(r"\\\1", html.escape(str(text), quote=False)) + return re.sub(r"[\r\n]+", " ", escaped.replace("@", "@\u200b")) + + +def image(url: str, title: str, width: int) -> str: + return f'{html.escape(title, quote=True)}' + + +def tour_section(manifest: dict[str, Any], urls: dict[str, str]) -> str: + steps = manifest.get("steps") or [] + counts = {status: sum(1 for step in steps if step.get("status") == status) + for status in ("ok", "failed", "unavailable")} + summary = ", ".join(f"{count} {status}" for status, count in counts.items() if count) + lines = [f"### {cell(manifest.get('title') or 'Tour')}: {summary or 'no steps ran'}", ""] + if manifest.get("error"): + lines += [f"> {cell(str(manifest['error']).splitlines()[0][:300])}", ""] + capture = str(manifest.get("capture_mode") or "") + if capture.startswith("none") and not any(urls.get(str(step.get("shot"))) for step in steps): + lines += [f"> No screenshots: screen capture is unavailable on this runner ({cell(capture[:400])}).", ""] + if urls.get("tour.gif"): + lines.append(image(urls["tour.gif"], "tour", 720)) + if urls.get("tour.mp4"): + lines += ["", f"[Video (mp4)]({urls['tour.mp4']})"] + lines += ["", "| # | Step | Result | Command |", "| --- | --- | --- | --- |"] + lines += [step_row(step) for step in steps] + failed = [step for step in steps if step.get("status") == "failed" and urls.get(str(step.get("shot", "")))] + if failed: + lines += ["", "Failed steps:", ""] + lines += [f"**{cell(step['index'])}. {cell(step['title'])}**
{image(urls[step['shot']], step['title'], 480)}
" + for step in failed] + rest = [step for step in steps if step not in failed and urls.get(str(step.get("shot", "")))] + if rest: + lines += ["", f"
Screenshots ({len(rest)})", ""] + lines += [f"**{cell(step['index'])}. {cell(step['title'])}**
{image(urls[step['shot']], step['title'], 480)}
" + for step in rest] + lines += ["", "
"] + return "\n".join(lines) + + +def render(sha: str, sections: list[str], run_url: str | None, note: str | None) -> str: + run = f" ([run]({run_url}))" if run_url else "" + lines = [COMMENT_MARKER, f"**cmux-next tour** of `{sha}`{run}", ""] + if note: + lines += [note, ""] + lines += sections or ["The tour produced no media."] + lines += ["", f"Tours are data in `{TOURS_DIR}`: add a step or a tour file to show your feature."] + return "\n".join(lines) + "\n" + + +def gh(args: list[str], stdin: str | None = None) -> str: + done = subprocess.run(["gh", *args], input=stdin, capture_output=True, text=True) + if done.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {done.stderr.strip()}") + return done.stdout + + +def upsert_comment(repo: str, pr: int, body: str) -> None: + ids = gh(["api", f"repos/{repo}/issues/{pr}/comments", "--paginate", "--jq", + f'.[] | select(.user.login == "{BOT}" and (.body | contains("{COMMENT_MARKER}"))) | .id']) + existing = ids.split() + payload = json.dumps({"body": body}) + if existing: + gh(["api", "-X", "PATCH", f"repos/{repo}/issues/comments/{existing[0]}", "--input", "-"], payload) + else: + gh(["api", "-X", "POST", f"repos/{repo}/issues/{pr}/comments", "--input", "-"], payload) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--media", type=Path, required=True) + parser.add_argument("--pr", type=int, required=True) + parser.add_argument("--sha", required=True) + parser.add_argument("--repo", default="manaflow-ai/cmux") + parser.add_argument("--run-url") + parser.add_argument("--note", help="why the tour did not run or is incomplete") + parser.add_argument("--dry-run", action="store_true", help="print the comment; upload and post nothing") + args = parser.parse_args(argv) + + if not args.dry_run and head_moved(args): + return 0 + + pr_media = load_pr_media() + sections = [] + with tempfile.TemporaryDirectory() as scratch: + for directory, manifest in manifests(args.media) if args.media.is_dir() else []: + try: + sections.append(publish_tour(directory, manifest, args, pr_media, Path(scratch))) + except (OSError, ValueError, TypeError, KeyError, subprocess.SubprocessError, pr_media.MediaError) as error: + print(f"warning: tour {directory.name}: {error}", file=sys.stderr) + sections.append(f"### {cell(directory.name)}: the media could not be published ({cell(str(error)[:200])})") + + body = render(args.sha, sections, args.run_url, args.note) + if args.dry_run: + print(body) + return 0 + # Uploads take minutes; a push in the meantime posts its own media. + if head_moved(args): + return 0 + upsert_comment(args.repo, args.pr, body) + print(f"Posted the tour media on #{args.pr}") + return 0 + + +def head_moved(args: argparse.Namespace) -> bool: + head = gh(["api", f"repos/{args.repo}/pulls/{args.pr}", "--jq", ".head.sha"]).strip() + if head != args.sha: + print(f"#{args.pr} moved to {head[:8]}; its own run posts the media") + return True + return False + + +def publish_tour(directory: Path, manifest: dict[str, Any], args: argparse.Namespace, pr_media: Any, + scratch: Path) -> str: + """Upload one tour's media; its comment section.""" + name = directory.name + folder = f"{args.pr}/{args.sha[:8]}/next-{name}" + files: dict[str, Path] = {} + for step in manifest["steps"]: + shot = shot_file(directory, step.get("shot")) + if shot and shot.stat().st_size <= pr_media.INLINE_MAX_BYTES \ + and pr_media.family_of_content(shot) == "png": + files[step["shot"]] = shot + into = scratch / name + into.mkdir() + mp4, gif = make_video(directory, manifest, into, pr_media) + for local in (mp4, gif): + if local: + files[local.name] = local + urls = {} + for key, local in files.items(): + remote = f"{folder}/{key}" + if not args.dry_run: + pr_media.put_file(args.repo, pr_media.BRANCH, remote, local, + f"cmux-next tour media for #{args.pr} at {args.sha[:8]}") + urls[key] = pr_media.raw_url(args.repo, pr_media.BRANCH, remote) + return tour_section(manifest, urls) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/cmux-next/media/tour.py b/scripts/cmux-next/media/tour.py new file mode 100755 index 000000000000..8ebe98b517d6 --- /dev/null +++ b/scripts/cmux-next/media/tour.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python3 +"""Run cmux-next UI tours against a built app and capture each step. + + tour.py --app "/cmux DEV ci-media.app" --out "$RUNNER_TEMP/media" tours/core.json [...] + +Each tour launches the app fresh on a private control socket (the knobs +scripts/smoke-signed-app-cli.sh uses), drives it only through the bundled +`cmux` CLI (plans/cmux-next/cli.md), and writes //: + + manifest.json steps with their command, status, output and screenshot + shots/NN-*.png one screenshot per step (a failed step is captured too) + frames/*.jpg the window about twice a second, for the video + +A tour is a JSON file with a `title` and `steps`. A step has a `title` and +optionally: + + cmux argv after `cmux`; app scopes get --app-socket, see `daemon` + daemon true for a daemon verb (notify, terminal ...): it gets + --session cmux-app- instead of the app socket + wait seconds to let the UI settle before the screenshot (default 1.5) + shot false to skip the screenshot + optional why the step may fail (a feature not ported yet); its failure is + reported as unavailable and does not fail the tour + +Screenshots cover the main window's rectangle on screen, so child windows +(browser pages, overlays) are in them; with no window frame they cover the +main display. Capture runs in the console user's session, as the E2E +recorder on main does (scripts/ci/preflight-e2e-screen-capture.py), so it +works when the runner user is not the console user. + +Exit status: 0 when every required step passed, 1 otherwise. The media is +written either way. +""" + +from __future__ import annotations + +import argparse +import getpass +import json +import os +from pathlib import Path +import re +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +from typing import Any + +STEP_KEYS = {"title", "cmux", "daemon", "wait", "shot", "optional"} +TOUR_KEYS = {"title", "steps"} +DEFAULT_WAIT = 1.5 +STEP_TIMEOUT = 20.0 +SOCKET_TIMEOUT = 60.0 +FRAME_INTERVAL = 0.5 +# Five minutes of frames; a tour that runs longer keeps its screenshots but its +# video stops here, so a stuck step cannot fill the runner's disk. +MAX_FRAMES = 600 +WINDOW_TIMEOUT = 20.0 +# scripts/release-media/host_agent.py on main looks in the same places. +HELPER_CANDIDATES = (Path.home() / "Applications/CuaSshScreenCapture.app", + Path("/Applications/CuaSshScreenCapture.app")) +# A long tour would make a GIF past the inline limit; split it instead. +MAX_STEPS = 30 +# Inherited caller context a cmux terminal exports. Left in place, a local run +# inside cmux would route the tour's commands at the caller's own workspace. +SCRUBBED_ENV = ("CMUX_WORKSPACE_ID", "CMUX_SURFACE_ID", "CMUX_TAB_ID", "CMUX_PANEL_ID", + "CMUX_SOCKET_PATH", "CMUX_SOCKET_PASSWORD", "CMUX_TUI_SOCKET", "CMUX_TAG", + "CMUX_BUNDLE_ID", "CMUX_SOCKET") + + +class TourError(Exception): + pass + + +def slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:40] or "step" + + +def load_tour(path: Path) -> dict[str, Any]: + """The tour in `path`, checked; TourError says what is wrong with it.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise TourError(f"{path}: {error}") from error + if not isinstance(data, dict) or set(data) - TOUR_KEYS: + raise TourError(f"{path}: a tour is an object with only {sorted(TOUR_KEYS)}") + steps = data.get("steps") + if not isinstance(data.get("title"), str) or not data["title"].strip(): + raise TourError(f"{path}: `title` must be a non-empty string") + if not isinstance(steps, list) or not steps: + raise TourError(f"{path}: `steps` must be a non-empty list") + if len(steps) > MAX_STEPS: + raise TourError(f"{path}: {len(steps)} steps; split tours past {MAX_STEPS}") + for number, step in enumerate(steps, 1): + where = f"{path}: step {number}" + if not isinstance(step, dict): + raise TourError(f"{where} is not an object") + unknown = set(step) - STEP_KEYS + if unknown: + raise TourError(f"{where} has unknown keys {sorted(unknown)}") + if not isinstance(step.get("title"), str) or not step["title"].strip(): + raise TourError(f"{where} needs a `title`") + command = step.get("cmux") + if command is not None and (not isinstance(command, list) or not command + or not all(isinstance(word, str) for word in command)): + raise TourError(f"{where}: `cmux` must be a non-empty list of strings") + if step.get("daemon") and command is None: + raise TourError(f"{where}: `daemon` needs a `cmux` command") + wait = step.get("wait", DEFAULT_WAIT) + if isinstance(wait, bool) or not isinstance(wait, (int, float)) or not 0 <= wait <= 30: + raise TourError(f"{where}: `wait` must be 0 to 30 seconds") + for key in ("daemon", "shot"): + if key in step and not isinstance(step[key], bool): + raise TourError(f"{where}: `{key}` must be true or false") + if "optional" in step and (not isinstance(step["optional"], str) or not step["optional"].strip()): + raise TourError(f"{where}: `optional` is the reason the step may fail") + return {"name": slug(path.stem), "title": data["title"].strip(), "steps": steps} + + +def command_line(argv: list[str]) -> str: + return " ".join(word if re.fullmatch(r"[\w@%+=:,./-]+", word) else json.dumps(word) for word in argv) + + +class Session: + """Runs commands in the console user's GUI session. + + `prefix` reaches that session from another user (app launch, kill). + Capture needs Screen Recording, which a job's process may not have, so + three routes are probed once and the first that writes an image wins: + screencapture from this process, screencapture through `launchctl + asuser` (main's E2E recorder; needs passwordless sudo), and an approved + CuaSshScreenCapture.app (main's scripts/release-media helper backend). + """ + + def __init__(self) -> None: + self.prefix: list[str] = [] + self.user = getpass.getuser() + console = subprocess.run(["stat", "-f", "%Su", "/dev/console"], + capture_output=True, text=True).stdout.strip() + asuser: list[str] = [] + if console and console not in ("root", "loginwindow"): + uid = subprocess.run(["id", "-u", console], capture_output=True, text=True).stdout.strip() + asuser = ["sudo", "-n", "launchctl", "asuser", uid, "sudo", "-n", "-H", "-u", console] + if console != self.user: + self.prefix = asuser + self.user = console + self.capture_prefix = self.prefix + self.capture_mode = "unprobed" + self.capture_routes = [("direct", self.prefix)] + ([("launchctl asuser", asuser)] if asuser and asuser != self.prefix else []) + self.helper: Path | None = None + + def run(self, argv: list[str], timeout: float = 15) -> subprocess.CompletedProcess[str]: + return subprocess.run(self.prefix + argv, capture_output=True, text=True, timeout=timeout) + + def capture(self, argv: list[str], timeout: float = 15) -> subprocess.CompletedProcess[str]: + return subprocess.run(self.capture_prefix + argv, capture_output=True, text=True, timeout=timeout) + + def probe_capture(self, scratch: Path) -> None: + """Pick the capture route that writes a display image; notes every attempt.""" + notes = [] + for mode, prefix in self.capture_routes: + image = scratch / f"probe-{len(notes)}.jpg" + try: + done = subprocess.run(prefix + ["/usr/sbin/screencapture", "-x", "-t", "jpg", "-D", "1", str(image)], + capture_output=True, text=True, timeout=15) + said = (done.stderr or done.stdout).strip() or f"exit {done.returncode}" + except (OSError, subprocess.SubprocessError) as error: + said = str(error) + if image.is_file() and image.stat().st_size > 0: + self.capture_prefix, self.capture_mode = prefix, mode + return + notes.append(f"{mode}: {said}") + for helper in HELPER_CANDIDATES: + if not helper.is_dir(): + continue + image = scratch / "probe-helper.png" + if self.helper_capture(helper, ["desktop"], image) is None: + self.helper, self.capture_mode = helper, "helper" + return + notes.append(f"{helper.name}: wrote no image (not approved for Screen Recording?)") + if not any(helper.is_dir() for helper in HELPER_CANDIDATES): + notes.append("no CuaSshScreenCapture.app") + self.capture_mode = "none (" + "; ".join(notes) + ")" + + def helper_capture(self, helper: Path, what: list[str], path: Path) -> str | None: + """One helper capture (`desktop` or `window `) to a PNG; None on success.""" + try: + path.unlink(missing_ok=True) + done = subprocess.run(self.prefix + ["/usr/bin/open", "-g", "-n", "-W", str(helper), "--args", *what, str(path)], + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError) as error: + return str(error) + if path.is_file() and path.stat().st_size > 0: + return None + return (done.stderr or done.stdout).strip() or "the helper wrote no image" + + def shareable(self, path: Path) -> Path: + """A directory the session's user can write when that user is someone else.""" + path.mkdir(parents=True, exist_ok=True) + if self.prefix or self.capture_prefix: + path.chmod(0o777) + return path + + +class Screen: + """Screenshots of a rectangle in global top-left coordinates (screencapture -R).""" + + def __init__(self, session: Session, scratch: Path | None = None) -> None: + self.session = session + self.scratch = scratch or Path(tempfile.gettempdir()) + self.rect: tuple[int, int, int, int] | None = None + self.primary_height: float | None = None + self.primary_width: float | None = None + script = ('ObjC.import("AppKit");' + 'var f = $.NSScreen.screens.objectAtIndex(0).frame;' + 'JSON.stringify([f.size.width, f.size.height])') + try: + done = session.capture(["osascript", "-l", "JavaScript", "-e", script]) + self.primary_width, self.primary_height = (float(value) for value in json.loads(done.stdout)[:2]) + except (subprocess.SubprocessError, ValueError, IndexError, TypeError): + self.primary_width = self.primary_height = None + + def follow(self, frame: list[float] | None) -> None: + """Aim at a window frame (NSWindow coordinates: bottom-left origin on the primary screen).""" + # Without a usable window frame (none, gone, too small, off the + # display) nothing is captured: the old rect would show what is behind. + self.rect = None + if not frame or self.primary_height is None or len(frame) != 4: + return + x, y, width, height = frame + if width < 50 or height < 50: + return + top = self.primary_height - (y + height) + left, right = max(0.0, x), x + width + bottom = min(self.primary_height, top + height) + if self.primary_width: + right = min(self.primary_width, right) + top = max(0.0, top) + if right - left < 50 or bottom - top < 50: + return + self.rect = (int(left), int(top), int(right - left), int(bottom - top)) + + def capture(self, path: Path, kind: str = "png") -> str | None: + """Write a screenshot of the app window's rect; None on success, else what went wrong. + + Never the whole display: the runner's console is shared, and these + images are published, so with no window rect there is no shot. + """ + if not self.rect: + return "no app window on screen to capture" + if self.session.helper: + return self.helper_capture(path, kind) + argv = ["/usr/sbin/screencapture", "-x", "-t", kind, f"-R{','.join(map(str, self.rect))}", str(path)] + try: + done = self.session.capture(argv) + except subprocess.TimeoutExpired: + return "screencapture timed out" + if path.is_file() and path.stat().st_size > 0: + return None + return (done.stderr or done.stdout or f"screencapture exited {done.returncode}").strip() + + + def helper_capture(self, path: Path, kind: str) -> str | None: + """The whole display through the helper, cropped to the rect with sips.""" + # The whole desktop lands outside shots/ and frames/, so a kill + # mid-capture cannot publish it, in a directory the helper's user can write. + raw = self.scratch / f"raw-{threading.get_ident()}.png" + problem = self.session.helper_capture(self.session.helper, ["desktop"], raw) + if problem: + return problem + if not self.primary_width: + raw.unlink(missing_ok=True) + return "unknown display size; cannot crop the helper's desktop image" + # The rect is clamped to the primary display (follow()), which the + # helper's desktop image starts with. + pixels = subprocess.run(["sips", "-g", "pixelWidth", str(raw)], capture_output=True, text=True).stdout + match = re.search(r"pixelWidth:\s*(\d+)", pixels) + scale = int(match.group(1)) / self.primary_width if match else 1.0 + x, y, width, height = (round(value * scale) for value in self.rect) + argv = ["sips", "-s", "format", "jpeg" if kind == "jpg" else "png", + "--cropOffset", str(y), str(x), "-c", str(height), str(width)] + done = subprocess.run(argv + [str(raw), "--out", str(path)], capture_output=True, text=True) + raw.unlink(missing_ok=True) + if path.is_file() and path.stat().st_size > 0: + return None + return (done.stderr or done.stdout).strip() or "sips wrote no image" + + +class Recorder(threading.Thread): + """Frames of the screen rectangle about every FRAME_INTERVAL seconds, tagged with the step.""" + + def __init__(self, screen: Screen, directory: Path) -> None: + super().__init__(daemon=True) + self.screen = screen + self.directory = directory + self.step = 0 + self.frames: list[dict[str, Any]] = [] + self.error: str | None = None + self.stopping = threading.Event() + self.lock = threading.Lock() + + def run(self) -> None: + while not self.stopping.is_set(): + started = time.monotonic() + with self.lock: + if self.stopping.is_set(): + break + if len(self.frames) >= MAX_FRAMES: + self.error = f"the video stops at {MAX_FRAMES} frames" + break + path = self.directory / f"frame-{len(self.frames):05d}.jpg" + problem = self.screen.capture(path, "jpg") + with self.lock: + if self.stopping.is_set(): + path.unlink(missing_ok=True) + break + if problem: + self.error = problem + else: + self.frames.append({"file": f"frames/{path.name}", "step": self.step}) + self.stopping.wait(max(0.0, FRAME_INTERVAL - (time.monotonic() - started))) + + def stop(self) -> list[dict[str, Any]]: + """Stop recording; the frames kept, which nothing appends to afterwards.""" + with self.lock: + self.stopping.set() + frames = list(self.frames) + if self.is_alive(): + # A capture in flight is bounded (60 s for the helper); its frame is dropped. + self.join(timeout=65) + return frames + + +def clean_tag(tag: Any) -> str: + """The tag as DaemonLauncher.sessionName reduces it to one path component.""" + return re.sub(r"[^A-Za-z0-9_.-]", "-", str(tag or "")).strip("-.") + + +class App: + """One launch of the app on a private socket, driven through its bundled CLI.""" + + def __init__(self, app_path: Path, session: Session, workdir: Path, *, fresh_state: bool = False) -> None: + self.app_path = app_path + self.session = session + self.fresh_state = fresh_state + self.cli_path = app_path / "Contents/Resources/bin/cmux" + plist = app_path / "Contents/Info.plist" + executable = subprocess.run(["/usr/libexec/PlistBuddy", "-c", "Print :CFBundleExecutable", str(plist)], + capture_output=True, text=True, check=True).stdout.strip() + self.executable = app_path / "Contents/MacOS" / executable + tag = subprocess.run(["/usr/libexec/PlistBuddy", "-c", "Print :LSEnvironment:CMUX_TAG", str(plist)], + capture_output=True, text=True).stdout.strip() + self.bundle_tag = clean_tag(tag) + # sun_path holds 104 bytes, so the socket lives in a short private directory. + self.socket_path = session.shareable(workdir) / "s.sock" + self.log_path = workdir / "app.log" + self.process: subprocess.Popen[bytes] | None = None + self.identity: dict[str, Any] = {} + + def environment(self) -> dict[str, str]: + env = {key: value for key, value in os.environ.items() if key not in SCRUBBED_ENV} + return env + + def clear_leftovers(self) -> None: + """Stop a copy of this bundle and its daemon a cancelled run left behind.""" + for pattern in (f"{self.app_path}/Contents/Resources/bin/", str(self.executable)): + self.session.run(["pkill", "-f", re.escape(pattern)]) + + def state_directory(self) -> Path | None: + """DaemonLauncher.tagStateDirectory: where a tagged build keeps its workspaces.""" + if not self.bundle_tag: + return None + home = Path(os.path.expanduser(f"~{self.session.user}")) + return home / "Library/Application Support/cmux/tags" / self.bundle_tag / "tui" + + def stop_daemon(self) -> None: + """End the tagged build's daemon and its terminals through its own socket. + + The daemon outlives the app and writes its workspaces back to the state + directory, so deleting that directory under a live daemon changes nothing. + An untagged build's session is the user's own, so it is never stopped. + """ + if self.bundle_tag: + self.cli(["server", "stop", "--end-terminals"], daemon=True, timeout=20) + + def clear_state(self) -> None: + """Start from no workspaces, not the ones an earlier tour on this tag left.""" + directory = self.state_directory() + if self.fresh_state and directory: + self.session.run(["rm", "-rf", str(directory)]) + print(f"fresh state: cleared {directory}", flush=True) + + def launch(self) -> None: + self.stop_daemon() + self.clear_leftovers() + self.clear_state() + knobs = [f"CMUX_NEXT_SOCKET_PATH={self.socket_path}", "CMUX_NEXT_SOCKET_MODE=allowAll"] + argv = self.session.prefix + ["env", *knobs, str(self.executable), "-ApplePersistenceIgnoreState", "YES"] + with self.log_path.open("wb") as log: + self.process = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT, + env=self.environment(), start_new_session=True) + deadline = time.monotonic() + SOCKET_TIMEOUT + while True: + if self.process.poll() is not None: + raise TourError(f"the app exited ({self.process.returncode}) before answering: {self.log_tail()}") + if self.socket_path.exists() and self.cli(["app", "ping"], timeout=5).returncode == 0: + break + if time.monotonic() > deadline: + raise TourError(f"no answer on {self.socket_path} within {SOCKET_TIMEOUT:.0f}s: {self.log_tail()}") + time.sleep(0.5) + done = self.cli(["--json", "app", "identify"]) + try: + self.identity = json.loads(done.stdout) + except ValueError: + self.identity = {} + bundle = self.identity.get("bundle_id") + if bundle: + # Best effort: the tour reads better with the app in front. + self.session.run(["osascript", "-e", f'tell application id "{bundle}" to activate']) + + def daemon_session(self) -> str: + """DaemonLauncher.sessionName: cmux-app, or cmux-app-.""" + cleaned = clean_tag(self.identity.get("tag")) or getattr(self, "bundle_tag", "") + return f"cmux-app-{cleaned}" if cleaned else "cmux-app" + + def cli(self, args: list[str], *, daemon: bool = False, timeout: float = STEP_TIMEOUT) -> subprocess.CompletedProcess[str]: + scope = ["--session", self.daemon_session()] if daemon else ["--app-socket", str(self.socket_path)] + # Global flags (--json) stay in front of the scope flags. + leading = [word for word in args[:1] if word in ("--json", "--jsonl", "--quiet")] + argv = [str(self.cli_path), *leading, *scope, *args[len(leading):]] + try: + return subprocess.run(argv, capture_output=True, text=True, timeout=timeout, env=self.environment()) + except subprocess.TimeoutExpired as expired: + return subprocess.CompletedProcess(argv, 124, expired.stdout or "", f"timed out after {timeout:.0f}s") + + def request(self, method: str, params: dict[str, Any] | None = None) -> Any: + """One control-socket request with no CLI verb (debug.layers has none).""" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(10) + connection.connect(str(self.socket_path)) + connection.sendall((json.dumps({"id": 1, "method": method, "params": params or {}}) + "\n").encode()) + reply = b"" + while not reply.endswith(b"\n"): + chunk = connection.recv(65536) + if not chunk: + break + reply += chunk + answer = json.loads(reply.decode() or "{}") + if not answer.get("ok"): + raise TourError(f"{method}: {answer.get('error')}") + return answer.get("result") + + def main_window_frame(self) -> list[float] | None: + try: + windows = (self.request("debug.layers") or {}).get("windows") or [] + except (OSError, ValueError, TourError): + return None + frames = [window.get("frame") for window in windows if isinstance(window, dict)] + frames = [frame for frame in frames if isinstance(frame, list) and len(frame) == 4] + return max(frames, key=lambda frame: frame[2] * frame[3]) if frames else None + + def alive(self) -> bool: + pid = self.identity.get("pid") + if isinstance(pid, int): + return self.session.run(["kill", "-0", str(pid)]).returncode == 0 + return self.process is not None and self.process.poll() is None + + def log_tail(self, lines: int = 30) -> str: + try: + return "\n".join(self.log_path.read_text(errors="replace").splitlines()[-lines:]) + except OSError: + return "" + + def stop(self) -> None: + pid = self.identity.get("pid") + if isinstance(pid, int): + self.session.run(["kill", str(pid)]) + if self.process and self.process.poll() is None: + try: + os.killpg(self.process.pid, signal.SIGTERM) + self.process.wait(timeout=10) + except (ProcessLookupError, PermissionError, subprocess.TimeoutExpired): + pass + if isinstance(pid, int): + self.session.run(["kill", "-9", str(pid)]) + # The app leaves its daemon running by design; this tour started it. + self.stop_daemon() + self.clear_leftovers() + self.clear_state() + + +def run_step(app: App, screen: Screen, step: dict[str, Any], index: int, shots: Path) -> dict[str, Any]: + started = time.monotonic() + result: dict[str, Any] = {"index": index, "title": step["title"], "status": "ok"} + command = step.get("cmux") + if command: + result["command"] = "cmux " + command_line(command) + done = app.cli(command, daemon=bool(step.get("daemon"))) + output = (done.stdout.strip() + "\n" + done.stderr.strip()).strip() + result["output"] = output[-1500:] + if done.returncode != 0: + result["status"] = "unavailable" if step.get("optional") else "failed" + result["detail"] = f"exited {done.returncode}" + if step.get("optional"): + result["detail"] += f" ({step['optional']})" + time.sleep(step.get("wait", DEFAULT_WAIT)) + if not app.alive(): + result["status"] = "failed" + result["detail"] = (result.get("detail", "") + " the app is no longer running").strip() + screen.follow(app.main_window_frame()) + if step.get("shot", True) or result["status"] == "failed": + suffix = "-failed" if result["status"] == "failed" else "" + path = shots / f"{index:02d}-{slug(step['title'])}{suffix}.png" + problem = screen.capture(path) + if problem: + result["shot_error"] = problem + else: + result["shot"] = f"shots/{path.name}" + result["seconds"] = round(time.monotonic() - started, 2) + return result + + +def run_tour(app_path: Path, tour: dict[str, Any], out: Path, session: Session, *, + fresh_state: bool = False) -> dict[str, Any]: + directory = out / tour["name"] + if directory.exists(): + shutil.rmtree(directory) + shots = session.shareable(directory / "shots") + frames = session.shareable(directory / "frames") + manifest: dict[str, Any] = {"name": tour["name"], "title": tour["title"], "steps": [], "frames": [], + "frame_interval": FRAME_INTERVAL, "console_user": session.user, + "capture_mode": session.capture_mode} + workdir = Path(tempfile.mkdtemp(prefix="cmux-tour.", dir="/tmp")) + app = App(app_path, session, workdir, fresh_state=fresh_state) + screen = Screen(session, session.shareable(workdir / "raw")) + recorder = Recorder(screen, frames) + try: + try: + app.launch() + except (TourError, OSError, subprocess.SubprocessError) as error: + manifest["error"] = f"launch: {error}" + manifest["steps"].append({"index": 0, "title": "Launch", "status": "failed", "detail": str(error)[:1500]}) + return manifest + manifest["app"] = {key: app.identity.get(key) for key in ("app", "version", "build", "bundle_id", "tag")} + # Frames and shots cover the window only, so the recording starts once it is on screen. + deadline = time.monotonic() + WINDOW_TIMEOUT + while screen.rect is None and time.monotonic() < deadline: + screen.follow(app.main_window_frame()) + if screen.rect is None: + time.sleep(0.5) + if screen.rect is None: + manifest["frames_error"] = f"no app window on screen within {WINDOW_TIMEOUT:.0f}s" + else: + recorder.start() + for index, step in enumerate(tour["steps"], 1): + recorder.step = index + result = run_step(app, screen, step, index, shots) + manifest["steps"].append(result) + print(f"[{tour['name']}] {index:02d} {result['status']:<11} {step['title']}" + + (f": {result['detail']}" if result.get("detail") else ""), flush=True) + if not app.alive(): + manifest["error"] = "the app exited during the tour: " + app.log_tail() + break + finally: + # A second signal must not cut the cleanup short. + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + # No frame after this point: once the app is gone, the rect shows + # whatever was behind it on the shared console. + with recorder.lock: + recorder.stopping.set() + # Then the app: a leftover app or daemon would serve the next run on this Mac. + app.stop() + manifest["frames"] = recorder.stop() if recorder.ident else [] + if recorder.error and not manifest["frames"]: + manifest.setdefault("frames_error", recorder.error) + manifest["capture_rect"] = screen.rect + if app.log_path.exists(): + shutil.copy(app.log_path, directory / "app.log") + shutil.rmtree(workdir, ignore_errors=True) + (directory / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return manifest + + +def terminated(signum: int, frame: Any) -> None: + raise SystemExit(128 + signum) + + +def passed(manifest: dict[str, Any]) -> bool: + return "error" not in manifest and all(step["status"] != "failed" for step in manifest["steps"]) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("tours", nargs="+", type=Path, help="tour JSON files or directories of them") + parser.add_argument("--app", type=Path, help="the built app bundle") + parser.add_argument("--out", type=Path, help="where each tour's folder goes") + parser.add_argument("--check", action="store_true", help="only validate the tour files") + parser.add_argument("--fresh-state", action="store_true", + help="delete the tagged build's saved workspaces before and after each tour") + args = parser.parse_args(argv) + files: list[Path] = [] + for path in args.tours: + files.extend(sorted(path.glob("*.json")) if path.is_dir() else [path]) + try: + tours = [load_tour(path) for path in files] + except TourError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + names = [tour["name"] for tour in tours] + if len(set(names)) != len(names): + print(f"error: tour names collide: {names}", file=sys.stderr) + return 2 + if args.check: + print(f"ok: {len(tours)} tour(s), {sum(len(t['steps']) for t in tours)} steps") + return 0 + if not args.app or not args.out: + parser.error("--app and --out are required unless --check") + # A cancelled job gets SIGINT, then SIGTERM; both unwind through + # run_tour's cleanup, so the app and its daemon do not outlive the job. + signal.signal(signal.SIGTERM, terminated) + session = Session() + args.out.mkdir(parents=True, exist_ok=True) + probe = Path(tempfile.mkdtemp(prefix="cmux-tour-probe.", dir="/tmp")) + try: + session.probe_capture(session.shareable(probe)) + finally: + # The probe captured the whole display; it never leaves the runner. + shutil.rmtree(probe, ignore_errors=True) + print(f"screen capture: {session.capture_mode}", flush=True) + ok = True + for tour in tours: + signal.signal(signal.SIGINT, signal.default_int_handler) + signal.signal(signal.SIGTERM, terminated) + ok = passed(run_tour(args.app, tour, args.out, session, fresh_state=args.fresh_state)) and ok + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/cmux-next/media/tours/core.json b/scripts/cmux-next/media/tours/core.json new file mode 100644 index 000000000000..f3c67db84727 --- /dev/null +++ b/scripts/cmux-next/media/tours/core.json @@ -0,0 +1,21 @@ +{ + "title": "Core UI", + "steps": [ + {"title": "Launch", "wait": 2}, + {"title": "Type in the terminal", "daemon": true, + "cmux": ["terminal", "current", "write", "--text", "echo hello from the cmux-next tour\r"]}, + {"title": "New workspace", "cmux": ["workspace", "new", "--name", "Tour"]}, + {"title": "Split right", "cmux": ["action", "run", "splitRight"]}, + {"title": "Split down", "cmux": ["action", "run", "splitDown"]}, + {"title": "New terminal tab", "cmux": ["action", "run", "newSurface"]}, + {"title": "Browser tab", "cmux": ["tab", "new-browser", "--url", "https://example.com"], "wait": 5}, + {"title": "Hide the sidebar", "cmux": ["action", "run", "toggleSidebar"]}, + {"title": "Show the sidebar", "cmux": ["action", "run", "toggleSidebar"], "shot": false}, + {"title": "Notification", "daemon": true, + "cmux": ["notify", "--title", "Tour", "--body", "Hello from CI"], "wait": 2}, + {"title": "Notifications panel", "cmux": ["action", "run", "showNotifications"], + "optional": "the notifications panel is not ported yet"}, + {"title": "Agent chat", "cmux": ["agent", "new-chat"], "wait": 4, + "optional": "agent chat views are not ported yet"} + ] +} diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 021ca4d3f171..4002f0faed5a 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -915,3 +915,7 @@ lane = "linux-guard" [[test]] path = "tests/test_iroh_drift_issue.py" lane = "linux-guard" + +[[test]] +path = "tests/test_cmux_next_media.py" +lane = "linux-guard" diff --git a/tests/test_cmux_next_media.py b/tests/test_cmux_next_media.py new file mode 100644 index 000000000000..c0b48bd753b9 --- /dev/null +++ b/tests/test_cmux_next_media.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""cmux-next tour media: the runner pick, the tour format and the PR comment.""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts" / "ci")) + +import cmux_next_media_runner as runner # noqa: E402 + + +def load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +MEDIA = ROOT / "scripts" / "cmux-next" / "media" +tour = load("cmux_next_tour", MEDIA / "tour.py") +publish = load("cmux_next_publish", MEDIA / "publish.py") + +SLOTS = json.dumps({"glaeda-std-xcode-26.6": 42, "glaeda-root-std-xcode-26.6": 19, + "glaeda-gui-std-xcode-26.6": 10}) +XCODE = "/Applications/Xcode_26.6.app" + + +class RunnerPick(unittest.TestCase): + def test_a_same_repository_head_takes_the_gui_runner(self) -> None: + label, reason = runner.pick(same_repo=True, owned="1", owned_slots=SLOTS, pr_xcode_app=XCODE) + self.assertEqual(label, "glaeda-gui-std-xcode-26.6") + self.assertIn("10 gui runner", reason) + + def test_never_blacksmith_and_never_a_fork(self) -> None: + cases = { + "fork": dict(same_repo=False, owned="1", owned_slots=SLOTS, pr_xcode_app=XCODE), + "owned off": dict(same_repo=True, owned="0", owned_slots=SLOTS, pr_xcode_app=XCODE), + "no gui slots": dict(same_repo=True, owned="1", pr_xcode_app=XCODE, + owned_slots=json.dumps({"glaeda-std-xcode-26.6": 42})), + "other xcode": dict(same_repo=True, owned="1", owned_slots=SLOTS, + pr_xcode_app="/Applications/Xcode_26.3.app"), + "malformed slots": dict(same_repo=True, owned="1", owned_slots="{", pr_xcode_app=XCODE), + } + for name, kwargs in cases.items(): + with self.subTest(name): + label, reason = runner.pick(**kwargs) + self.assertEqual(label, "") + self.assertTrue(reason) + + +class TourFormat(unittest.TestCase): + def write(self, data) -> Path: + directory = Path(tempfile.mkdtemp()) + path = directory / "sample.json" + path.write_text(json.dumps(data), encoding="utf-8") + return path + + def test_every_checked_in_tour_loads(self) -> None: + tours = sorted((MEDIA / "tours").glob("*.json")) + self.assertTrue(tours) + for path in tours: + with self.subTest(path.name): + loaded = tour.load_tour(path) + self.assertTrue(loaded["steps"]) + + def test_bad_steps_are_refused_with_the_step_number(self) -> None: + bad = { + "unknown key": {"title": "x", "run": ["a"]}, + "no title": {"cmux": ["a"]}, + "string command": {"title": "x", "cmux": "pane split-right"}, + "daemon without command": {"title": "x", "daemon": True}, + "long wait": {"title": "x", "wait": 120}, + "empty optional": {"title": "x", "optional": ""}, + } + for name, step in bad.items(): + with self.subTest(name): + path = self.write({"title": "T", "steps": [{"title": "ok"}, step]}) + with self.assertRaisesRegex(tour.TourError, "step 2"): + tour.load_tour(path) + + def test_a_tour_needs_a_title_and_steps(self) -> None: + for data in ({"steps": [{"title": "x"}]}, {"title": "T", "steps": []}, {"title": "T", "steps": [], "x": 1}): + with self.subTest(data=data), self.assertRaises(tour.TourError): + tour.load_tour(self.write(data)) + + def test_check_mode_validates_without_an_app(self) -> None: + self.assertEqual(tour.main(["--check", str(MEDIA / "tours")]), 0) + self.assertEqual(tour.main(["--check", str(self.write({"title": "T", "steps": [{}]}))]), 2) + + def test_a_tour_fails_only_on_a_required_step(self) -> None: + manifest = {"steps": [{"status": "ok"}, {"status": "unavailable"}]} + self.assertTrue(tour.passed(manifest)) + self.assertFalse(tour.passed({"steps": [{"status": "failed"}]})) + self.assertFalse(tour.passed({"steps": [], "error": "launch: no socket"})) + + +class CaptureRect(unittest.TestCase): + def screen(self, width=1920.0, height=1080.0): + screen = tour.Screen.__new__(tour.Screen) + screen.rect, screen.primary_width, screen.primary_height = None, width, height + return screen + + def test_a_window_frame_becomes_a_top_left_rect(self) -> None: + screen = self.screen() + screen.follow([100, 200, 800, 600]) + self.assertEqual(screen.rect, (100, 280, 800, 600)) + + def test_a_window_partly_off_screen_is_clamped(self) -> None: + screen = self.screen() + screen.follow([-50, 900, 400, 300]) + self.assertEqual(screen.rect, (0, 0, 350, 180)) + screen.follow([1800, 0, 400, 300]) + self.assertEqual(screen.rect, (1800, 780, 120, 300)) + + def test_a_window_that_leaves_the_display_stops_the_capture(self) -> None: + screen = self.screen() + screen.follow([100, 200, 800, 600]) + screen.follow([5000, 200, 800, 600]) + self.assertIsNone(screen.rect) + + def test_a_window_that_disappears_stops_the_capture(self) -> None: + for gone in (None, [0, 0, 10, 10], [1, 2]): + with self.subTest(gone=gone): + screen = self.screen() + screen.follow([100, 200, 800, 600]) + screen.follow(gone) + self.assertIsNone(screen.rect) + + def test_no_window_means_no_rect_and_no_shot(self) -> None: + screen = self.screen() + screen.follow(None) + screen.follow([0, 0, 10, 10]) + self.assertIsNone(screen.rect) + self.assertEqual(screen.capture(Path("/tmp/never.png")), "no app window on screen to capture") + + +class Bounds(unittest.TestCase): + def test_the_recorder_stops_at_the_frame_cap(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + class Screen: + def capture(self, path: Path, kind: str) -> None: + return None + recorder = tour.Recorder(Screen(), Path(scratch)) + recorder.frames = [{"file": "x", "step": 0}] * tour.MAX_FRAMES + recorder.run() + self.assertEqual(len(recorder.frames), tour.MAX_FRAMES) + self.assertIn("stops at", recorder.error) + + def test_the_publisher_shares_the_cap(self) -> None: + self.assertEqual(publish.MAX_FRAMES, tour.MAX_FRAMES) + + +class FreshState(unittest.TestCase): + def app(self, tag: str, *, fresh: bool) -> object: + app = tour.App.__new__(tour.App) + app.session = type("S", (), {"user": "cmux"})() + app.bundle_tag = tour.clean_tag(tag) + app.fresh_state = fresh + return app + + def test_the_tag_state_directory_matches_the_daemon_launcher(self) -> None: + directory = self.app("ci-media", fresh=True).state_directory() + self.assertEqual(directory.parts[-4:], ("cmux", "tags", "ci-media", "tui")) + self.assertEqual(tour.clean_tag("a/b c."), "a-b-c") + + def test_an_untagged_build_never_loses_its_state(self) -> None: + app = self.app("", fresh=True) + self.assertIsNone(app.state_directory()) + app.session.run = lambda argv: self.fail(f"ran {argv}") + app.clear_state() + + def test_an_untagged_build_never_stops_the_users_daemon(self) -> None: + app = self.app("", fresh=True) + app.cli = lambda *a, **k: self.fail(f"ran {a}") + app.stop_daemon() + + def test_the_tagged_daemon_stops_through_its_session(self) -> None: + app = self.app("ci-media", fresh=False) + app.identity = {} + calls = [] + app.cli = lambda args, **options: calls.append((args, options.get("daemon"))) + app.stop_daemon() + self.assertEqual(app.daemon_session(), "cmux-app-ci-media") + self.assertEqual(calls, [(["server", "stop", "--end-terminals"], True)]) + + def test_state_is_kept_unless_asked(self) -> None: + app = self.app("ci-media", fresh=False) + app.session.run = lambda argv: self.fail(f"ran {argv}") + app.clear_state() + + +class Comment(unittest.TestCase): + MANIFEST = { + "name": "core", "title": "Core UI", + "steps": [ + {"index": 1, "title": "Launch", "status": "ok", "shot": "shots/01-launch.png"}, + {"index": 2, "title": "Split | right", "status": "failed", "detail": "exited 2", + "command": "cmux pane split-right", "shot": "shots/02-split-right-failed.png"}, + {"index": 3, "title": "Agent chat", "status": "unavailable", + "detail": "exited 1 (agent chat views are not ported yet)"}, + ], + } + URLS = {"shots/01-launch.png": "https://raw/01.png", "shots/02-split-right-failed.png": "https://raw/02.png", + "tour.gif": "https://raw/tour.gif", "tour.mp4": "https://raw/tour.mp4"} + + def test_failed_steps_show_inline_and_the_rest_fold(self) -> None: + section = publish.tour_section(self.MANIFEST, self.URLS) + self.assertIn("### Core UI: 1 ok, 1 failed, 1 unavailable", section) + failed, _, folded = section.partition("
") + self.assertIn("https://raw/02.png", failed) + self.assertNotIn("https://raw/01.png", failed) + self.assertIn("https://raw/01.png", folded) + self.assertIn("Split \\| right", section) + self.assertIn("[Video (mp4)](https://raw/tour.mp4)", section) + + def test_the_comment_carries_the_marker_and_a_note(self) -> None: + body = publish.render("a" * 40, [], "https://run", "The tour did not run: owned Macs are off.") + self.assertTrue(body.startswith(publish.COMMENT_MARKER)) + self.assertIn("The tour did not run", body) + self.assertIn(publish.TOURS_DIR, body) + + def test_dry_run_renders_from_tour_output(self) -> None: + media = Path(tempfile.mkdtemp()) + (media / "core" / "shots").mkdir(parents=True) + (media / "core" / "shots" / "01-launch.png").write_bytes(b"\x89PNG\r\n\x1a\n") + (media / "core" / "manifest.json").write_text(json.dumps(self.MANIFEST), encoding="utf-8") + self.assertEqual(publish.main(["--media", str(media), "--pr", "1", "--sha", "b" * 40, "--dry-run"]), 0) + + +class ArtifactIsData(unittest.TestCase): + """The tour artifact comes from a job that ran the pull request's code.""" + + def tour(self, manifest, name="core") -> Path: + media = Path(tempfile.mkdtemp()) + (media / name / "shots").mkdir(parents=True) + (media / name / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + return media + + def test_only_shot_paths_tour_py_writes_are_uploaded(self) -> None: + media = self.tour({"steps": []}) + directory = media / "core" + (directory / "shots" / "01-launch.png").write_bytes(b"x") + (directory / "secret.png").write_bytes(b"x") + os.symlink("/etc/hosts", directory / "shots" / "02-link.png") + self.assertEqual(publish.shot_file(directory, "shots/01-launch.png"), directory / "shots/01-launch.png") + for bad in ("../core/shots/01-launch.png", "secret.png", "shots/02-link.png", "/etc/hosts", + "shots/../../x.png", ["shots/01-launch.png"], None): + with self.subTest(bad=bad): + self.assertIsNone(publish.shot_file(directory, bad)) + + def test_folders_that_are_not_tour_slugs_are_skipped(self) -> None: + self.assertEqual(publish.manifests(self.tour({"steps": []}, name="Bad Name")), []) + self.assertEqual(len(publish.manifests(self.tour({"steps": [1, {"title": "x"}]}))), 1) + self.assertEqual(publish.manifests(self.tour(["not", "an", "object"])), []) + + def test_manifest_text_is_escaped(self) -> None: + section = publish.tour_section({"title": "", "steps": [ + {"index": 1, "title": "