cmux NEXT agent engine bench: WebKit modes stay WebKit, and --shots captures each engine over the window glass

scripts/cmux-next/agent-engine-bench.py

@@ -34,13 +34,14 @@
                     help="launch through LaunchServices (`open -n`), for a shell outside the console's GUI session (ssh)")
 parser.add_argument("--out", default="/tmp")
 parser.add_argument("--json")
+parser.add_argument("--shots", help="a folder: light and dark screen captures of the open session in each mode, with the window glass on")
 opts = parser.parse_args()
 # LaunchServices reports the real path (/private/tmp for /tmp), so match processes by it.
 TAG, APP = opts.tag, os.path.realpath(opts.app.rstrip("/"))
 SOCKET = f"/tmp/cmux-debug-{TAG}.sock"
 CLI = os.path.join(APP, "Contents/Resources/bin/cmux")
 BINARY = next((p for p in sorted(glob.glob(os.path.join(APP, "Contents/MacOS/*"))) if not p.endswith(".dylib")), None)
-TUNABLES = {"own": {}, "page": {"agent.pageHost": True}, "chromium": {"agent.chromium": True}}
+TUNABLES = {"own": {"agent.chromium": False}, "page": {"agent.pageHost": True, "agent.chromium": False}, "chromium": {"agent.chromium": True}}
 app = None
 
 
@@ -136,6 +137,25 @@ def typing():
             "composer_text_length": pane("readiness").get("composer_text_length"), **pane("typing_stats")}
 
 
+def screen_shots(mode):
+    """The screen with the chat's session open, light then dark: the screen, not the app's own snapshot,
+    since only the window server composites a Chromium page window over the glass under it."""
+    if not opts.shots:
+        return None
+    os.makedirs(opts.shots, exist_ok=True)
+    taken = {}
+    for look in ("light", "dark"):
+        rpc("debug.appearance", {"mode": look})
+        time.sleep(3)  # test harness: the theme reload and the page's repaint land
+        # screencapture takes one file per display; a single-display mini writes only d1.
+        names = [os.path.join(opts.shots, f"{mode}-{look}-d{n}.png") for n in (1, 2)]
+        done = subprocess.run(["/usr/sbin/screencapture", "-x", "-o", *names], capture_output=True, text=True, timeout=30)
+        taken[look] = {"files": [n for n in names if os.path.exists(n)], "error": done.stderr.strip() or None}
+    rpc("debug.appearance", {"mode": "system"})
+    taken["windows"] = rpc("debug.window_list")
+    return taken
+
+
 def focus_probe():
     if not opts.activate:
         return None
@@ -167,7 +187,11 @@ def launch(mode, scratch):
     json.dump({"browser": {"defaultEngine": "chromium"}}, open(config, "w"))
     tunables = os.path.join(scratch, "debug-tunables.json")
     json.dump({"version": 1, "values": TUNABLES[mode]}, open(tunables, "w"))
-    open(os.path.join(scratch, "ghostty"), "w").close()
+    with open(os.path.join(scratch, "ghostty"), "w") as ghostty:
+        if opts.shots:
+            # The window glass (Liquid Glass under a 0.7 theme tint), so the captures show what a page does over it;
+            # an empty config is an opaque window.
+            ghostty.write("background-opacity = 0.7\nbackground-blur = macos-glass-regular\n")
     env = {"HOME": os.environ["HOME"], "USER": os.environ.get("USER", ""), "TMPDIR": os.environ.get("TMPDIR", "/tmp"),
            "PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "CMUX_NEXT_SOCKET_MODE": "automation", "CMUX_DEV_BACKEND_MODE": "local",
            "CMUX_NEXT_AGENT_PANE_MOCK": "1", "CMUX_NEXT_CONFIG_FILE": config, "CMUX_NEXT_DEBUG_TUNABLES_FILE": tunables,
@@ -224,6 +248,7 @@ def run_mode(mode):
     result["one_chat"] = memory("one chat", 1)
     result["session"] = open_session()
     result["focus"] = focus_probe()
+    result["shots"] = screen_shots(mode)
     result["typing"] = typing()
     pane("seed_rows", count=5000, timeout=60)
     time.sleep(3)  # test harness: the seeded transcript lays out