perf: add workspace scaling budget harness

.github/workflows/cmux-next-perf-budgets.yml

@@ -70,7 +70,7 @@ on:
         required: false
         default: ""
       parts:
-        description: "GUI parts (transcript-scroll,sidebar,startup) or pane-overflow-fade; manual Blacksmith mode only"
+        description: "GUI parts (transcript-scroll,sidebar,workspace-scaling,startup) or pane-overflow-fade; manual Blacksmith mode only"
         required: false
         default: ""
       product_sha:
@@ -235,6 +235,7 @@ jobs:
             case "$part" in
               transcript-scroll) args=(--app-cache --vdisplay=120 --suite=budgets --only=acp --raw=acp) ;;
               sidebar) args=(--app-cache --vdisplay=120 --suite=budgets --only=sidebar) ;;
+              workspace-scaling) args=(--app-cache --vdisplay=120 --vdisplay-hold=3600 --suite=workspace-scaling) ;;
               startup) args=(--app-cache --vdisplay=120 --suite=launch --profile empty --mode warm --runs 5 --timeout 20) ;;
               *) echo "unsupported part: $part" >&2; exit 2 ;;
             esac

scripts/cmux-next/automation-typing-bench.py

@@ -63,6 +63,9 @@ def load(name, file):
 parser.add_argument("--json")
 parser.add_argument("--layouts", default="light,stress", help="comma list of light and stress (STRESS session)")
 parser.add_argument("--agents", type=int, default=10)
+parser.add_argument("--workspace-count", type=int, default=0,
+                    help="seed this total workspace count with empty rows; --agents remains the bounded active set")
+parser.add_argument("--no-browser", action="store_true", help="skip the optional browser agent (used for exact one-workspace typing baseline)")
 parser.add_argument("--agent-hz", type=float, default=40.0, help="terminal agent writes per second, all agents together")
 parser.add_argument("--samples", type=int, default=120)
 parser.add_argument("--interval", type=float, default=0.12, help="seconds between keys (wider than one echo)")
@@ -92,7 +95,7 @@ def pct(values):
         return None
     s = sorted(values)
     pick = lambda p: s[min(len(s) - 1, int(round((len(s) - 1) * p)))]
-    return {"n": len(s), "p50": round(pick(0.5), 2), "p95": round(pick(0.95), 2), "max": round(s[-1], 2),
+    return {"n": len(s), "p50": round(pick(0.5), 2), "p95": round(pick(0.95), 2), "p99": round(pick(0.99), 2), "max": round(s[-1], 2),
             "mean": round(statistics.fmean(s), 2)}
 
 
@@ -153,6 +156,40 @@ def terminal_of(workspace_name):
     return None
 
 
+def seed_empty_workspaces(count):
+    """Fill the requested topology with empty workspaces so large runs do not exhaust PTYs."""
+    if count <= 0:
+        return {"requested": 0, "created": 0, "failures": []}
+    # The probe adds one focused typing workspace, optional browser workspace and agent
+    # workspaces. Account for any row the app restored before seeding so the final topology
+    # remains exactly the requested size instead of silently becoming count+1.
+    existing = len(topology().get("workspaces") or [])
+    fixed = opts.agents + 1 + (0 if opts.no_browser else 1)
+    empty = max(0, count - existing - fixed)
+    failures = []
+    created = 0
+    for index in range(empty):
+        done = cli("workspace", "create", "--name", f"scale empty {index}", "--empty")
+        if done.returncode == 0:
+            created += 1
+        else:
+            failures.append({"index": index, "returncode": done.returncode,
+                             "stderr": (done.stderr or "")[-300:]})
+    return {"requested": empty, "created": created, "existing": existing, "failures": failures}
+
+
+def reset_restored_workspaces():
+    """Keep a restored row from making the requested topology count+1."""
+    existing = len(topology().get("workspaces") or [])
+    if not existing:
+        return {"existing": 0, "closed": 0, "failures": []}
+    done = cli("workspace", "close-all-in-all-windows", "--confirm", timeout=120)
+    if done.returncode != 0 or not wait(lambda: len(topology().get("workspaces") or []) == 0, 30):
+        return {"existing": existing, "closed": 0,
+                "failures": [{"returncode": done.returncode, "stderr": (done.stderr or "")[-300:]}]}
+    return {"existing": existing, "closed": existing, "failures": []}
+
+
 # ---------------------------------------------------------------- load
 class Load(threading.Thread):
     def __init__(self, name, step, hz):
@@ -357,28 +394,48 @@ def main():
     pid = APP.process.pid
     summary = {}
     try:
+        # Existing typing benches preserve their normal startup semantics. Scaling runs opt in
+        # with --workspace-count and need a deterministic empty topology before seeding.
+        reset = reset_restored_workspaces() if opts.workspace_count else {"existing": 0, "closed": 0, "failures": []}
+        results["reset_restored_workspaces"] = reset
+        if reset["failures"]:
+            results["invalid"] = {"restored_workspaces": reset}
+            return 2
+        seeded = seed_empty_workspaces(opts.workspace_count)
+        results["seeded_empty_workspaces"] = seeded
         names = [f"agent {i:02d}" for i in range(opts.agents)]
         for name in names:
             cli("workspace", "create", "--name", name, "--shell", "exec cat")
         tabs = [t for t in (wait(lambda n=n: terminal_of(n), 20) for n in names) if t]
         note(f"agent terminals: {len(tabs)} of {opts.agents}")
-        cli("workspace", "create", "--name", "browser agent", "--focus", "--empty")
-        page = os.path.join(OUT, "agent-page.html")
-        open(page, "w").write(PAGE.format(rows="".join(f"<li>row {i} {'lorem ipsum ' * 4}</li>" for i in range(opts.rows))))
-        rpc("action.run", {"action": "openBrowser.chromium", "args": {"url": pathlib.Path(page).as_uri()}, "origin": "script"}, timeout=120)
-        browser_tab = wait(lambda: (lambda r: r.get("tab") if isinstance(r, dict) and r.get("value") is True else None)(
-            rpc("browser.page.eval", {"script": "document.readyState === 'complete' && !!document.getElementById('b')"}, timeout=20)), 60, 0.5)
-        note(f"browser agent tab: {browser_tab}")
-        cef = rpc("debug.cef", timeout=20)
-        results["cef"] = cef
-        not_chromium = chromium_problems(cef) if browser_tab else ["the browser agent's page never loaded"]
-        if not_chromium:
-            results["invalid"] = not_chromium
-            note(f"not measured: the browser agent's page is not Chromium: {not_chromium}")
-            return 2
+        browser_tab = None
+        if not opts.no_browser:
+            cli("workspace", "create", "--name", "browser agent", "--focus", "--empty")
+            page = os.path.join(OUT, "agent-page.html")
+            open(page, "w").write(PAGE.format(rows="".join(f"<li>row {i} {'lorem ipsum ' * 4}</li>" for i in range(opts.rows))))
+            rpc("action.run", {"action": "openBrowser.chromium", "args": {"url": pathlib.Path(page).as_uri()}, "origin": "script"}, timeout=120)
+            browser_tab = wait(lambda: (lambda r: r.get("tab") if isinstance(r, dict) and r.get("value") is True else None)(
+                rpc("browser.page.eval", {"script": "document.readyState === 'complete' && !!document.getElementById('b')"}, timeout=20)), 60, 0.5)
+            note(f"browser agent tab: {browser_tab}")
+            cef = rpc("debug.cef", timeout=20)
+            results["cef"] = cef
+            not_chromium = chromium_problems(cef) if browser_tab else ["the browser agent's page never loaded"]
+            if not_chromium:
+                results["invalid"] = not_chromium
+                note(f"not measured: the browser agent's page is not Chromium: {not_chromium}")
+                return 2
         cli("workspace", "create", "--name", "typing", "--focus", "--shell", "exec cat")
         typing = terminal_of("typing")
         results["setup"] = {"agent_terminals": len(tabs), "browser_tab": browser_tab, "typing": typing}
+        actual_workspaces = len(topology().get("workspaces") or [])
+        results["setup"]["workspace_count"] = actual_workspaces
+        if seeded["failures"] or (opts.workspace_count and actual_workspaces != opts.workspace_count):
+            results["invalid"] = {
+                "requested_workspace_count": opts.workspace_count,
+                "actual_workspace_count": actual_workspaces,
+                "seed_failures": seeded["failures"],
+            }
+            return 2
         for layout in LAYOUTS:
             if layout == "stress":
                 session = STRESS.Stress(rpc, log=note).build()

scripts/cmux-next/instant-new-surfaces-bench.py

@@ -60,6 +60,10 @@
 parser.add_argument("--phases", default="idle,load")
 parser.add_argument("--commands", default="split_right,split_down,new_tab,new_terminal,new_workspace")
 parser.add_argument("--workspaces", default="", help="comma-separated workspace counts to run every phase at (empty: as launched)")
+parser.add_argument("--empty-workspaces", action="store_true",
+                    help="seed additional workspaces without terminals (safe for 1000-workspace scaling runs)")
+parser.add_argument("--exact-workspaces", action="store_true",
+                    help="close restored workspaces before each count so the topology is exactly the requested size")
 parser.add_argument("--budget-frames", type=float, default=0, help="fail when a command's p95 display frame of visibility is above this (1: the first frame; 0: report only)")
 parser.add_argument("--budget-key-ms", type=float, default=16.7)
 parser.add_argument("--hang-stacks", action="store_true",
@@ -344,17 +348,29 @@ def workspace_count():
 
 
 def grow_workspaces(count):
-    """At least `count` terminal workspaces, the bench's still shown (scripts make them unfocused)."""
+    """At least `count` workspaces; optionally seed empty rows to avoid opening 1000 PTYs."""
     have = workspace_count()
     for index in range(have, count):
-        rpc("action.run", {"action": "newTab", "args": {"name": f"bench-{index}"}, "wait": True})
+        if opts.empty_workspaces:
+            run("workspace", "create", "--empty", "--name", f"bench-{index}", timeout=120)
+        else:
+            rpc("action.run", {"action": "newTab", "args": {"name": f"bench-{index}"}, "wait": True})
     now = workspace_count()
     print(f"workspaces: {have} -> {now}", flush=True)
     if now < count:
         sys.exit(f"could not reach {count} workspaces (have {now})")
     return now
 
 
+def reset_workspaces():
+    """Drop restored rows in this isolated bench profile before an exact topology run."""
+    if workspace_count() == 0:
+        return
+    done = run("workspace", "close-all-in-all-windows", "--confirm", timeout=120)
+    if done.returncode != 0 or not wait(lambda: workspace_count() == 0, 30):
+        sys.exit(f"could not clear restored workspaces: {done.stderr[-400:]}")
+
+
 def frame_of(row, key="visible_ms"):
     """Which display frame after the key showed it (1 = the first frame the display link drew).
     The app's `frames_to_visible` divides by the refresh period, which counts a late first frame
@@ -380,7 +396,7 @@ def summary(rows):
             out[f"{phase}/{command}"] = {
                 "n": len(sel), "missed": sum(v is None for v in vis),
                 "key_p50": pct([r.get("key_ms") for r in sel], .5), "key_max": pct([r.get("key_ms") for r in sel], 1),
-                "visible_p50": pct(vis, .5), "visible_p95": pct(vis, .95), "visible_max": pct(vis, 1),
+                "visible_p50": pct(vis, .5), "visible_p95": pct(vis, .95), "visible_p99": pct(vis, .99), "visible_max": pct(vis, 1),
                 "frames_p50": pct([frame_of(r) for r in sel], .5),
                 "frames_p95": pct([frame_of(r) for r in sel], .95),
                 "frames_max": pct([frame_of(r) for r in sel], 1),
@@ -457,8 +473,12 @@ def relaunch():
     for count in counts:
         suffix = ""
         if count is not None:
+            if opts.exact_workspaces:
+                reset_workspaces()
             ensure_terminal()
             suffix = f"@{grow_workspaces(count)}ws"
+            if opts.exact_workspaces and workspace_count() != count:
+                sys.exit(f"exact workspace count failed: requested {count}, have {workspace_count()}")
         for phase in opts.phases.split(","):
             if phase == "load":
                 start_load()

scripts/cmux-next/perf-cron.py

@@ -13,7 +13,8 @@
                     nightly or dev build waits for a Mac on GitHub.
   measure (gateway) runs only on its own runner label (vars.CI_PERF_CRON_GATEWAY), skips unless the part's host
                     (the measurement's, else cmux20-bench) is idle and no dev build waits in the controller queue,
-                    then runs ONE part (its run_seconds: 15 minutes, the build 20, STRESS and sidebar 25) as a
+                    then runs ONE part (its run_seconds: 15 minutes, the build 20, STRESS and sidebar 25, workspace
+                    scaling 60 because it seeds and measures a 1000-row topology) as a
                     background (lowest priority) measurement step on that host, and cancels it when a foreground
                     job queues that the host could run (its labels, its name, or a target worker).
   request (controller) with no gateway label and vars.CMUX_NEXT_PERF_CRON_CONTROLLER=1: a job named for the part;
@@ -157,6 +158,10 @@
     # 400 workspaces, 6 scenarios: not timed against 900 s yet (perf-budgets' own per-bench cap is 900 s).
     {"name": "sidebar", "args": [*MEASURE, "--suite=budgets", "--only=sidebar"],
      "artifact": "artifacts/split-spring/report.json", "kind": "budgets", "run_seconds": 1500},
+    # 1/100/1000 workspace topology: warning-only constant-scaling trend budget. This is the one 60-minute
+    # part: seeding and measuring all three topologies is intentionally longer than the normal 15-minute tick.
+    {"name": "workspace-scaling", "args": [*MEASURE, "--vdisplay-hold=3600", "--suite=workspace-scaling"],
+     "artifact": "artifacts/split-spring/report.json", "kind": "budgets", "needs": "workspace-scaling", "run_seconds": 3600},
     {"name": "home-idle", "args": [*MEASURE, "--suite=budgets", "--only=home,idle"],
      "artifact": "artifacts/split-spring/report.json", "kind": "budgets"},
     {"name": "chat-anim", "args": [*MEASURE, "--suite=budgets", "--only=acp,anim"],
@@ -171,6 +176,7 @@
 ROW_PARTS = [
     (r"^panes\.[^.]+\.stress\.", "stress"), (r"^panes\.", "motion"), (r"^(cmd_n|cmd_t|ctrl_backtick|cmd_d|cmd_shift_d)\.", "motion"),
     (r"^(tab_switch|palette)\.", "switch"), (r"^perf_gate\.", "perf-gate"), (r"^(sidebar|workspace_switch)\.", "sidebar"),
+    (r"^workspace_scaling\.", "workspace-scaling"),
     (r"^(home|idle)\.", "home-idle"), (r"^(acp|anim)\.", "chat-anim"), (r"^launch\.", "launch"),
     (r"^diff\.", "diff-viewer"), (r"^webui\.", "web-ui"),
 ]
@@ -603,6 +609,10 @@ def target_of(name, row, refresh_ms):
     refresh_ms = row_refresh(row, refresh_ms)
     unit = row.get("unit") or "ms"
     name = base_row(name)
+    if name.startswith("workspace_scaling."):
+        # Scaling rows are judged only by their 1000-vs-baseline ratio. Their absolute
+        # latency is intentionally advisory and must not inherit the generic 10 ms target.
+        return None
     if name.startswith(("diff.", "launch.", "webui.")):
         return row.get("budget")
     if unit in ("frames", "checks") or name.endswith(".dropped"):
@@ -1455,6 +1465,12 @@ def supported_parts_at(sha):
             text = subprocess.run(["git", "-C", ROOT, "show", f"{sha}:scripts/cmux-next/split-spring-terminal.py"],
                                   capture_output=True, text=True).stdout
             ok = f"--suite={need}" in text
+        elif need == "workspace-scaling":
+            text = subprocess.run(["git", "-C", ROOT, "show", f"{sha}:scripts/cmux-next/split-spring-terminal.py"],
+                                  capture_output=True, text=True).stdout
+            ok = "--suite=workspace-scaling" in text and subprocess.run(
+                ["git", "-C", ROOT, "cat-file", "-e", f"{sha}:scripts/cmux-next/workspace-scaling-bench.py"],
+                capture_output=True).returncode == 0
         else:
             ok = True
         if ok:

scripts/cmux-next/split-spring-terminal.py

@@ -86,6 +86,10 @@ def _flag(name, default=None):
 if "--suite=budgets" in sys.argv:
     suite = os.path.join(os.path.dirname(os.path.abspath(__file__)), "perf-budgets.py")
     os.execv(sys.executable, [sys.executable, suite] + [a for a in sys.argv[1:] if a != "--suite=budgets"])
+# `--suite=workspace-scaling`: the warning-only 1/100/1000 workspace trend budget.
+if "--suite=workspace-scaling" in sys.argv:
+    suite = os.path.join(os.path.dirname(os.path.abspath(__file__)), "workspace-scaling-bench.py")
+    os.execv(sys.executable, [sys.executable, suite] + [a for a in sys.argv[1:] if a != "--suite=workspace-scaling"])
 # `--suite=config-parity`: is the bench build (split-spring.sh --config=bench builds --app) as fast as Release?
 # Builds a Release app of this checkout (tag cfgparity-rel), then scripts/cmux-next/bench-config-parity.py
 # runs the same actions on both, interleaved (--runs, --rounds, --idle pass through).

scripts/cmux-next/tests/test_perf_cron.py

@@ -149,6 +149,8 @@ def test_a_frame_row_target_is_one_display_refresh(self):
         self.assertTrue(cron.over_target("cmd_t.idle.key_to_visible", row(12.0), 16.67))
         self.assertTrue(cron.over_target("perf_gate.tabs.failed_checks", row(1, "checks"), 16.67))
         self.assertIsNone(cron.target_of("anim.tab.over_8_33", row(3.0), 16.67))
+        self.assertIsNone(cron.target_of("workspace_scaling.type.1000", row(99.0), 16.67),
+                          "workspace scaling is advisory ratio-only")
 
     def test_a_new_perf_gate_failure_regresses(self):
         regs = cron.find_regressions({"perf_gate.tabs.failed_checks": row(1, "checks", 0.5)},