bench(acpmux): bench=build-turn, an agent turn that runs a build, at each CPU class

scripts/cmux-next/acpmux-overhead-bench.py

@@ -40,9 +40,18 @@
                    person typing does
   warm_first       the same for a prompt right after, the agent awake
 
+--mode build-turn: an agent turn that runs a build (cpu_class.rs: the CPU class a session's harness
+starts at). Claude Code only, through acpmux, each run a fresh session: the stand-in answers the
+prompt with one Bash call running --build-cmd, then "ok". Per run:
+  end     ms from the prompt's send to the turn's end (the build is most of it)
+  pri     the base priority (`ps -o pri=`) of the shell Claude Code ran the build in: 4 is
+          background (darwinbg), 31 the default class
+  build_s the build's own seconds, as the shell measured them
+
 Usage:
   acpmux-overhead-bench.py --acpmux PATH [--harness claude,codex,opencode] [-n 20]
-                           [--mode overhead|prewarm|hibernate] [--json OUT] [--md OUT]
+                           [--mode overhead|prewarm|hibernate|build-turn] [--build-cmd CMD]
+                           [--json OUT] [--md OUT]
 """
 
 from __future__ import annotations
@@ -74,6 +83,10 @@ def __init__(self):
         # Model requests and their body bytes, for each run's share (`Counter`).
         self.requests = 0
         self.bytes = 0
+        # --mode build-turn: the Bash command a prompt naming BUILD_TURN gets, and every tool
+        # result the harness sent back, in order.
+        self.build_cmd: str | None = None
+        self.tool_results: list[str] = []
 
     def hit(self, tag: str, size: int = 0):
         with self.lock:
@@ -145,6 +158,9 @@ def do_POST(self):
     def anthropic(self, body):
         model = body.get("model", "claude-bench")
         usage = {"input_tokens": 10, "output_tokens": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0}
+        call = build_call(self.server, body)
+        if call is not None and body.get("stream"):
+            return self.reply(200, sse(tool_use_events(model, usage, call)), "text/event-stream")
         if not body.get("stream"):
             msg = {"id": "msg_bench", "type": "message", "role": "assistant", "model": model,
                    "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn",
@@ -202,6 +218,53 @@ def chat(self, body):
         self.reply(200, sse(events), "text/event-stream")
 
 
+# The prompt of a --mode build-turn run: the stand-in answers it with one Bash call.
+BUILD_TURN = "BUILD_TURN: run the build."
+
+
+def build_call(server, body):
+    """The Bash input a main-loop request of a build turn gets, or None (answer "ok").
+
+    Only a request that offers the Bash tool (a side request, such as a title, offers none) and
+    whose last user message is the build prompt itself, not the tool result that follows it.
+    Claude Code may end its messages with a system one (its environment), so the last user
+    message is looked for, never the last message."""
+    if not server.build_cmd or not any(t.get("name") == "Bash" for t in body.get("tools") or []):
+        return None
+    users = [m for m in body.get("messages") or [] if m.get("role") == "user"]
+    content = users[-1].get("content") if users else None
+    blocks = content if isinstance(content, list) else [{"type": "text", "text": content or ""}]
+    results = [b for b in blocks if isinstance(b, dict) and b.get("type") == "tool_result"]
+    if results:
+        for r in results:
+            inner = r.get("content")
+            text = inner if isinstance(inner, str) else "".join(
+                c.get("text", "") for c in inner or [] if isinstance(c, dict))
+            with server.lock:
+                server.tool_results.append(text)
+        return None
+    if not any(BUILD_TURN in (b.get("text") or "") for b in blocks if isinstance(b, dict)):
+        return None
+    return {"command": server.build_cmd, "description": "Run the build", "timeout": 600000}
+
+
+def tool_use_events(model, usage, call):
+    tid = "toolu_bench_" + uuid.uuid4().hex[:12]
+    return [
+        ("message_start", {"type": "message_start", "message": {"id": "msg_bench", "type": "message", "role": "assistant",
+                                                                 "model": model, "content": [], "stop_reason": None,
+                                                                 "stop_sequence": None, "usage": usage}}),
+        ("content_block_start", {"type": "content_block_start", "index": 0,
+                                 "content_block": {"type": "tool_use", "id": tid, "name": "Bash", "input": {}}}),
+        ("content_block_delta", {"type": "content_block_delta", "index": 0,
+                                 "delta": {"type": "input_json_delta", "partial_json": json.dumps(call)}}),
+        ("content_block_stop", {"type": "content_block_stop", "index": 0}),
+        ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None},
+                           "usage": {"output_tokens": 1}}),
+        ("message_stop", {"type": "message_stop"}),
+    ]
+
+
 # ---------------------------------------------------------------- line pipes
 
 
@@ -683,7 +746,8 @@ def main():
     ap.add_argument("--harness", default="claude,codex,opencode")
     ap.add_argument("-n", type=int, default=20)
     ap.add_argument("--warmup", type=int, default=2)
-    ap.add_argument("--mode", choices=["overhead", "prewarm", "hibernate"], default="overhead")
+    ap.add_argument("--mode", choices=["overhead", "prewarm", "hibernate", "build-turn"], default="overhead")
+    ap.add_argument("--build-cmd", help="--mode build-turn: the shell command the turn's Bash call runs")
     ap.add_argument("--idle-wait", type=int, default=135, help="--mode hibernate: seconds the chats sit idle")
     ap.add_argument("--json")
     ap.add_argument("--md")
@@ -708,6 +772,12 @@ def main():
     if args.no_agent_tools:
         extra["ACPMUX_AGENT_TOOLS"] = "0"
     daemon = Daemon(os.path.abspath(args.acpmux), env, home, extra)
+    if args.mode == "build-turn":
+        try:
+            return build_turn_main(args, daemon, stand, scratch)
+        finally:
+            daemon.close()
+            shutil.rmtree(scratch, ignore_errors=True)
     if args.mode in ("prewarm", "hibernate"):
         try:
             return (prewarm_main if args.mode == "prewarm" else hibernate_main)(args, daemon, scratch)
@@ -776,6 +846,68 @@ def main():
     shutil.rmtree(scratch, ignore_errors=True)
 
 
+def build_turn_main(args, daemon: Daemon, stand: StandIn, scratch):
+    """--mode build-turn: n turns, each in a fresh session, whose one tool call runs the build."""
+    stand.build_cmd = args.build_cmd
+    rows, errors = [], []
+    for k in range(args.warmup + args.n):
+        cwd = os.path.join(scratch, f"build-{k}")
+        os.makedirs(cwd)
+        sid = None
+        try:
+            _, r = daemon.request("session/new", {"cwd": cwd, "mcpServers": [],
+                                                   "_meta": {"acpmux": {"harness": "claude", "policy": "approve-all",
+                                                                        "model": MODELS["claude"]}}})
+            sid = r["sessionId"]
+            io = daemon.io
+            daemon.next += 1
+            rid, start, seen = daemon.next, io.mark(), len(stand.tool_results)
+            ts = time.perf_counter()
+            io.send({"jsonrpc": "2.0", "id": rid, "method": "session/prompt",
+                     "params": {"sessionId": sid, "prompt": [{"type": "text", "text": BUILD_TURN}]}})
+            te, m, _ = io.wait(lambda m: m.get("id") == rid, timeout=1200, start=start)
+            if "error" in m:
+                raise RuntimeError(f"session/prompt: {m['error']}")
+            out = "\n".join(stand.tool_results[seen:])
+            row = {"end": ms(ts, te)}
+            for key, name in (("pri", "BUILD-PRI"), ("build_s", "BUILD-SECONDS"), ("exit", "BUILD-EXIT")):
+                hit = [line.split()[1] for line in out.splitlines() if line.startswith(name + " ") and len(line.split()) > 1]
+                row[key] = float(hit[-1]) if hit else None
+            if row["exit"] != 0:
+                raise RuntimeError(f"the build did not pass: {out[-400:]!r}")
+            print(f"build-turn run {k + 1}/{args.warmup + args.n}: {row}", file=sys.stderr, flush=True)
+            if k >= args.warmup:
+                rows.append(row)
+        except Exception as e:  # noqa: BLE001: a bench reports and goes on
+            errors.append(f"run {k}: {e}")
+            print(f"build-turn run {k}: {e}", file=sys.stderr)
+        finally:
+            if sid:
+                try:
+                    daemon.request("session/close", {"sessionId": sid}, timeout=20)
+                except Exception:
+                    pass
+    summary = {m: {"p50": pct([r[m] for r in rows if r[m] is not None], 0.5),
+                   "min": min((r[m] for r in rows if r[m] is not None), default=None),
+                   "max": max((r[m] for r in rows if r[m] is not None), default=None)} for m in ("end", "build_s", "pri")}
+    results = {"claude": {"summary": summary, "runs": rows, "errors": errors},
+               "_meta": {"label": args.label, "acpmux": args.acpmux, "n": args.n, "build_cmd": args.build_cmd,
+                         "host": socket.gethostname(), "load": os.getloadavg()}}
+    if args.json:
+        with open(args.json, "w") as f:
+            json.dump(results, f, indent=1)
+    lines = [f"### build-turn {args.label or 'run'} ({socket.gethostname()}, n={len(rows)}, load {os.getloadavg()[0]:.1f})", "",
+             "| metric | p50 | min | max |", "|---|---|---|---|"]
+    for m, v in summary.items():
+        lines.append(f"| {m} | {v['p50']} | {v['min']} | {v['max']} |")
+    text = "\n".join(lines) + "\n"
+    print(text)
+    if args.md:
+        with open(args.md, "a") as f:
+            f.write(text + "\n")
+    return 1 if errors else 0
+
+
 def prewarm_main(args, daemon: Daemon, scratch):
     results = {}
     for h in args.harness.split(","):
@@ -832,4 +964,4 @@ def prewarm_main(args, daemon: Daemon, scratch):
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

scripts/measure/acpmux-overhead.sh

@@ -40,6 +40,17 @@
 # run and the next prompt's time to its first answer text. With before=SHA both builds run.
 #   cmux-ci run --class exclusive --script scripts/measure/acpmux-overhead.sh --ref SHA \
 #     --arg=bench=hibernate --arg=harness=claude,codex [--arg=n=4] [--arg=before=SHA] --timeout 5400
+#
+# bench=build-turn: an agent turn that runs a build (acpmux cpu_class.rs, the CPU class a session's
+# harness starts at): Claude Code through acpmux, each run a fresh session whose one Bash call is a
+# cold debug build of crate= (default cmux-history) in a fresh target dir. Per acpmux build: the
+# turn's time, the build's own seconds and the base priority of the shell it ran in (4: background
+# class). Then the same build run directly at each class (ACPMUX-BUILD-RAW lines): default,
+# `taskpolicy -c utility` and `taskpolicy -b`, interleaved. With before=SHA both builds run.
+# Output: ACPMUX-OVERHEAD and ACPMUX-BUILD-RAW lines. Exit 1 when a run failed.
+#   cmux-ci run --class exclusive --script scripts/measure/acpmux-overhead.sh --ref SHA \
+#     --arg=bench=build-turn --arg=harness=claude [--arg=n=3] [--arg=crate=cmux-history] \
+#     [--arg=before=SHA] --timeout 5400
 set -euo pipefail
 n=20
 harness="claude,codex,opencode"
@@ -49,6 +60,7 @@ codex_version="0.163.0"
 opencode_version="1.19.3"
 guard_ms=5
 bench=harness
+crate=cmux-history
 for arg in "$@"; do
   case "$arg" in
     bench=*) bench="${arg#bench=}" ;;
@@ -59,6 +71,7 @@ for arg in "$@"; do
     codex=*) codex_version="${arg#codex=}" ;;
     opencode=*) opencode_version="${arg#opencode=}" ;;
     guard_ms=*) guard_ms="${arg#guard_ms=}" ;;
+    crate=*) crate="${arg#crate=}" ;;
     *) echo "acpmux-overhead.sh: unknown argument $arg" >&2; exit 2 ;;
   esac
 done
@@ -69,7 +82,9 @@ for v in "$claude_version" "$codex_version" "$opencode_version"; do
   [[ "$v" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "acpmux-overhead.sh: versions are X.Y.Z" >&2; exit 2; }
 done
 [[ "$guard_ms" =~ ^[0-9]{1,4}$ ]] || { echo "acpmux-overhead.sh: guard_ms is a number" >&2; exit 2; }
-[[ "$bench" =~ ^(harness|daemon|prewarm|hibernate)$ ]] || { echo "acpmux-overhead.sh: bench=harness, daemon, prewarm or hibernate" >&2; exit 2; }
+[[ "$bench" =~ ^(harness|daemon|prewarm|hibernate|build-turn)$ ]] || { echo "acpmux-overhead.sh: bench=harness, daemon, prewarm, hibernate or build-turn" >&2; exit 2; }
+[[ "$crate" =~ ^[a-z0-9_-]{1,64}$ ]] || { echo "acpmux-overhead.sh: crate= is a cmux-tui package name" >&2; exit 2; }
+[[ "$bench" != build-turn || "$harness" == claude ]] || { echo "acpmux-overhead.sh: bench=build-turn runs harness=claude only" >&2; exit 2; }
 # Every tool this bench always calls, checked before anything runs, so a worker without one says
 # which at once instead of failing after a build (cmux20-bench had no npm). Build tools are checked
 # where a build happens: cargo for the cmux fallback below, cargo and rustup in build-acpmux.sh when
@@ -84,6 +99,9 @@ need_tools() {  # need_tools WHAT TOOL...: exit 2 naming every TOOL not on PATH.
 }
 if [[ "$bench" == daemon ]]; then
   need_tools "bench=daemon" git sysctl sw_vers cargo
+elif [[ "$bench" == build-turn ]]; then
+  # The turn's build runs cargo and perl (its clock); ps reads its shell's priority.
+  need_tools "bench=build-turn" git sysctl sw_vers curl shasum awk python3 npm ps cargo perl
 elif [[ "$bench" == hibernate ]]; then
   # Hibernation counts the agent's processes with ps (acpmux-overhead-bench.py).
   need_tools "bench=hibernate" git sysctl sw_vers curl shasum awk python3 npm ps
@@ -185,6 +203,52 @@ echo "ACPMUX-OVERHEAD-BIN claude $(claude --version) codex $(codex --version) op
 status=0
 mode=overhead
 [[ "$bench" == prewarm ]] && mode=prewarm
+if [[ "$bench" == build-turn ]]; then
+  # The bench gives the harness a scratch HOME: the build names the step's own toolchain.
+  cargo_home="${CARGO_HOME:-$HOME/.cargo}" rustup_home="${RUSTUP_HOME:-$HOME/.rustup}"
+  (cd cmux-tui && CARGO_HOME="$cargo_home" RUSTUP_HOME="$rustup_home" cargo fetch --locked >/dev/null)
+  mkdir -p "$work/build-targets"
+  # The turn's Bash call runs this script: its shell's priority, then the build's seconds and exit.
+  cat > "$work/build-turn.sh" <<SH
+echo "BUILD-PRI \$(ps -o pri= -p \$\$)"
+cd "$root/cmux-tui" || exit 1
+t="\$(mktemp -d "$work/build-targets/t.XXXXXX")"
+s="\$(perl -MTime::HiRes=time -e 'printf "%.2f", time')"
+CARGO_HOME="$cargo_home" RUSTUP_HOME="$rustup_home" PATH="$(dirname "$(command -v cargo)"):\$PATH" \\
+  CARGO_TARGET_DIR="\$t" cargo build --offline --locked -q -p "$crate" >/dev/null 2>&1
+e=\$?
+echo "BUILD-SECONDS \$(perl -MTime::HiRes=time -e 'printf "%.2f", time - \$ARGV[0]' "\$s")"
+echo "BUILD-EXIT \$e"
+rm -rf "\$t"
+exit "\$e"
+SH
+  build_cmd="/bin/sh $(printf '%q' "$work/build-turn.sh")"
+  turn() {
+    local label="$1" bin="$2"
+    python3 -I scripts/cmux-next/acpmux-overhead-bench.py --acpmux "$bin" --mode build-turn --build-cmd "$build_cmd" \
+      -n "$n" --warmup 1 --label "$label" --json "$work/$label.json" --md "$work/table.md" 2> "$work/$label.log" | sed "s/^/ACPMUX-OVERHEAD /"
+    tail -n 8 "$work/$label.log" | sed "s/^/ACPMUX-OVERHEAD-LOG $label /"
+    python3 -I -c 'import json,sys; sys.exit(1 if json.load(open(sys.argv[1]))["claude"]["errors"] else 0)' "$work/$label.json"
+  }
+  [[ -n "$before" ]] && { turn before "$work/before/acpmux" || status=1; }
+  turn after "$work/after/acpmux" || status=1
+  # The same build, run directly at each class, interleaved.
+  for k in $(seq 1 "$n"); do
+    for cls in default utility background; do
+      case "$cls" in
+        default) wrap=() ;;
+        utility) wrap=(/usr/sbin/taskpolicy -c utility) ;;
+        background) wrap=(/usr/sbin/taskpolicy -b) ;;
+      esac
+      rc=0
+      out="$(${wrap[@]+"${wrap[@]}"} /bin/sh -c "$build_cmd")" || rc=$?
+      echo "ACPMUX-BUILD-RAW $cls run=$k $(echo "$out" | tr '\n' ' ')"
+      (( rc == 0 )) || { echo "ACPMUX-BUILD-RAW-FAIL $cls run=$k exit=$rc"; status=1; }
+    done
+  done
+  echo "ACPMUX-OVERHEAD-LOAD end $(sysctl -n vm.loadavg)"
+  exit "$status"
+fi
 [[ "$bench" == hibernate ]] && mode=hibernate
 run() {
   local label="$1" bin="$2"