diff --git a/scripts/cmux-next/acpmux-overhead-bench.py b/scripts/cmux-next/acpmux-overhead-bench.py index ca17b437752f..8036a48f559d 100755 --- a/scripts/cmux-next/acpmux-overhead-bench.py +++ b/scripts/cmux-next/acpmux-overhead-bench.py @@ -40,9 +40,18 @@ person typing does warm_first the same for a prompt right after, the agent awake +--mode build-turn: an agent turn that runs a build (cpu_class.rs: the CPU class a session's harness +starts at). Claude Code only, through acpmux, each run a fresh session: the stand-in answers the +prompt with one Bash call running --build-cmd, then "ok". Per run: + end ms from the prompt's send to the turn's end (the build is most of it) + pri the base priority (`ps -o pri=`) of the shell Claude Code ran the build in: 4 is + background (darwinbg), 31 the default class + build_s the build's own seconds, as the shell measured them + Usage: acpmux-overhead-bench.py --acpmux PATH [--harness claude,codex,opencode] [-n 20] - [--mode overhead|prewarm|hibernate] [--json OUT] [--md OUT] + [--mode overhead|prewarm|hibernate|build-turn] [--build-cmd CMD] + [--json OUT] [--md OUT] """ from __future__ import annotations @@ -74,6 +83,10 @@ def __init__(self): # Model requests and their body bytes, for each run's share (`Counter`). self.requests = 0 self.bytes = 0 + # --mode build-turn: the Bash command a prompt naming BUILD_TURN gets, and every tool + # result the harness sent back, in order. + self.build_cmd: str | None = None + self.tool_results: list[str] = [] def hit(self, tag: str, size: int = 0): with self.lock: @@ -145,6 +158,9 @@ def do_POST(self): def anthropic(self, body): model = body.get("model", "claude-bench") usage = {"input_tokens": 10, "output_tokens": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} + call = build_call(self.server, body) + if call is not None and body.get("stream"): + return self.reply(200, sse(tool_use_events(model, usage, call)), "text/event-stream") if not body.get("stream"): msg = {"id": "msg_bench", "type": "message", "role": "assistant", "model": model, "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn", @@ -202,6 +218,53 @@ def chat(self, body): self.reply(200, sse(events), "text/event-stream") +# The prompt of a --mode build-turn run: the stand-in answers it with one Bash call. +BUILD_TURN = "BUILD_TURN: run the build." + + +def build_call(server, body): + """The Bash input a main-loop request of a build turn gets, or None (answer "ok"). + + Only a request that offers the Bash tool (a side request, such as a title, offers none) and + whose last user message is the build prompt itself, not the tool result that follows it. + Claude Code may end its messages with a system one (its environment), so the last user + message is looked for, never the last message.""" + if not server.build_cmd or not any(t.get("name") == "Bash" for t in body.get("tools") or []): + return None + users = [m for m in body.get("messages") or [] if m.get("role") == "user"] + content = users[-1].get("content") if users else None + blocks = content if isinstance(content, list) else [{"type": "text", "text": content or ""}] + results = [b for b in blocks if isinstance(b, dict) and b.get("type") == "tool_result"] + if results: + for r in results: + inner = r.get("content") + text = inner if isinstance(inner, str) else "".join( + c.get("text", "") for c in inner or [] if isinstance(c, dict)) + with server.lock: + server.tool_results.append(text) + return None + if not any(BUILD_TURN in (b.get("text") or "") for b in blocks if isinstance(b, dict)): + return None + return {"command": server.build_cmd, "description": "Run the build", "timeout": 600000} + + +def tool_use_events(model, usage, call): + tid = "toolu_bench_" + uuid.uuid4().hex[:12] + return [ + ("message_start", {"type": "message_start", "message": {"id": "msg_bench", "type": "message", "role": "assistant", + "model": model, "content": [], "stop_reason": None, + "stop_sequence": None, "usage": usage}}), + ("content_block_start", {"type": "content_block_start", "index": 0, + "content_block": {"type": "tool_use", "id": tid, "name": "Bash", "input": {}}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, + "delta": {"type": "input_json_delta", "partial_json": json.dumps(call)}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 1}}), + ("message_stop", {"type": "message_stop"}), + ] + + # ---------------------------------------------------------------- line pipes @@ -683,7 +746,8 @@ def main(): ap.add_argument("--harness", default="claude,codex,opencode") ap.add_argument("-n", type=int, default=20) ap.add_argument("--warmup", type=int, default=2) - ap.add_argument("--mode", choices=["overhead", "prewarm", "hibernate"], default="overhead") + ap.add_argument("--mode", choices=["overhead", "prewarm", "hibernate", "build-turn"], default="overhead") + ap.add_argument("--build-cmd", help="--mode build-turn: the shell command the turn's Bash call runs") ap.add_argument("--idle-wait", type=int, default=135, help="--mode hibernate: seconds the chats sit idle") ap.add_argument("--json") ap.add_argument("--md") @@ -708,6 +772,12 @@ def main(): if args.no_agent_tools: extra["ACPMUX_AGENT_TOOLS"] = "0" daemon = Daemon(os.path.abspath(args.acpmux), env, home, extra) + if args.mode == "build-turn": + try: + return build_turn_main(args, daemon, stand, scratch) + finally: + daemon.close() + shutil.rmtree(scratch, ignore_errors=True) if args.mode in ("prewarm", "hibernate"): try: return (prewarm_main if args.mode == "prewarm" else hibernate_main)(args, daemon, scratch) @@ -776,6 +846,68 @@ def main(): shutil.rmtree(scratch, ignore_errors=True) +def build_turn_main(args, daemon: Daemon, stand: StandIn, scratch): + """--mode build-turn: n turns, each in a fresh session, whose one tool call runs the build.""" + stand.build_cmd = args.build_cmd + rows, errors = [], [] + for k in range(args.warmup + args.n): + cwd = os.path.join(scratch, f"build-{k}") + os.makedirs(cwd) + sid = None + try: + _, r = daemon.request("session/new", {"cwd": cwd, "mcpServers": [], + "_meta": {"acpmux": {"harness": "claude", "policy": "approve-all", + "model": MODELS["claude"]}}}) + sid = r["sessionId"] + io = daemon.io + daemon.next += 1 + rid, start, seen = daemon.next, io.mark(), len(stand.tool_results) + ts = time.perf_counter() + io.send({"jsonrpc": "2.0", "id": rid, "method": "session/prompt", + "params": {"sessionId": sid, "prompt": [{"type": "text", "text": BUILD_TURN}]}}) + te, m, _ = io.wait(lambda m: m.get("id") == rid, timeout=1200, start=start) + if "error" in m: + raise RuntimeError(f"session/prompt: {m['error']}") + out = "\n".join(stand.tool_results[seen:]) + row = {"end": ms(ts, te)} + for key, name in (("pri", "BUILD-PRI"), ("build_s", "BUILD-SECONDS"), ("exit", "BUILD-EXIT")): + hit = [line.split()[1] for line in out.splitlines() if line.startswith(name + " ") and len(line.split()) > 1] + row[key] = float(hit[-1]) if hit else None + if row["exit"] != 0: + raise RuntimeError(f"the build did not pass: {out[-400:]!r}") + print(f"build-turn run {k + 1}/{args.warmup + args.n}: {row}", file=sys.stderr, flush=True) + if k >= args.warmup: + rows.append(row) + except Exception as e: # noqa: BLE001: a bench reports and goes on + errors.append(f"run {k}: {e}") + print(f"build-turn run {k}: {e}", file=sys.stderr) + finally: + if sid: + try: + daemon.request("session/close", {"sessionId": sid}, timeout=20) + except Exception: + pass + summary = {m: {"p50": pct([r[m] for r in rows if r[m] is not None], 0.5), + "min": min((r[m] for r in rows if r[m] is not None), default=None), + "max": max((r[m] for r in rows if r[m] is not None), default=None)} for m in ("end", "build_s", "pri")} + results = {"claude": {"summary": summary, "runs": rows, "errors": errors}, + "_meta": {"label": args.label, "acpmux": args.acpmux, "n": args.n, "build_cmd": args.build_cmd, + "host": socket.gethostname(), "load": os.getloadavg()}} + if args.json: + with open(args.json, "w") as f: + json.dump(results, f, indent=1) + lines = [f"### build-turn {args.label or 'run'} ({socket.gethostname()}, n={len(rows)}, load {os.getloadavg()[0]:.1f})", "", + "| metric | p50 | min | max |", "|---|---|---|---|"] + for m, v in summary.items(): + lines.append(f"| {m} | {v['p50']} | {v['min']} | {v['max']} |") + text = "\n".join(lines) + "\n" + print(text) + if args.md: + with open(args.md, "a") as f: + f.write(text + "\n") + return 1 if errors else 0 + + def prewarm_main(args, daemon: Daemon, scratch): results = {} for h in args.harness.split(","): @@ -832,4 +964,4 @@ def prewarm_main(args, daemon: Daemon, scratch): if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/measure/acpmux-overhead.sh b/scripts/measure/acpmux-overhead.sh index c501e809485c..773aadde74eb 100755 --- a/scripts/measure/acpmux-overhead.sh +++ b/scripts/measure/acpmux-overhead.sh @@ -40,6 +40,17 @@ # run and the next prompt's time to its first answer text. With before=SHA both builds run. # cmux-ci run --class exclusive --script scripts/measure/acpmux-overhead.sh --ref SHA \ # --arg=bench=hibernate --arg=harness=claude,codex [--arg=n=4] [--arg=before=SHA] --timeout 5400 +# +# bench=build-turn: an agent turn that runs a build (acpmux cpu_class.rs, the CPU class a session's +# harness starts at): Claude Code through acpmux, each run a fresh session whose one Bash call is a +# cold debug build of crate= (default cmux-history) in a fresh target dir. Per acpmux build: the +# turn's time, the build's own seconds and the base priority of the shell it ran in (4: background +# class). Then the same build run directly at each class (ACPMUX-BUILD-RAW lines): default, +# `taskpolicy -c utility` and `taskpolicy -b`, interleaved. With before=SHA both builds run. +# Output: ACPMUX-OVERHEAD and ACPMUX-BUILD-RAW lines. Exit 1 when a run failed. +# cmux-ci run --class exclusive --script scripts/measure/acpmux-overhead.sh --ref SHA \ +# --arg=bench=build-turn --arg=harness=claude [--arg=n=3] [--arg=crate=cmux-history] \ +# [--arg=before=SHA] --timeout 5400 set -euo pipefail n=20 harness="claude,codex,opencode" @@ -49,6 +60,7 @@ codex_version="0.163.0" opencode_version="1.19.3" guard_ms=5 bench=harness +crate=cmux-history for arg in "$@"; do case "$arg" in bench=*) bench="${arg#bench=}" ;; @@ -59,6 +71,7 @@ for arg in "$@"; do codex=*) codex_version="${arg#codex=}" ;; opencode=*) opencode_version="${arg#opencode=}" ;; guard_ms=*) guard_ms="${arg#guard_ms=}" ;; + crate=*) crate="${arg#crate=}" ;; *) echo "acpmux-overhead.sh: unknown argument $arg" >&2; exit 2 ;; esac done @@ -69,7 +82,9 @@ for v in "$claude_version" "$codex_version" "$opencode_version"; do [[ "$v" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "acpmux-overhead.sh: versions are X.Y.Z" >&2; exit 2; } done [[ "$guard_ms" =~ ^[0-9]{1,4}$ ]] || { echo "acpmux-overhead.sh: guard_ms is a number" >&2; exit 2; } -[[ "$bench" =~ ^(harness|daemon|prewarm|hibernate)$ ]] || { echo "acpmux-overhead.sh: bench=harness, daemon, prewarm or hibernate" >&2; exit 2; } +[[ "$bench" =~ ^(harness|daemon|prewarm|hibernate|build-turn)$ ]] || { echo "acpmux-overhead.sh: bench=harness, daemon, prewarm, hibernate or build-turn" >&2; exit 2; } +[[ "$crate" =~ ^[a-z0-9_-]{1,64}$ ]] || { echo "acpmux-overhead.sh: crate= is a cmux-tui package name" >&2; exit 2; } +[[ "$bench" != build-turn || "$harness" == claude ]] || { echo "acpmux-overhead.sh: bench=build-turn runs harness=claude only" >&2; exit 2; } # Every tool this bench always calls, checked before anything runs, so a worker without one says # which at once instead of failing after a build (cmux20-bench had no npm). Build tools are checked # where a build happens: cargo for the cmux fallback below, cargo and rustup in build-acpmux.sh when @@ -84,6 +99,9 @@ need_tools() { # need_tools WHAT TOOL...: exit 2 naming every TOOL not on PATH. } if [[ "$bench" == daemon ]]; then need_tools "bench=daemon" git sysctl sw_vers cargo +elif [[ "$bench" == build-turn ]]; then + # The turn's build runs cargo and perl (its clock); ps reads its shell's priority. + need_tools "bench=build-turn" git sysctl sw_vers curl shasum awk python3 npm ps cargo perl elif [[ "$bench" == hibernate ]]; then # Hibernation counts the agent's processes with ps (acpmux-overhead-bench.py). need_tools "bench=hibernate" git sysctl sw_vers curl shasum awk python3 npm ps @@ -185,6 +203,52 @@ echo "ACPMUX-OVERHEAD-BIN claude $(claude --version) codex $(codex --version) op status=0 mode=overhead [[ "$bench" == prewarm ]] && mode=prewarm +if [[ "$bench" == build-turn ]]; then + # The bench gives the harness a scratch HOME: the build names the step's own toolchain. + cargo_home="${CARGO_HOME:-$HOME/.cargo}" rustup_home="${RUSTUP_HOME:-$HOME/.rustup}" + (cd cmux-tui && CARGO_HOME="$cargo_home" RUSTUP_HOME="$rustup_home" cargo fetch --locked >/dev/null) + mkdir -p "$work/build-targets" + # The turn's Bash call runs this script: its shell's priority, then the build's seconds and exit. + cat > "$work/build-turn.sh" </dev/null 2>&1 +e=\$? +echo "BUILD-SECONDS \$(perl -MTime::HiRes=time -e 'printf "%.2f", time - \$ARGV[0]' "\$s")" +echo "BUILD-EXIT \$e" +rm -rf "\$t" +exit "\$e" +SH + build_cmd="/bin/sh $(printf '%q' "$work/build-turn.sh")" + turn() { + local label="$1" bin="$2" + python3 -I scripts/cmux-next/acpmux-overhead-bench.py --acpmux "$bin" --mode build-turn --build-cmd "$build_cmd" \ + -n "$n" --warmup 1 --label "$label" --json "$work/$label.json" --md "$work/table.md" 2> "$work/$label.log" | sed "s/^/ACPMUX-OVERHEAD /" + tail -n 8 "$work/$label.log" | sed "s/^/ACPMUX-OVERHEAD-LOG $label /" + python3 -I -c 'import json,sys; sys.exit(1 if json.load(open(sys.argv[1]))["claude"]["errors"] else 0)' "$work/$label.json" + } + [[ -n "$before" ]] && { turn before "$work/before/acpmux" || status=1; } + turn after "$work/after/acpmux" || status=1 + # The same build, run directly at each class, interleaved. + for k in $(seq 1 "$n"); do + for cls in default utility background; do + case "$cls" in + default) wrap=() ;; + utility) wrap=(/usr/sbin/taskpolicy -c utility) ;; + background) wrap=(/usr/sbin/taskpolicy -b) ;; + esac + rc=0 + out="$(${wrap[@]+"${wrap[@]}"} /bin/sh -c "$build_cmd")" || rc=$? + echo "ACPMUX-BUILD-RAW $cls run=$k $(echo "$out" | tr '\n' ' ')" + (( rc == 0 )) || { echo "ACPMUX-BUILD-RAW-FAIL $cls run=$k exit=$rc"; status=1; } + done + done + echo "ACPMUX-OVERHEAD-LOAD end $(sysctl -n vm.loadavg)" + exit "$status" +fi [[ "$bench" == hibernate ]] && mode=hibernate run() { local label="$1" bin="$2"