diff --git a/.agents/skills/stuck-crewmate-recovery/SKILL.md b/.agents/skills/stuck-crewmate-recovery/SKILL.md index c5209051a44..1af893b44e6 100644 --- a/.agents/skills/stuck-crewmate-recovery/SKILL.md +++ b/.agents/skills/stuck-crewmate-recovery/SKILL.md @@ -59,6 +59,13 @@ Never restart, stop, or update the shared daemon on a crewmate's claim. It is one instance serving every lane and home, so a restart kills other lanes' in-flight runs. Only positive socket refusal or absence is a daemon-down finding; escalate that finding, or a failed run record that names a daemon error, to the captain. +## A worker parked on a provider quota wall + +`bin/fm-crew-state.sh` reports `state: quota` when a live harness is stalled on a provider usage-limit retry modal instead of advancing: the process is alive and painting, but the submitted turn cannot run. +Treat it as neither a wedge nor a declared external wait. +The retry modal leaves the composer unreadable, so an in-place `fm-control.sh relaunch` cannot fix it and a fresh worker on the same provider would hit the same wall. +Preserve the worktree and its unlanded work, and bring the work back under a new task id chosen for a provider with headroom rather than relaunching in place; a provider limit is not something the fleet can clear, so escalate it to the captain when it blocks delivery. + ## Live-endpoint escalation Escalate in order: diff --git a/bin/fm-busy-lib.sh b/bin/fm-busy-lib.sh index d8f7a0ee111..547a9638d29 100755 --- a/bin/fm-busy-lib.sh +++ b/bin/fm-busy-lib.sh @@ -43,10 +43,10 @@ # fm-recovery a documented recovery reset after relaunch # Classifier-only sources (never written into a record): # endpoint-gone, herdr-native, grok-regex, rovo-regex, agy-regex, muse-session-log, -# cursor-transcript, missing, malformed, gen-mismatch, source-mismatch, +# cursor-transcript, quota-wall, missing, malformed, gen-mismatch, source-mismatch, # kimi-unverified, codex-unverified, capture-failed, no-target # -# Classification (fm_busy_classify): busy | idle | unknown | dead, always +# Classification (fm_busy_classify): busy | idle | unknown | dead | quota, always # with the producing source as the second token. Precedence: # 1. dead endpoint (fm_busy_classify_live only) -> dead endpoint-gone # 2. standalone Kimi before verification -> unknown kimi-unverified @@ -57,12 +57,18 @@ # Grok/Rovo/AGY temporary regex fallbacks classify a grok, rovo, or agy # task from its rendered tail, then unknown missing # 5. malformed, stale, or untrusted records -> unknown, never a fallback -# Grok, Rovo, and AGY are the ONLY rendered-text classifications that survive the -# redesign, because none of their structured lifecycles was credited-live-verified -# in the approved audit (Rovo's clean ACP stopReason lives outside the TUI -# path firstmate drives, see references/harness/rovo.md; agy 1.2.0 exposes no -# hook surface at all, see references/harness/agy.md); each is scoped to -# its own harness= and can never classify another adapter. The delivery +# 6. any busy verdict over a rendered provider quota wall -> quota quota-wall +# (the wall section below owns the two-signal rule) +# Grok, Rovo, and AGY are the ONLY per-harness rendered-text sources that +# survive the redesign, because none of their structured lifecycles was +# credited-live-verified in the approved audit (Rovo's clean ACP stopReason +# lives outside the TUI path firstmate drives, see references/harness/rovo.md; +# agy 1.2.0 exposes no hook surface at all, see references/harness/agy.md); +# each is scoped to its own harness= and can never classify another adapter. +# The quota wall above is the one cross-harness rendered override: it never +# invents a verdict from a rendered surface alone, it only DOWNGRADES an +# already-busy semantic verdict to quota, so a missing or misread signal leaves +# the semantic verdict intact. The delivery # guards in bin/fm-composer-lib.sh match rendered footers for submit # acknowledgement and away-mode supervisor injection only; neither is a # recorded worker state source. @@ -867,13 +873,60 @@ fm_busy_agy_tail_busy() { | grep -qiE 'esc[[:space:]]+to[[:space:]]+cancel' } +# --------------------------------------------------------------------------- +# Provider quota / usage wall +# +# A worker parked on a provider quota wall is the one case where a live, +# painting harness is NOT advancing: the retry modal keeps the process and the +# TUI busy while the submitted turn cannot run. The measured fleet incident +# (workers on one provider, all stopped at the same weekly limit) had every one +# of them classified busy from its semantic record and reported working, so the +# wall must override the semantic busy verdict. +# +# The signal is rendered and provider-specific, so it is deliberately built +# from TWO independent wall-phrase families and requires both: +# limit - the wall names a spent usage/rate/quota limit +# wait - the wall names a scheduled retry or reset/backoff +# A single vendor string therefore cannot carry the verdict, and the phrases are +# wall-shaped rather than bare words (`quota`, `retry`) so a worker writing or +# discussing a quota-retry feature does not match. The check is scoped to the +# last few non-empty lines, where a blocking modal renders in the harness +# chrome the composer otherwise occupies, so scrollback output is never +# mistaken for a wall. +FM_BUSY_QUOTA_LIMIT_RE='(usage|rate)[ -]?limit|quota (exceed|exhaust|reached|limit)|exhausted (your )?(capacity|quota)|too many requests|out of (credits|quota)' +FM_BUSY_QUOTA_WAIT_RE='retry(ing)? (in|after)|will reset|resets? (in|at|after)|try again|attempt #[0-9]|get more access|upgrade your plan' + +# fm_busy_quota_tail_wall: consumes a rendered tail on stdin; 0 when the tail +# shows a provider quota wall (both families present within the bounded tail). +fm_busy_quota_tail_wall() { + local tail + tail=$(grep -v '^[[:space:]]*$' | tail -6) + [ -n "$tail" ] || return 1 + printf '%s\n' "$tail" | grep -qiE "$FM_BUSY_QUOTA_LIMIT_RE" || return 1 + printf '%s\n' "$tail" | grep -qiE "$FM_BUSY_QUOTA_WAIT_RE" || return 1 + return 0 +} + +# fm_busy_quota_wall_verdict: 0 when a busy pane's captured tail shows a quota +# wall. Captures the pane itself when the caller passed no tail, bounded by +# fm_backend_capture; every other outcome is a no, never a false wall. +fm_busy_quota_wall_verdict() { # [tail] + local backend=$1 target=$2 tail=${3-} + if [ -z "$tail" ]; then + command -v fm_backend_capture >/dev/null 2>&1 || return 1 + tail=$(fm_backend_capture "$backend" "$target" 40 2>/dev/null) || return 1 + fi + [ -n "$tail" ] || return 1 + printf '%s' "$tail" | fm_busy_quota_tail_wall +} + # fm_busy_classify: semantic classification for a task whose endpoint the # caller has already established as present. Prints " ": -# busy|idle|unknown plus the producing source (see header). Never probes +# busy|idle|unknown|quota plus the producing source (see header). Never probes # process state. is optional pre-captured plain output used only by # the grok, rovo, and agy arms; when absent each captures through # fm_backend_capture if available, else reports unknown capture-failed. -fm_busy_classify() { # [tail40] +fm_busy_classify_raw() { # [tail40] local backend=$1 target=$2 harness=$3 id=$4 state=$5 tail40=${6-} local out rc r_state r_source native log case "$harness" in @@ -1020,6 +1073,21 @@ fm_busy_classify() { # [tail40] printf 'unknown missing' } +# fm_busy_classify (public): fm_busy_classify_raw plus the one rendered +# override - a semantic busy verdict over a provider quota retry modal is not +# advancing, so it reports `quota` instead of busy. Every other verdict passes +# through unchanged. +fm_busy_classify() { # [tail40] + local backend=$1 target=$2 verdict tail40=${6-} + verdict=$(fm_busy_classify_raw "$@") + if [ "${verdict%% *}" = busy ] \ + && fm_busy_quota_wall_verdict "$backend" "$target" "$tail40"; then + printf 'quota quota-wall' + return 0 + fi + printf '%s' "$verdict" +} + # fm_busy_classify_live: fm_busy_classify behind the one process-level # override - a gone endpoint is dead, never busy. Requires fm-backend.sh to # be sourced for fm_backend_target_exists. @@ -1055,9 +1123,9 @@ fm_busy_classify_meta() { # [tail40] # fm_busy_is_busy: boolean view for callers that only gate on provable # activity. 0 iff the classification verdict is exactly busy; idle, unknown, -# and dead all return 1, so an unknown can never be silently promoted to -# either boolean pole - callers that must distinguish idle from unknown read -# the full classification instead. +# dead, and quota all return 1, so an unknown or a quota-parked worker can +# never be silently promoted to working - callers that must distinguish idle +# from unknown or quota read the full classification instead. fm_busy_is_busy() { # [tail40] local verdict verdict=$(fm_busy_classify "$@") diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 0752e52f370..7a6f5260c2d 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -1945,6 +1945,9 @@ crew_absorb_class() { # src=${line#*source: }; src=${src%% *} case "$src" in run-step|pane) printf 'working'; return ;; esac fi + # `quota` (a live harness parked on a provider quota wall) is deliberately + # NOT absorbed: the worker is neither advancing nor a declared external wait, + # so it surfaces for firstmate to preserve and replace. printf 'none' } diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index 160c729ed67..660a290b7aa 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -22,7 +22,7 @@ # Output is one stable, parseable, token-tight line firstmate can read every # heartbeat: # -# state: · source: · +# state: · source: · # # Logic, in order: # 1. Resolve worktree + backend target + kind from state/.meta. A meta @@ -90,7 +90,11 @@ # proven historical head, or kind=scout): fall back to the recorded # backend's pane busy state, then the resolved status declaration # when its verb maps to a recognized run-state. Decision-only events such as -# `resolved` never become current state or detail. +# `resolved` never become current state or detail. A busy pane whose text +# shows a provider quota wall reports `quota` instead of working +# (bin/fm-busy-lib.sh owns the wall signal): the harness is alive but the +# agent is not advancing, so recovery is preserve-and-replace under a new +# id, never an in-place relaunch into the same unreadable retry modal. # 5. Missing meta or torn-down worktree: report unknown · none. If no run is # attributed to this crew, a dead endpoint also reports unknown · none rather # than trusting a stale status log. On tmux and herdr, which own a @@ -253,12 +257,14 @@ pane_readable() { # esac } # crew_busy_verdict: the crew's semantic busy state from the one contract -# owner (bin/fm-busy-lib.sh), as " ". A converted -# adapter answers from its own lifecycle record; Grok answers from its -# isolated rendered-tail fallback; a herdr crew's native `busy` is accepted +# owner (bin/fm-busy-lib.sh), as " ". A +# converted adapter answers from its own lifecycle record; Grok answers from +# its isolated rendered-tail fallback; a herdr crew's native `busy` is accepted # when no record exists, but its native `idle` is NOT, because agent.get # reports generation state (idle while a crew blocks on its own long-running -# foreground tool call) rather than turn state. +# foreground tool call) rather than turn state. A `quota` verdict is a busy +# record reclassified by the rendered provider-wall signal, which the contract +# owner captures itself when the caller passes no tail. crew_busy_verdict() { # local tail40='' case "$HARNESS" in @@ -1010,13 +1016,15 @@ fi # Secondmates idle on their own watcher (idle pane = healthy), so the busy # state is not meaningful for them; read their state from the status log only. -# Only an exact busy verdict reports working here, and only an exact idle -# verdict permits the status-log fallback below. Missing, malformed, stale, or -# unverified semantic state remains unknown. +# Only an exact busy verdict reports working here, an exact quota verdict +# reports quota (a live harness stalled on a provider wall, not advancing), and +# only an exact idle verdict permits the status-log fallback below. Missing, +# malformed, stale, or unverified semantic state remains unknown. if [ "$KIND" != secondmate ]; then BUSY_VERDICT=$(crew_busy_verdict "$BACKEND_TARGET") case "${BUSY_VERDICT%% *}" in busy) emit working pane "harness busy (${BUSY_VERDICT#* })" ;; + quota) emit quota pane "provider quota wall: not advancing (preserve and replace under a new id, not relaunch)" ;; idle) ;; *) emit unknown pane "harness state unavailable ($BUSY_VERDICT)" ;; esac diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 44e8da93bcc..14a1dcd3174 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -359,6 +359,7 @@ family_for_basename() { fm-pr-state-live-e2e.test.sh|\ fm-sessionstart-hook-live-e2e.test.sh|fm-sessionstart-instruction-refresh-live-e2e.test.sh|\ fm-quota-array-dispatch-live-e2e.test.sh|fm-send-secondmate-marker-herdr-e2e.test.sh|\ + fm-quota-wall-live-e2e.test.sh|\ fm-send-inbox-doorbell-live-e2e.test.sh|\ fm-calm-claude-mod-plugin.test.sh|fm-calm-claude-mod-live-e2e.test.sh|\ fm-herdr-submit-confirm-live-e2e.test.sh) diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index 2ec6d91f5b7..fd3412c9e52 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -2095,3 +2095,30 @@ A throwaway scout was spawned through `bin/fm-spawn.sh --scout --harness omp --m 6. `bin/fm-control.sh exit` stopped the agent and `bin/fm-teardown.sh` returned the worktree and closed the item. `FM_OMP_LIVE_E2E=1 tests/fm-omp-primary-live-e2e.test.sh` refreshes the primary evidence; the worker path above is refreshed by repeating the scout dispatch after any omp upgrade. + +## Provider quota-wall classification + +A worker parked on a provider quota wall must not read as working: the harness is alive and painting a retry modal while the submitted turn cannot advance. +`bin/fm-busy-lib.sh` classifies that rendered wall as `quota` over an otherwise-busy task, and `bin/fm-crew-state.sh` surfaces it as `state: quota` (source `pane`) instead of `working`, so supervision sees a stalled worker rather than a healthy one. +The signal is built from two independent rendered families - a limit phrase and a retry/reset phrase - within the last few non-empty lines, so no single vendor string is load-bearing and ordinary worker output does not match. + +Verified on 2026-09-20 with opencode 1.18.31 on Linux, driving the real installed OpenCode TUI against a local 429 stub provider so its own retry modal renders with no model tokens spent: + +```sh +bin/fm-test-run.sh tests/fm-quota-wall-live-e2e.test.sh +``` + +Observed output: + +```text +ok - a busy OpenCode worker without a rendered wall reads working +ok - OpenCode 1.18.31 real 429 quota retry modal classifies as quota, not working +``` + +The real modal OpenCode painted for the stub's quota error, captured from the pane: + +```text +⬝⬝⬝⬝■■■■ weekly usage limit reached. It will reset in 1 day 14 hours [retrying attempt #1] esc interrupt +``` + +`tests/fm-crew-state.test.sh` pins the logic portably over a synthetic pane transcript, including the divergence cases where only one family, or ordinary worker prose, never reads `quota`. diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index a51574f0f14..ed1f2209f63 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -1699,6 +1699,100 @@ test_no_run_busy_pane() { pass "no run + a busy semantic record reads working, attributed to its source" } +# (f2) A busy record over a rendered provider quota wall reads quota, never +# working. The wall is recognized from TWO independent rendered families (a +# limit phrase and a retry/reset phrase), so the near-miss cases below prove +# neither family alone - and no ordinary worker prose - can carry the verdict. +# The synthetic transcripts stand in for real captured panes; the live guard +# (tests/fm-quota-wall-live-e2e.test.sh) proves the same matcher against the +# real installed OpenCode retry modal. +test_no_run_busy_quota_wall_reads_quota() { + reset_fakes + local d; d=$(new_case quota-wall) + make_repo_on_branch "$d/wt" fm/feat-q + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-q.meta" "window=fm:fm-feat-q" "worktree=$d/wt" "kind=ship" "harness=opencode" + FM_FAKE_AXI_STATUS="" + FM_FAKE_RUNS_LIST="" + FM_FAKE_BUSY=1 + local label text gen out + while IFS='|' read -r label text; do + [ -n "$label" ] || continue + FM_FAKE_BUSY_TEXT=$text + export FM_FAKE_BUSY_TEXT + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$d/state" feat-q) + "$ROOT/bin/fm-busy-event.sh" apply "$d/state" feat-q busy --gen "$gen" \ + --source opencode-plugin --event session-status + out=$(run_crew_state "$d" feat-q) + assert_contains "$out" "state: quota" "$label wall reads quota" + assert_contains "$out" "source: pane" "$label wall is attributed to the pane" + assert_not_contains "$out" "state: working" "$label wall never reads working" + done <<'EOF' +opencode-go|weekly usage limit reached. It will reset in 1 day 14 hours [retrying in ~1 day, attempt #1] +claude|Claude usage limit reached. Your limit will reset at 3pm. +gemini|You have exhausted your capacity. Your quota will reset after 20h. +codex|You've hit your usage limit. Please try again later. +generic-429|429 Too Many Requests: rate limit exceeded, retry after 120 seconds. +EOF + pass "a busy record over any provider quota wall reads quota, not working" +} + +# The two-signal rule, asserted as a divergence so it cannot go quietly vacuous: +# a pane carrying only ONE family stays busy (working), never quota. +test_quota_wall_requires_both_signals() { + reset_fakes + local d; d=$(new_case quota-near-miss) + make_repo_on_branch "$d/wt" fm/feat-qn + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-qn.meta" "window=fm:fm-feat-qn" "worktree=$d/wt" "kind=ship" "harness=opencode" + FM_FAKE_AXI_STATUS="" + FM_FAKE_RUNS_LIST="" + FM_FAKE_BUSY=1 + local label text gen out + while IFS='|' read -r label text; do + [ -n "$label" ] || continue + FM_FAKE_BUSY_TEXT=$text + export FM_FAKE_BUSY_TEXT + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$d/state" feat-qn) + "$ROOT/bin/fm-busy-event.sh" apply "$d/state" feat-qn busy --gen "$gen" \ + --source opencode-plugin --event session-status + out=$(run_crew_state "$d" feat-qn) + assert_not_contains "$out" "state: quota" "$label is not a wall" + assert_contains "$out" "state: working" "$label stays busy" + done <<'EOF' +limit-only|model weekly usage limit reached +wait-only|connection lost, retrying in 30s attempt #2 +ordinary-prose|working: refactoring the quota reset path +code-prose|working: adding a usage-limit retry path +EOF + pass "one family, or ordinary prose, never reads quota" +} + +# The watcher's absorb proof goes through the real reader, so a quota-parked +# worker must not be provably working - otherwise its stale wake would be +# swallowed exactly as the measured incident was. +test_quota_wall_not_provably_working() { + reset_fakes + local d gen; d=$(new_case quota-not-provable) + make_repo_on_branch "$d/wt" fm/feat-qp + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-qp.meta" "window=fm:fm-feat-qp" "worktree=$d/wt" "kind=ship" "harness=opencode" + FM_FAKE_AXI_STATUS="$(run_running fm/other-crew)" + FM_FAKE_RUNS_LIST="$(cat <<'EOF' + running fm/other-crew aaaaaaa 2026-07-02 22:10 +EOF +)" + FM_FAKE_BUSY=1 + FM_FAKE_BUSY_TEXT='weekly usage limit reached. It will reset in 1 day 14 hours [retrying in 8s attempt #3]' + export FM_FAKE_BUSY_TEXT + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$d/state" feat-qp) + "$ROOT/bin/fm-busy-event.sh" apply "$d/state" feat-qp busy --gen "$gen" \ + --source opencode-plugin --event session-status + PATH="$d/fakebin:$PATH" FM_STATE_OVERRIDE="$d/state" crew_is_provably_working feat-qp \ + && fail "a quota-parked worker must not be absorbed as provably working" + pass "crew_is_provably_working surfaces a quota-parked worker" +} + # A converted adapter must NOT read working from rendered footer text: the # redesign removed that dependency, so a pane painting "esc to interrupt" with # no semantic record is unknown, never working and never silently idle. @@ -3587,6 +3681,9 @@ test_terminal_run_without_live_sibling_is_unchanged test_coarse_run_does_not_probe_other_branch_ci_log_for_ready_status test_other_branch_run_ignored test_no_run_busy_pane +test_no_run_busy_quota_wall_reads_quota +test_quota_wall_requires_both_signals +test_quota_wall_not_provably_working test_no_run_footer_text_alone_is_not_working test_no_run_grok_uses_isolated_fallback test_no_run_herdr_unknown_uses_backend_capture diff --git a/tests/fm-quota-wall-live-e2e.test.sh b/tests/fm-quota-wall-live-e2e.test.sh new file mode 100644 index 00000000000..36c744dc010 --- /dev/null +++ b/tests/fm-quota-wall-live-e2e.test.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Live guard for the rendered provider quota-wall signal that +# bin/fm-busy-lib.sh classifies and bin/fm-crew-state.sh surfaces as `quota`. +# +# The wall is a vendor-rendered surface, so a synthetic transcript alone cannot +# prove it exists (the portable regression in tests/fm-crew-state.test.sh pins +# the logic; this guard proves the rendering). It drives the REAL installed +# OpenCode TUI against a local 429 stub provider whose error body carries a +# quota message, so OpenCode paints its own retry modal - the exact shape the +# fleet incident measured - without spending any real model tokens. It then +# proves the same task reads `working` from its busy record before the modal +# renders and `quota` once it does. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +fail() { + printf 'not ok - %s\n' "$1" >&2 + exit 1 +} + +pass() { + printf 'ok - %s\n' "$1" +} + +fm_live_gate default-on FM_QUOTA_WALL_LIVE opencode tmux node + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OPENCODE_BIN=$(command -v opencode 2>/dev/null || true) +REAL_TMUX=$(command -v tmux 2>/dev/null || true) +NODE_BIN=$(command -v node 2>/dev/null || true) +SOCKET="fm-quota-wall-live-$$" +SESSION=quota-wall-live +ID=quota-wall-live +LAB= +NODE_PID= + +[ -n "$OPENCODE_BIN" ] || fail "opencode is not installed" +[ -n "$REAL_TMUX" ] || fail "tmux is not installed" +[ -n "$NODE_BIN" ] || fail "node is not installed" +OPENCODE_VERSION=$("$OPENCODE_BIN" --version) || fail "opencode --version failed" + +lab_pid_is_safe() { # + local pid=$1 cmd + cmd=$(ps -p "$pid" -o command= 2>/dev/null || true) + case "$cmd" in + *"$LAB"*) return 0 ;; + esac + return 1 +} + +cleanup() { + [ -n "$REAL_TMUX" ] && "$REAL_TMUX" -L "$SOCKET" kill-server >/dev/null 2>&1 || true + if [ -n "$NODE_PID" ] && lab_pid_is_safe "$NODE_PID"; then + kill -TERM "$NODE_PID" 2>/dev/null || true + fi + sleep 0.3 + [ -z "$LAB" ] || rm -rf -- "$LAB" +} + +trap cleanup EXIT + +capture() { + "$REAL_TMUX" -L "$SOCKET" capture-pane -p -t "$SESSION" 2>/dev/null || true +} + +crew_state() { # read the injected task's current state through the real reader + env -u FM_CREW_STATE_META_OVERRIDE -u FM_CREW_STATE_STATUS_OVERRIDE \ + PATH="$LAB/bin:$PATH" FM_STATE_OVERRIDE="$LAB/state" \ + "$ROOT/bin/fm-crew-state.sh" "$ID" +} + +# shellcheck source=/dev/null +. "$ROOT/bin/fm-busy-lib.sh" + +LAB=$(mktemp -d "${TMPDIR:-/tmp}/fm-quota-wall-live.XXXXXX") || fail "could not create the isolated lab" +mkdir -p "$LAB/workspace" "$LAB/config/opencode" "$LAB/data" "$LAB/cache" "$LAB/state" "$LAB/bin" \ + || fail "could not lay out the isolated lab" +git -C "$LAB/workspace" init -q || fail "could not initialize the isolated workspace" +git -C "$LAB/workspace" config user.email "guard@local" || fail "could not configure the isolated workspace" +git -C "$LAB/workspace" config user.name "guard" || fail "could not configure the isolated workspace" +git -C "$LAB/workspace" commit -q --allow-empty -m init || fail "could not seed the isolated workspace" +git -C "$LAB/workspace" checkout -q -b fm/quota-wall-live || fail "could not branch the isolated workspace" +WORKSPACE=$(cd "$LAB/workspace" && pwd -P) || fail "could not resolve the isolated workspace" + +# A local stub answers every provider request with 429 and a quota message, so +# the real OpenCode CLI renders its own retry modal with no model tokens spent. +PORTFILE="$LAB/port" +"$NODE_BIN" -e ' +const http = require("http"); +const fs = require("fs"); +const server = http.createServer((req, res) => { + res.writeHead(429, { "content-type": "application/json" }); + res.end(JSON.stringify({ + error: { + message: "weekly usage limit reached. It will reset in 1 day 14 hours", + type: "rate_limit_error" + } + })); +}); +server.listen(0, "127.0.0.1", () => fs.writeFileSync(process.argv[1], String(server.address().port))); +' "$PORTFILE" & +NODE_PID=$! +for _ in $(seq 1 60); do + [ -s "$PORTFILE" ] && break + sleep 0.1 +done +[ -s "$PORTFILE" ] || fail "the 429 stub never reported its port" +PORT=$(cat "$PORTFILE") + +# shellcheck disable=SC2016 # "$schema" is a literal JSON key, not a shell expansion. +printf '{"$schema":"https://opencode.ai/config.json","provider":{"openai":{"options":{"baseURL":"http://127.0.0.1:%s/v1","apiKey":"stub-key"}}},"model":"openai/gpt-4o-mini"}\n' "$PORT" \ + > "$LAB/config/opencode/opencode.json" || fail "could not write the isolated OpenCode config" + +# The reader resolves the guard's tmux server through the PATH shim, exactly +# as it resolves a task's own backend socket in production. +printf '#!/usr/bin/env bash\nexec "%s" -L "%s" "$@"\n' "$REAL_TMUX" "$SOCKET" > "$LAB/bin/tmux" \ + || fail "could not write the tmux shim" +chmod +x "$LAB/bin/tmux" || fail "could not make the tmux shim executable" + +"$REAL_TMUX" -L "$SOCKET" new-session -d -s "$SESSION" -x 120 -y 40 -c "$WORKSPACE" \ + "env XDG_CONFIG_HOME='$LAB/config' XDG_DATA_HOME='$LAB/data' XDG_CACHE_HOME='$LAB/cache' XDG_STATE_HOME='$LAB/state' OPENCODE_DISABLE_AUTOUPDATE=1 OPENCODE_DISABLE_LSP_DOWNLOAD=1 '$OPENCODE_BIN'" \ + || fail "could not start the isolated OpenCode TUI" +for _ in $(seq 1 120); do + capture | grep -Fq "$OPENCODE_VERSION" && break + sleep 0.5 +done +capture | grep -Fq "$OPENCODE_VERSION" || fail "the isolated OpenCode TUI never reached its composer" + +printf 'kind=scout\nharness=opencode\nbackend=tmux\nwindow=%s\nworktree=%s\n' "$SESSION" "$WORKSPACE" \ + > "$LAB/state/$ID.meta" || fail "could not write the task metadata" +"$ROOT/bin/fm-busy-event.sh" arm "$LAB/state" "$ID" --state busy --source opencode-plugin --event session-status \ + >/dev/null || fail "could not seed the busy record" + +# Negative control: a busy record with no wall rendered reads working, so the +# quota verdict below cannot be vacuous. +before=$(crew_state) +case "$before" in + *"state: working"*) pass "a busy OpenCode worker without a rendered wall reads working" ;; + *) fail "a busy OpenCode worker without a wall should read working, got: $before" ;; +esac + +"$REAL_TMUX" -L "$SOCKET" send-keys -t "$SESSION" -l "Say hi" || fail "could not type the prompt" +"$REAL_TMUX" -L "$SOCKET" send-keys -t "$SESSION" Enter || fail "could not submit the prompt" + +after= +for _ in $(seq 1 180); do + after=$(crew_state) + case "$after" in *"state: quota"*) break ;; esac + sleep 0.5 +done +case "$after" in + *"state: quota"*) ;; + *) capture >&2; fail "the real OpenCode quota retry modal never classified as quota, got: $after" ;; +esac +case "$after" in + *"source: pane"*) ;; + *) fail "the quota verdict must be attributed to the pane, got: $after" ;; +esac + +# Prove the verdict came from the live vendor surface: the captured pane tail +# itself must match the rendered matcher. +capture | fm_busy_quota_tail_wall \ + || { capture >&2; fail "the live OpenCode wall pane did not match fm_busy_quota_tail_wall"; } + +pass "OpenCode $OPENCODE_VERSION real 429 quota retry modal classifies as quota, not working"