diff --git a/.agents/skills/operational-home-layout/SKILL.md b/.agents/skills/operational-home-layout/SKILL.md index bf17f233ea3..f51340058bb 100644 --- a/.agents/skills/operational-home-layout/SKILL.md +++ b/.agents/skills/operational-home-layout/SKILL.md @@ -39,6 +39,7 @@ config/trace-context optional presence flag enabling default-off native W3C tra config/lavish-axi-host optional one-line per-machine Lavish server address; LOCAL, gitignored, inherited by secondmate homes, and exported into every worker launch; see docs/configuration.md "Lavish server address" for opening versus polling config/brief-include.md optional standing worker instructions appended verbatim as the last section of every ship and scout scaffold; LOCAL, gitignored, and not inherited; keep its text out of `## Firstmate spec`; see docs/configuration.md "Home brief include" config/fleet-ledger optional presence flag opting this home in to the default-off fleet activity ledger state/fleet-ledger.jsonl that outside tools can follow; LOCAL, gitignored, and not inherited; see docs/fleet-ledger.md +config/wait-no-turns optional presence flag opting this home into default-off waiting-worker behavior (brief waiting section, foreground pipeline drive, pending-reply hold, one fire-and-forget retry ring); LOCAL, gitignored, and not inherited; see docs/configuration.md "Waiting worker spends no turns" config/turnend-churn-absorb optional presence flag opting this home into the default-off absorb of bare turn-end wakes on pane churn; LOCAL, gitignored, and not inherited; see docs/configuration.md "Turn-end pane-churn absorb" config/wedge-defer-parked-gate optional presence flag opting this home into the default-off deferral of a wedge escalation for a lane parked at a validation gate awaiting the supervisor's own still-open decision; LOCAL, gitignored, and not inherited; see docs/configuration.md "Parked-gate wait deferral" config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index 864b2c857df..2ff9c727762 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -350,16 +350,46 @@ INBOX_DIR=$(shell_quote "$STATE/$ID.inbox") # scaffold kind. The record format, doorbell line, and re-ring ladder are # owned by bin/fm-task-inbox-lib.sh. The doorbell names the inbox as # "$FM_TASK_INBOX", which bin/fm-spawn.sh exports into every launch; the full -# path here remains the fallback for a worker launched without that export, -# plus the natural-checkpoint habit. +# path here remains the fallback for a worker launched without that export. +# The doorbell itself is self-describing, so this section is reinforcement +# for the natural-checkpoint habit, not the only carrier of the instruction. +# config/wait-no-turns (docs/configuration.md) adds the line that a waiting +# worker does not poll the inbox: checkpoint checks happen during active work, +# so waiting still spends no turns. IFS= read -r -d '' INBOX_SECTION < --watch`, or `until ; do sleep 30; done` for anything else. +Never spend turns on `sleep` followed by a status check, and never background a command in order to poll it. +In Claude Code that `until` loop in a single Bash call is the sanctioned foreground wait: when the harness refuses a sleep-then-check command and points you at backgrounding instead, reissue the wait as the loop rather than accepting the background. +Bound that command by what your harness lets one command run: in Pi pass the bash tool a `timeout` of at most 2700 seconds, because Pi sets none by default; in Claude Code pass the Bash tool its maximum `timeout` of 600000 ms, because its default is 2 minutes; in Codex keep waiting on a still-running command with empty `write_stdin` polls of up to 300000 ms; elsewhere pass your shell tool its largest timeout and assume at most 10 minutes. +Give any `--wait` a duration a little under that bound. +When the bound passes with nothing changed, run the same blocking command again, with no status check in between. +The one exception is `respond`: it sent its answer before it began waiting, so reattach with `no-mistakes axi run --wait` instead, and never send the same `respond` again, because it would answer whichever gate parks next without you reading it. +A wait your shell can watch this way needs no `paused:` line, except your own pipeline run, a long foreground command, or your own validation round, which you declare once just before its blocking hold: append `paused:` once just before its first blocking command, then stay in the command, and never append it again as you reissue that command. +EOF +WAIT_SECTION=${WAIT_SECTION%$'\n'} +WAIT_BLOCK= +if [ -e "$CONFIG/wait-no-turns" ]; then + WAIT_BLOCK="$WAIT_SECTION"$'\n\n' +fi + if [ "$KIND" = secondmate ]; then SECONDMATE_PROJECTS="" idx=1 @@ -577,7 +607,7 @@ $CREWMATE_PAUSE_INSTRUCTIONS Firstmate's reply normally writes that closing line at answer time; when a blocker or wait clears WITHOUT a firstmate reply, append \`resolved [at=]: {how it cleared}\` yourself (same \`[key=]\` if you opened it with one) as you resume. $SHARED_INFRA_RULE -$INBOX_SECTION +$WAIT_BLOCK$INBOX_SECTION # Definition of done Write your findings to \`$DATA/$ID/report.md\`. @@ -655,7 +685,7 @@ $ASK_USER_BLOCK Firstmate's reply normally writes that closing line at answer time; when a blocker or wait clears WITHOUT a firstmate reply, append \`resolved [at=]: {how it cleared}\` yourself (same \`[key=]\` if you opened it with one) as you resume. $SHARED_INFRA_RULE -$INBOX_SECTION +$WAIT_BLOCK$INBOX_SECTION # Project memory A project's \`AGENTS.md\` or \`CLAUDE.md\` is loaded into every agent session in that project, so edit it only to correct information that is factually wrong - including information your own change made wrong - and never to add knowledge because it is missing. diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 3d4e1f62282..11cc7f24cfb 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -922,6 +922,25 @@ EOF printf '%s\n' "$current" } +# The subset of status_open_decisions the task raised about its own work: a +# reserved-namespace key is raised by a supervisor library about the task (a +# pending-reply escalation), a `remote-reply-continuity-` key is the parent's +# own blocker about a broken remote reply mirror +# (bin/fm-procevent-remote-reply.sh), and a `captain-hold-` key relays a child +# decision a secondmate escalated to the captain (bin/fm-captain-hold.sh) while +# it keeps working, so the task is not waiting on any of them. Pending-reply +# recovery and a fire-and-forget retry ring consult this set and leave a task +# alone while it is non-empty. +status_own_open_decisions() { # + local line prefix + status_open_decisions "$1" | while IFS= read -r line || [ -n "$line" ]; do + for prefix in ${FM_CLASSIFY_RESERVED_KEY_PREFIXES:-$FM_CLASSIFY_RESERVED_KEY_PREFIXES_DEFAULT} remote-reply-continuity- captain-hold-; do + case "$line" in "$prefix"*) continue 2 ;; esac + done + printf '%s\n' "$line" + done +} + # 0 when the fold above still holds at least one decision OPENED by # `needs-decision` - the status side's own record that a human was asked # something and has not answered. A `blocked` record is deliberately not this: a diff --git a/bin/fm-dod-lib.sh b/bin/fm-dod-lib.sh index 52cc1cc1c14..aed6f5d0d22 100755 --- a/bin/fm-dod-lib.sh +++ b/bin/fm-dod-lib.sh @@ -280,12 +280,28 @@ EOF # Written once; only the two sentences about a green PR depend on the forge, # because on gerrit the ci step is skipped and there is no PR to report. fm_nm_driving_block() { # - local pr_return_line='' pr_reattach_clause=';' + local pr_return_line='' pr_reattach_clause=';' drive_block wait_cfg if [ "$1" != gerrit ]; then pr_return_line="Only a drive call's return reports the green PR: \`no-mistakes axi status\` shows progress but never reports \`checks-passed\` while the ci step is still monitoring the PR for merge, so never wait on a status poll for the next gate or outcome. " pr_reattach_clause="; once checks are green it returns \`checks-passed\` immediately, and" fi + # config/wait-no-turns selects the foreground drive. Absent, the text matches + # the backgrounded drive a home had before that flag. + wait_cfg=${CONFIG:-${FM_CONFIG_OVERRIDE:-${FM_HOME:-}/config}} + if [ -e "$wait_cfg/wait-no-turns" ]; then + drive_block="Drive the run with ONE foreground \`no-mistakes axi run\` and let it block. +It bounds its own hold for you: \`--wait\` (default 8m) exists precisely so a harness with a ten-minute command cap gets a structured return instead of being killed mid-hold. +Declare that wait using the brief's status-reporting rule before the foreground drive call. +Never background a wait, and never arm a timer to stand in for one: a backgrounded call returns in milliseconds, so it does not wait at all, and every timer left behind fires later as a paid wake for nothing. +${pr_return_line}Whenever a drive call returns without a gate or an outcome - its own wait elapsed, or it was killed or timed out - that is not a failure: reattach at once by re-running \`no-mistakes axi run\` without flags, and issue the same foreground call again, one at a time, until a gate or outcome comes back${pr_reattach_clause} if it refuses because no run is active, read the finished outcome from \`no-mistakes axi status\`." + else + drive_block="One drive call blocks until the next gate or outcome, which routinely outlives what your harness lets a single command run: Claude Code kills a command at ten minutes maximum, while one fix round is capped around thirty minutes and up to three rounds chain. +So background the drive call instead of sitting in one blocking hold your harness will kill, and read its return when it finishes. +Declare that wait using the brief's status-reporting rule before waiting on the backgrounded drive call. +Where a harness's own command limit is not established, assume it bounds commands and use that same backgrounded shape. +${pr_return_line}Whenever a drive call returns without a gate or an outcome - its own wait elapsed, or it was killed or timed out - reattach at once by re-running \`no-mistakes axi run\` without flags, backgrounded the same way${pr_reattach_clause} if it refuses because no run is active, read the finished outcome from \`no-mistakes axi status\`." + fi cat < task_id=$(fm_pending_reply_get "$rec" task_id) # A remote mate's report may exist and simply not have been mirrored yet. fm_pending_reply_missing_report_is_evidence "$state" "$task_id" "$completed" || return 1 + # config/wait-no-turns: a mate waiting on its own open decision or blocker + # is never poked. The recovery stays unattempted until the answer lands. + if [ -e "${FM_CONFIG_OVERRIDE:-${FM_HOME:-}/config}/wait-no-turns" ]; then + [ -z "$(status_own_open_decisions "$state/$task_id.status")" ] || return 1 + fi status_file=$(fm_pending_reply_get "$rec" parent_status) parent_home=$(fm_pending_reply_get "$rec" parent_home) msg=$(fm_pending_reply_recovery_message "$rec") diff --git a/bin/fm-send.sh b/bin/fm-send.sh index af09392a4d0..19562680313 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -51,7 +51,9 @@ # watcher re-rings an unacknowledged message while its endpoint remains # available, escalates after the bounded ladder, and instead routes a positively # dead or missing endpoint directly to recovery without typing. An explicit -# fire-and-forget record is excluded from that ladder. +# fire-and-forget record is excluded from that ladder; when config/wait-no-turns +# is present and its ring here was skipped or failed, the watcher rings it +# exactly once more. # bin/fm-task-inbox-lib.sh owns the record format, the doorbell line, and the # re-ring ladder. The composer pre-check before the ring is ADVISORY only: when # the composer visibly holds pending text the ring is skipped with a notice and @@ -1085,9 +1087,22 @@ else # bounded re-ring ladder or direct unavailable-endpoint recovery. ring_rc=0 fm_task_inbox_ring "$TARGET_BACKEND" "$T" "$INBOX_RECORD" "$EXPECTED_LABEL" || ring_rc=$? + ring_retry="the watcher will re-ring" + if [ -n "$FIRE_AND_FORGET_ID" ] \ + && [ -e "${FM_CONFIG_OVERRIDE:-$FM_HOME/config}/wait-no-turns" ]; then + case "$ring_rc" in + 1|2) + if fm_task_inbox_mark_retry "$STATE" "$INBOX_TASK_ID" "$INBOX_RECORD"; then + ring_retry="the watcher will ring it once more" + else + ring_retry="its one retry ring could not be recorded, so nothing will ring it again" + fi + ;; + esac + fi case "$ring_rc" in - 1) echo "fm-send: doorbell skipped (composer visibly holds pending text); the steer is durably recorded at $INBOX_RECORD and the watcher will re-ring" >&2 ;; - 2) echo "fm-send: doorbell did not reach $T; the steer is durably recorded at $INBOX_RECORD and the watcher will re-ring" >&2 ;; + 1) echo "fm-send: doorbell skipped (composer visibly holds pending text); the steer is durably recorded at $INBOX_RECORD and $ring_retry" >&2 ;; + 2) echo "fm-send: doorbell did not reach $T; the steer is durably recorded at $INBOX_RECORD and $ring_retry" >&2 ;; 3) echo "fm-send: doorbell not typed because the agent in $T has exited; the steer is durably recorded at $INBOX_RECORD for recovery (stuck-crewmate-recovery), and the watcher will not re-ring a dead pane" >&2 ;; esac exit 0 diff --git a/bin/fm-task-inbox-lib.sh b/bin/fm-task-inbox-lib.sh index c1d48689975..257da1dd54d 100644 --- a/bin/fm-task-inbox-lib.sh +++ b/bin/fm-task-inbox-lib.sh @@ -30,11 +30,14 @@ # .inbox/.ring-state watcher re-ring ladder: "\t\t" # .inbox/.escalated oldest-message name already surfaced as stale, # so later polls suppress another escalation +# .inbox/.retry-ring name of a fire-and-forget record still owed its +# one retry ring (fm_task_inbox_mark_retry) # # Record format (fm_task_inbox_write / fm_task_inbox_body): # schema=fm-task-inbox.v1 # at= # delivery=fire-and-forget present only when the re-ring ladder must ignore it +# (it still gets one retry ring; see below) # -- # @@ -59,6 +62,19 @@ # crash or marker failure may produce a rare duplicate rather than silently lose # a wake. # +# Retry ring (fm_task_inbox_mark_retry): only while config/wait-no-turns is +# present. A fire-and-forget record never enters the ladder, but when +# fm-send's ring at enqueue did not land +# (fm_task_inbox_ring returned 1 or 2) it marks the record, and one grace later +# the due action is `retry`: once the worker has no open decision of its own, +# the watcher rings once more and spends the mark +# whatever the result, so the record never rings a third time and never +# escalates. A waiting worker does not poll its inbox (bin/fm-brief.sh), so +# without this retry the record could sit unread until a checkpoint. A pending ordinary record's +# ladder rings the same inbox, so the retry waits behind it, and an +# acknowledged record drops its mark. The remote steer leg has no watcher +# ladder and owes no retry. +# # Inbox names containing bytes outside printable ASCII are unsupported. The # doorbell refuses them rather than sending terminal control bytes to a pane. # @@ -369,11 +385,30 @@ fm_task_inbox_oldest_unhandled() { # printf '%s' "$best" } +# Owe a fire-and-forget record its one retry ring (see the header). A newer +# mark replaces an older one: a ring names the whole inbox, not one record. +fm_task_inbox_mark_retry() { # + local dir + dir=$(fm_task_inbox_dir "$1" "$2") + { printf '%s\n' "${3##*/}" > "$dir/.retry-ring"; } 2>/dev/null +} + +# Spend the retry mark after its ring, only while it still names that record: +# a newer mark written meanwhile is owed its own retry and survives. Fails only +# when the processed record's mark stays behind. +fm_task_inbox_clear_retry() { # + local dir + dir=$(fm_task_inbox_dir "$1" "$2") + [ "$(cat "$dir/.retry-ring" 2>/dev/null)" = "${3##*/}" ] || return 0 + rm -f "$dir/.retry-ring" 2>/dev/null +} + # The re-ring ladder decision for one task. Prints exactly one of: # quiet nothing due (healthy, within grace or spacing, # or already escalated for the current oldest) # ring one doorbell re-ring is due # escalate attempt budget spent; surface as stale +# retry a fire-and-forget record's one retry ring is due # An empty inbox also resets the ladder bookkeeping so the next message starts # a fresh ladder. fm_task_inbox_due_action() { # @@ -381,6 +416,17 @@ fm_task_inbox_due_action() { # dir=$(fm_task_inbox_dir "$1" "$2") if ! oldest=$(fm_task_inbox_oldest_unhandled "$1" "$2"); then rm -f "$dir/.ring-state" "$dir/.escalated" 2>/dev/null || true + # The one retry ring exists only while config/wait-no-turns is present. + # Absent, a mark is left untouched and the inbox stays quiet, as before. + if [ -e "${FM_CONFIG_OVERRIDE:-${FM_HOME:-}/config}/wait-no-turns" ]; then + base=$(cat "$dir/.retry-ring" 2>/dev/null || true) + if ! fm_task_inbox_seq_of "$base" >/dev/null || [ ! -f "$dir/$base" ]; then + rm -f "$dir/.retry-ring" 2>/dev/null || true + elif [ "$(fm_path_age "$dir/.retry-ring")" -ge "$(fm_task_inbox_grace_secs)" ]; then + printf 'retry %s' "$dir/$base" + return 0 + fi + fi printf 'quiet' return 0 fi diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 96bae225fa5..35c5a9f1b75 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -529,7 +529,10 @@ inbox_steer_escalate_unavailable() { # # stale path instead of silently re-ringing forever; acknowledgement or teardown # still makes the race quiet. The attempt is data-plane typing or a # composer-protected skip, never a wake, so normal retries keep the watcher -# blocking. Runs for secondmates +# blocking. A fire-and-forget record's one retry ring follows the same busy +# wait, also waits while the worker has an open decision or blocker of its own +# (status_own_open_decisions), and never escalates: a dead pane just spends it. +# Runs for secondmates # too: their pane-staleness exemption is about quiet panes being healthy, # while an unacknowledged instruction past the ladder is a stuck steer. inbox_steer_check() { # @@ -537,6 +540,9 @@ inbox_steer_check() { # action=$(fm_task_inbox_due_action "$STATE" "$task") || return 0 verb=${action%% *} [ "$verb" != quiet ] || return 0 + if [ "$verb" = retry ] && [ -n "$(status_own_open_decisions "$STATE/$task.status")" ]; then + return 0 + fi rec=${action#* } count= case "$verb" in @@ -549,7 +555,11 @@ inbox_steer_check() { # agent_state=$(fm_backend_agent_state "$backend" "$w" 2>/dev/null || true) case "$agent_state" in dead|missing) - inbox_steer_escalate_unavailable "$w" "$task" "$rec" + if [ "$verb" = retry ]; then + fm_task_inbox_clear_retry "$STATE" "$task" "$rec" || true + else + inbox_steer_escalate_unavailable "$w" "$task" "$rec" + fi return 0 ;; esac @@ -578,6 +588,16 @@ inbox_steer_check() { # fi triage_log "steer-inbox delivery attempt: $task ${rec##*/} result=$ring_rc" ;; + retry) + ring_rc=0 + fm_task_inbox_ring "$backend" "$w" "$rec" "$(window_label "$w")" || ring_rc=$? + if ! fm_task_inbox_clear_retry "$STATE" "$task" "$rec" && [ -f "$rec" ]; then + reason="stale: $w (steering-inbox retry mark unremovable: ${rec%/*}/.retry-ring cannot be removed, so $rec would ring on every poll - inspect the inbox directory)" + fm_wake_append stale "$w" "$reason" || exit 1 + wake "$reason" + fi + triage_log "steer-inbox retry ring: $task ${rec##*/} result=$ring_rc" + ;; escalate) reason="stale: $w (unread firstmate instruction: $rec still unhandled after $count doorbell delivery attempts with an idle pane; inspect the worker)" if [ ! -d "${rec%/*}" ] || [ ! -f "$rec" ]; then diff --git a/docs/configuration.md b/docs/configuration.md index 5dc5b92f47e..965ddbce5a9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -573,6 +573,13 @@ See [`trace-context.md`](trace-context.md) for carrier semantics, supported rout See [`fleet-ledger.md`](fleet-ledger.md) for the opt-in setup, record contract, and limits. +## Waiting worker spends no turns (config/wait-no-turns) + +The optional local, gitignored `config/wait-no-turns` presence flag opts this home into keeping a waiting worker from spending turns until it is answered. +With it present, ship and scout briefs gain the `# Waiting` section and the foreground no-mistakes drive text, every brief's inbox section keeps the natural-checkpoint check and adds that a waiting worker does not poll or list its inbox because a waiting instruction rings, a pending-reply recovery waits while that mate has its own open decision or blocker, and a fire-and-forget steer whose doorbell did not land gets one later ring. +With the file absent, generated briefs omit the waiting section and the no-poll inbox line, the drive text backgrounds the call, recovery sends during an open decision, and a fire-and-forget steer is not owed a retry ring. +The flag is a home-local preference and is not inherited by secondmate homes. + ## Turn-end pane-churn absorb (config/turnend-churn-absorb) The optional local, gitignored `config/turnend-churn-absorb` presence flag opts this home into a default-off third form of positive work evidence in watcher triage. diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index 42d496acb39..eb7537cf84b 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -487,6 +487,7 @@ When deduplication finds that the worker already moved the matching record into The remote host runs no doorbell re-ring ladder of its own. A swallowed doorbell for an ordinary reply-bearing request surfaces through the parent's pending-reply recovery and escalation. Its recovery request rings the doorbell again when it is enqueued. +A fire-and-forget record, such as a reconcile ask, gets its single retry ring only on the local plane, and only when `config/wait-no-turns` is present: the remote steer leg owes no re-ring, so a swallowed remote doorbell for one waits for the next ring into that inbox, and a remote-side retry is known follow-up scope. ### Remote reads diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index 28593006c4a..bef0095bb91 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -844,6 +844,43 @@ ok - opencode (1.18.33): the doorbell reached a real worker, which acted and ack OpenCode needed `FM_SEND_INBOX_LIVE_TIMEOUT=560` because its configured model was still mid-turn at the default 240 seconds. Pi 0.87.1 was installed but not verified: its configured model returned an account error (`The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account`) before it read the inbox. +## Waiting-worker command ceilings + +The `# Waiting` section of the ship and scout briefs (`bin/fm-brief.sh`) has a worker hold every external wait inside one blocking shell command, bounded by what its harness lets one command run. +That section is generated only when `config/wait-no-turns` is present. +Those bounds were read from the installed vendor code on 2026-09-11, macOS arm64, with Pi 0.85.1, codex-cli 0.154.0, and Claude Code 2.1.268. + +```sh +grep -n "Timeout in seconds" "$(npm root -g)/@earendil-works/pi-coding-agent/dist/core/tools/bash.js" +strings -n 20 "$(readlink -f "$(command -v codex)")" | grep -o "Non-empty writes default to [^.]*; empty polls wait [^.]*\." +strings -n 8 "$(readlink -f "$(command -v claude)")" | grep -oE '=120000,[A-Za-z0-9_$]+=600000;' | head -1 +``` + +Observed output: + +```text +28: timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), +Non-empty writes default to 250 ms and cap at 30000 ms; empty polls wait 5000-300000 ms by default. +=120000,ARo=600000; +``` + +Pi's bash tool runs a command with no time limit unless the call passes `timeout`, so the brief asks for at most 2700 seconds, which stays under the watcher's 3600-second busy-turn bound. +Codex yields a still-running command back to the model, and one empty `write_stdin` poll then waits up to 300000 ms. +Claude Code's Bash tool defaults to 120000 ms and accepts at most 600000 ms; `BASH_DEFAULT_TIMEOUT_MS` and `BASH_MAX_TIMEOUT_MS` override those two values. + +Claude Code also constrains the shape of a wait, not only its length, so the brief has to name the shape that is allowed rather than only forbid the ones that are not. +Run as separate Bash tool calls on 2026-09-14 with Claude Code 2.1.268: + +```sh +until [ -e /tmp/fm-wait-probe ]; do sleep 30; done # ran to completion, rc=0 +sleep 61; echo "rc=$?" # rc=0 +sleep 40; echo "checked at $(date +%s)" # rc=0 +``` + +An earlier `sleep 60` chained ahead of a status check was refused before execution, with a message pointing at `Monitor` with an until-loop and at `run_in_background: true`, and adding "Do not chain shorter sleeps to work around this block". +The blocking foreground `until` loop is therefore the wait a Claude Code worker may use, and it is what the brief names, because the refusal's own `run_in_background` suggestion is the one shape a waiting worker must not take: a backgrounded call returns at once and so does not wait at all. +The brief's portable regression is `tests/fm-brief.test.sh`; rerun these commands after upgrading any of the three harnesses and update the numbers in the brief when they move. + ## Gemini The Gemini crewmate adapter was verified on 2026-09-04 with gemini-cli 0.58.0 on Linux, Node v24.20.0, tmux 3.4. diff --git a/tests/fm-brief.test.sh b/tests/fm-brief.test.sh index 7234ae435ce..225d1c0975c 100755 --- a/tests/fm-brief.test.sh +++ b/tests/fm-brief.test.sh @@ -920,7 +920,8 @@ SIGNALS test_ship_and_scout_teach_validation_round_pause() { local home kind id brief home="$TMP_ROOT/validation-round-pause-home" - mkdir -p "$home/data" + mkdir -p "$home/data" "$home/config" + : > "$home/config/wait-no-turns" for kind in ship scout; do id="brief-validation-round-pause-$kind" @@ -930,6 +931,12 @@ test_ship_and_scout_teach_validation_round_pause() { FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" firstmate --mode no-mistakes >/dev/null 2>&1 fi brief="$home/data/$id/brief.md" + assert_grep "your own validation round, which you declare once just before its blocking hold" "$brief" \ + "$kind brief did not teach workers to declare their validation-round wait before holding it" + assert_grep "append \`paused:\` once just before its first blocking command, then stay in the command" "$brief" \ + "$kind brief's Waiting section does not declare the validation round once and then hold it" + assert_no_grep "is not a \`paused:\` wait" "$brief" \ + "$kind brief still tells workers never to declare a wait they hold in a command" assert_grep "your own validation round" "$brief" \ "$kind brief did not teach workers to declare their validation-round wait" assert_grep 'Before ending your turn with your own background shell or monitor still running' "$brief" \ @@ -941,7 +948,7 @@ test_ship_and_scout_teach_validation_round_pause() { assert_grep 'Do not declare active implementation or reasoning as a wait' "$brief" \ "$kind brief did not limit the declaration to actual waits" done - pass "fm-brief.sh: ship and scout scaffolds teach validation-round pauses" + pass "fm-brief.sh: ship and scout scaffolds declare a validation-round pause once, then hold it" } test_scout_and_secondmate_load_decision_hold_policy() { @@ -1026,6 +1033,68 @@ test_scout_and_secondmate_scaffold() { pass "fm-brief: scout and secondmate code paths still scaffold well-formed briefs" } +# Contract: a waiting worker spends no turns. A decision wait ends the turn, an +# external wait sleeps in one bounded blocking shell command sized per harness, +# and a waiting worker neither polls its inbox nor polls a pipeline between holds. +test_workers_wait_without_spending_turns() { + local home id brief + home="$TMP_ROOT/wait-home" + mkdir -p "$home/data" "$home/config" + : > "$home/config/wait-no-turns" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" brief-wait-ship some-proj --mode no-mistakes >/dev/null 2>&1 \ + || fail "fm-brief.sh ship scaffold exited non-zero" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" brief-wait-scout some-proj --scout >/dev/null 2>&1 \ + || fail "fm-brief.sh scout scaffold exited non-zero" + for id in brief-wait-ship brief-wait-scout; do + brief="$home/data/$id/brief.md" + assert_grep "end your turn at once" "$brief" "$id: a decision wait must end the turn" + assert_grep "with ONE blocking shell command that returns when the state changes" "$brief" \ + "$id: an external wait must sleep in one blocking shell command" + assert_grep "gh pr checks --watch" "$brief" "$id: the CI wait primitive is missing" + assert_grep "a \`timeout\` of at most 2700 seconds" "$brief" "$id: the Pi ceiling is missing" + assert_grep "its maximum \`timeout\` of 600000 ms" "$brief" "$id: the Claude Code ceiling is missing" + assert_grep "empty \`write_stdin\` polls of up to 300000 ms" "$brief" "$id: the Codex ceiling is missing" + assert_grep "is the sanctioned foreground wait" "$brief" \ + "$id: the wait a Claude Code worker may use is not named" + assert_grep "reattach with \`no-mistakes axi run --wait\` instead, and never send the same \`respond\` again" "$brief" \ + "$id: a timed-out respond must reattach with axi run, never resend its answer" + assert_grep "Do not poll or list the inbox while waiting; a waiting instruction rings." "$brief" \ + "$id: polling the inbox while waiting is not forbidden" + assert_grep "natural checkpoint" "$brief" "$id: the flag dropped the natural-checkpoint inbox check" + done + brief="$home/data/brief-wait-ship/brief.md" + assert_grep "issue the same foreground call again" "$brief" \ + "the no-mistakes DOD must reattach with the same foreground call" + assert_no_grep "background the drive call" "$brief" "the no-mistakes DOD still backgrounds the drive call" + + FM_SECONDMATE_CHARTER='Supervise the alpha domain.' \ + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" brief-wait-sm --secondmate --no-projects >/dev/null 2>&1 \ + || fail "fm-brief.sh secondmate scaffold exited non-zero" + brief="$home/data/brief-wait-sm/brief.md" + assert_grep "Do not poll or list the inbox while waiting; a waiting instruction rings." "$brief" \ + "secondmate: polling the inbox while waiting is not forbidden" + assert_grep "natural checkpoint" "$brief" "secondmate: the flag dropped the natural-checkpoint inbox check" + pass "fm-brief: workers end the turn on a decision, wait in one bounded shell command, and never poll" +} + +# Without config/wait-no-turns the scaffold matches the pre-flag brief and drive text. +test_wait_no_turns_absent_keeps_the_previous_brief() { + local home brief + home="$TMP_ROOT/wait-off" + mkdir -p "$home/data" + [ ! -e "$home/config/wait-no-turns" ] + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" brief-wait-off some-proj --mode no-mistakes >/dev/null 2>&1 \ + || fail "fm-brief.sh ship scaffold exited non-zero" + brief="$home/data/brief-wait-off/brief.md" + assert_no_grep "end your turn at once" "$brief" "an absent flag still added the waiting section" + assert_grep "natural checkpoint" "$brief" "an absent flag dropped the unprompted inbox check" + assert_no_grep "Do not poll or list the inbox while waiting" "$brief" "an absent flag still added the no-poll inbox line" + assert_grep "background the drive call" "$brief" "an absent flag replaced the backgrounded drive text" + assert_no_grep "issue the same foreground call again" "$brief" \ + "an absent flag still asked for the foreground reattach" + pass "fm-brief: without config/wait-no-turns the brief and drive text stay as they were" +} + test_worker_role_scope() { local kind home brief home="$TMP_ROOT/worker-role" @@ -1352,6 +1421,8 @@ test_ship_and_scout_teach_validation_round_pause test_scout_and_secondmate_load_decision_hold_policy test_scout_and_secondmate_scaffold test_scout_lavish_line_follows_presentation_floor +test_workers_wait_without_spending_turns +test_wait_no_turns_absent_keeps_the_previous_brief test_home_brief_include_is_appended_last test_ship_branch_prefix_defaults_to_legacy_fm test_ship_branch_prefix_override_is_consistent_across_modes diff --git a/tests/fm-pending-reply.test.sh b/tests/fm-pending-reply.test.sh index 350741fca2a..777d1c97c57 100755 --- a/tests/fm-pending-reply.test.sh +++ b/tests/fm-pending-reply.test.sh @@ -201,6 +201,75 @@ test_completed_turn_no_report_triggers_one_recovery() { pass "completed turn with no report triggers exactly one recovery" } +# A mate waiting on its own open decision is never poked by the recovery; the +# recovery stays unattempted and runs once the decision closes. +test_recovery_waits_while_the_mate_has_an_open_decision() { + local home state corr hook_log + home=$(setup_parent decision-wait) + state="$home/state" + hook_log="$TMP_ROOT/decision-wait-hook.log" + : > "$hook_log" + export FM_PENDING_REPLY_NOW=2500 + mkdir -p "$home/config" + : > "$home/config/wait-no-turns" + FM_CONFIG_OVERRIDE="$home/config" + # Invoked indirectly through FM_PENDING_REPLY_SEND_HOOK. + # shellcheck disable=SC2329 + decision_wait_hook() { + printf '%s\n' "$1" >> "$hook_log" + } + export -f decision_wait_hook + export FM_PENDING_REPLY_SEND_HOOK=decision_wait_hook + + corr=$(fm_pending_reply_create "$home" "$state" "hibit" "status of phase 8") + fm_pending_reply_mark_delivered "$state" "$corr" + fm_pending_reply_observe_busy "$state" "$corr" busy + fm_pending_reply_observe_busy "$state" "$corr" idle + printf 'needs-decision [key=scope]: narrow or wide?\n' >> "$state/hibit.status" + if fm_pending_reply_send_recovery "$state" "$corr" 2>/dev/null; then + fail "recovery must wait while the mate waits on its own decision" + fi + [ ! -s "$hook_log" ] || fail "recovery poked a mate waiting on its decision" + [ "$(phase_of "$state" "$corr")" = awaiting_report ] \ + || fail "a deferred recovery must stay unattempted, got $(phase_of "$state" "$corr")" + + printf 'resolved [key=scope]: answered: narrow\n' >> "$state/hibit.status" + fm_pending_reply_send_recovery "$state" "$corr" || fail "recovery should send once the decision closes" + [ "$(wc -l < "$hook_log" | tr -d ' ')" = 1 ] || fail "expected exactly one recovery send" + unset FM_PENDING_REPLY_SEND_HOOK + unset FM_CONFIG_OVERRIDE + pass "recovery never pokes a mate waiting on its own decision, and runs once it closes" +} + +# Without the flag, an open decision does not hold the recovery. +test_recovery_sends_during_an_open_decision_without_the_flag() { + local home state corr hook_log + home=$(setup_parent decision-wait-off) + state="$home/state" + hook_log="$TMP_ROOT/decision-wait-off-hook.log" + : > "$hook_log" + mkdir -p "$home/config" + FM_CONFIG_OVERRIDE="$home/config" + export FM_PENDING_REPLY_NOW=2500 + # shellcheck disable=SC2329 + decision_wait_off_hook() { + printf '%s\n' "$1" >> "$hook_log" + } + export -f decision_wait_off_hook + export FM_PENDING_REPLY_SEND_HOOK=decision_wait_off_hook + corr=$(fm_pending_reply_create "$home" "$state" "hibit" "status of phase 8") + fm_pending_reply_mark_delivered "$state" "$corr" + fm_pending_reply_observe_busy "$state" "$corr" busy + fm_pending_reply_observe_busy "$state" "$corr" idle + printf 'needs-decision [key=scope]: narrow or wide?\n' >> "$state/hibit.status" + fm_pending_reply_send_recovery "$state" "$corr" \ + || fail "recovery should send while a decision is open when the flag is absent" + [ "$(wc -l < "$hook_log" | tr -d ' ')" = 1 ] || fail "expected the recovery to send" + unset FM_PENDING_REPLY_SEND_HOOK + unset FM_CONFIG_OVERRIDE + pass "recovery sends during an open decision when config/wait-no-turns is absent" +} + test_recovery_grace_measures_from_turn_completion() { local home state corr hook_log lines home=$(setup_parent grace-from-completion) @@ -1927,6 +1996,8 @@ test_escalated_undelivered_correlation_stays_retryable() { test_normal_correlated_reply_resolves_once test_completed_turn_no_report_triggers_one_recovery +test_recovery_waits_while_the_mate_has_an_open_decision +test_recovery_sends_during_an_open_decision_without_the_flag test_recovery_grace_measures_from_turn_completion test_recovery_fresh_status_read_resolves_before_firing test_partial_resolve_write_blocks_firing diff --git a/tests/fm-remote-reply.test.sh b/tests/fm-remote-reply.test.sh index 47f75a75593..d6b00c7cead 100755 --- a/tests/fm-remote-reply.test.sh +++ b/tests/fm-remote-reply.test.sh @@ -694,6 +694,9 @@ pass "source-line identity survives commit failure and cursor-loss recapture" # the reserved key over. # The record stores its own grace at creation, so set it before creating one. export FM_PENDING_REPLY_GRACE_SECS=0 +# Answer the mate's earlier decisions and blocker first: a recovery repost waits +# while the mate has one of its own open (tests/fm-pending-reply.test.sh). +printf 'resolved [key=%s]: answered\n' rough-cut-version ctl default >> "$PARENT/state/ios.status" ESCALATED_CORR=$(fm_pending_reply_create "$PARENT" "$PARENT/state" ios 'confirm the notarization') [ -n "$ESCALATED_CORR" ] || fail "could not create the pending-reply record to escalate" fm_pending_reply_mark_delivered "$PARENT/state" "$ESCALATED_CORR" \ diff --git a/tests/fm-send-inbox.test.sh b/tests/fm-send-inbox.test.sh index c0c61f62ae5..2367771311d 100644 --- a/tests/fm-send-inbox.test.sh +++ b/tests/fm-send-inbox.test.sh @@ -14,7 +14,8 @@ # 4. The composer pre-check is advisory: visibly pending text skips the ring # with a notice, and the steer is still durably sent (exit 0). # 5. A failed doorbell is still a sent steer (exit 0, record durable): the -# watcher's re-ring ladder owns delivery from the record on. +# watcher's re-ring ladder owns delivery from the record on. A +# fire-and-forget record whose ring did not land is owed one retry ring. # 6. Carve-outs keep the typed plane: a leading "/" (any harness), a leading # "$" to codex, an explicit backend target, and the --key path. # 7. A marked secondmate steer carries its marker + corr token in the record @@ -241,6 +242,56 @@ test_failed_ring_is_still_sent() { pass "fm-send inbox: a failed doorbell is still a durably sent steer" } +# Contract: a fire-and-forget record stays outside the re-ring ladder, so a +# ring that did not land at enqueue is owed exactly one retry by the watcher. +test_fire_and_forget_unlanded_ring_owes_one_retry() { + local dir err rc + dir=$(setup_case faf-retry) + mkdir -p "$dir/home/config" + : > "$dir/home/config/wait-no-turns" + err="$dir/send.err" + # The stub lists only window fm-t1, so the secondmate takes it over. + rm -f "$dir/home/state/t1.meta" + fm_write_secondmate_meta "$dir/home/state/domain.meta" "$dir/home" "sess:fm-t1" alpha claude + run_send "$dir" "$err" FM_FAKE_TMUX_COMPOSER=pending -- \ + fm-domain --fire-and-forget 0123456789abcdef "reconcile your books"; rc=$? + expect_code 0 "$rc" "a skipped fire-and-forget ring is still a sent steer" + [ "$(cat "$dir/home/state/domain.inbox/.retry-ring" 2>/dev/null)" = 001.msg ] \ + || fail "a skipped fire-and-forget ring did not owe its one retry" + assert_contains "$(cat "$err")" "the watcher will ring it once more" \ + "the skip notice should promise exactly one retry" + + run_send "$dir" "$err" -- fm-domain --fire-and-forget 1123456789abcdef "reconcile again"; rc=$? + expect_code 0 "$rc" "a rung fire-and-forget steer should succeed" + [ "$(cat "$dir/home/state/domain.inbox/.retry-ring" 2>/dev/null)" = 001.msg ] \ + || fail "a ring that landed must not owe a retry for its own record" + + dir=$(setup_case ordinary-no-retry) + err="$dir/send.err" + run_send "$dir" "$err" FM_FAKE_TMUX_COMPOSER=pending -- t1 "ordinary steer" + [ ! -e "$dir/home/state/t1.inbox/.retry-ring" ] \ + || fail "an ordinary record rides the ladder and must not owe a separate retry" + pass "fm-send inbox: a fire-and-forget ring that did not land owes one retry ring" +} + +# Without the flag a skipped fire-and-forget ring is not owed a retry. +test_fire_and_forget_retry_stays_off_without_the_flag() { + local dir err rc + dir=$(setup_case faf-retry-off) + err="$dir/send.err" + [ ! -e "$dir/home/config/wait-no-turns" ] + rm -f "$dir/home/state/t1.meta" + fm_write_secondmate_meta "$dir/home/state/domain.meta" "$dir/home" "sess:fm-t1" alpha claude + run_send "$dir" "$err" FM_FAKE_TMUX_COMPOSER=pending -- \ + fm-domain --fire-and-forget 0123456789abcdef "reconcile your books"; rc=$? + expect_code 0 "$rc" "a skipped fire-and-forget ring is still a sent steer" + [ ! -e "$dir/home/state/domain.inbox/.retry-ring" ] \ + || fail "an absent flag still owed a fire-and-forget retry" + assert_contains "$(cat "$err")" "the watcher will re-ring" \ + "an absent flag should keep the ordinary re-ring notice" + pass "fm-send inbox: without config/wait-no-turns a fire-and-forget ring is not retried" +} + test_harness_invocations_stay_typed() { local dir err typed # A slash command must reach the harness's own parser, on any harness. @@ -459,6 +510,8 @@ test_multiline_steer_is_legal test_resend_enqueues_new_sequence test_pending_composer_skips_ring_advisorily test_failed_ring_is_still_sent +test_fire_and_forget_unlanded_ring_owes_one_retry +test_fire_and_forget_retry_stays_off_without_the_flag test_harness_invocations_stay_typed test_explicit_target_stays_typed test_key_path_never_touches_inbox diff --git a/tests/fm-task-inbox.test.sh b/tests/fm-task-inbox.test.sh index 7a188c76422..3c9c8dce4d4 100644 --- a/tests/fm-task-inbox.test.sh +++ b/tests/fm-task-inbox.test.sh @@ -26,6 +26,9 @@ # 6. Dead panes: the doorbell line is a shell no-op when executed by a bare # shell, the ring skips an agent the backend classifies dead, and the # watcher surfaces such a record exactly once instead of re-ringing. +# 7. A fire-and-forget record stays outside the ladder, but one whose first +# ring did not land gets exactly one retry ring and never escalates. The +# retry waits while the worker has an open decision of its own. set -u # shellcheck source=tests/wake-helpers.sh @@ -79,6 +82,10 @@ case "${1:-}" in if [ -n "${FM_ACK_RECORD:-}" ] && [ -f "$FM_ACK_RECORD" ]; then mv "$FM_ACK_RECORD" "${FM_ACK_RECORD%/*}/handled/" fi + # A concurrent fire-and-forget send marking its newer record mid-ring. + if [ -n "${FM_RING_MARKS_RETRY:-}" ]; then + printf '%s\n' "${FM_RING_MARKS_RETRY##*/}" > "${FM_RING_MARKS_RETRY%/*}/.retry-ring" + fi fi exit 0 ;; display-message) @@ -548,6 +555,59 @@ test_fire_and_forget_records_never_enter_the_ladder() { pass "inbox: fire-and-forget records stay durable and outside the ladder" } +test_fire_and_forget_retry_is_owed_once() { + local state fire tracked action + state="$TMP_ROOT/faf-retry/state"; mkdir -p "$state" "$TMP_ROOT/faf-retry/config" + : > "$TMP_ROOT/faf-retry/config/wait-no-turns" + export FM_CONFIG_OVERRIDE="$TMP_ROOT/faf-retry/config" + fire=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "one-shot steer" fire-and-forget) + age_path "$fire" + inbox_lib "$state" fm_task_inbox_mark_retry "$state" t1 "$fire" + action=$(FM_TASK_INBOX_GRACE_SECS=3600 inbox_lib "$state" fm_task_inbox_due_action "$state" t1) + [ "$action" = quiet ] || fail "a retry inside grace should be quiet, got: $action" + age_path "$state/t1.inbox/.retry-ring" + action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1) + [ "$action" = "retry $fire" ] || fail "an aged retry mark should be due its ring, got: $action" + # An ordinary record's ladder rings the same inbox, so the retry waits behind it. + tracked=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "tracked steer") + age_path "$tracked" + action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1) + [ "$action" = "ring $tracked" ] || fail "a pending ordinary record should own the ring, got: $action" + mv "$tracked" "$state/t1.inbox/handled/" + action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1) + [ "$action" = "retry $fire" ] || fail "the retry should resume once the ordinary record is handled, got: $action" + # Once spent, the record is quiet for good: no second retry and no escalation. + inbox_lib "$state" fm_task_inbox_clear_retry "$state" t1 "$fire" + action=$(FM_TASK_INBOX_GRACE_SECS=0 FM_TASK_INBOX_RING_MAX=0 \ + inbox_lib "$state" fm_task_inbox_due_action "$state" t1) + [ "$action" = quiet ] || fail "a spent retry rang or escalated again: $action" + # An acknowledged record drops its mark. + inbox_lib "$state" fm_task_inbox_mark_retry "$state" t1 "$fire" + age_path "$state/t1.inbox/.retry-ring" + mv "$fire" "$state/t1.inbox/handled/" + action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1) + [ "$action" = quiet ] || fail "an acknowledged record's retry should be dropped, got: $action" + [ ! -e "$state/t1.inbox/.retry-ring" ] || fail "an acknowledged record kept its retry mark" + unset FM_CONFIG_OVERRIDE + pass "inbox: a fire-and-forget record whose ring did not land is owed exactly one retry" +} + +# A retry mark is ignored while config/wait-no-turns is absent. +test_fire_and_forget_retry_is_quiet_without_the_flag() { + local state fire action + state="$TMP_ROOT/faf-retry-off/state"; mkdir -p "$state" "$TMP_ROOT/faf-retry-off/config" + export FM_CONFIG_OVERRIDE="$TMP_ROOT/faf-retry-off/config" + fire=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "one-shot steer" fire-and-forget) + age_path "$fire" + inbox_lib "$state" fm_task_inbox_mark_retry "$state" t1 "$fire" + age_path "$state/t1.inbox/.retry-ring" + action=$(FM_TASK_INBOX_GRACE_SECS=60 inbox_lib "$state" fm_task_inbox_due_action "$state" t1) + [ "$action" = quiet ] || fail "an absent flag still owed a retry ring, got: $action" + [ -e "$state/t1.inbox/.retry-ring" ] || fail "an absent flag removed a retry mark it should have left" + unset FM_CONFIG_OVERRIDE + pass "inbox: without config/wait-no-turns a fire-and-forget retry mark stays quiet" +} + test_ring_ladder_policy() { local state rec action state="$TMP_ROOT/ladder/state"; mkdir -p "$state" @@ -725,6 +785,101 @@ test_watcher_surfaces_unwritable_ladder() { pass "watcher: unwritable ladder bookkeeping surfaces a stale wake after the doorbell" } +test_watcher_pays_fire_and_forget_retry_once() { + local dir state out log pid fire rings i=0 + dir=$(setup_watch_case faf-retry) + mkdir -p "$dir/config" + : > "$dir/config/wait-no-turns" + state="$dir/state"; out="$dir/watch.out"; log="$dir/send.log"; : > "$log" + fire=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "one-shot steer" fire-and-forget) + age_path "$fire" + inbox_lib "$state" fm_task_inbox_mark_retry "$state" t1 "$fire" + age_path "$state/t1.inbox/.retry-ring" + watch_bg "$state" "$dir/fakebin" "$out" \ + FM_CONFIG_OVERRIDE="$dir/config" \ + FM_SEND_LOG="$log" FM_FAKE_TMUX_CAPTURE="$(idle_capture "$dir")" \ + FM_TASK_INBOX_RING_MAX=1 + pid=$! + while [ "$i" -lt 100 ]; do + grep -qF 'Firstmate instruction waiting' "$log" 2>/dev/null && break + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + i=$((i + 1)) + done + sleep 3 + kill -0 "$pid" 2>/dev/null \ + || fail "a fire-and-forget retry must not wake firstmate (watcher exited):"$'\n'"$(cat "$out")" + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null + rings=$(grep -cF 'Firstmate instruction waiting' "$log" || true) + [ "$rings" = 1 ] || fail "expected exactly one retry ring, got $rings:"$'\n'"$(cat "$log")" + [ ! -s "$state/.wake-queue" ] || fail "a fire-and-forget retry queued a wake:"$'\n'"$(cat "$state/.wake-queue")" + [ ! -e "$state/t1.inbox/.retry-ring" ] || fail "the watcher did not spend the retry mark" + [ ! -e "$state/t1.inbox/.ring-state" ] || fail "a fire-and-forget retry entered the re-ring ladder" + [ -f "$fire" ] || fail "the retry ring removed the durable record" + pass "watcher: a fire-and-forget record's owed retry rings exactly once and never escalates" +} + +# One watcher inbox check against an idle pane, through the production watcher +# functions, so a status log the case writes is not also read as a wake. +steer_check_once() { # + PATH="$1/fakebin:$PATH" FM_STATE_OVERRIDE="$1/state" FM_SEND_LOG="$1/send.log" \ + FM_FAKE_TMUX_CAPTURE="$(idle_capture "$1")" FM_TASK_INBOX_GRACE_SECS=1 \ + bash -c '. "$1" && inbox_steer_check sess:fm-t1 t1' _ "$WATCH" >/dev/null 2>&1 +} + +test_watcher_holds_retry_while_the_worker_decides() { + local dir state log fire rings + dir=$(setup_watch_case faf-retry-decision) + mkdir -p "$dir/config" + : > "$dir/config/wait-no-turns" + export FM_CONFIG_OVERRIDE="$dir/config" + state="$dir/state"; log="$dir/send.log"; : > "$log" + fire=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "one-shot steer" fire-and-forget) + age_path "$fire" + inbox_lib "$state" fm_task_inbox_mark_retry "$state" t1 "$fire" + age_path "$state/t1.inbox/.retry-ring" + printf 'needs-decision [key=pick]: ship alpha or beta?\n' > "$state/t1.status" + steer_check_once "$dir" + steer_check_once "$dir" + [ ! -s "$log" ] || fail "the retry rang a worker waiting on its own decision:"$'\n'"$(cat "$log")" + [ -e "$state/t1.inbox/.retry-ring" ] || fail "the held retry lost its mark" + + printf 'resolved [key=pick]: alpha\n' >> "$state/t1.status" + steer_check_once "$dir" + steer_check_once "$dir" + rings=$(grep -cF 'Firstmate instruction waiting' "$log" || true) + [ "$rings" = 1 ] || fail "expected exactly one retry ring once the decision closed, got $rings:"$'\n'"$(cat "$log")" + [ ! -e "$state/t1.inbox/.retry-ring" ] || fail "the watcher did not spend the retry mark" + unset FM_CONFIG_OVERRIDE + pass "watcher: a fire-and-forget retry waits out the worker's own decision, then rings once" +} + +test_watcher_retry_keeps_a_newer_mark() { + local dir state log fire newer rings + dir=$(setup_watch_case faf-retry-newer) + mkdir -p "$dir/config" + : > "$dir/config/wait-no-turns" + export FM_CONFIG_OVERRIDE="$dir/config" + state="$dir/state"; log="$dir/send.log"; : > "$log" + fire=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "one-shot steer" fire-and-forget) + age_path "$fire" + inbox_lib "$state" fm_task_inbox_mark_retry "$state" t1 "$fire" + age_path "$state/t1.inbox/.retry-ring" + newer=$(inbox_lib "$state" fm_task_inbox_write "$state" t1 "newer steer" fire-and-forget) + FM_RING_MARKS_RETRY="$newer" steer_check_once "$dir" + rings=$(grep -cF 'Firstmate instruction waiting' "$log" || true) + [ "$rings" = 1 ] || fail "expected the owed retry to ring once, got $rings:"$'\n'"$(cat "$log")" + [ "$(cat "$state/t1.inbox/.retry-ring" 2>/dev/null)" = "${newer##*/}" ] \ + || fail "the spent retry removed a newer record's mark written during its ring" + age_path "$state/t1.inbox/.retry-ring" + steer_check_once "$dir" + rings=$(grep -cF 'Firstmate instruction waiting' "$log" || true) + [ "$rings" = 2 ] || fail "the newer record's retry did not ring, got $rings:"$'\n'"$(cat "$log")" + [ ! -e "$state/t1.inbox/.retry-ring" ] || fail "the watcher did not spend the newer retry mark" + unset FM_CONFIG_OVERRIDE + pass "watcher: spending a retry keeps a newer record's mark written during its ring" +} + test_watcher_escalates_once_after_budget() { local dir state out log pid rec rings dir=$(setup_watch_case escalate) @@ -812,12 +967,17 @@ test_concurrent_writers_never_clobber test_writer_retries_after_a_vanished_lock_collision test_ladder_writes_ignore_vanished_inbox test_fire_and_forget_records_never_enter_the_ladder +test_fire_and_forget_retry_is_owed_once +test_fire_and_forget_retry_is_quiet_without_the_flag test_ring_ladder_policy test_watcher_rerings_idle_pane_quietly test_watcher_waits_on_busy_pane test_watcher_quiet_on_healthy_inbox test_watcher_ack_silences_unwritable_ladder test_watcher_surfaces_unwritable_ladder +test_watcher_pays_fire_and_forget_retry_once +test_watcher_holds_retry_while_the_worker_decides +test_watcher_retry_keeps_a_newer_mark test_watcher_escalates_once_after_budget test_watcher_dead_pane_escalates_once_without_ringing test_watcher_dead_pane_ignores_stale_busy_state