From 7c7643d93470ace7e14cca71894343f4a46e9890 Mon Sep 17 00:00:00 2001 From: Sungin Kim Date: Sat, 1 Aug 2026 23:14:55 +0000 Subject: [PATCH 1/3] fix(watch): suppress parked-decision stale wakes on pane repaint A crew that correctly parks on a keyed `needs-decision:` is genuinely not working, so `crew_is_provably_working` rightly refuses to absorb its stale pane. The one-shot stale suppressor was keyed on the pane content hash, but an idle harness pane is not static - its footer carries a live context percentage and quota readout - so every repaint changed the hash and re-surfaced the identical, already-escalated waiting state as a fresh stale wake. One parked pane escalated twenty-two consecutive times while healthy, each costing a full handling turn. Key the terminal stale path's suppressor on the open-decision identity from `status_open_decisions` (bin/fm-classify-lib.sh's existing keyed fold) whenever the captain-relevant status is a still-open keyed decision, so a repaint alone is silent. Detection is not weakened: a new status line, a new decision key, or the decision resolving all change the open set and surface normally, and a parked crew whose backend confidently reports its agent dead still escalates through the shared wedge timer. Only a confident `dead` verdict counts, so an ambiguous or unreadable backend read cannot manufacture a wedge alarm. Tests cover all three properties and fail against the previous behavior. --- bin/fm-watch.sh | 52 ++++++++++- docs/architecture.md | 2 + tests/fm-watch-triage.test.sh | 159 ++++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 9f17cbc357b..cf704fe8361 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -406,6 +406,33 @@ pause_state_class() { # printf '%s' "$class" } +# Identity of the captain decisions a task currently has OPEN, from the durable +# keyed fold owned by fm-classify-lib.sh (status_open_decisions): a +# needs-decision/blocked line opens a key, and only a matching resolution or +# verified captain-held transfer closes it. Prints a stable digest of that open +# set, or nothing when no decision is open. This is what the terminal stale path +# suppresses a repeat surface on, INSTEAD of the pane hash: a crew parked on a +# captain decision keeps the same open set no matter how its pane repaints, +# while a new status line, a new decision key, and the decision resolving all +# change it. +open_decision_id() { # + local task=$1 open + [ -n "$task" ] || return 0 + open=$(status_open_decisions "$STATE/$task.status") + [ -n "$open" ] || return 0 + printf '%s' "$open" | hash_pane +} + +# 0 when a parked crew has stopped responding: its recorded endpoint no longer +# carries a live agent. Only the confident `dead` verdict counts (an ambiguous, +# unreadable, or unverified backend read stays absorbed), so suppressing the +# repeat surface above never costs wedge detection for a parked crew that dies. +parked_agent_is_dead() { # + local win=$1 state + state=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || return 1 + [ "$state" = dead ] +} + surface_nonterminal_stale() { # local win=$1 h=$2 key task last key=$(printf '%s' "$win" | tr ':/.' '___') @@ -918,7 +945,29 @@ EOF # line. On a NEW hash, give an active run/busy pane (the same # authoritative source fm-crew-state.sh itself already prioritizes # over the log) a chance to override before trusting the log. - if [ "$(cat "$sf" 2>/dev/null || true)" != "$h" ]; then + # + # The one-shot suppressor is keyed on the OPEN DECISION, not the pane + # hash, whenever the captain-relevant status is a still-open keyed + # decision. An idle harness pane is not static - a live context and + # quota footer repaints on its own - so a hash-keyed one-shot let the + # identical already-escalated waiting state re-surface on every + # repaint, costing a full handling turn each time (twenty-two + # consecutive surfaces on one parked pane on 2026-08-01). Detection is + # unchanged: the open set moves on a new status line, a new decision + # key, or the decision resolving, and a parked crew whose agent has + # died still escalates through the shared wedge timer below. + did=$(open_decision_id "$task") + dsf="$STATE/.stale-decision-$key" + [ -n "$did" ] || rm -f "$dsf" + if [ -n "$did" ] && [ "$(cat "$dsf" 2>/dev/null || true)" = "$did" ]; then + printf '%s' "$h" > "$sf" + if parked_agent_is_dead "$w"; then + wedge_timer_check "$w" "$ssf" "stale (parked on an open decision, agent gone)" "$ewf" + else + rm -f "$ssf" "$ewf" + triage_log "absorbed stale (open decision already surfaced): $w" + fi + elif [ "$(cat "$sf" 2>/dev/null || true)" != "$h" ]; then if crew_is_provably_working "$(window_to_task "$w" "$STATE")"; then printf '%s' "$h" > "$sf" date +%s > "$ssf" @@ -926,6 +975,7 @@ EOF else fm_wake_append stale "$w" "stale: $w" || exit 1 printf '%s' "$h" > "$sf" + [ -n "$did" ] && printf '%s' "$did" > "$dsf" rm -f "$ssf" mark_surfaced "$STATE/$(window_to_task "$w" "$STATE").status" wake "stale: $w" diff --git a/docs/architecture.md b/docs/architecture.md index 95b0c2ff133..e1504bff76c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -23,6 +23,8 @@ For an ordinary crew that has stopped, the normal-mode watcher first surfaces on Live or inconclusive liveness remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. Fresh stale panes use the same current-state read before trusting the status log, so an active run or a proven busy worker outranks an old captain-relevant status-log line left behind before validation. +When that captain-relevant status is a still-open keyed decision, the one-shot suppressor is keyed on the open-decision identity from `bin/fm-classify-lib.sh`'s `status_open_decisions` fold rather than on the pane hash, because an idle harness pane repaints on its own and a hash-keyed suppressor re-surfaced the identical already-escalated waiting state on every repaint. +Such a crew therefore surfaces at most once per open decision, a new status line, a new decision key, or the decision resolving still surfaces normally, and a parked crew whose backend confidently reports its agent dead still escalates through the same wedge timer. No-change heartbeats are also benign. Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. After each drain, `fm-wake-drain.sh` runs the same liveness guard as the supervision scripts, so a lapsed watcher chain surfaces even on a turn that only drains and handles queued wakes. diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index c6d7928bb78..e86a91fe165 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -515,6 +515,162 @@ test_stale_terminal_status_overridden_by_active_run() { pass "a stale terminal-looking status is overridden and absorbed while a run is actively working, then wedge-escalated" } +# --- a parked keyed decision is surfaced once, not once per pane repaint ------- +# Regression for the 2026-08-01 parked-decision noise: a crew that correctly +# appends "needs-decision [key=...]: ..." and stops is genuinely not working, so +# crew_is_provably_working rightly refuses to absorb its stale pane. The one-shot +# suppressor used to be keyed on the pane hash, and an idle harness pane is not +# static (a live context/quota footer repaints on its own), so every repaint +# re-surfaced the identical already-escalated waiting state - twenty-two +# consecutive times on one pane. Suppression must key on the OPEN DECISION +# (fm-classify-lib.sh's status_open_decisions fold) so a repaint alone is silent. +# +# Fixture note: each phase primes .hash-/.count- so the FIRST poll already sees a +# stably stale pane, exactly as the terminal-stale cases above do. +prime_stale_pane() { # + local state=$1 window=$2 text=$3 capture=$4 key + printf '%s' "$text" > "$capture" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text "$text")" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" +} + +# Prime the .seen-* signal suppressor for a status file so only the STALE path +# under test can surface a wake (a status write otherwise fires its own signal). +prime_status_seen() { # + local state=$1 task=$2 + printf '%s' "$(seen_sig "$state/$task.status")" > "$state/.seen-${task}_status" +} + +test_parked_decision_survives_pane_repaint() { + local dir state fakebin out drain_out capture window key pid + dir=$(make_case parked-decision-repaint); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture="$dir/pane.txt" + window="test:fm-parked" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf 'window=%s\nkind=ship\n' "$window" > "$state/parked.meta" + printf 'needs-decision [key=review-gate]: ship as-is or split the migration\n' \ + > "$state/parked.status" + prime_status_seen "$state" parked + # A live agent: this crew is waiting, not wedged. + export FM_FAKE_TMUX_CURRENT_COMMAND=claude + + # Phase A: first sighting of the parked decision surfaces exactly as before. + prime_stale_pane "$state" "$window" 'awaiting decision · context 41%' "$capture" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "watcher did not surface a newly parked captain decision" + grep -Fx "stale: $window" "$out" >/dev/null || fail "the parked decision did not print a stale wake" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the parked decision failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null \ + || fail "the parked decision was not queued on its first sighting" + + # Phase B: the pane repaints (a moving context percentage) while the SAME + # decision stays open. The hash changes; the decision does not. Nothing may + # surface. + : > "$out" + prime_stale_pane "$state" "$window" 'awaiting decision · context 39%' "$capture" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "a pane repaint re-surfaced an already-escalated parked decision: $(cat "$out")" + fi + [ ! -s "$out" ] || fail "a pane repaint printed a wake for an unchanged parked decision" + [ ! -s "$state/.wake-queue" ] || fail "a pane repaint queued a wake for an unchanged parked decision" + reap "$pid" + unset FM_FAKE_TMUX_CURRENT_COMMAND + pass "a parked keyed decision surfaces once, not once per pane repaint" +} + +# The suppressor must be scoped to the decision that was surfaced, never to the +# pane: a genuinely NEW decision on the same (still repainting) pane is new work +# firstmate has not acted on and must wake normally. The status write's own +# signal wake is suppressed here so only the stale path can surface it. +test_new_keyed_decision_on_parked_pane_surfaces() { + local dir state fakebin out drain_out capture window pid + dir=$(make_case parked-decision-new-key); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture="$dir/pane.txt" + window="test:fm-parked2" + printf 'window=%s\nkind=ship\n' "$window" > "$state/parked2.meta" + printf 'needs-decision [key=review-gate]: ship as-is or split the migration\n' \ + > "$state/parked2.status" + prime_status_seen "$state" parked2 + export FM_FAKE_TMUX_CURRENT_COMMAND=claude + + prime_stale_pane "$state" "$window" 'awaiting decision · context 41%' "$capture" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "watcher did not surface the first parked decision" + FM_STATE_OVERRIDE="$state" "$DRAIN" > /dev/null 2>&1 || true + + # A second, different decision opens on the same pane, which also repaints. + printf 'needs-decision [key=schema-choice]: single table or one per tenant\n' \ + >> "$state/parked2.status" + prime_status_seen "$state" parked2 + prime_stale_pane "$state" "$window" 'awaiting decision · context 38%' "$capture" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a NEW keyed decision on a parked pane did not surface" + grep -Fx "stale: $window" "$out" >/dev/null || fail "the new keyed decision did not print a stale wake" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the new keyed decision failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null \ + || fail "the new keyed decision was not queued" + unset FM_FAKE_TMUX_CURRENT_COMMAND + pass "a new keyed decision on an already-parked pane still surfaces" +} + +# Suppressing the repeat surface must not cost wedge detection: a crew parked on +# an open decision whose agent then dies is a wedge suspect and must escalate +# through the shared timer. Only a confident dead verdict counts, so the live +# agent above stays absorbed while a shell-at-the-prompt endpoint escalates. +test_parked_decision_with_dead_agent_wedge_escalates() { + local dir state fakebin out capture window key pid + dir=$(make_case parked-decision-dead); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture="$dir/pane.txt" + window="test:fm-parked3" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf 'window=%s\nkind=ship\n' "$window" > "$state/parked3.meta" + printf 'needs-decision [key=review-gate]: ship as-is or split the migration\n' \ + > "$state/parked3.status" + prime_status_seen "$state" parked3 + export FM_FAKE_TMUX_CURRENT_COMMAND=claude + + prime_stale_pane "$state" "$window" 'awaiting decision · context 41%' "$capture" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "watcher did not surface the parked decision before the wedge case" + FM_STATE_OVERRIDE="$state" "$DRAIN" > /dev/null 2>&1 || true + + # The agent exits, leaving a bare shell at the endpoint, and the pane repaints + # once more. The decision is unchanged, so the repeat surface stays suppressed, + # but the wedge timer - backdated past the threshold - must still escalate. + export FM_FAKE_TMUX_CURRENT_COMMAND=zsh + prime_stale_pane "$state" "$window" 'awaiting decision · context 37%' "$capture" + echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a parked crew whose agent died was not detected as a wedge suspect" + grep -F "stale: $window" "$out" >/dev/null || fail "the dead parked crew did not print a stale wake" + grep -F "possible wedge" "$out" >/dev/null || fail "the dead parked crew was not flagged as a possible wedge" + unset FM_FAKE_TMUX_CURRENT_COMMAND + pass "a parked crew that stops responding is still detected as a wedge suspect" +} + # --- non-terminal stale, crew provably working: absorbed, then wedge-escalated --- # A provably-working crew (an actively-running pipeline) legitimately sits on a # static pane (e.g. waiting on CI), so a non-terminal stale is absorbed and only @@ -1567,6 +1723,9 @@ test_working_note_not_working_surfaced test_actionable_signal_surfaced test_terminal_stale_surfaced test_stale_terminal_status_overridden_by_active_run +test_parked_decision_survives_pane_repaint +test_new_keyed_decision_on_parked_pane_surfaces +test_parked_decision_with_dead_agent_wedge_escalates test_nonterminal_stale_provably_working_absorbed_then_escalated test_wedge_escalation_marks_demand_deep_inspection_after_threshold test_wedge_escalation_resets_when_pane_becomes_active From 66bb1ef6e865ba5d32ada6fdb6e15b2d98fe5903 Mon Sep 17 00:00:00 2001 From: Sungin Kim Date: Tue, 4 Aug 2026 02:01:46 +0000 Subject: [PATCH 2/3] no-mistakes(review): Captain, reconcile surfaced open-decision markers --- bin/fm-push-transition-lib.sh | 36 ++++++++++++++++++++- bin/fm-watch.sh | 22 ++----------- tests/fm-watch-triage.test.sh | 59 ++++++++++++++++++++++++++++++----- 3 files changed, 88 insertions(+), 29 deletions(-) diff --git a/bin/fm-push-transition-lib.sh b/bin/fm-push-transition-lib.sh index 75b8faf6c87..95860005a26 100644 --- a/bin/fm-push-transition-lib.sh +++ b/bin/fm-push-transition-lib.sh @@ -45,10 +45,44 @@ _hb_surfaced_path() { printf '%s/.hb-surfaced-%s' "$STATE" "$(printf '%s' "$1" | tr ':/.' '___')" } -# Record a captain-relevant status after its durable wake has been enqueued. +_fm_surface_digest() { + if command -v md5 >/dev/null 2>&1; then md5 -q; else md5sum | cut -d ' ' -f1; fi +} + +open_decision_id() { # + local task=$1 open + [ -n "$task" ] || return 0 + open=$(status_open_decisions "$STATE/$task.status") + [ -n "$open" ] || return 0 + printf '%s' "$open" | _fm_surface_digest +} + +_stale_decision_marker_path() { # + local task=$1 target key + target=$(fm_backend_target_of_meta "$STATE/$task.meta") + [ -n "$target" ] || return 0 + key=$(printf '%s' "$target" | tr ':/.' '___') + printf '%s/.stale-decision-%s' "$STATE" "$key" +} + +_reconcile_surfaced_open_decision() { # + local task=$1 marker decision + marker=$(_stale_decision_marker_path "$task") + [ -n "$marker" ] || return 0 + decision=$(open_decision_id "$task") + if [ -n "$decision" ]; then + printf '%s' "$decision" > "$marker" + else + rm -f "$marker" + fi +} + +# Record a surfaced status after its durable wake has been enqueued. mark_surfaced() { # local f=$1 task last task=$(basename "$f"); task="${task%.status}" + case "$f" in *.status) ;; *) return 0 ;; esac + _reconcile_surfaced_open_decision "$task" last=$(last_status_line "$f") [ -n "$last" ] || return 0 status_is_captain_relevant "$last" || return 0 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index cf704fe8361..9031c05f045 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -173,7 +173,7 @@ _event_cap_fails=0 afk_present() { [ -e "$STATE/.afk" ]; } hash_pane() { - if command -v md5 >/dev/null 2>&1; then md5 -q; else md5sum | cut -d' ' -f1; fi + _fm_surface_digest } # window_is_busy: 0 (busy) iff the task's harness is PROVABLY working, through @@ -406,23 +406,6 @@ pause_state_class() { # printf '%s' "$class" } -# Identity of the captain decisions a task currently has OPEN, from the durable -# keyed fold owned by fm-classify-lib.sh (status_open_decisions): a -# needs-decision/blocked line opens a key, and only a matching resolution or -# verified captain-held transfer closes it. Prints a stable digest of that open -# set, or nothing when no decision is open. This is what the terminal stale path -# suppresses a repeat surface on, INSTEAD of the pane hash: a crew parked on a -# captain decision keeps the same open set no matter how its pane repaints, -# while a new status line, a new decision key, and the decision resolving all -# change it. -open_decision_id() { # - local task=$1 open - [ -n "$task" ] || return 0 - open=$(status_open_decisions "$STATE/$task.status") - [ -n "$open" ] || return 0 - printf '%s' "$open" | hash_pane -} - # 0 when a parked crew has stopped responding: its recorded endpoint no longer # carries a live agent. Only the confident `dead` verdict counts (an ambiguous, # unreadable, or unverified backend read stays absorbed), so suppressing the @@ -571,7 +554,7 @@ mark_all_captain_relevant_surfaced() { local f task last while IFS=$(printf '\t') read -r f task last; do [ -n "$f" ] || continue - printf '%s' "$last" > "$(_hb_surfaced_path "$task")" + mark_surfaced "$f" done < <(scan_captain_relevant_statuses "$STATE") } @@ -975,7 +958,6 @@ EOF else fm_wake_append stale "$w" "stale: $w" || exit 1 printf '%s' "$h" > "$sf" - [ -n "$did" ] && printf '%s' "$did" > "$dsf" rm -f "$ssf" mark_surfaced "$STATE/$(window_to_task "$w" "$STATE").status" wake "stale: $w" diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index e86a91fe165..7cee836f892 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -543,29 +543,28 @@ prime_status_seen() { # } test_parked_decision_survives_pane_repaint() { - local dir state fakebin out drain_out capture window key pid + local dir state fakebin out drain_out capture window pid dir=$(make_case parked-decision-repaint); state="$dir/state"; fakebin="$dir/fakebin" out="$dir/watch.out"; drain_out="$dir/drain.out"; capture="$dir/pane.txt" window="test:fm-parked" - key=$(printf '%s' "$window" | tr ':/.' '___') printf 'window=%s\nkind=ship\n' "$window" > "$state/parked.meta" printf 'needs-decision [key=review-gate]: ship as-is or split the migration\n' \ > "$state/parked.status" - prime_status_seen "$state" parked # A live agent: this crew is waiting, not wedged. export FM_FAKE_TMUX_CURRENT_COMMAND=claude - # Phase A: first sighting of the parked decision surfaces exactly as before. - prime_stale_pane "$state" "$window" 'awaiting decision · context 41%' "$capture" + # Phase A: the normal first sighting is the status signal. + printf '%s' 'awaiting decision · context 41%' > "$capture" PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 40 || fail "watcher did not surface a newly parked captain decision" - grep -Fx "stale: $window" "$out" >/dev/null || fail "the parked decision did not print a stale wake" + grep -F "signal: $state/parked.status" "$out" >/dev/null \ + || fail "the parked decision did not print its initial status signal" FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the parked decision failed" - grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null \ - || fail "the parked decision was not queued on its first sighting" + grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "parked.status" >/dev/null \ + || fail "the parked decision signal was not queued on its first sighting" # Phase B: the pane repaints (a moving context percentage) while the SAME # decision stays open. The hash changes; the decision does not. Nothing may @@ -586,6 +585,49 @@ test_parked_decision_survives_pane_repaint() { pass "a parked keyed decision surfaces once, not once per pane repaint" } +test_resolved_decision_can_reopen_identically() { + local dir state fakebin out capture window pid + dir=$(make_case parked-decision-reopen); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture="$dir/pane.txt"; window="test:fm-reopen" + printf 'window=%s\nkind=ship\n' "$window" > "$state/reopen.meta" + printf 'needs-decision [key=review-gate]: ship as-is or split the migration\n' \ + > "$state/reopen.status" + prime_status_seen "$state" reopen + export FM_FAKE_TMUX_CURRENT_COMMAND=claude + + prime_stale_pane "$state" "$window" 'awaiting decision · context 41%' "$capture" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "watcher did not surface the original keyed decision" + FM_STATE_OVERRIDE="$state" "$DRAIN" > /dev/null 2>&1 || true + + printf 'resolved [key=review-gate]: migration will ship as-is\n' >> "$state/reopen.status" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "the decision resolution did not surface" + grep -F "signal: $state/reopen.status" "$out" >/dev/null || fail "the resolution did not print a signal wake" + FM_STATE_OVERRIDE="$state" "$DRAIN" > /dev/null 2>&1 || true + + printf 'needs-decision [key=review-gate]: ship as-is or split the migration\n' \ + >> "$state/reopen.status" + prime_status_seen "$state" reopen + prime_stale_pane "$state" "$window" 'awaiting decision again · context 38%' "$capture" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "an identical decision did not surface after resolving and reopening" + grep -Fx "stale: $window" "$out" >/dev/null || fail "the identically reopened decision did not print a stale wake" + unset FM_FAKE_TMUX_CURRENT_COMMAND + pass "a resolved keyed decision can reopen identically and surface again" +} + # The suppressor must be scoped to the decision that was surfaced, never to the # pane: a genuinely NEW decision on the same (still repainting) pane is new work # firstmate has not acted on and must wake normally. The status write's own @@ -1724,6 +1766,7 @@ test_actionable_signal_surfaced test_terminal_stale_surfaced test_stale_terminal_status_overridden_by_active_run test_parked_decision_survives_pane_repaint +test_resolved_decision_can_reopen_identically test_new_keyed_decision_on_parked_pane_surfaces test_parked_decision_with_dead_agent_wedge_escalates test_nonterminal_stale_provably_working_absorbed_then_escalated From 6e0c8ff5b5b30c66b7fcd58a09d65451b382e5f3 Mon Sep 17 00:00:00 2001 From: Sungin Kim Date: Tue, 4 Aug 2026 02:18:19 +0000 Subject: [PATCH 3/3] no-mistakes(document): Clarify parked-decision stale wake documentation --- bin/fm-watch.sh | 59 ++++++++++++++++++----------------- docs/architecture.md | 8 +++-- docs/configuration.md | 2 +- tests/fm-watch-triage.test.sh | 17 ++++------ 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 9031c05f045..862777ad7cd 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -2,13 +2,16 @@ # Firstmate watcher. # Classifies supervision wakes in bash. In normal mode it absorbs benign wakes # and keeps blocking; it queues and exits only for actionable wakes. -# The no-verb signal and stale path is absorb-only-when-provably-working: a wake -# is absorbed only when the crew shows POSITIVE evidence it is still working (an -# actively-running no-mistakes step, or a backend busy signal), and surfaced -# otherwise, so a crew that finishes (or stops and waits) without a current -# working signal is never silently swallowed. A declared external-wait pause is -# the separate idle absorb case and re-surfaces only on its long bounded cadence, -# although its initial no-verb status signal still surfaces in normal mode. +# The no-verb signal and first-sighting stale paths are +# absorb-only-when-provably-working: a wake is absorbed only when the crew shows +# POSITIVE evidence it is still working (an actively-running no-mistakes step, +# or a backend busy signal), and surfaced otherwise, so a crew that finishes +# without a current working signal is never silently swallowed. A declared +# external-wait pause is the separate idle absorb case and re-surfaces only on +# its long bounded cadence. A repaint-only repeat of an already-surfaced keyed +# open-decision set is also absorbed, but a confidently dead parked agent still +# enters the wedge timer. The initial no-verb status signal still surfaces in +# normal mode. # While state/.afk exists, the daemon owns triage and this watcher queues and exits # on every wake. Printed reason lines: # signal: ... status/turn-end signals, surfaced when a listed status @@ -20,8 +23,11 @@ # line, since the crew's own log gets no new entry once # firstmate hands it to a no-mistakes validation. A declared # external-wait pause is absorbed instead with its own long -# re-surface cadence, never as a wedge. Only when neither -# absorb class applies does the log's last line decide: +# re-surface cadence, never as a wedge. An unchanged keyed +# open-decision set is deduped across pane repaint, unless +# the backend confidently reports the parked agent dead and +# routes it through the wedge timer. Only when none of these +# absorb classes applies does the log's last line decide: # terminal (captain-relevant) or non-terminal (no verb), # both surfaced at once. A provably-working stale past the # wedge threshold also surfaces, with an "escalation N" @@ -119,20 +125,22 @@ SIGNAL_GRACE=${FM_SIGNAL_GRACE:-30} # seconds to linger after a signal so trai # than wake firstmate's LLM for each, this watcher classifies every wake in bash # and ABSORBS the benign majority - it advances the suppression marker, logs to a # debug log, and keeps blocking WITHOUT enqueuing or exiting. The no-verb signal -# / stale path is absorb-only-when-provably-working: such a wake is absorbed ONLY -# while the crew shows positive evidence it is still working (an actively-running -# no-mistakes step, or a busy pane, via crew_is_provably_working over -# fm-crew-state.sh); a crew that stopped its turn with no running pipeline and no +# / first-sighting stale path is absorb-only-when-provably-working: such a wake +# is absorbed ONLY while the crew shows positive evidence it is still working +# (an actively-running no-mistakes step, or a busy pane, via +# crew_is_provably_working over fm-crew-state.sh); a crew that stopped its turn +# with no running pipeline and no # busy pane is SURFACED, so a finish reported only through interactive pane menus # (no done: status) is never swallowed. An ACTIONABLE wake (a captain-relevant -# signal, a no-verb signal whose crew is not provably working, any check, a stale -# pane whose crew is not provably working, a provably-working stale past the -# threshold, or anything unknown) is written to the durable queue and exits, which -# is what wakes the LLM through the background-task completion. The same classifier +# signal, a no-verb signal whose crew is not provably working, any check, a +# first-sighting stale pane whose crew is not provably working, a +# provably-working stale past the threshold, or anything unknown) is written to +# the durable queue and exits, which is what wakes the LLM through the +# background-task completion. The same classifier # (fm-classify-lib.sh) backs the away-mode daemon; while state/.afk exists the # daemon owns triage, so this watcher reverts to one-shot (enqueue + exit on every # wake) and never double-triages - and never runs the costly provably-working read. -STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provably-working stale escalates as a possible wedge +STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provably-working stale or dead parked-decision repeat escalates as a possible wedge # A busy pane is unconditional proof of liveness with no built-in duration bound, # so a hung foreground call can remain hidden even while its rendered busy # footer changes every poll. BUSY_TURN_MAX_SECS bounds how long any busy pane @@ -929,16 +937,11 @@ EOF # authoritative source fm-crew-state.sh itself already prioritizes # over the log) a chance to override before trusting the log. # - # The one-shot suppressor is keyed on the OPEN DECISION, not the pane - # hash, whenever the captain-relevant status is a still-open keyed - # decision. An idle harness pane is not static - a live context and - # quota footer repaints on its own - so a hash-keyed one-shot let the - # identical already-escalated waiting state re-surface on every - # repaint, costing a full handling turn each time (twenty-two - # consecutive surfaces on one parked pane on 2026-08-01). Detection is - # unchanged: the open set moves on a new status line, a new decision - # key, or the decision resolving, and a parked crew whose agent has - # died still escalates through the shared wedge timer below. + # Key this one-shot on the complete open-decision set, not volatile + # pane bytes. mark_surfaced reconciles this marker for every surfaced + # status, while a confidently dead parked agent still advances the + # shared wedge timer below. docs/architecture.md owns the full wake + # contract. did=$(open_decision_id "$task") dsf="$STATE/.stale-decision-$key" [ -n "$did" ] || rm -f "$dsf" diff --git a/docs/architecture.md b/docs/architecture.md index e1504bff76c..340c98db22e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,8 @@ firstmate's always-loaded operating contract and routing index for conditional p ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. -Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, first-sighting stale panes whose crew is not provably working, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Normal mode separately absorbs repaint-only repeats of an already-surfaced keyed open-decision set unless the backend confidently reports that the parked agent is dead. Repeated provably-working stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. A busy pane is otherwise exempt from staleness, but only until its latest `state/.turn-ended` marker reaches `FM_BUSY_TURN_MAX_SECS`, or its `state/.meta` spawn record reaches that age before any turn completes; past that bound it is routed through the same wedge escalation, with the identical reason, escalation count, and `demand-deep-inspection` marker, for inspection only - never an automatic interrupt, signal, or restart. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) before detector state advances, so a missed process exit can be recovered by draining the queue. @@ -23,8 +24,9 @@ For an ordinary crew that has stopped, the normal-mode watcher first surfaces on Live or inconclusive liveness remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. Fresh stale panes use the same current-state read before trusting the status log, so an active run or a proven busy worker outranks an old captain-relevant status-log line left behind before validation. -When that captain-relevant status is a still-open keyed decision, the one-shot suppressor is keyed on the open-decision identity from `bin/fm-classify-lib.sh`'s `status_open_decisions` fold rather than on the pane hash, because an idle harness pane repaints on its own and a hash-keyed suppressor re-surfaced the identical already-escalated waiting state on every repaint. -Such a crew therefore surfaces at most once per open decision, a new status line, a new decision key, or the decision resolving still surfaces normally, and a parked crew whose backend confidently reports its agent dead still escalates through the same wedge timer. +When that captain-relevant status includes a still-open keyed decision, the one-shot suppressor is keyed on a digest of the complete open-decision set returned by `bin/fm-classify-lib.sh`'s `status_open_decisions` fold rather than on the pane hash, because an idle harness pane repaints on its own and a hash-keyed suppressor re-surfaced an unchanged already-escalated waiting state on every repaint. +While the backend reports the agent live or returns inconclusive liveness, an unchanged open-decision set therefore surfaces only once despite pane repaint, while a fresh actionable status signal, a changed decision key or set, and resolution still surface normally. +A parked crew whose backend confidently reports its agent dead still escalates through the same wedge timer. No-change heartbeats are also benign. Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. After each drain, `fm-wake-drain.sh` runs the same liveness guard as the supervision scripts, so a lapsed watcher chain surfaces even on a turn that only drains and handles queued wakes. diff --git a/docs/configuration.md b/docs/configuration.md index 4925337e007..aa462011e93 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -470,7 +470,7 @@ FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watche FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked -FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb +FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane or a confidently dead parked-decision repeat escalates; other first-sighting stale states surface immediately unless they declare the pause verb FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 7cee836f892..cf5ad9f3583 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -516,14 +516,9 @@ test_stale_terminal_status_overridden_by_active_run() { } # --- a parked keyed decision is surfaced once, not once per pane repaint ------- -# Regression for the 2026-08-01 parked-decision noise: a crew that correctly -# appends "needs-decision [key=...]: ..." and stops is genuinely not working, so -# crew_is_provably_working rightly refuses to absorb its stale pane. The one-shot -# suppressor used to be keyed on the pane hash, and an idle harness pane is not -# static (a live context/quota footer repaints on its own), so every repaint -# re-surfaced the identical already-escalated waiting state - twenty-two -# consecutive times on one pane. Suppression must key on the OPEN DECISION -# (fm-classify-lib.sh's status_open_decisions fold) so a repaint alone is silent. +# A parked decision is genuinely not working, so crew_is_provably_working rightly +# refuses to absorb its first stale pane. The separate repaint suppressor must key +# on the complete open-decision set so volatile footer changes alone stay silent. # # Fixture note: each phase primes .hash-/.count- so the FIRST poll already sees a # stably stale pane, exactly as the terminal-stale cases above do. @@ -628,9 +623,9 @@ test_resolved_decision_can_reopen_identically() { pass "a resolved keyed decision can reopen identically and surface again" } -# The suppressor must be scoped to the decision that was surfaced, never to the -# pane: a genuinely NEW decision on the same (still repainting) pane is new work -# firstmate has not acted on and must wake normally. The status write's own +# The suppressor must be scoped to the open-decision set that was surfaced, +# never to the pane: a genuinely NEW decision on the same repainting pane is new +# work firstmate has not acted on and must wake normally. The status write's own # signal wake is suppressed here so only the stale path can surface it. test_new_keyed_decision_on_parked_pane_surfaces() { local dir state fakebin out drain_out capture window pid