From f0f7b72022a700e0982f2076ed2579e480b332f7 Mon Sep 17 00:00:00 2001 From: sdivanl Date: Sun, 20 Sep 2026 17:47:09 +0800 Subject: [PATCH 01/12] fix: park deliberately stopped tasks --- bin/fm-control-lib.sh | 39 +++++++++++-- bin/fm-control.sh | 33 +++++++---- bin/fm-spawn.sh | 10 ++++ bin/fm-teardown.sh | 5 +- bin/fm-watch.sh | 51 ++++++++++++++++- tests/fm-control-relaunch.test.sh | 19 ++++++- tests/fm-control.test.sh | 39 +++++++++++++ tests/fm-teardown.test.sh | 5 +- tests/fm-watch-triage.test.sh | 93 +++++++++++++++++++++++++++++++ 9 files changed, 273 insertions(+), 21 deletions(-) diff --git a/bin/fm-control-lib.sh b/bin/fm-control-lib.sh index 6e6be0d5c3a..e1c99bb79c9 100644 --- a/bin/fm-control-lib.sh +++ b/bin/fm-control-lib.sh @@ -13,11 +13,13 @@ # here rather than improvised per harness in agent prose. # # This file owns three capability tables plus their pure artifact-path tables, -# and ONE named exception to that purity - fm_control_endpoint_absence_verdict, -# the single owner of the per-backend endpoint-absence proof, which does run -# backend reads. Everything else has no side effects, runs no backend command, -# and reads no state, so sourcing this file is still free and the tables can be -# read by a test as a pure contract: +# and TWO named exceptions to that purity. fm_control_endpoint_absence_verdict +# is the single owner of the per-backend endpoint-absence proof and does run +# backend reads; the fm_control_deliberate_stop_* helpers own the durable +# deliberate-stop marker and read or write exactly one per-task state file +# (state/.deliberate-stop) with no backend command. Everything else has no +# side effects, runs no backend command, and reads no state, so sourcing this +# file is still free and the tables can be read by a test as a pure contract: # # 1. Verb allowlist. There is no arbitrary-text and no generic raw-key entry # point on the control plane; a caller either names an allowlisted verb or @@ -348,3 +350,30 @@ fm_control_harness_turnend_auth_path() { # *) return 0 ;; esac } + +# The durable deliberate-stop marker: state/.deliberate-stop. Written by +# bin/fm-control.sh's exit verb (the stop path itself, never inferred later +# from status prose), cleared by a relaunch (bin/fm-spawn.sh --relaunch) and by +# teardown, and read by bin/fm-watch.sh. Presence means the task's worker was +# deliberately stopped, so its idle endpoint is a parked task: the watcher gives +# it the declared-pause treatment - a long bounded recheck cadence, never a +# stale or wedge escalation - instead of treating it as a worker that stopped +# responding on its own. The record body is the epoch second of the stop, so a +# re-stop after a relaunch starts a fresh recheck window. A failed stop attempt +# never writes it, so a refused or unattributed endpoint keeps escalating exactly +# as it always did. +fm_control_deliberate_stop_marker() { # + printf '%s/%s.deliberate-stop' "$1" "$2" +} + +fm_control_deliberate_stop_record() { # + printf '%s\n' "$(date +%s)" > "$(fm_control_deliberate_stop_marker "$1" "$2")" +} + +fm_control_deliberate_stop_clear() { # + rm -f -- "$(fm_control_deliberate_stop_marker "$1" "$2")" +} + +fm_control_deliberate_stop_present() { # -> 0 when the marker exists + [ -e "$(fm_control_deliberate_stop_marker "$1" "$2")" ] +} diff --git a/bin/fm-control.sh b/bin/fm-control.sh index 1b73aa644ac..46c47e7d587 100755 --- a/bin/fm-control.sh +++ b/bin/fm-control.sh @@ -470,15 +470,32 @@ retire_busy_incarnation() { fi } +# finish_stop : record the durable deliberate-stop marker and print the +# stop outcome. The marker is the whole point of the exit verb's "deliberate" +# guarantee: it lets the watcher treat this parked task as a deliberate stop (a +# long bounded recheck, never a stale or wedge escalation) instead of a worker +# that went quiet on its own. Written only here, on the verified success paths, +# so a refused or failed stop never records one. +finish_stop() { # + # A verified stop ends this busy incarnation even if it was already dead + # before the control command arrived. + retire_busy_incarnation + fm_control_deliberate_stop_record "$STATE" "$ID" \ + || die "could not record task $ID's deliberate stop" + printf '%s' "$1" + return 0 +} + # do_exit: stop the running agent, preserving endpoint and worktree. Prints -# `already-stopped`, `endpoint-gone`, or `stopped`. +# `already-stopped`, `endpoint-gone`, or `stopped`, and leaves the durable +# deliberate-stop marker behind on every verified stop (bin/fm-control-lib.sh). do_exit() { local state cmd verdict composer_state cancel absence interrupt_result=not-needed require_state_verified_backend exit state=$(agent_state) case "$state" in dead) - printf 'already-stopped' + finish_stop already-stopped return 0 ;; alive) ;; @@ -496,7 +513,7 @@ do_exit() { # verb normally preserves did not survive. The worktree and every # uncommitted change are untouched, and `relaunch` re-creates the # endpoint from here. - printf 'endpoint-gone' + finish_stop endpoint-gone return 0 ;; dead) @@ -504,7 +521,7 @@ do_exit() { # no agent - a herdr pane whose session server was merely stopped is # the common case. Nothing is gone, so this is the ordinary # already-stopped outcome. - printf 'already-stopped' + finish_stop already-stopped return 0 ;; alive) @@ -525,8 +542,7 @@ do_exit() { state=$(agent_state) case "$state" in dead) - retire_busy_incarnation - printf 'stopped' + finish_stop stopped return 0 ;; alive) interrupt_result="delivered verified=agent-alive cancel=$cancel" ;; @@ -560,10 +576,7 @@ do_exit() { state=$(wait_agent_state "$EXIT_WAIT" dead) || { die "exit-delivered $ID interrupt=$interrupt_result exit-command=delivered agent-state=$state exit=unconfirmed; the agent did not stop within ${EXIT_WAIT}s" } - # The incarnation is over: retire its busy wiring so no stale record or - # orphaned generation survives the agent that produced it. - retire_busy_incarnation - printf 'stopped' + finish_stop stopped } # --- transactional relaunch ------------------------------------------------- diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 518e905f60f..864e9387e8d 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -4974,6 +4974,16 @@ if [ -n "$SPAWN_DEFERRED_SIGNAL" ]; then echo "error: spawn of $ID was interrupted after launch delivery began; $SPAWN_PRESERVED_CLAIM" >&2 exit "$SPAWN_DEFERRED_SIGNAL_STATUS" fi +if [ "$RELAUNCH" -eq 1 ]; then + # All launch delivery and the relaunch record publication now succeeded, so + # the replacement supersedes the deliberately stopped incarnation. Clear its + # parked-task marker only at this commit point: an earlier launch failure + # leaves the stopped task parked rather than reviving the stale/wedge ladder. + fm_control_deliberate_stop_clear "$STATE_REAL" "$ID" || { + echo "error: replacement for $ID was launched, but its deliberate-stop marker could not be cleared" >&2 + exit 1 + } +fi fm_lock_release "$SPAWN_META_LOCK" SPAWN_META_LOCK_HELD=0 diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index ec792392b98..272f7dd5cdb 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -3143,8 +3143,8 @@ cleanup_firstmate_home_children() { "$sub_state/$child_id.pi-ext.ts" "$sub_state/$child_id.omp-ext.ts" \ "$sub_state/$child_id.grok-turnend-token" "$sub_state/$child_id.kimi-turnend-token" \ "$sub_state/$child_id.muse-session" "$sub_state/$child_id.muse-session-current" \ - "$sub_state/$child_id.cursor-session" "$sub_state/$child_id.reconcile-nudged" \ - "$sub_state/.$child_id.branch-outcome-index" + "$sub_state/$child_id.cursor-session" "$sub_state/$child_id.deliberate-stop" \ + "$sub_state/$child_id.reconcile-nudged" "$sub_state/.$child_id.branch-outcome-index" done } @@ -3593,6 +3593,7 @@ rm -f "$STATE/$ID.turn-ended" "$STATE/$ID.progress" \ "$STATE/$ID.muse-session-current" "$STATE/$ID.cursor-session" \ "$STATE/$ID.control-relaunch" "$STATE/$ID.control-relaunch.meta-prior" \ "$STATE/$ID.control-relaunch.brief-prior" "$STATE/$ID.control-relaunch.note" \ + "$STATE/$ID.deliberate-stop" \ "$STATE/$ID.reconcile-nudged" "$STATE/$ID.gemini-settings.json" \ "$STATE/.$ID.branch-outcome-index" # The steering inbox (bin/fm-task-inbox-lib.sh) is runtime state for the diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 31cf64aa43f..c6e36b45e60 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -29,7 +29,11 @@ # external-wait pause or verified captain-held transfer is # absorbed instead with its own long re-surface cadence, # never as a wedge, and that recheck reason names which -# human the wait is on. Only when neither absorb class +# human the wait is on. A task whose worker firstmate +# deliberately stopped (the durable state/.deliberate-stop +# marker, owned by bin/fm-control-lib.sh) is parked on that +# same long recheck cadence regardless of its last status +# line, never wedge-escalated. Only when neither absorb class # applies does the log's latest recognized status event decide: # terminal (captain-relevant) or non-terminal (no verb), # both surfaced at once. A provably-working stale past the @@ -188,6 +192,12 @@ mkdir -p "$STATE" # watcher reads only its presence (afk_record_present below). # shellcheck source=bin/fm-afk-contract.sh . "$SCRIPT_DIR/fm-afk-contract.sh" +# The durable deliberate-stop marker (state/.deliberate-stop) is owned by +# bin/fm-control-lib.sh; this watcher reads only its presence +# (fm_control_deliberate_stop_present) so a deliberately parked worker takes the +# declared-pause treatment instead of the stale/wedge ladder. +# shellcheck source=bin/fm-control-lib.sh +. "$SCRIPT_DIR/fm-control-lib.sh" WATCH_LOCK="$STATE/.watch.lock" WATCH_PATH="$SCRIPT_DIR/fm-watch.sh" @@ -403,7 +413,8 @@ window_label() { # The ONE derivation of a window's per-window marker key: `:`, `/` and `.` become # `_` so a window name is usable as a filename suffix. Every per-window file the # watcher keeps is named by it (.hash-, .count-, .stale-, .stale-since-, -# .wedge-escalations-, .paused-*, .writing-*, .waiting-*), and live homes hold those markers on +# .wedge-escalations-, .paused-*, .writing-*, .waiting-*, +# .deliberate-stop-resurfaced-*), and live homes hold those markers on # disk under the current format, so the format lives here alone: a second copy is # how a future change to it silently orphans a window's markers instead of clearing # them. The helpers below take the derived key rather than re-deriving it, so one @@ -1358,6 +1369,34 @@ handle_paused_stale() { # triage_log "absorbed stale ($detail, age ${age}s): $win" } +# Absorb a stale pane whose task carries the durable deliberate-stop marker +# (state/.deliberate-stop, owned by bin/fm-control-lib.sh): firstmate stopped +# this worker on purpose, so an idle endpoint is a parked task, not a wedge +# suspect. Same bounded recheck cadence as handle_paused_stale - re-surface once +# per PAUSE_RESURFACE_SECS so a forgotten parked task cannot rot invisibly - and +# never the wedge ladder. The re-surface age is anchored on the marker's own +# mtime (when the stop was recorded) rather than the status file or pane hash, so +# a churny idle pane cannot reset the cadence. The scope is the marker's content +# (the stop epoch), so a re-stop after a relaunch starts a fresh window instead +# of inheriting the previous one's throttle. Uses its own .deliberate-stop-*- +# throttle rather than the .paused-* flag, because the .paused-* machinery is +# cleared whenever the last status line stops declaring a wait - a deliberately +# stopped worker's last line is routinely `done:`, not a wait declaration. +handle_deliberate_stop_stale() { # + local win=$1 task=$2 h=$3 key marker age scope + key=$(window_key "$win") + printf '%s' "$h" > "$STATE/.stale-$key" + rm -f "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key" + clear_write_tracking "$key" + marker=$(fm_control_deliberate_stop_marker "$STATE" "$task") + age=$(age_of "$marker") + scope="deliberate-stop:$(cat "$marker" 2>/dev/null || true)" + resurface_absorbed "$win" "$STATE/.deliberate-stop-resurfaced-$key" "$age" \ + "stale: $win (deliberately stopped ${age}s ago, rechecked on a long cadence not a wedge; relaunch the worker or clean up the finished task)" \ + "$scope" + triage_log "absorbed stale (deliberate stop, age ${age}s): $win" +} + # Apply the busy-pane completed-turn bound to a window whose bound has already # crossed, honoring the worker's OWN declared external wait. Prints/queues # nothing itself; it only chooses which absorber owns the crossed bound. @@ -2660,6 +2699,14 @@ EOF paused) handle_paused_stale "$w" "$task" "$h" ;; *) clear_pause_tracking "$key" ;; esac + elif fm_control_deliberate_stop_present "$STATE" "$task"; then + # Firstmate stopped this worker on purpose, so its idle endpoint is a + # parked task: the declared-pause treatment (bounded recheck, never a + # wedge escalation), regardless of whether the last status line is + # terminal (`done:`) or non-terminal. Read before the afk gate so a + # parked worker stays parked in every supervision shape instead of + # depending on the away-mode daemon knowing this control-plane marker. + handle_deliberate_stop_stale "$w" "$task" "$h" elif afk_present; then # Daemon owns triage: one-shot per distinct stale hash, as before, # except that a captain-held pane is never handed over while the diff --git a/tests/fm-control-relaunch.test.sh b/tests/fm-control-relaunch.test.sh index 5631488cebd..fbf80fa2cdc 100755 --- a/tests/fm-control-relaunch.test.sh +++ b/tests/fm-control-relaunch.test.sh @@ -387,6 +387,20 @@ test_same_harness_relaunch_keeps_identity_and_reuses_the_endpoint() { pass "fm-control relaunch: a same-harness relaunch replaces the agent in the same endpoint and worktree" } +test_relaunch_clears_the_deliberate_stop_marker() { + local dir out rc + dir=$(new_case deliberate-stop-clear rl1) + add_ship_task "$dir" rl1 claude + # A prior deliberate stop left this marker (bin/fm-control-lib.sh owns it); + # the relaunch must clear it so the replacement is supervised normally again. + printf '%s\n' "$(date +%s)" > "$dir/home/state/rl1.deliberate-stop" + out=$(run_control "$dir" rl1 relaunch --note "resume after a deliberate stop"); rc=$? + expect_code 0 "$rc" "the relaunch should succeed"$'\n'"$out" + [ ! -e "$dir/home/state/rl1.deliberate-stop" ] \ + || fail "a relaunch must clear the deliberate-stop marker so the replacement is supervised normally" + pass "fm-control relaunch: clears the durable deliberate-stop marker left by a prior stop" +} + test_relaunch_refuses_before_exit_when_the_composer_holds_pending_text() { local dir out rc dir=$(new_case pending-exit rl43) @@ -818,7 +832,9 @@ test_wiring_removal_failure_refuses_before_replacement_arm() { || fail "the transaction should record the partial launch failure" [ "$(journal_field "$dir" rl29 rollback)" = prior-record-kept ] \ || fail "unpublished rollback should retain the live durable record" - pass "fm-control relaunch: wiring cleanup failure refuses replacement arming" + [ -e "$dir/home/state/rl29.deliberate-stop" ] \ + || fail "an aborted relaunch must retain the parked deliberate-stop marker" + pass "fm-control relaunch: wiring cleanup failure refuses replacement arming and retains the parked stop" } test_turnend_auth_paths_are_owned_by_the_control_adapter() { @@ -2198,6 +2214,7 @@ test_relaunch_moves_a_drifted_item_back_in_flight() { } test_same_harness_relaunch_keeps_identity_and_reuses_the_endpoint +test_relaunch_clears_the_deliberate_stop_marker test_relaunch_refuses_before_exit_when_the_composer_holds_pending_text test_relaunch_refuses_before_exit_when_the_composer_state_is_unproven test_relaunch_from_linked_home_preserves_recorded_worktree diff --git a/tests/fm-control.test.sh b/tests/fm-control.test.sh index 67105bcfd99..fba416ebf34 100755 --- a/tests/fm-control.test.sh +++ b/tests/fm-control.test.sh @@ -634,6 +634,44 @@ test_already_stopped_exit_is_idempotent() { pass "fm-control exit: an already-stopped agent is idempotent success with no bytes sent" } +test_exit_records_the_deliberate_stop_marker() { + local dir out rc marker gen + # A live agent: the ordinary stop path records the marker. + dir=$(new_case deliberate-stop-live) + add_task "$dir" t1 claude + alive_as "$dir" claude + out=$(run_control "$dir" t1 exit); rc=$? + expect_code 0 "$rc" "exit should succeed"$'\n'"$out" + assert_contains "$out" "stopped t1" "exit should report the stop" + marker="$dir/home/state/t1.deliberate-stop" + [ -f "$marker" ] || fail "exit did not record the deliberate-stop marker" + [ -n "$(cat "$marker")" ] || fail "the deliberate-stop marker is empty" + + # An already-stopped agent is still a deliberate stop (idempotent success). + dir=$(new_case deliberate-stop-idempotent) + add_task "$dir" t1 claude + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$dir/home/state" t1) + alive_as "$dir" zsh + out=$(run_control "$dir" t1 exit); rc=$? + expect_code 0 "$rc" "exiting an already-stopped agent should succeed"$'\n'"$out" + assert_contains "$out" "already-stopped t1" "the outcome should say it was already stopped" + [ -f "$dir/home/state/t1.deliberate-stop" ] \ + || fail "an already-stopped exit did not record the deliberate-stop marker" + [ ! -e "$dir/home/state/t1.busy-gen" ] && [ ! -e "$dir/home/state/t1.busy-state" ] \ + || fail "an already-stopped exit did not retire busy generation $gen" + + # A refused stop (unprovable endpoint) must NOT record the marker: the agent + # was not proven stopped, so it keeps escalating exactly as it always did. + dir=$(new_case deliberate-stop-refused) + add_task "$dir" t1 claude + : > "$dir/fake/windows" + out=$(run_control "$dir" t1 exit); rc=$? + expect_code 1 "$rc" "an unprovable endpoint must refuse"$'\n'"$out" + [ ! -e "$dir/home/state/t1.deliberate-stop" ] \ + || fail "a refused stop recorded a deliberate-stop marker" + pass "fm-control exit: a verified stop records the deliberate-stop marker; a refused stop does not" +} + test_missing_tmux_endpoint_refuses_rather_than_claiming_a_stop() { local dir out rc dir=$(new_case gone) @@ -908,6 +946,7 @@ test_verb_allowlist_is_closed test_resume_is_refused_with_its_reason test_relaunch_only_flags_are_rejected_on_other_verbs test_already_stopped_exit_is_idempotent +test_exit_records_the_deliberate_stop_marker test_missing_tmux_endpoint_refuses_rather_than_claiming_a_stop test_interrupt_refuses_when_no_agent_runs test_ambiguous_endpoint_refuses diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index 04f7f096231..edb57e4aa47 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -1953,6 +1953,7 @@ test_teardown_missing_busy_sidecar_completes() { gen=$("$ROOT/bin/fm-busy-event.sh" arm "$case_dir/state" task-x1) printf 'busy_gen=%s\n' "$gen" >> "$case_dir/state/task-x1.meta" rm -f "$case_dir/state/task-x1.busy-gen" + printf '%s\n' "$(date +%s)" > "$case_dir/state/task-x1.deliberate-stop" set +e run_teardown "$case_dir" --force > "$case_dir/stdout" 2> "$case_dir/stderr" @@ -1962,9 +1963,11 @@ test_teardown_missing_busy_sidecar_completes() { expect_code 0 "$rc" "missing-busy-sidecar: teardown should treat the incarnation as already retired" assert_absent "$case_dir/state/task-x1.busy-state" \ "missing-busy-sidecar: teardown left the orphan busy record" + assert_absent "$case_dir/state/task-x1.deliberate-stop" \ + "missing-busy-sidecar: teardown left the deliberate-stop marker behind" assert_absent "$case_dir/state/task-x1.meta" \ "missing-busy-sidecar: teardown remained incomplete" - pass "teardown completes when an exact busy-state sidecar is already absent" + pass "teardown retires the deliberate-stop marker when an exact busy-state sidecar is already absent" } test_herdr_teardown_clears_escalation_marker() { diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 8a94ebdf22f..50c1234d076 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1984,6 +1984,97 @@ test_terminal_stale_surfaced() { pass "a stale pane sitting on a terminal status is surfaced (queue + exit)" } +# --- deliberate stop: a parked finished worker is absorbed, never wedge-escalated --- +# Regression for firstmate issue #5004: a finished worker firstmate stopped on +# purpose (its task record stays open) has no status-line declaration of the +# wait, so the stale path kept re-surfacing it as a possible wedge. The durable +# deliberate-stop marker (state/.deliberate-stop) must give it the same +# bounded recheck cadence a declared pause gets: absorbed on first sight, then +# re-surfaced once past PAUSE_RESURFACE_SECS as a recheck, never a wedge. +test_deliberately_stopped_finished_task_is_parked_not_stale() { + local dir state fakebin out drain_out capture_file window key pane_hash sig pid back + dir=$(make_case deliberate-stop-parked); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-parked" + printf 'finished, deliberately parked' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/parked.meta" + # A finished worker: its last status line is terminal (`done:`), not a wait. + printf 'done: investigation finished\n' > "$state/parked.status" + sig=$(seen_sig "$state/parked.status"); printf '%s' "$sig" > "$state/.seen-parked_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "finished, deliberately parked") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + # The stop path itself wrote this marker; the watcher only reads its presence. + printf '%s\n' "$(date +%s)" > "$state/parked.deliberate-stop" + + # Phase A: a fresh deliberate stop is absorbed - no wake, no wedge timer. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_poll_cycle "$state" "$pid"; then + reap "$pid"; fail "watcher exited for a deliberately parked finished task (should absorb): $(cat "$out")" + fi + [ ! -s "$out" ] || fail "a deliberately parked finished task printed a wake during absorb" + [ ! -s "$state/.wake-queue" ] || fail "a deliberately parked finished task enqueued a wake during absorb" + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on deliberate-stop absorb" + [ ! -e "$state/.stale-since-$key" ] || fail "a deliberate-stop absorb must not start the wedge timer" + [ ! -e "$state/.wedge-escalations-$key" ] || fail "a deliberate-stop absorb must not arm the wedge escalation counter" + reap "$pid" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional deliberate-stop phase-A watcher stop" + + # Phase B: age the marker past the cadence; the parked task re-surfaces once as + # a recheck - never a wedge - so a forgotten parked task cannot rot invisibly. + back=$(( $(date +%s) - 500 )) + set_mtime "$back" "$state/parked.deliberate-stop" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not re-surface a deliberately parked task past the threshold" + grep -F "stale: $window" "$out" >/dev/null || fail "re-surface did not print a stale wake" + grep -F "deliberately stopped" "$out" >/dev/null || fail "re-surface was not labeled a deliberate-stop recheck" + grep -F "possible wedge" "$out" >/dev/null && fail "a deliberately parked task was mislabeled a possible wedge" + [ -e "$state/.deliberate-stop-resurfaced-$key" ] || fail "the deliberate-stop re-surface throttle was not recorded" + [ ! -e "$state/.stale-since-$key" ] || fail "a deliberate-stop re-surface must not use the wedge timer" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the deliberate-stop re-surface failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "deliberate-stop re-surface was not queued" + pass "a deliberately stopped finished task is parked on the bounded recheck cadence, never wedge-escalated" +} + +# --- clearing the deliberate-stop marker returns the task to normal supervision --- +# Relaunch and cleanup clear the marker (fm-spawn/fm-teardown), and the next stale +# sighting of the now-marker-less task must take the ordinary path again: a +# finished worker whose last line is `done:` surfaces as a terminal stale exactly +# as it did before the marker existed. +test_deliberate_stop_marker_cleared_resumes_terminal_stale_surfacing() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case deliberate-stop-cleared); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-unparked" + printf 'finished, marker cleared' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/unparked.meta" + printf 'done: investigation finished\n' > "$state/unparked.status" + sig=$(seen_sig "$state/unparked.status"); printf '%s' "$sig" > "$state/.seen-unparked_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "finished, marker cleared") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + # The marker has been cleared by the relaunch/cleanup path; no marker exists. + [ ! -e "$state/unparked.deliberate-stop" ] || fail "fixture must start without the deliberate-stop marker" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a marker-less finished task as terminal stale" + grep -Fx "stale: $window" "$out" >/dev/null || fail "watcher did not print the terminal stale wake after the marker was cleared" + pass "clearing the deliberate-stop marker returns the finished task to ordinary terminal-stale supervision" +} + # --- stale pane, STALE terminal status overridden by an active run: absorbed --- # Regression for the 2026-07 herdr false-surface incidents: a crew's own status # log gets no new entry once firstmate hands it to a no-mistakes validation @@ -6000,6 +6091,8 @@ test_routine_appends_after_a_classified_event_stay_absorbed test_unreadable_status_reports_once_per_file_state test_permission_recovery_surfaces_preserved_status test_terminal_stale_surfaced +test_deliberately_stopped_finished_task_is_parked_not_stale +test_deliberate_stop_marker_cleared_resumes_terminal_stale_surfacing test_stale_terminal_status_overridden_by_active_run test_nonterminal_stale_provably_working_absorbed_then_escalated test_wedge_escalation_marks_demand_deep_inspection_after_threshold From 06899bfc33f98253a854e89cf598b8c22ca18f97 Mon Sep 17 00:00:00 2001 From: sdivanl Date: Sun, 20 Sep 2026 22:12:28 +0800 Subject: [PATCH 02/12] no-mistakes(review): Honor deliberate-stop marker in the away-mode daemon --- bin/fm-supervise-daemon.sh | 25 ++++++++++++++++++++++++- docs/agent-control.md | 2 +- tests/fm-daemon.test.sh | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index 472d19a20cb..dbd46618245 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -49,7 +49,10 @@ # fm-classify-lib.sh's combined predicate - instead gets its own longer # PAUSE_RESURFACE_SECS recheck, never a wedge escalation, whether its pane # reads idle or busy; only a status append that stops declaring the wait -# ends that routing. A captain-held transfer is not rechecked at all while +# ends that routing. A worker the control plane deliberately stopped +# (state/.deliberate-stop, written by bin/fm-control.sh's exit verb) is +# parked the same way, whatever its last status line says. A captain-held +# transfer is not rechecked at all while # the away-posture record (state/.afk-contract) exists: nobody is there to # answer it, and the return brief lists it. # Crewmates are autonomous, so a delayed stale response does not stall a @@ -181,6 +184,12 @@ FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" # for the captain is never rechecked (the watcher applies the same rule). # shellcheck source=bin/fm-afk-contract.sh . "$FM_DAEMON_DIR/fm-afk-contract.sh" +# The durable deliberate-stop marker (state/.deliberate-stop), owned by +# bin/fm-control-lib.sh. The daemon reads only its presence so a deliberately +# parked task takes the declared-pause cadence instead of the wedge ladder, in +# both classify_stale and the housekeeping stale recheck. +# shellcheck source=bin/fm-control-lib.sh +. "$FM_DAEMON_DIR/fm-control-lib.sh" # Supervisor-pane discovery (FM_SUPERVISOR_TARGET_DEFAULT, # FM_SUPERVISOR_BACKEND_DEFAULT, discover_supervisor_target, @@ -434,6 +443,14 @@ classify_stale() { # [ ] printf 'pause|paused (awaiting external), rechecked on a long cadence: %s' "$last" return fi + if fm_control_deliberate_stop_present "$state" "$task"; then + # Firstmate stopped this worker on purpose and its task record stayed open, + # so an idle endpoint is a parked task, not a wedge suspect: the same long + # recheck cadence as a declared pause, regardless of whether the last status + # line is a non-terminal leftover or a finished `done:`. + printf 'pause|deliberately stopped (parked task, rechecked on a long cadence): %s' "$task" + return + fi if [ -n "$last" ] && status_is_captain_relevant "$last"; then # Independent of free-text captain-relevant matching: a nonterminal progress # verb (working:) must never take the terminal stale path. Seen-status dedupe @@ -1067,6 +1084,12 @@ housekeeping() { # fi task=$(window_to_task "$win" "$state") last=$(last_status_line "$state/$task.status") + if fm_control_deliberate_stop_present "$state" "$task"; then + # A deliberately parked task never wedge-escalates: drop any stale marker + # left over from before the stop. The watcher owns the bounded recheck. + rm -f "$marker" + continue + fi if [ -n "$last" ] && status_is_paused_or_captain_held "$last"; then reconcile_pause_tracking "$win" "$state" "$last" continue diff --git a/docs/agent-control.md b/docs/agent-control.md index 19a0e4ad543..2fbac89f362 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -23,7 +23,7 @@ The failure repeated across harnesses and homes, and the workaround (remember to `bin/fm-send.sh`'s `--key` path reads the composer-clear table from this owner too, rather than keeping a second copy of it. - **Per-backend capability**: which named keys a runtime backend can deliver, and whether it has a recovery-grade agent-state classifier able to prove an agent stopped. -The one thing this file owns that is not a pure table is the [endpoint-absence proof](#reclaiming-a-task-whose-endpoint-is-gone) below, which does run backend reads; sourcing the file is still free. +The two things this file owns that are not pure tables are the [endpoint-absence proof](#reclaiming-a-task-whose-endpoint-is-gone) below, which does run backend reads, and the `fm_control_deliberate_stop_*` helpers, which read or write exactly one per-task state file (`state/.deliberate-stop`); sourcing the file is still free. A recorded `harness=` is not always an exact adapter name: a task launched from a raw command records that command's basename instead. `fm_control_harness_family` is the one place that prefix rule is stated, and an unrecognized value resolves to no adapter rather than being guessed into one. diff --git a/tests/fm-daemon.test.sh b/tests/fm-daemon.test.sh index 510a4320988..9d9bc49bdcb 100755 --- a/tests/fm-daemon.test.sh +++ b/tests/fm-daemon.test.sh @@ -1229,6 +1229,41 @@ test_housekeeping_captain_held_stale_marker_transitions_to_pause() { pass "housekeeping moves a captain hold's existing stale marker to pause before wedge escalation" } +# firstmate issue #5004 at the daemon boundary. The watcher absorbs a deliberately +# parked task before the away gate, but its durable re-surface wake is drained by +# the away-mode daemon, and a wedge marker can have been aged before the stop. Both +# the daemon's classifier and its housekeeping stale recheck must honor the same +# durable deliberate-stop marker, or a deliberately parked task still wedge-escalates +# in away mode. +test_housekeeping_deliberate_stop_marker_never_wedge_escalates() { + local dir state fakebin win pane key out reason + dir=$(make_supercase stale-to-deliberate-stop) + state="$dir/state"; fakebin="$dir/fakebin"; win="sess:fm-parked-w16"; pane="$dir/pane.txt" + printf 'working: investigation in progress\n' > "$state/parked-w16.status" + printf 'idle prompt $\n' > "$pane" + key=$(printf '%s' "parked-w16" | tr ':/.' '___') + # The stop path owns the marker; the daemon reads only its presence. + fm_control_deliberate_stop_record "$state" parked-w16 + + # classify_stale parks it on the pause action, exactly like a declared wait, so + # the wake itself never records a wedge marker. + out=$(FM_STATE_OVERRIDE="$state" classify_stale "$win" "$state") + case "$out" in pause\|*) ;; *) fail "a deliberately parked task did not classify as pause: $out" ;; esac + date +%s > "$state/.subsuper-stale-$key" + reason="stale: $win (deliberately stopped 500s ago, rechecked on a long cadence not a wedge; relaunch the worker or clean up the finished task)" + LOG="$dir/daemon.log" FM_STATE_OVERRIDE="$state" handle_wake "$reason" "$state" + [ ! -e "$state/.subsuper-stale-$key" ] || fail "the deliberate-stop wake left a wedge marker behind" + + # A wedge marker already aged before the stop must be dropped, not escalated. + echo $(( $(date +%s) - 5000 )) > "$state/.subsuper-stale-$key" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=240 FM_ESCALATE_BATCH_SECS=999999 \ + housekeeping "$state" + [ ! -e "$state/.subsuper-stale-$key" ] || fail "a deliberately parked task kept its wedge marker" + [ ! -s "$state/.subsuper-escalations" ] || fail "a deliberately parked task wedge-escalated: $(cat "$state/.subsuper-escalations")" + pass "the away-mode daemon parks a deliberately stopped task instead of wedge-escalating it" +} + test_housekeeping_pause_marker_transitions_to_clear() { local dir state fakebin win pane key dir=$(make_supercase paused-to-stale) @@ -2812,6 +2847,7 @@ test_housekeeping_paused_unpaused_cleared test_housekeeping_captain_held_resolved_cleared test_housekeeping_stale_marker_transitions_to_pause test_housekeeping_captain_held_stale_marker_transitions_to_pause +test_housekeeping_deliberate_stop_marker_never_wedge_escalates test_housekeeping_pause_marker_transitions_to_clear test_housekeeping_herdr_persistent_stale_resolves_meta test_housekeeping_herdr_idle_busy_record_clears_stale From 1a182158e42fe6cbb2681eed268b09e1cef11ee0 Mon Sep 17 00:00:00 2001 From: sdivanl Date: Sun, 20 Sep 2026 23:17:36 +0800 Subject: [PATCH 03/12] no-mistakes(review): Absorb deliberate-stop re-stops on first sight --- bin/fm-watch.sh | 12 +++---- tests/fm-watch-triage.test.sh | 60 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index c6e36b45e60..91a0af4d778 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -1376,24 +1376,22 @@ handle_paused_stale() { # # per PAUSE_RESURFACE_SECS so a forgotten parked task cannot rot invisibly - and # never the wedge ladder. The re-surface age is anchored on the marker's own # mtime (when the stop was recorded) rather than the status file or pane hash, so -# a churny idle pane cannot reset the cadence. The scope is the marker's content -# (the stop epoch), so a re-stop after a relaunch starts a fresh window instead -# of inheriting the previous one's throttle. Uses its own .deliberate-stop-*- +# a churny idle pane cannot reset the cadence. Each stop's marker mtime opens +# its own window, so a re-stop is absorbed on first sight rather than waking at +# once off the previous stop's throttle. Uses its own .deliberate-stop-*- # throttle rather than the .paused-* flag, because the .paused-* machinery is # cleared whenever the last status line stops declaring a wait - a deliberately # stopped worker's last line is routinely `done:`, not a wait declaration. handle_deliberate_stop_stale() { # - local win=$1 task=$2 h=$3 key marker age scope + local win=$1 task=$2 h=$3 key marker age key=$(window_key "$win") printf '%s' "$h" > "$STATE/.stale-$key" rm -f "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key" clear_write_tracking "$key" marker=$(fm_control_deliberate_stop_marker "$STATE" "$task") age=$(age_of "$marker") - scope="deliberate-stop:$(cat "$marker" 2>/dev/null || true)" resurface_absorbed "$win" "$STATE/.deliberate-stop-resurfaced-$key" "$age" \ - "stale: $win (deliberately stopped ${age}s ago, rechecked on a long cadence not a wedge; relaunch the worker or clean up the finished task)" \ - "$scope" + "stale: $win (deliberately stopped ${age}s ago, rechecked on a long cadence not a wedge; relaunch the worker or clean up the finished task)" triage_log "absorbed stale (deliberate stop, age ${age}s): $win" } diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 50c1234d076..c9ad0139b6b 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -2046,6 +2046,65 @@ test_deliberately_stopped_finished_task_is_parked_not_stale() { pass "a deliberately stopped finished task is parked on the bounded recheck cadence, never wedge-escalated" } +# --- deliberate stop: a re-stop absorbs on first sight despite a stale throttle --- +# Regression: the deliberate-stop absorb passed the marker's stop epoch as the +# resurface "scope", and resurface_absorbed treats a scope change as an +# unconditional re-surface. A task re-stopped after an earlier stop had recorded +# this window's throttle therefore woke on first sight ("deliberately stopped 0s +# ago") instead of absorbing. The marker's own mtime already opens each stop's +# window, so the scope is gone and the first sight absorbs again, while the +# bounded recheck still fires once the marker passes the cadence. +test_restopped_deliberate_task_absorbs_before_the_recheck_cadence() { + local dir state fakebin out capture_file window key pane_hash sig pid now throttle + dir=$(make_case deliberate-stop-restop); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-restop" + printf 'finished, deliberately parked twice' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/restop.meta" + printf 'done: investigation finished\n' > "$state/restop.status" + sig=$(seen_sig "$state/restop.status"); printf '%s' "$sig" > "$state/.seen-restop_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "finished, deliberately parked twice") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + # A prior stop recorded this window's re-surface throttle; the worker was then + # relaunched and stopped again, rewriting the marker with a fresh epoch. + now=$(date +%s) + throttle="$state/.deliberate-stop-resurfaced-$key" + printf 'deliberate-stop:%s' "$(( now - 1000 ))" > "$throttle" + set_mtime "$(( now - 1000 ))" "$throttle" + printf '%s\n' "$now" > "$state/restop.deliberate-stop" + + # The fresh re-stop must be absorbed on first sight: no wake, no wedge timer. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_poll_cycle "$state" "$pid"; then + reap "$pid"; fail "a re-stopped task woke on first sight off the previous stop's throttle: $(cat "$out")" + fi + [ ! -s "$out" ] || fail "a re-stopped task printed a wake during its first-sight absorb" + [ ! -s "$state/.wake-queue" ] || fail "a re-stopped task enqueued a wake during its first-sight absorb" + [ ! -e "$state/.stale-since-$key" ] || fail "a re-stop absorb must not start the wedge timer" + reap "$pid" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional re-stop absorb watcher stop" + + # Past the cadence the parked task still re-surfaces exactly once, so the scope + # removal did not silence the bounded recheck. + set_mtime "$(( $(date +%s) - 500 ))" "$state/restop.deliberate-stop" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || { reap "$pid"; fail "a re-stopped task did not re-surface past the recheck cadence"; } + grep -F "deliberately stopped" "$out" >/dev/null || fail "the re-stop recheck was not labeled a deliberate-stop recheck" + grep -F "possible wedge" "$out" >/dev/null && fail "a re-stopped task was mislabeled a possible wedge" + pass "a re-stopped deliberate task absorbs on first sight, then still re-surfaces on the bounded cadence" +} + # --- clearing the deliberate-stop marker returns the task to normal supervision --- # Relaunch and cleanup clear the marker (fm-spawn/fm-teardown), and the next stale # sighting of the now-marker-less task must take the ordinary path again: a @@ -6092,6 +6151,7 @@ test_unreadable_status_reports_once_per_file_state test_permission_recovery_surfaces_preserved_status test_terminal_stale_surfaced test_deliberately_stopped_finished_task_is_parked_not_stale +test_restopped_deliberate_task_absorbs_before_the_recheck_cadence test_deliberate_stop_marker_cleared_resumes_terminal_stale_surfacing test_stale_terminal_status_overridden_by_active_run test_nonterminal_stale_provably_working_absorbed_then_escalated From 5a5d34f7c5ee2d01a3a4f0777a529b42adec01b4 Mon Sep 17 00:00:00 2001 From: sdivanl Date: Sun, 20 Sep 2026 23:57:06 +0800 Subject: [PATCH 04/12] no-mistakes(document): Document the durable deliberate-stop marker across supervisors --- .agents/skills/afk/SKILL.md | 1 + AGENTS.md | 3 ++- docs/agent-control.md | 2 ++ docs/architecture.md | 2 +- docs/configuration.md | 4 ++-- docs/scripts.md | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index d089ed9dc65..73eef087528 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -163,6 +163,7 @@ Classify each wake this way: With no unreported actionable event, the wake self-handles, and the current declaration outranks an enriched possible-wedge reason so it never escalates on the `FM_STALE_ESCALATE_SECS` cadence. If a declared external wait is still declared past `FM_PAUSE_RESURFACE_SECS` (default four hours), housekeeping sends one recheck and resets the pause window; a captain-held transfer is never rechecked while the posture record exists. The window ages against the crew's own latest status line, so only a status append that stops declaring the wait ends this routing and restores wedge detection. + A `stale` whose task carries the durable deliberate-stop marker (`state/.deliberate-stop`, written by `bin/fm-control.sh exit`) self-handles the same way and is never wedge-escalated, whatever its last status line says; the watcher owns that bounded recheck so it holds in every supervision shape. - `check` -> always escalate. Check scripts print only when firstmate should wake. - `stale` with a terminal status or bare legacy captain-relevant line -> escalate. Nonterminal progress remains transient even when its prose contains a legacy free-text token or its seen-status marker already matches, so record a marker and self-handle. diff --git a/AGENTS.md b/AGENTS.md index b0a86720c3e..869b19611e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,6 +103,7 @@ state/ runtime records and signals; gitignored .turn-ended touched by turn-end hooks .progress touched for observed native-harness activity inside one Pi turn; bin/fm-busy-event.sh owns its generation binding and bin/fm-watch.sh reads it beside turn-ended for the busy-age bound only, never as a completed turn .busy-state .busy-gen semantic busy-state record (one line, atomically replaced) and its per-incarnation gen sidecar; bin/fm-busy-event.sh is the only writer and bin/fm-busy-lib.sh owns the record format and classification; arming again replaces the previous incarnation so late events carrying its gen are rejected as stale; removed by retire and teardown + .deliberate-stop durable marker (one epoch second) written by bin/fm-control.sh's exit verb for a verified stop, cleared by a successful relaunch (bin/fm-spawn.sh) and by teardown, and read by bin/fm-watch.sh and the away-mode daemon so the stopped task is parked on the declared-pause recheck cadence instead of escalated as a wedge .grok-turnend-token firstmate-owned grok hook registry token for the task; removed by teardown .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown .gemini-settings.json firstmate-owned per-task Gemini settings carrying the busy-state and turn-end hooks, reached through GEMINI_CLI_SYSTEM_SETTINGS_PATH so nothing is written into the project's own .gemini/; removed by teardown @@ -152,7 +153,7 @@ state/ runtime records and signals; gitignored .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .claude-autoarm-failure-notified .claude-autoarm-failure-alarmed .turnend-claude-blocks .turnend-claude-blocks.lock Claude Stop auto-arm single-flight, epoch, failure-episode, attended-alarm, guard-budget, and budget-lock records; never touch .cursor-park-owner .cursor-park-owner.lock .turnend-cursor-blocks Cursor stop-hook owner record, publication and commit lock, and bounded repair-nag budget; never touch - .hash-* .count-* .stale-* .stale-since-* .churn-since-* .paused-* .wedge-escalations-* .dead-reported-* .writing-* .waiting-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch + .hash-* .count-* .stale-* .stale-since-* .churn-since-* .deliberate-stop-* .paused-* .wedge-escalations-* .dead-reported-* .writing-* .waiting-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch .watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete .last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it .subsuper-* .supervise-daemon.* sub-supervisor internals; never touch diff --git a/docs/agent-control.md b/docs/agent-control.md index 2fbac89f362..3b5a4c751bd 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -37,6 +37,7 @@ A recorded `harness=` is not always an exact adapter name: a task launched from | `relaunch` | Replace the running agent with a new one in the same worktree - and the same endpoint whenever that endpoint still exists - on the exact recorded adapter or an explicitly chosen harness, model, and effort. | The new agent is alive on the endpoint the task's record now names, and that record names the harness that is actually running. | An exit that delivers lifecycle input but cannot prove the agent stopped fails with `exit=unconfirmed`, reports the observed agent state and any interrupt cancellation claim, and never claims that nothing changed. +Every verified `exit` - including the idempotent `already-stopped` outcome - records the durable deliberate-stop marker at `state/.deliberate-stop`, whose presence tells the watcher and the away-mode daemon to park the stopped task on the declared-pause recheck cadence instead of escalating its idle endpoint as a possible wedge; a refused or unconfirmed stop records nothing, and a successful `relaunch` or teardown clears it. Interrupt never rewrites busy state as proof of its own success. Claude exposes no lifecycle acknowledgement for a manual interrupt, so delivery succeeds with `cancel=unconfirmed` and its adapter-owned busy state remains as observed. muse's session log records `terminal=cancelled` for the interrupted run, so the control plane reports `cancel=confirmed` only after observing that exact acknowledgement. @@ -134,6 +135,7 @@ The worktree and the task's records are unaffected either way. - A refusal **before** the agent is stopped leaves the durable record and the instructions byte-identical. - A launch failure **after** the agent is stopped restores the prior durable record, keeps the progress note so a later recovery still has it, marks the journal `failed:launching`, and reports plainly that no agent is running and where the work is preserved. + The deliberate-stop marker the `exit` verb recorded remains, so the task stays parked rather than reviving the stale/wedge ladder until a later relaunch clears it. - If the launch owner already published the new record but no running agent can be confirmed, the new record is kept: the task is recorded on the new harness with no agent confirmed, which is exactly what recovery reconciles. Rewriting it back to the old harness would be a second, worse inaccuracy. diff --git a/docs/architecture.md b/docs/architecture.md index 08b6a195f8d..75112216293 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -83,7 +83,7 @@ The deferral is bounded per endpoint by `FM_TURNEND_CHURN_ABSORB_SECS`, tracked That bound is load-bearing rather than cosmetic: churn and staleness read the same pane, so a pane that renders continuously - a clock, a spinner, a shell heartbeat, or a harness that leaves a background renderer alive after its agent yields - never reaches the staleness backbone's two-identical-hashes test either, and an unbounded churn absorb would leave a genuinely stopped worker behind such a renderer with no path left to surface it. If two metadata records derive the same per-window marker key, including two records that name the same endpoint, that marker is not attributable churn evidence for either task, so the bare turn-ended wake surfaces without changing or migrating existing marker state. A `kind=secondmate` task's status signal is the parent-directed reply stream and is never absorbed as provably working; its bare turn-ended signal is absorbed only by the ordinary authoritative working proof because an active secondmate does not enter the staleness backbone that would resurface deferred pane-churn evidence. -A crew that declares `paused:` for a known external wait, or carries a verified `captain-held` transfer, is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge, except that a captain-held transfer is not rechecked while the away-posture record exists. +A crew that declares `paused:` for a known external wait, carries a verified `captain-held` transfer, or was deliberately stopped by the control plane (`state/.deliberate-stop`, written by `bin/fm-control.sh`'s `exit` verb) is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge, whatever its last status line says; a captain-held transfer is not rechecked while the away-posture record exists. For an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies that same cadence to an unchanged `paused:` or durable `captain-held` endpoint while attended; the pause classification itself is recovered only when the backend confidently reports its agent dead. Live or inconclusive liveness remains fail-open at that initial surface, so a worker genuinely waiting on a decision is never silenced. Its later sights are still held to that same bounded cadence rather than re-alarming on every pane-hash change, because the throttle is keyed to the declaration and not to the pane an idle parked worker keeps ticking. diff --git a/docs/configuration.md b/docs/configuration.md index 442092b75f8..fe82b9f3890 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1171,9 +1171,9 @@ FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals FM_TURNEND_CHURN_ABSORB_SECS=900 # longest one endpoint's bare turn-ends may be deferred on pane-churn evidence alone; only consulted when config/turnend-churn-absorb is present FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked -FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates, unless that pane's own worker declared a wait that has not elapsed, or, where config/wedge-defer-parked-gate arms it, that pane's crew is parked at a validation gate awaiting the supervisor's decision on it that the crew raised under that run's key and nobody has answered yet, either of which takes the FM_PAUSE_RESURFACE_SECS recheck below instead; stale panes whose crew is not provably working surface immediately unless admitted directly to the declared-wait cadence, while a live idle declared wait still surfaces once before that cadence bounds repeats; at that same escalation moment a recovery-grade agent-state probe (docs/architecture.md owns that dead-record contract) reports a pane whose endpoint is proven `dead` or `missing` once and stops re-escalating it while it stays that way +FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates, unless that pane's own worker declared a wait that has not elapsed, or, where config/wedge-defer-parked-gate arms it, that pane's crew is parked at a validation gate awaiting the supervisor's decision on it that the crew raised under that run's key and nobody has answered yet, or the task carries the durable deliberate-stop marker (`state/.deliberate-stop`, written by `bin/fm-control.sh exit`), any of which takes the FM_PAUSE_RESURFACE_SECS recheck below instead; stale panes whose crew is not provably working surface immediately unless admitted directly to the declared-wait cadence, while a live idle declared wait still surfaces once before that cadence bounds repeats; at that same escalation moment a recovery-grade agent-state probe (docs/architecture.md owns that dead-record contract) reports a pane whose endpoint is proven `dead` or `missing` once and stops re-escalating it while it stays that way FM_BUSY_TURN_MAX_SECS=3600 # maximum age without a completed turn or explicit native-harness progress (bin/fm-watch.sh owns marker selection), before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a declared external wait, an attended verified captain-held transfer, or - where config/wedge-defer-parked-gate arms it - a validation gate of the crew's own awaiting the supervisor's still-unanswered decision takes the FM_PAUSE_RESURFACE_SECS recheck below instead -FM_PAUSE_RESURFACE_SECS=14400 # four hours between bounded rechecks of a declared external wait or verified captain-held transfer, and between repeated new-hash stale alarms for an ordinary crew task with an open backlog captain call; a structured until time can make an external-wait recheck occur sooner but cannot extend this bound; this includes a live idle pane after its first inconclusive stale wake, a provably-working pane whose own unelapsed declared wait or, where config/wedge-defer-parked-gate arms it, unanswered supervisor-owed validation gate defers its FM_STALE_ESCALATE_SECS escalation, and a live busy pane past FM_BUSY_TURN_MAX_SECS, while the away-mode daemon uses the same setting and ages its window against the crew's own latest status line rather than pane busy state; a captain-held transfer is never rechecked while the away-posture record exists, while an armed validation gate awaiting the supervisor's decision keeps this recheck in either posture +FM_PAUSE_RESURFACE_SECS=14400 # four hours between bounded rechecks of a declared external wait, a verified captain-held transfer, or a deliberately stopped parked task (`state/.deliberate-stop`), and between repeated new-hash stale alarms for an ordinary crew task with an open backlog captain call; a structured until time can make an external-wait recheck occur sooner but cannot extend this bound; this includes a live idle pane after its first inconclusive stale wake, a provably-working pane whose own unelapsed declared wait or, where config/wedge-defer-parked-gate arms it, unanswered supervisor-owed validation gate defers its FM_STALE_ESCALATE_SECS escalation, and a live busy pane past FM_BUSY_TURN_MAX_SECS, while the away-mode daemon uses the same setting and ages its window against the crew's own latest status line rather than pane busy state; a captain-held transfer is never rechecked while the away-posture record exists, while an armed validation gate awaiting the supervisor's decision keeps this recheck in either posture FM_SECONDMATE_WAKE_STALL_SECS=180 # minimum interval with no change of the oldest actionable foreign wake-queue row (it advances as the mate drains, and a queue reprovisioned under the same task id starts a fresh interval at whatever sequence it restarts) before an endpoint-recorded local secondmate produces one durable parent wake-loop-stall notification for that no-progress episode; a mate that is provably inside an active turn (an exact busy verdict) does not escalate until that same no-progress interval reaches FM_BUSY_TURN_MAX_SECS above, declared external-wait pause rows are excluded, and zero or invalid values use 180 FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WORKTREE_WRITE_PRUNE='.git node_modules .venv venv __pycache__ .mypy_cache .pytest_cache .ruff_cache .tox target dist build .next .cache vendor' # directory names the wedge detector's task-worktree write probe skips; the default keeps .git out so a supervisor's own read-only git command can never look like crew progress; set it to the empty string to prune nothing, which widens the probe to the whole depth-bounded tree rather than disabling it diff --git a/docs/scripts.md b/docs/scripts.md index 1ff6f206419..b752cd517a3 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -118,7 +118,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-lease.sh` | Claim, release, inspect, and sweep per-task supervision leases | | `fm-lease-lib.sh` | One owner of the supervision lease contract and the main-only role-partition guards | | `fm-control.sh` | Agent lifecycle control plane: allowlisted `interrupt`, `exit`, and transactional `relaunch` verbs for an exact task id ([agent-control.md](agent-control.md)) | -| `fm-control-lib.sh` | One executable owner of the control-plane verb allowlist, per-harness interrupt/exit mechanics, per-backend capability, and the endpoint-absence proof both `exit` and `relaunch` read | +| `fm-control-lib.sh` | One executable owner of the control-plane verb allowlist, per-harness interrupt/exit mechanics, per-backend capability, the endpoint-absence proof both `exit` and `relaunch` read, and the durable deliberate-stop marker that `bin/fm-watch.sh` and the away-mode daemon read to park a stopped task | | `fm-busy-lib.sh` | Single owner of the semantic busy-state contract: verdicts, source attribution, and per-harness sources | | `fm-busy-event.sh` | The only writer of a task's semantic busy-state record and native-harness progress marker; arms an incarnation and applies lifecycle events | | `fm-tmux-lib.sh` | Shared tmux pane primitives for composer capture, verified submit, and the submit-time busy check | From c7a3d1511d7324318666284394fd977c7ebd4a8c Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 00:47:47 +0800 Subject: [PATCH 05/12] no-mistakes(review): Re-surface deliberate-stop rechecks in away-mode daemon housekeeping --- .agents/skills/afk/SKILL.md | 2 +- bin/fm-supervise-daemon.sh | 55 +++++++++++++++++++++++++----------- tests/fm-daemon.test.sh | 56 +++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index 73eef087528..7808221917e 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -163,7 +163,7 @@ Classify each wake this way: With no unreported actionable event, the wake self-handles, and the current declaration outranks an enriched possible-wedge reason so it never escalates on the `FM_STALE_ESCALATE_SECS` cadence. If a declared external wait is still declared past `FM_PAUSE_RESURFACE_SECS` (default four hours), housekeeping sends one recheck and resets the pause window; a captain-held transfer is never rechecked while the posture record exists. The window ages against the crew's own latest status line, so only a status append that stops declaring the wait ends this routing and restores wedge detection. - A `stale` whose task carries the durable deliberate-stop marker (`state/.deliberate-stop`, written by `bin/fm-control.sh exit`) self-handles the same way and is never wedge-escalated, whatever its last status line says; the watcher owns that bounded recheck so it holds in every supervision shape. + A `stale` whose task carries the durable deliberate-stop marker (`state/.deliberate-stop`, written by `bin/fm-control.sh exit`) self-handles the same way and is never wedge-escalated, whatever its last status line says; its bounded recheck is the same `FM_PAUSE_RESURFACE_SECS` pause window, re-surfaced by the watcher while it triages and by housekeeping in away mode. - `check` -> always escalate. Check scripts print only when firstmate should wake. - `stale` with a terminal status or bare legacy captain-relevant line -> escalate. Nonterminal progress remains transient even when its prose contains a legacy free-text token or its seen-status marker already matches, so record a marker and self-handle. diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index dbd46618245..f14285b1d19 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -551,6 +551,12 @@ reconcile_pause_tracking() { # if status_is_paused_or_captain_held "$last"; then stale_marker_remove "$win" "$state" pause_marker_record "$win" "$state" + elif fm_control_deliberate_stop_present "$state" "$task"; then + # A deliberately parked task takes the same bounded pause cadence, but its + # last status line is routinely `done:` rather than a wait declaration, so + # the declaration test above cannot be the only one that keeps it parked. + stale_marker_remove "$win" "$state" + pause_marker_record "$win" "$state" elif [ -e "$marker" ] || [ -e "$state/.paused-$watcher_key" ]; then clear_pause_tracking "$win" "$state" fi @@ -1037,7 +1043,7 @@ _oldest_line_age() { # -> seconds since the oldest buffered item first ar # 3) heartbeat scan: every HEARTBEAT_SCAN_SECS, grep state/*.status for a # captain-relevant line the per-wake classifier missed and escalate it. housekeeping() { # - local state=$1 now due f key task win marker age last max_defer oldest pause_secs marker_epoch until bounded_until pause_reason + local state=$1 now due f key task win marker age last max_defer oldest pause_secs marker_epoch until bounded_until pause_reason deliberate now=$(_now) migrate_watcher_pause_markers "$state" @@ -1086,7 +1092,7 @@ housekeeping() { # last=$(last_status_line "$state/$task.status") if fm_control_deliberate_stop_present "$state" "$task"; then # A deliberately parked task never wedge-escalates: drop any stale marker - # left over from before the stop. The watcher owns the bounded recheck. + # left over from before the stop. Its bounded recheck is the pause loop's. rm -f "$marker" continue fi @@ -1110,11 +1116,15 @@ housekeeping() { # # status_is_paused_or_captain_held owns which declarations qualify), so it is # rechecked on a much longer cadence than a wedge (PAUSE_RESURFACE_SECS) and never # escalated as one - but it MUST re-surface, so neither a forgotten pause nor a - # forgotten captain hold can rot invisibly. Past the window: gone -> drop; still - # declaring the wait -> escalate a recheck digest and reset the marker so the window + # forgotten captain hold can rot invisibly. A task the control plane deliberately + # stopped (state/.deliberate-stop) is parked the same way, whatever its last + # status line says, because the watcher hands that bounded recheck to the away-mode + # daemon and it would otherwise be swallowed here. Past the window: gone -> drop; + # still parked -> escalate a recheck digest and reset the marker so the window # repeats. The digest names WHICH human the wait is on, because the captain is the - # one reading it: an external dependency for a paused: declaration, and the captain - # themself for a verified hold transfer. + # one reading it: an external dependency for a paused: declaration, the captain + # themself for a verified hold transfer, and firstmate for a deliberately stopped + # task. # Pane busy state does NOT end the wait. A declared wait can legitimately hold a # pane busy - a worker parked on a long foreground call it keeps live for as long # as the wait lasts - so reading busy as "the crew resumed" retires the window of @@ -1131,7 +1141,12 @@ housekeeping() { # fi task=$(window_to_task "$win" "$state") last=$(last_status_line "$state/$task.status") - if [ -z "$last" ] || ! status_is_paused_or_captain_held "$last"; then + deliberate=0 + if [ -n "$last" ] && status_is_paused_or_captain_held "$last"; then + : + elif fm_control_deliberate_stop_present "$state" "$task"; then + deliberate=1 + else reconcile_pause_tracking "$win" "$state" "$last" continue fi @@ -1141,15 +1156,19 @@ housekeeping() { # due="$state/.subsuper-pause-until-due-$key" until= bounded_until=0 - if status_is_captain_held "$last" && fm_afk_contract_present "$state"; then - continue - fi - if until=$(status_paused_until "$last"); then - if [ "$now" -lt "$until" ] && [ "$age" -lt "$pause_secs" ]; then + if [ "$deliberate" -eq 0 ]; then + if status_is_captain_held "$last" && fm_afk_contract_present "$state"; then continue - elif [ "$now" -lt "$until" ]; then - bounded_until=1 - elif [ "$(cat "$due" 2>/dev/null || true)" = "$until" ]; then + fi + if until=$(status_paused_until "$last"); then + if [ "$now" -lt "$until" ] && [ "$age" -lt "$pause_secs" ]; then + continue + elif [ "$now" -lt "$until" ]; then + bounded_until=1 + elif [ "$(cat "$due" 2>/dev/null || true)" = "$until" ]; then + [ "$age" -ge "$pause_secs" ] || continue + fi + else [ "$age" -ge "$pause_secs" ] || continue fi else @@ -1166,7 +1185,11 @@ housekeeping() { # 2) rm -f "$marker" ;; *) last=$(last_status_line "$state/$task.status") - if [ -n "$last" ] && status_is_captain_held "$last"; then + if [ "$deliberate" -eq 1 ]; then + if escalate_add "$state" "deliberately stopped ${age}s (parked task, recheck whether to relaunch the worker or clean up the finished task): $win"; then + _now > "$marker" + fi + elif [ -n "$last" ] && status_is_captain_held "$last"; then if escalate_add "$state" "captain-held ${age}s (awaiting the captain, answer the held decision or release the hold): $win"; then _now > "$marker" fi diff --git a/tests/fm-daemon.test.sh b/tests/fm-daemon.test.sh index 9d9bc49bdcb..f65b793cdc8 100755 --- a/tests/fm-daemon.test.sh +++ b/tests/fm-daemon.test.sh @@ -1264,6 +1264,61 @@ test_housekeeping_deliberate_stop_marker_never_wedge_escalates() { pass "the away-mode daemon parks a deliberately stopped task instead of wedge-escalating it" } +# firstmate issue #5004 at the away-mode re-surface boundary. The watcher absorbs a +# deliberately parked task and hands the away-mode daemon its bounded recheck wake; +# the daemon classifies it as a pause and records the pause marker, but the task's +# last status line is routinely `done:`, not a wait declaration, so the housekeeping +# pause loop must keep aging that marker off the deliberate-stop marker and emit the +# recheck digest instead of clearing it un-escalated (which let a parked task rot +# invisibly in away mode). +test_housekeeping_deliberate_stop_resurfaces_on_the_pause_cadence() { + local dir state fakebin win pane key reason age + dir=$(make_supercase deliberate-stop-resurface) + state="$dir/state"; fakebin="$dir/fakebin"; win="sess:fm-parked-w17"; pane="$dir/pane.txt" + printf 'done: investigation finished\n' > "$state/parked-w17.status" + seen_through "$state" parked-w17 + printf 'idle prompt $\n' > "$pane" + key=$(printf '%s' "parked-w17" | tr ':/.' '___') + fm_control_deliberate_stop_record "$state" parked-w17 + + # The watcher's bounded recheck wake is drained by the away-mode daemon, which + # classifies it as a pause and records the pause marker. + reason="stale: $win (deliberately stopped 500s ago, rechecked on a long cadence not a wedge; relaunch the worker or clean up the finished task)" + LOG="$dir/daemon.log" FM_STATE_OVERRIDE="$state" handle_wake "$reason" "$state" + [ -e "$state/.subsuper-paused-$key" ] || fail "the deliberate-stop recheck wake did not record a pause marker" + + # Inside the cadence the marker must neither escalate nor be recreated fresh. + echo $(( $(date +%s) - 100 )) > "$state/.subsuper-paused-$key" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_ESCALATE_BATCH_SECS=999999 FM_PAUSE_RESURFACE_SECS=3600 \ + housekeeping "$state" + [ ! -s "$state/.subsuper-escalations" ] || fail "a deliberate stop re-surfaced inside its cadence: $(cat "$state/.subsuper-escalations")" + [ -e "$state/.subsuper-paused-$key" ] || fail "a deliberate stop lost its pause marker inside the window" + + # Past the cadence it MUST re-surface a recheck naming the parked task, never a + # possible wedge, and reset its window so the cadence repeats. + echo $(( $(date +%s) - 5000 )) > "$state/.subsuper-paused-$key" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_ESCALATE_BATCH_SECS=999999 FM_PAUSE_RESURFACE_SECS=240 \ + housekeeping "$state" + grep -F "deliberately stopped" "$state/.subsuper-escalations" >/dev/null 2>&1 \ + || fail "a deliberately parked task did not re-surface on the pause cadence: $(cat "$state/.subsuper-escalations" 2>/dev/null || true)" + grep -F "possible wedge" "$state/.subsuper-escalations" >/dev/null 2>&1 \ + && fail "a deliberately parked task re-surfaced as a possible wedge" + [ -e "$state/.subsuper-paused-$key" ] || fail "deliberate-stop pause marker cleared instead of reset for the next window" + age=$(( $(date +%s) - $(cat "$state/.subsuper-paused-$key" 2>/dev/null || echo 0) )) + [ "$age" -lt 60 ] || fail "deliberate-stop pause marker was not reset to now on re-surface (age ${age}s)" + + # Clearing the deliberate-stop marker (relaunch/teardown) returns the task to + # ordinary supervision: the stale pause marker the cadence left must clear. + fm_control_deliberate_stop_clear "$state" parked-w17 + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_ESCALATE_BATCH_SECS=999999 FM_PAUSE_RESURFACE_SECS=240 \ + housekeeping "$state" + [ ! -e "$state/.subsuper-paused-$key" ] || fail "clearing the deliberate-stop marker left the pause marker behind" + pass "the away-mode daemon re-surfaces a deliberately stopped task on the bounded pause cadence" +} + test_housekeeping_pause_marker_transitions_to_clear() { local dir state fakebin win pane key dir=$(make_supercase paused-to-stale) @@ -2848,6 +2903,7 @@ test_housekeeping_captain_held_resolved_cleared test_housekeeping_stale_marker_transitions_to_pause test_housekeeping_captain_held_stale_marker_transitions_to_pause test_housekeeping_deliberate_stop_marker_never_wedge_escalates +test_housekeeping_deliberate_stop_resurfaces_on_the_pause_cadence test_housekeeping_pause_marker_transitions_to_clear test_housekeeping_herdr_persistent_stale_resolves_meta test_housekeeping_herdr_idle_busy_record_clears_stale From 31cabc1da58fa54b4bedfd53bea010420dd98720 Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 01:13:29 +0800 Subject: [PATCH 06/12] no-mistakes(review): Honor deliberate-stop marker on the busy-turn path --- bin/fm-control-lib.sh | 5 ++-- bin/fm-watch.sh | 26 ++++++++++++----- docs/architecture.md | 2 +- docs/configuration.md | 2 +- tests/fm-watch-triage.test.sh | 53 +++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 11 deletions(-) diff --git a/bin/fm-control-lib.sh b/bin/fm-control-lib.sh index e1c99bb79c9..55f71cee9fc 100644 --- a/bin/fm-control-lib.sh +++ b/bin/fm-control-lib.sh @@ -358,8 +358,9 @@ fm_control_harness_turnend_auth_path() { # # deliberately stopped, so its idle endpoint is a parked task: the watcher gives # it the declared-pause treatment - a long bounded recheck cadence, never a # stale or wedge escalation - instead of treating it as a worker that stopped -# responding on its own. The record body is the epoch second of the stop, so a -# re-stop after a relaunch starts a fresh recheck window. A failed stop attempt +# responding on its own. The file's mtime, refreshed by each stop so a re-stop +# after a relaunch starts a fresh recheck window, opens that cadence; the body +# records the stop's epoch second. A failed stop attempt # never writes it, so a refused or unattributed endpoint keeps escalating exactly # as it always did. fm_control_deliberate_stop_marker() { # diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 91a0af4d778..01dde66ab62 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -67,8 +67,9 @@ # only up to BUSY_TURN_MAX_SECS with no completed turn # (state/.turn-ended, or the spawn record before any # turn completes). Past that bound, a declared external -# wait or verified captain-held transfer uses the long -# pause recheck cadence; under daemon-backed afk an +# wait or verified captain-held transfer, or a worker the +# control plane deliberately stopped, uses the long pause +# recheck cadence; under daemon-backed afk an # external wait is instead handed to the daemon as this # plain reason once per declaration, while captain-held # work stays silent until return @@ -285,7 +286,8 @@ STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provabl # non-busy stale - so it escalates via the existing stale reason, escalation # counter, and demand-deep-inspection marker for human inspection only, never an # automatic interrupt, signal, or restart - unless the crew declared the wait -# itself, which takes the long pause cadence instead. Set generously above +# itself or was deliberately stopped by the control plane, either of which takes +# the long pause cadence instead. Set generously above # any legitimate interval without observable progress, including silent long # tool calls, builds, or test runs. BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600} @@ -1403,10 +1405,12 @@ handle_deliberate_stop_stale() { # # A busy pane past BUSY_TURN_MAX_SECS is normally a wedge suspect because a hung # foreground call can hide behind a busy signature. A `paused:` declaration or # verified captain-held transfer instead identifies that live foreground call as -# the expected external wait. The caller has already confirmed liveness through -# the busy verdict, so this exception does not suppress undeclared wedges or -# alter the separate non-busy classification. handle_paused_stale keeps the -# exception bounded by re-surfacing it once per PAUSE_RESURFACE_SECS. +# the expected external wait, and a worker firstmate deliberately stopped +# (state/.deliberate-stop) is parked the same way whether its pane reads idle +# or busy. The caller has already confirmed liveness through the busy verdict, so +# this exception does not suppress undeclared wedges or alter the separate +# non-busy classification. handle_paused_stale and handle_deliberate_stop_stale +# keep the exception bounded by re-surfacing it once per PAUSE_RESURFACE_SECS. # A pane that declared nothing falls through to the shared wedge timer, which, # in a home that armed config/wedge-defer-parked-gate, applies the same rule to # the one wait a busy pane cannot declare: a validation gate of its own awaiting @@ -1420,6 +1424,14 @@ handle_deliberate_stop_stale() { # busy_turn_bound_check() { # local win=$1 task=$2 h=$3 since_file=$4 escalation_file=$5 key statusf declared statusf="$STATE/$task.status" + if fm_control_deliberate_stop_present "$STATE" "$task"; then + # A deliberately stopped worker is parked whether its pane reads idle or busy: + # the same bounded recheck the idle path gives it, never the busy-turn wedge + # ladder. Handled in both postures, mirroring that path's branch before the + # afk gate. + handle_deliberate_stop_stale "$win" "$task" "$h" + return 0 + fi if status_is_paused_or_captain_held "$(last_status_line "$statusf")"; then if afk_present; then # Away mode is daemon-owned, so this bound hands off the PLAIN wake identity diff --git a/docs/architecture.md b/docs/architecture.md index 75112216293..16c854dbfdd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -83,7 +83,7 @@ The deferral is bounded per endpoint by `FM_TURNEND_CHURN_ABSORB_SECS`, tracked That bound is load-bearing rather than cosmetic: churn and staleness read the same pane, so a pane that renders continuously - a clock, a spinner, a shell heartbeat, or a harness that leaves a background renderer alive after its agent yields - never reaches the staleness backbone's two-identical-hashes test either, and an unbounded churn absorb would leave a genuinely stopped worker behind such a renderer with no path left to surface it. If two metadata records derive the same per-window marker key, including two records that name the same endpoint, that marker is not attributable churn evidence for either task, so the bare turn-ended wake surfaces without changing or migrating existing marker state. A `kind=secondmate` task's status signal is the parent-directed reply stream and is never absorbed as provably working; its bare turn-ended signal is absorbed only by the ordinary authoritative working proof because an active secondmate does not enter the staleness backbone that would resurface deferred pane-churn evidence. -A crew that declares `paused:` for a known external wait, carries a verified `captain-held` transfer, or was deliberately stopped by the control plane (`state/.deliberate-stop`, written by `bin/fm-control.sh`'s `exit` verb) is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge, whatever its last status line says; a captain-held transfer is not rechecked while the away-posture record exists. +A crew that declares `paused:` for a known external wait, carries a verified `captain-held` transfer, or was deliberately stopped by the control plane (`state/.deliberate-stop`, written by `bin/fm-control.sh`'s `exit` verb) is separately absorbed, whether its pane reads idle or busy, and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge, whatever its last status line says; a captain-held transfer is not rechecked while the away-posture record exists. For an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies that same cadence to an unchanged `paused:` or durable `captain-held` endpoint while attended; the pause classification itself is recovered only when the backend confidently reports its agent dead. Live or inconclusive liveness remains fail-open at that initial surface, so a worker genuinely waiting on a decision is never silenced. Its later sights are still held to that same bounded cadence rather than re-alarming on every pane-hash change, because the throttle is keyed to the declaration and not to the pane an idle parked worker keeps ticking. diff --git a/docs/configuration.md b/docs/configuration.md index fe82b9f3890..f725c41a92a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1172,7 +1172,7 @@ FM_TURNEND_CHURN_ABSORB_SECS=900 # longest one endpoint's bare turn-ends may b FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates, unless that pane's own worker declared a wait that has not elapsed, or, where config/wedge-defer-parked-gate arms it, that pane's crew is parked at a validation gate awaiting the supervisor's decision on it that the crew raised under that run's key and nobody has answered yet, or the task carries the durable deliberate-stop marker (`state/.deliberate-stop`, written by `bin/fm-control.sh exit`), any of which takes the FM_PAUSE_RESURFACE_SECS recheck below instead; stale panes whose crew is not provably working surface immediately unless admitted directly to the declared-wait cadence, while a live idle declared wait still surfaces once before that cadence bounds repeats; at that same escalation moment a recovery-grade agent-state probe (docs/architecture.md owns that dead-record contract) reports a pane whose endpoint is proven `dead` or `missing` once and stops re-escalating it while it stays that way -FM_BUSY_TURN_MAX_SECS=3600 # maximum age without a completed turn or explicit native-harness progress (bin/fm-watch.sh owns marker selection), before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a declared external wait, an attended verified captain-held transfer, or - where config/wedge-defer-parked-gate arms it - a validation gate of the crew's own awaiting the supervisor's still-unanswered decision takes the FM_PAUSE_RESURFACE_SECS recheck below instead +FM_BUSY_TURN_MAX_SECS=3600 # maximum age without a completed turn or explicit native-harness progress (bin/fm-watch.sh owns marker selection), before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a declared external wait, a deliberately stopped parked task (`state/.deliberate-stop`), an attended verified captain-held transfer, or - where config/wedge-defer-parked-gate arms it - a validation gate of the crew's own awaiting the supervisor's still-unanswered decision takes the FM_PAUSE_RESURFACE_SECS recheck below instead FM_PAUSE_RESURFACE_SECS=14400 # four hours between bounded rechecks of a declared external wait, a verified captain-held transfer, or a deliberately stopped parked task (`state/.deliberate-stop`), and between repeated new-hash stale alarms for an ordinary crew task with an open backlog captain call; a structured until time can make an external-wait recheck occur sooner but cannot extend this bound; this includes a live idle pane after its first inconclusive stale wake, a provably-working pane whose own unelapsed declared wait or, where config/wedge-defer-parked-gate arms it, unanswered supervisor-owed validation gate defers its FM_STALE_ESCALATE_SECS escalation, and a live busy pane past FM_BUSY_TURN_MAX_SECS, while the away-mode daemon uses the same setting and ages its window against the crew's own latest status line rather than pane busy state; a captain-held transfer is never rechecked while the away-posture record exists, while an armed validation gate awaiting the supervisor's decision keeps this recheck in either posture FM_SECONDMATE_WAKE_STALL_SECS=180 # minimum interval with no change of the oldest actionable foreign wake-queue row (it advances as the mate drains, and a queue reprovisioned under the same task id starts a fresh interval at whatever sequence it restarts) before an endpoint-recorded local secondmate produces one durable parent wake-loop-stall notification for that no-progress episode; a mate that is provably inside an active turn (an exact busy verdict) does not escalate until that same no-progress interval reaches FM_BUSY_TURN_MAX_SECS above, declared external-wait pause rows are excluded, and zero or invalid values use 180 FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index c9ad0139b6b..dc37f9d505c 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -2134,6 +2134,58 @@ test_deliberate_stop_marker_cleared_resumes_terminal_stale_surfacing() { pass "clearing the deliberate-stop marker returns the finished task to ordinary terminal-stale supervision" } +# --- deliberate stop + busy pane: the busy-turn bound must park, not wedge --- +# The deliberate-stop marker was honored on the idle stale path but not on the +# busy-turn path: a stopped worker whose pane still rendered a recognized busy +# signature, with no completed turn past BUSY_TURN_MAX_SECS, went to +# wedge_timer_check and escalated as a possible wedge - contradicting the +# daemon's "parked whether idle or busy" contract. This fixture pins that the +# busy-turn bound routes it to the same bounded recheck the idle path uses. +test_busy_deliberate_stop_is_rechecked_not_wedge_escalated() { + local dir state fakebin out capture_file window key sig pid + dir=$(make_case busy-deliberate-stop); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-parked-busy" + printf 'Working... (7200.4s)' > "$capture_file" + printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/parked-busy.meta" + record_pi_busy "$state" parked-busy + printf 'done: investigation finished\n' > "$state/parked-busy.status" + sig=$(seen_sig "$state/parked-busy.status"); printf '%s' "$sig" > "$state/.seen-parked-busy_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + # No completed turn: age the spawn record itself, past the busy-turn bound. + touch -t 200001010000 "$state/parked-busy.meta" + printf '%s\n' "$(date +%s)" > "$state/parked-busy.deliberate-stop" + + # Phase A: past the bound, the deliberately stopped busy pane is absorbed on + # the long cadence and never starts a wedge. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=999 \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a deliberately stopped busy pane was escalated: $(cat "$out")"; } + reap "$pid" + [ ! -s "$out" ] || fail "a deliberately stopped busy pane printed a wake reason: $(cat "$out")" + [ ! -e "$state/.stale-since-$key" ] || fail "a deliberately stopped busy pane started the wedge timer" + [ ! -e "$state/.wedge-escalations-$key" ] || fail "a deliberately stopped busy pane incremented the escalation counter" + ack_stopped_cycle "$state" || fail "could not acknowledge the intentional busy deliberate-stop phase-A stop" + + # Phase B: age the stop past the cadence; the parked task re-surfaces once as a + # deliberate-stop recheck - never a possible wedge. + set_mtime "$(( $(date +%s) - 500 ))" "$state/parked-busy.deliberate-stop" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=240 \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || { reap "$pid"; fail "a deliberately stopped busy pane did not re-surface past the cadence"; } + grep -F "deliberately stopped" "$out" >/dev/null || fail "the busy-turn recheck was not labeled a deliberate-stop recheck: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null && fail "a deliberately stopped busy pane was mislabeled a possible wedge: $(cat "$out")" + [ -e "$state/.deliberate-stop-resurfaced-$key" ] || fail "the deliberate-stop re-surface throttle was not recorded" + [ ! -e "$state/.stale-since-$key" ] || fail "a deliberately stopped busy pane used the wedge timer" + pass "a deliberately stopped busy pane is parked on the bounded recheck cadence, never wedge-escalated" +} + # --- stale pane, STALE terminal status overridden by an active run: absorbed --- # Regression for the 2026-07 herdr false-surface incidents: a crew's own status # log gets no new entry once firstmate hands it to a no-mistakes validation @@ -6153,6 +6205,7 @@ test_terminal_stale_surfaced test_deliberately_stopped_finished_task_is_parked_not_stale test_restopped_deliberate_task_absorbs_before_the_recheck_cadence test_deliberate_stop_marker_cleared_resumes_terminal_stale_surfacing +test_busy_deliberate_stop_is_rechecked_not_wedge_escalated test_stale_terminal_status_overridden_by_active_run test_nonterminal_stale_provably_working_absorbed_then_escalated test_wedge_escalation_marks_demand_deep_inspection_after_threshold From 376d16aba0def533ec715e539a4bedb7593c06a1 Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 02:08:58 +0800 Subject: [PATCH 07/12] no-mistakes(review): Recheck deliberately parked tasks when their idle pane churns --- bin/fm-watch.sh | 4 ++- tests/fm-watch-triage.test.sh | 56 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 01dde66ab62..87d4c8f7563 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -2865,7 +2865,9 @@ EOF clear_write_tracking "$key" fi task=$(window_to_task "$w" "$STATE") - if ! afk_present && status_is_paused_or_captain_held "$(last_status_line "$STATE/$task.status")" && [ "$busy_now" -ne 0 ]; then + if [ "$busy_now" -ne 0 ] && fm_control_deliberate_stop_present "$STATE" "$task"; then + handle_deliberate_stop_stale "$w" "$task" "$h" + elif ! afk_present && status_is_paused_or_captain_held "$(last_status_line "$STATE/$task.status")" && [ "$busy_now" -ne 0 ]; then case "$(pause_state_class "$w" "$task")" in paused) handle_paused_stale "$w" "$task" "$h" ;; # Inconclusive, but the declared wait itself still stands, so only the diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index dc37f9d505c..06e0d5166e6 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -2186,6 +2186,61 @@ test_busy_deliberate_stop_is_rechecked_not_wedge_escalated() { pass "a deliberately stopped busy pane is parked on the bounded recheck cadence, never wedge-escalated" } +# --- deliberate stop: a churning idle pane still gets the bounded recheck --- +# The deliberate-stop marker was honored only on the stable-hash idle branch. An +# idle parked pane whose display keeps ticking (a clock, a token counter) changes +# hash every poll, so it never reaches that branch; the new-hash path must still +# keep its bounded recheck alive, or the parked task rots invisibly in both +# postures. +test_churning_deliberate_stop_still_rechecked() { + local dir state fakebin out capture_file window key sig pid churn_pid i tmp + dir=$(make_case deliberate-stop-churn); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-churn-parked" + printf 'finished, deliberately parked' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/churn-parked.meta" + printf 'done: investigation finished\n' > "$state/churn-parked.status" + sig=$(seen_sig "$state/churn-parked.status"); printf '%s' "$sig" > "$state/.seen-churn-parked_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + # The previous poll recorded DIFFERENT pane content, and a background writer + # keeps rewriting it, so every poll is a new hash and the pane can never + # become a stable stale pane. + printf '%s' "$(hash_text 'an earlier tick')" > "$state/.hash-$key" + printf '0\n' > "$state/.count-$key" + # Age the stop past the cadence so the recheck must fire on this very poll. + printf '%s\n' "$(date +%s)" > "$state/churn-parked.deliberate-stop" + set_mtime "$(( $(date +%s) - 500 ))" "$state/churn-parked.deliberate-stop" + tmp="$capture_file.tmp" + ( + i=0 + while :; do + i=$((i + 1)) + printf 'finished, footer tick %s' "$i" > "$tmp" + mv -f "$tmp" "$capture_file" + sleep 0.1 + done + ) & + churn_pid=$! + sleep 0.2 + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_for_exit "$pid" 100; then + reap "$pid"; kill "$churn_pid" 2>/dev/null || true; wait "$churn_pid" 2>/dev/null || true + fail "a churning deliberately parked task was never rechecked (it rotted invisibly): $(cat "$out")" + fi + kill "$churn_pid" 2>/dev/null || true + wait "$churn_pid" 2>/dev/null || true + grep -F "deliberately stopped" "$out" >/dev/null || fail "the churning-pane recheck was not labeled a deliberate-stop recheck: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null && fail "a churning deliberately parked task was mislabeled a possible wedge: $(cat "$out")" + [ -e "$state/.deliberate-stop-resurfaced-$key" ] || fail "the deliberate-stop re-surface throttle was not recorded for the churning pane" + [ ! -e "$state/.stale-since-$key" ] || fail "a churning deliberate-stop recheck must not use the wedge timer" + pass "a churning deliberately parked task still gets the bounded recheck, never a wedge" +} + # --- stale pane, STALE terminal status overridden by an active run: absorbed --- # Regression for the 2026-07 herdr false-surface incidents: a crew's own status # log gets no new entry once firstmate hands it to a no-mistakes validation @@ -6206,6 +6261,7 @@ test_deliberately_stopped_finished_task_is_parked_not_stale test_restopped_deliberate_task_absorbs_before_the_recheck_cadence test_deliberate_stop_marker_cleared_resumes_terminal_stale_surfacing test_busy_deliberate_stop_is_rechecked_not_wedge_escalated +test_churning_deliberate_stop_still_rechecked test_stale_terminal_status_overridden_by_active_run test_nonterminal_stale_provably_working_absorbed_then_escalated test_wedge_escalation_marks_demand_deep_inspection_after_threshold From 45f2a5d665b33a855a94ffbf5256805a5e8a339c Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 03:22:39 +0800 Subject: [PATCH 08/12] no-mistakes(review): Clear deliberate-stop marker at relaunch delivery commit point --- bin/fm-spawn.sh | 20 ++++++++++---------- tests/fm-control-relaunch.test.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 864e9387e8d..d0d3158ff6e 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -4914,6 +4914,16 @@ fi # This is the commit point: all endpoint and harness delivery that can reject # the spawn has succeeded. Re-read and transition while holding the same # per-task lock as metadata publication, then and only then report success. +if [ "$RELAUNCH" -eq 1 ]; then + # All launch delivery and the relaunch record publication now succeeded, so + # the replacement supersedes the deliberately stopped incarnation. Clear its + # parked-task marker only at this commit point: an earlier launch failure + # leaves the stopped task parked rather than reviving the stale/wedge ladder. + fm_control_deliberate_stop_clear "$STATE_REAL" "$ID" || { + echo "error: replacement for $ID was launched, but its deliberate-stop marker could not be cleared" >&2 + exit 1 + } +fi if [ "$SPAWN_META_LOCK_HELD" != 1 ]; then SPAWN_META_LOCK=$(fm_meta_lock_path "$STATE/$ID.meta") || exit 1 fm_lock_acquire_wait "$SPAWN_META_LOCK" @@ -4974,16 +4984,6 @@ if [ -n "$SPAWN_DEFERRED_SIGNAL" ]; then echo "error: spawn of $ID was interrupted after launch delivery began; $SPAWN_PRESERVED_CLAIM" >&2 exit "$SPAWN_DEFERRED_SIGNAL_STATUS" fi -if [ "$RELAUNCH" -eq 1 ]; then - # All launch delivery and the relaunch record publication now succeeded, so - # the replacement supersedes the deliberately stopped incarnation. Clear its - # parked-task marker only at this commit point: an earlier launch failure - # leaves the stopped task parked rather than reviving the stale/wedge ladder. - fm_control_deliberate_stop_clear "$STATE_REAL" "$ID" || { - echo "error: replacement for $ID was launched, but its deliberate-stop marker could not be cleared" >&2 - exit 1 - } -fi fm_lock_release "$SPAWN_META_LOCK" SPAWN_META_LOCK_HELD=0 diff --git a/tests/fm-control-relaunch.test.sh b/tests/fm-control-relaunch.test.sh index fbf80fa2cdc..a82a486e884 100755 --- a/tests/fm-control-relaunch.test.sh +++ b/tests/fm-control-relaunch.test.sh @@ -401,6 +401,31 @@ test_relaunch_clears_the_deliberate_stop_marker() { pass "fm-control relaunch: clears the durable deliberate-stop marker left by a prior stop" } +test_relaunch_clears_the_deliberate_stop_marker_when_the_backlog_commit_fails() { + local dir out rc=0 + command -v tasks-axi >/dev/null 2>&1 || { + pass "skipped: tasks-axi is not installed, so the backlog transition is inert" + return 0 + } + dir=$(new_case deliberate-stop-clear-backlog rl77) + add_ship_task "$dir" rl77 claude + seed_backlog "$dir" rl77 queued + break_tasks_axi_start "$dir" + printf '%s\n' "$(date +%s)" > "$dir/home/state/rl77.deliberate-stop" + + out=$(run_control "$dir" rl77 relaunch --note "resume after a deliberate stop") || rc=$? + + # The replacement is delivered before the deferred backlog commit, so even + # though that commit fails and the relaunch reports failure, the running + # replacement must not stay classified as a deliberately parked task. + expect_code 1 "$rc" "a failed backlog commit should fail the relaunch"$'\n'"$out" + assert_grep "encode launch-brief" "$dir/fake/literal" \ + "the replacement should have been delivered before the backlog commit failed" + [ ! -e "$dir/home/state/rl77.deliberate-stop" ] \ + || fail "a delivered relaunch must clear the deliberate-stop marker even when the backlog commit fails" + pass "fm-control relaunch: a failed post-launch backlog commit still clears the deliberate-stop marker" +} + test_relaunch_refuses_before_exit_when_the_composer_holds_pending_text() { local dir out rc dir=$(new_case pending-exit rl43) @@ -2215,6 +2240,7 @@ test_relaunch_moves_a_drifted_item_back_in_flight() { test_same_harness_relaunch_keeps_identity_and_reuses_the_endpoint test_relaunch_clears_the_deliberate_stop_marker +test_relaunch_clears_the_deliberate_stop_marker_when_the_backlog_commit_fails test_relaunch_refuses_before_exit_when_the_composer_holds_pending_text test_relaunch_refuses_before_exit_when_the_composer_state_is_unproven test_relaunch_from_linked_home_preserves_recorded_worktree From 1aa5133c906d39c6b241ad399bcd37f868f50cbd Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 03:48:07 +0800 Subject: [PATCH 09/12] no-mistakes(review): Anchor away-mode deliberate-stop recheck on the stop epoch --- bin/fm-supervise-daemon.sh | 19 +++++++++++++++---- tests/fm-daemon.test.sh | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index f14285b1d19..7449031881c 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -516,12 +516,23 @@ stale_marker_remove() { # # PAUSE_RESURFACE_SECS (much longer than a wedge) and re-surfaces the wait once # per window. Recording is create-if-absent so the timestamp is stable across a # churny pane (many distinct stale hashes map to one marker), keeping the cadence -# hash-immune. +# hash-immune. A deliberately stopped task's wait was declared by the stop +# itself, so its first window is anchored on the durable stop marker's mtime +# rather than on when this daemon first observed it. pause_marker_record() { # - create if absent - local win=$1 state=$2 key marker - key=$(_stale_key "$(window_to_task "$win" "$state")") + local win=$1 state=$2 key marker task stop_epoch + task=$(window_to_task "$win" "$state") + key=$(_stale_key "$task") marker="$state/.subsuper-paused-$key" - [ -e "$marker" ] || _now > "$marker" + [ -e "$marker" ] && return 0 + stop_epoch= + if fm_control_deliberate_stop_present "$state" "$task"; then + stop_epoch=$(_stat_file_mtime "$(fm_control_deliberate_stop_marker "$state" "$task")") + fi + case "$stop_epoch" in + ''|*[!0-9]*) _now > "$marker" ;; + *) printf '%s\n' "$stop_epoch" > "$marker" ;; + esac } pause_marker_remove() { # diff --git a/tests/fm-daemon.test.sh b/tests/fm-daemon.test.sh index f65b793cdc8..828fdd40d0f 100755 --- a/tests/fm-daemon.test.sh +++ b/tests/fm-daemon.test.sh @@ -1319,6 +1319,40 @@ test_housekeeping_deliberate_stop_resurfaces_on_the_pause_cadence() { pass "the away-mode daemon re-surfaces a deliberately stopped task on the bounded pause cadence" } +# The deliberate-stop first recheck must be anchored on the stop itself, not on +# when the away-mode daemon first drained the watcher's wake. Otherwise the +# daemon starts a second full PAUSE_RESURFACE_SECS window and the parked task +# first reaches the captain at roughly twice the declared-pause cadence. +test_housekeeping_deliberate_stop_first_recheck_is_anchored_on_the_stop() { + local dir state fakebin win pane key reason stop_epoch + dir=$(make_supercase deliberate-stop-first-window) + state="$dir/state"; fakebin="$dir/fakebin"; win="sess:fm-parked-w18"; pane="$dir/pane.txt" + printf 'done: investigation finished\n' > "$state/parked-w18.status" + seen_through "$state" parked-w18 + printf 'idle prompt $\n' > "$pane" + key=$(printf '%s' "parked-w18" | tr ':/.' '___') + fm_control_deliberate_stop_record "$state" parked-w18 + stop_epoch=$(( $(date +%s) - 500 )) + fm_touch_epoch "$stop_epoch" "$(fm_control_deliberate_stop_marker "$state" parked-w18)" + + # The watcher's bounded recheck wake reaches the daemon already past the first + # window, so the pause marker it records inherits that window. + reason="stale: $win (deliberately stopped 500s ago, rechecked on a long cadence not a wedge; relaunch the worker or clean up the finished task)" + LOG="$dir/daemon.log" FM_STATE_OVERRIDE="$state" handle_wake "$reason" "$state" + [ -e "$state/.subsuper-paused-$key" ] || fail "the deliberate-stop recheck wake did not record a pause marker" + + # The very next housekeeping tick must re-surface the parked task; a second + # full window here is the doubling this test guards against. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \ + FM_STATE_OVERRIDE="$state" FM_ESCALATE_BATCH_SECS=999999 FM_PAUSE_RESURFACE_SECS=240 \ + housekeeping "$state" + grep -F "deliberately stopped" "$state/.subsuper-escalations" >/dev/null 2>&1 \ + || fail "a deliberately parked task did not re-surface in its first window: $(cat "$state/.subsuper-escalations" 2>/dev/null || true)" + grep -F "possible wedge" "$state/.subsuper-escalations" >/dev/null 2>&1 \ + && fail "a deliberately parked task re-surfaced as a possible wedge" + pass "the away-mode daemon anchors a deliberate stop's first recheck on the stop itself" +} + test_housekeeping_pause_marker_transitions_to_clear() { local dir state fakebin win pane key dir=$(make_supercase paused-to-stale) @@ -2904,6 +2938,7 @@ test_housekeeping_stale_marker_transitions_to_pause test_housekeeping_captain_held_stale_marker_transitions_to_pause test_housekeeping_deliberate_stop_marker_never_wedge_escalates test_housekeeping_deliberate_stop_resurfaces_on_the_pause_cadence +test_housekeeping_deliberate_stop_first_recheck_is_anchored_on_the_stop test_housekeeping_pause_marker_transitions_to_clear test_housekeeping_herdr_persistent_stale_resolves_meta test_housekeeping_herdr_idle_busy_record_clears_stale From 675613a60407ab20e98a8767849ca2221c7d1938 Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 04:30:32 +0800 Subject: [PATCH 10/12] no-mistakes(document): Document deliberate-stop parking across watcher and away-mode docs --- .agents/skills/afk/SKILL.md | 2 +- docs/agent-control.md | 2 +- docs/architecture.md | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index 7808221917e..3907250e63a 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -149,7 +149,7 @@ The daemon still clears its buffer only on the backend's `empty` success verdict The daemon wraps `fm-watch.sh`, runs the watcher as a child, presents every durable wake after each actionable watcher close, classifies each presented record in bash, and acknowledges the presented generation only after routing completes. It self-handles the routine majority without consuming a firstmate turn. -Captain-relevant events, plus a bounded recheck of a declared external wait that is still declared, escalate to firstmate's context as one pre-read, single-line, batched digest. +Captain-relevant events, plus a bounded recheck of a declared external wait that is still declared or of a deliberately stopped parked task, escalate to firstmate's context as one pre-read, single-line, batched digest. The captain-relevant verb set, declared-wait vocabulary, status-span classifier, and presentation-marker contract live in shared `bin/fm-classify-lib.sh`, while each supervisor owns its routing and fleet scan as a consumer of that policy. While `state/.afk` exists the daemon owns the watcher, so the watcher reverts to one-shot and lets the daemon do the triage - the two never run their triage at the same time. diff --git a/docs/agent-control.md b/docs/agent-control.md index 3b5a4c751bd..64b2e6c16b4 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -37,7 +37,7 @@ A recorded `harness=` is not always an exact adapter name: a task launched from | `relaunch` | Replace the running agent with a new one in the same worktree - and the same endpoint whenever that endpoint still exists - on the exact recorded adapter or an explicitly chosen harness, model, and effort. | The new agent is alive on the endpoint the task's record now names, and that record names the harness that is actually running. | An exit that delivers lifecycle input but cannot prove the agent stopped fails with `exit=unconfirmed`, reports the observed agent state and any interrupt cancellation claim, and never claims that nothing changed. -Every verified `exit` - including the idempotent `already-stopped` outcome - records the durable deliberate-stop marker at `state/.deliberate-stop`, whose presence tells the watcher and the away-mode daemon to park the stopped task on the declared-pause recheck cadence instead of escalating its idle endpoint as a possible wedge; a refused or unconfirmed stop records nothing, and a successful `relaunch` or teardown clears it. +Every verified `exit` - including the idempotent `already-stopped` outcome - records the durable deliberate-stop marker at `state/.deliberate-stop`, whose presence tells the watcher and the away-mode daemon to park the stopped task on the declared-pause recheck cadence instead of escalating its idle or busy endpoint as a possible wedge; a refused or unconfirmed stop records nothing, and a successful `relaunch` or teardown clears it. Interrupt never rewrites busy state as proof of its own success. Claude exposes no lifecycle acknowledgement for a manual interrupt, so delivery succeeds with `cancel=unconfirmed` and its adapter-owned busy state remains as observed. muse's session log records `terminal=cancelled` for the interrupted run, so the control plane reports `cancel=confirmed` only after observing that exact acknowledgement. diff --git a/docs/architecture.md b/docs/architecture.md index 16c854dbfdd..72ea2f94b40 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ firstmate's supervisor contract and routing index for conditional procedures is ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. -Actionable wakes include captain-relevant status signals, no-verb signals without positive evidence that their crew is still executing, authenticated check output such as PR merge polling or a Relay mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS` with no wait their own worker declared, no writes to their own task worktree, and - in a home that armed `config/wedge-defer-parked-gate` - no validation gate of their own awaiting an unanswered supervisor decision, declared external waits and attended captain-held transfers that remain declared past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Actionable wakes include captain-relevant status signals, no-verb signals without positive evidence that their crew is still executing, authenticated check output such as PR merge polling or a Relay mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS` with no wait their own worker declared and no control-plane deliberate-stop marker, no writes to their own task worktree, and - in a home that armed `config/wedge-defer-parked-gate` - no validation gate of their own awaiting an unanswered supervisor decision, declared external waits, attended captain-held transfers, and deliberately stopped parked tasks that remain past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. For an ordinary crew task, a wait is read from both of its records: the status line a worker declared, and the backlog hold `bin/fm-captain-hold.sh` recorded once firstmate handed the work to the captain. So a delivered ordinary crew task whose last line stays a `done` PR-ready line bounds repeated alarms from new pane hashes to the `FM_PAUSE_RESURFACE_SECS` cadence for the length of the captain's decision. The first hash still alarms, each new hash inside that window is absorbed, and a new hash after the window re-surfaces the hold; a terminal pane hash that never changes stays inert after its first alarm exactly as it did before this bound. @@ -56,8 +56,9 @@ The report decides nothing about the record's fate, because such a lane routinel The once-marker records the agent incarnation it was reported for - the task's per-incarnation busy gen (`state/.busy-gen`, minted by `bin/fm-busy-event.sh arm`, which changes exactly when the agent is replaced) - together with the verdict, so it re-arms when that endpoint reads live again and when the agent is replaced: a successor dying in the same window is reported again even when no threshold probe reads it alive in between and its dead display hashes identically to the one already reported. When no busy incarnation token is readable for the task (it was never armed, or its sidecar is unreadable), the marker falls back to keying on the pane hash: that keeps the once-per-display absorb for a record-less task rather than re-reporting on every threshold, at the residual cost that such a successor dying into a byte-identical dead display stays absorbed. A busy pane is otherwise exempt from staleness, but only until its last completed turn or explicit native-harness progress reaches `FM_BUSY_TURN_MAX_SECS` (`bin/fm-watch.sh` owns marker selection); past that bound it is routed through the same wedge escalation, with the identical reason, escalation count, worktree-write deferral, and `demand-deep-inspection` marker for a live agent and the same dead-record report when the endpoint is proven gone, for inspection only - never an automatic interrupt, signal, or restart. -A crew that declared an external wait (`paused:`) or a verified captain-held transfer is the first exception to that bound: its busy verdict supplies liveness while identifying the long-running foreground call as the declared wait, so it takes the bounded `FM_PAUSE_RESURFACE_SECS` recheck instead of a wedge escalation, except that a captain-held transfer is not rechecked while the away-posture record exists. -In a home that armed `config/wedge-defer-parked-gate`, a crew whose own validation gate awaits the supervisor's still-open decision for that run is the second, reached through the shared wedge timer rather than the declaration branch, because who owes that answer does not depend on what the pane is rendering; it takes the same bounded recheck, including while the away-posture record exists. +A crew that declared an external wait (`paused:`) or a verified captain-held transfer is an exception to that bound: its busy verdict supplies liveness while identifying the long-running foreground call as the declared wait, so it takes the bounded `FM_PAUSE_RESURFACE_SECS` recheck instead of a wedge escalation, except that a captain-held transfer is not rechecked while the away-posture record exists. +A task the control plane deliberately stopped (`state/.deliberate-stop`, written by `bin/fm-control.sh`'s `exit` verb) is checked before that declaration branch and takes the same bounded `FM_PAUSE_RESURFACE_SECS` recheck, whether its pane reads idle or busy, because its parked state is a control-plane record rather than a status line the worker wrote. +In a home that armed `config/wedge-defer-parked-gate`, a crew whose own validation gate awaits the supervisor's still-open decision for that run is another exception, reached through the shared wedge timer rather than the declaration branch, because who owes that answer does not depend on what the pane is rendering; it takes the same bounded recheck, including while the away-posture record exists. Lifting the declaration restores the unchanged busy-pane wedge path, while a pane that is no longer busy returns to the existing idle declared-wait classification. While the legacy daemon flag is active, a busy pane that crosses the bound under a declared external wait is handed to the daemon as the plain wake identity instead of taking that recheck in the watcher, because the daemon owns triage there and a wake already decorated as a possible wedge would override the daemon's own declared-wait verdict; an undeclared busy pane past the bound still takes the wedge escalation. That handoff is keyed on the declaration itself (the status log's signature) rather than on the pane capture, so a harness footer that ticks on every poll wakes the daemon once per declaration instead of once per poll, and it clears the wedge timer, escalation count, and worktree-write deferral exactly as the normal-mode absorber does, so an undeclared busy phase's timer does not resume when the declaration lifts. @@ -98,7 +99,7 @@ A secondmate home's terminal child ledger lines, PR registrations, captain holds Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. Each `fm-wake-drain.sh` presentation runs the same liveness guard as the supervision scripts, so a lapsed watcher chain surfaces even on a turn that only handles queued wakes. Routine watcher polling, supervision no-ops, elapsed waiting time, and absorbed benign wakes stay silent. -A declared external wait or an attended verified captain-held transfer trades that silence for one bounded recheck per pause window, naming which human the wait is on; while the away-posture record exists, captain-held work waits without rechecks and remains visible in the return brief. +A declared external wait, an attended verified captain-held transfer, or a deliberately stopped parked task trades that silence for one bounded recheck per pause window, naming which human the wait is on; while the away-posture record exists, captain-held work waits without rechecks and remains visible in the return brief. Crew status files are append-only wake-event logs, not current-state fields. Because of that, a per-wake read of only the latest line can bury an earlier still-open `needs-decision`/`blocked` under later unrelated appends; `fm-wake-drain.sh` prints a separate, fleet-wide OPEN DECISIONS section on every presentation (including the empty-queue path session-start relies on), built through `fm-classify-lib.sh`'s cursor-backed incremental scan using the authoritative `status_open_decisions` fold semantics so the buried decision keeps surfacing until that fold closes it while each presentation folds only new status-log appends. The drain coordinates that fold and its annotations through a locked fleet-wide snapshot whose `.status-presentation-cursor` manifest records each status file's identity plus independent annotation and outcome-backstop byte offsets. @@ -187,11 +188,11 @@ The watcher's `.seen-*` and `.hb-surfaced-` markers and the daemon's `.sub A keyed `needs-decision` or `blocked` transition accepted by the whole-file decision fold is retired only when that fold retires it - an explicit close for its exact key, or a terminal declaration by the ship or scout that owns the log - while a reserved-key transition the fold rejects surfaces as a reconciliation signal without becoming an open decision. The fold remains the sole owner of open/closed semantics, including same-key reopening and reserved-key handling, shared with the durable OPEN DECISIONS surface. The always-on watcher also uses that library's absorb classification on no-verb signals and first-sighting stale panes before status-log terminality is trusted, while the daemon maintains distinct wedge and declared-wait recheck cadences. -The daemon's declared-wait window ages against the crew's own latest status line rather than against pane busy state, because a declared wait can legitimately hold a pane busy, and only a status append that stops declaring the wait ends that routing and restores wedge detection. +The daemon's declared-wait window ages against the crew's own latest status line rather than against pane busy state, because a declared wait can legitimately hold a pane busy, and only a status append that stops declaring the wait ends that routing and restores wedge detection; a deliberately stopped parked task shares that window but is anchored on its stop marker's mtime instead, and only clearing that marker (a relaunch or teardown) returns it to ordinary supervision. A wake already decorated as a possible wedge does not override the daemon's own declared-wait verdict either, so a declaration keeps its pane on the recheck cadence instead of the wedge cadence. In away mode, seen-status dedupe does not clear possible-wedge aging for nonterminal progress, so housekeeping still re-escalates an unchanged idle pane at the configured bound. Away-mode housekeeping has no worktree-write deferral of its own, so while `state/.afk` exists a quiet crew that is writing its own worktree still escalates as a possible wedge at that bound. -The daemon escalates captain-relevant events, plus a bounded recheck for a declared external wait that is still declared, as one batched, single-line digest using the canonical `away-supervisor` kind from `bin/fm-operational-input.sh` so firstmate can distinguish it structurally from real messages; captain-held transfers remain silent until return while the posture record exists. +The daemon escalates captain-relevant events, plus a bounded recheck for a declared external wait that is still declared or for a deliberately stopped parked task, as one batched, single-line digest using the canonical `away-supervisor` kind from `bin/fm-operational-input.sh` so firstmate can distinguish it structurally from real messages; captain-held transfers remain silent until return while the posture record exists. Its supervisor injection path supports tmux and herdr panes, with `FM_SUPERVISOR_BACKEND` and `FM_SUPERVISOR_TARGET` resolved independently from the task-spawn backend. Pane existence, busy checks, composer checks, capture, and verified submit route through `bin/fm-backend.sh`: tmux keeps the same submit core used by the tmux send backend, while herdr uses native agent-state submit confirmation on idle baselines, a composer empty fallback when native stays idle, and a pre-Enter rendered-footer transition when that baseline is unavailable. The retries-exhausted queued-Enter decision is owned by `fm_composer_queued_enter_verdict` in `bin/fm-composer-lib.sh`; tmux and herdr provide only their backend-specific busy signals. From 9e83c8209b1ce1347ce7d0c82b2aab1f679cd3eb Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 19:01:17 +0800 Subject: [PATCH 11/12] no-mistakes(test): Add real-tmux live coverage for deliberate-stop parking --- tests/fm-deliberate-stop-live-e2e.test.sh | 404 ++++++++++++++++++++++ 1 file changed, 404 insertions(+) create mode 100755 tests/fm-deliberate-stop-live-e2e.test.sh diff --git a/tests/fm-deliberate-stop-live-e2e.test.sh b/tests/fm-deliberate-stop-live-e2e.test.sh new file mode 100755 index 00000000000..3d6cfb90cb4 --- /dev/null +++ b/tests/fm-deliberate-stop-live-e2e.test.sh @@ -0,0 +1,404 @@ +#!/usr/bin/env bash +# tests/fm-deliberate-stop-live-e2e.test.sh - LIVE coverage for the durable +# deliberate-stop marker (firstmate issue #5004) against a REAL tmux server on a +# private socket. The hermetic suites (fm-control.test.sh, fm-watch-triage.test.sh, +# fm-daemon.test.sh, fm-teardown.test.sh, fm-control-relaunch.test.sh) pin the +# same contracts with a stubbed endpoint; this guard is the one place the marker's +# end-to-end behavior is driven against the real product + real tmux. +# +# Its reason to exist is the validation gap issue #5004 surfaced: a marker that +# only a stub can't see, or a watcher that parks a marker but still escalates the +# real pane, would pass every hermetic test. The scenarios below stand the real +# control plane, watcher, and endpoint up and assert on the marker file and the +# watcher's actual output. +# +# Real tmux only; no model tokens are spent, so the guard is default-on wherever +# tmux exists and self-skips otherwise (fm_live_gate). +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +fm_live_gate default-on tmux + +CONTROL="$ROOT/bin/fm-control.sh" +WATCH="$ROOT/bin/fm-watch.sh" + +REAL_TMUX=$(command -v tmux) +SOCKET="fm-deliberate-stop-live-$$" +SESSION="live" +SHIM_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-deliberate-stop-live.XXXXXX") +TMP=$(mktemp -d "${TMPDIR:-/tmp}/fm-deliberate-stop-state.XXXXXX") + +cleanup() { + "$REAL_TMUX" -L "$SOCKET" kill-server >/dev/null 2>&1 || true + rm -rf "$SHIM_DIR" "$TMP" +} +trap cleanup EXIT + +# Transparent `tmux` shim: every bare `tmux ...` the product runs targets the +# private socket, never the host's real sessions. +cat > "$SHIM_DIR/tmux" < [command] + local w=$1 cmd=${2:-"bash -c 'printf \"done: investigation finished\\n\$ \"; exec bash'"} + "$REAL_TMUX" -L "$SOCKET" new-window -d -t "$SESSION:" -n "$w" "$cmd" \ + || fail "could not create pane $w" +} + +# --------------------------------------------------------------------------- +# Scenario A/B: the real control plane records the durable marker on a verified +# stop and refuses to record one on an unprovable endpoint. +# --------------------------------------------------------------------------- +write_meta() { # + local home=$1 id=$2 window=$3 + mkdir -p "$home/state" "$home/data/$id" + printf '# brief\n' > "$home/data/$id/brief.md" + { + echo "window=$window" + echo "endpoint_task_id=$id" + echo "worktree=$TMP" + echo "project=$TMP" + echo "harness=claude" + echo "kind=ship" + } > "$home/state/$id.meta" +} + +test_control_exit_records_marker_on_a_verified_stop_and_refuses_without_one() { + local home out rc + new_pane fm-task1 + + home="$TMP/home-stop" + write_meta "$home" task1 "$SESSION:fm-task1" + out=$(FM_HOME="$home" FM_CONTROL_POLL=0.05 FM_CONTROL_SETTLE_WAIT=0.2 \ + FM_CONTROL_EXIT_WAIT=0.2 "$CONTROL" task1 exit 2>&1); rc=$? + expect_code 0 "$rc" "the verified stop should succeed"$'\n'"$out" + [ -f "$home/state/task1.deliberate-stop" ] \ + || fail "a verified stop did not record the durable deliberate-stop marker" + [ -n "$(cat "$home/state/task1.deliberate-stop")" ] \ + || fail "the deliberate-stop marker is empty" + + # An absent tmux endpoint is unprovable and must be refused: no marker. + home="$TMP/home-refuse" + new_pane fm-task2 + write_meta "$home" task2 "$SESSION:fm-task2" + # Remove the window so the endpoint is genuinely absent/unprovable. + "$REAL_TMUX" -L "$SOCKET" kill-window -t "$SESSION:fm-task2" >/dev/null 2>&1 || true + out=$(FM_HOME="$home" FM_CONTROL_POLL=0.05 FM_CONTROL_SETTLE_WAIT=0.2 \ + FM_CONTROL_EXIT_WAIT=0.2 "$CONTROL" task2 exit 2>&1); rc=$? + expect_code 1 "$rc" "an unprovable endpoint must refuse"$'\n'"$out" + [ ! -e "$home/state/task2.deliberate-stop" ] \ + || fail "a refused stop recorded a deliberate-stop marker" + pass "live: a verified tmux stop records the deliberate-stop marker; an unprovable endpoint records none" +} + +# --------------------------------------------------------------------------- +# Scenario C-F: the real watcher parks a deliberately stopped finished pane on +# the bounded recheck cadence, never the wedge ladder - idle, busy, and churning. +# --------------------------------------------------------------------------- +watch_state() { # + local name=$1 w=$2 st=$3 state="$TMP/state-$1" + mkdir -p "$state" + printf 'window=%s\nkind=ship\nbackend=tmux\nharness=pi\n' "$w" > "$state/$name.meta" + printf '%s\n' "$st" > "$state/$name.status" + # Declare the pre-existing status already seen through the production + # signature owner, so the per-poll signal scan does not fire on it. + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1/bin/fm-wake-lib.sh" + fm_wake_status_mark_current "$2" "$3" + ' _ "$ROOT" "$state" "$state/$name.status" + printf '%s\n' "$state" +} + +run_watch() { # + FM_STATE_OVERRIDE="$1" FM_HOME="$TMP/home" FM_ROOT_OVERRIDE="$TMP" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + FM_WEDGE_ALARM_EXEC=discard FM_PAUSE_RESURFACE_SECS="$2" \ + exec "$WATCH" > "$3" 2> "$4" +} + +# Run the watcher for a bounded number of cycles, then stop it and report +# whether it was still alive (an absorb keeps it blocking). +watch_absorb() { # + local pid i + run_watch "$1" "$2" "$3" "$4" & + pid=$! + local alive=1 + i=0 + while [ "$i" -lt $(( ${5} * 10 )) ]; do + kill -0 "$pid" 2>/dev/null || { alive=0; break; } + sleep 0.1 + i=$((i + 1)) + done + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + [ "$alive" -eq 1 ] +} + +# Wait for the watcher to exit on a wake, bounded. +watch_until_exit() { # + local pid i + run_watch "$1" "$2" "$3" "$4" & + pid=$! + i=0 + while [ "$i" -lt "$5" ]; do + kill -0 "$pid" 2>/dev/null || { wait "$pid" 2>/dev/null || true; return 0; } + sleep 0.1 + i=$((i + 1)) + done + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + return 1 +} + +KEY_STOP="live_fm-live-stop" +KEY_BUSY="live_fm-live-busy" +KEY_CHURN="live_fm-live-churn" + +test_watcher_parks_a_deliberately_stopped_idle_pane() { + local state out err + new_pane fm-live-stop + state=$(watch_state live-stop "$SESSION:fm-live-stop" "done: investigation finished") + printf '%s\n' "$(date +%s)" > "$state/live-stop.deliberate-stop" + + # Fresh stop: absorbed, no wake, no wedge timer. + out="$TMP/a.out"; err="$TMP/a.err" + watch_absorb "$state" 999 "$out" "$err" 6 \ + || fail "the watcher exited for a freshly parked finished task (should absorb): $(cat "$out")" + [ ! -s "$out" ] || fail "a freshly parked finished task printed a wake during absorb" + [ ! -e "$state/.stale-since-$KEY_STOP" ] || fail "a fresh deliberate-stop absorb started the wedge timer" + + # Past the cadence: re-surfaces once as a bounded recheck, never a wedge. + rm -f "$state/.deliberate-stop-resurfaced-$KEY_STOP" "$state/.stale-$KEY_STOP" \ + "$state/.stale-since-$KEY_STOP" "$state/.watcher-down" "$state/.wake-queue" \ + "$state/.wake-queue.seq" "$state/.watch-deliveries.log" + touch -d "@$(( $(date +%s) - 500 ))" "$state/live-stop.deliberate-stop" + out="$TMP/b.out"; err="$TMP/b.err" + watch_until_exit "$state" 240 "$out" "$err" 200 \ + || fail "the watcher did not re-surface a parked task past the cadence" + grep -F "deliberately stopped" "$out" >/dev/null \ + || fail "the re-surface was not labeled a deliberate-stop recheck: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null \ + && fail "a deliberately parked task was mislabeled a possible wedge: $(cat "$out")" + [ ! -e "$state/.stale-since-$KEY_STOP" ] || fail "a deliberate-stop recheck used the wedge timer" + pass "live: the watcher absorbs a fresh deliberate stop and re-surfaces it on the bounded cadence" +} + +test_watcher_parks_a_deliberately_stopped_busy_pane() { + local state out err gen + new_pane fm-live-busy "bash -c 'printf \"Working... (7200.4s)\\n\"; exec bash'" + state=$(watch_state live-busy "$SESSION:fm-live-busy" "done: investigation finished") + # No completed turn: age the spawn record past the busy-turn bound, and record + # a real busy incarnation so the semantic verdict is busy. + touch -t 200001010000 "$state/live-busy.meta" + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$state" live-busy) + "$ROOT/bin/fm-busy-event.sh" apply "$state" live-busy busy --gen "$gen" \ + --source pi-ext --event agent-start + printf '%s\n' "$(date +%s)" > "$state/live-busy.deliberate-stop" + + out="$TMP/c.out"; err="$TMP/c.err" + watch_absorb "$state" 999 "$out" "$err" 8 \ + || fail "a deliberately stopped busy pane was escalated: $(cat "$out")" + [ ! -s "$out" ] || fail "a deliberately stopped busy pane printed a wake: $(cat "$out")" + [ ! -e "$state/.stale-since-$KEY_BUSY" ] || fail "a deliberately stopped busy pane started the wedge timer" + [ ! -e "$state/.wedge-escalations-$KEY_BUSY" ] || fail "a deliberately stopped busy pane incremented the escalation counter" + + # Past the cadence the parked busy pane still re-surfaces on the pause cadence. + rm -f "$state/.deliberate-stop-resurfaced-$KEY_BUSY" "$state/.stale-$KEY_BUSY" \ + "$state/.stale-since-$KEY_BUSY" "$state/.watcher-down" "$state/.wake-queue" \ + "$state/.wake-queue.seq" "$state/.watch-deliveries.log" + touch -d "@$(( $(date +%s) - 500 ))" "$state/live-busy.deliberate-stop" + out="$TMP/d.out"; err="$TMP/d.err" + watch_until_exit "$state" 240 "$out" "$err" 200 \ + || fail "a deliberately stopped busy pane did not re-surface past the cadence" + grep -F "deliberately stopped" "$out" >/dev/null \ + || fail "the busy recheck was not labeled a deliberate-stop recheck: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null \ + && fail "a deliberately stopped busy pane was mislabeled a possible wedge: $(cat "$out")" + pass "live: the watcher parks a deliberately stopped busy pane on the bounded recheck cadence" +} + +test_watcher_rechecks_a_deliberately_stopped_churning_pane() { + local state out err + cat > "$TMP/churn.sh" <<'CHURN' +#!/usr/bin/env bash +i=0 +while :; do + i=$((i + 1)) + printf 'finished, footer tick %s\n' "$i" + sleep 0.1 +done +CHURN + chmod +x "$TMP/churn.sh" + new_pane fm-live-churn "bash -c 'exec \"$TMP/churn.sh\"'" + state=$(watch_state live-churn "$SESSION:fm-live-churn" "done: investigation finished") + printf '%s\n' "$(date +%s)" > "$state/live-churn.deliberate-stop" + touch -d "@$(( $(date +%s) - 500 ))" "$state/live-churn.deliberate-stop" + + out="$TMP/e.out"; err="$TMP/e.err" + watch_until_exit "$state" 240 "$out" "$err" 200 \ + || fail "a churning deliberately parked task was never rechecked: $(cat "$out")" + grep -F "deliberately stopped" "$out" >/dev/null \ + || fail "the churning recheck was not labeled a deliberate-stop recheck: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null \ + && fail "a churning deliberately parked task was mislabeled a possible wedge: $(cat "$out")" + [ ! -e "$state/.stale-since-$KEY_CHURN" ] || fail "a churning deliberate-stop recheck used the wedge timer" + pass "live: a churning deliberately parked pane still gets the bounded deliberate-stop recheck" +} + +test_watcher_still_surfaces_a_markerless_finished_pane() { + local state out err + new_pane fm-live-nomarker + state=$(watch_state live-nomarker "$SESSION:fm-live-nomarker" "done: investigation finished") + # No deliberate-stop marker: the same finished idle pane must still take the + # ordinary terminal-stale path, so the marker is what parks it. + out="$TMP/f.out"; err="$TMP/f.err" + watch_until_exit "$state" 240 "$out" "$err" 200 \ + || fail "the watcher did not surface a marker-less finished task as terminal stale" + grep -Fx "stale: $SESSION:fm-live-nomarker" "$out" >/dev/null \ + || fail "a marker-less finished task did not surface as an ordinary terminal stale: $(cat "$out")" + pass "live: removing the marker returns the finished task to ordinary terminal-stale supervision" +} + +# --------------------------------------------------------------------------- +# Scenario G: teardown retires the marker for a real task, so no stale marker +# survives the task it parked. +# --------------------------------------------------------------------------- +test_teardown_clears_the_deliberate_stop_marker() { + local c fb out rc + new_pane fm-task-x1 + c="$TMP/td-case"; fb="$c/fakebin" + mkdir -p "$c/state" "$c/config" "$c/data/task-x1" "$fb" + # Stub only the external providers (worktree pool + forge); tmux and git are real. + cat > "$fb/treehouse" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + cat > "$fb/gh-axi" <<'SH' +#!/usr/bin/env bash +case "${1:-} ${2:-}" in + "pr list") printf 'count: 0 (showing first 0)\npull_requests[]: []\n'; exit 0 ;; + "pr view") echo "error: pull request not found" >&2; exit 1 ;; +esac +exit 0 +SH + cat > "$fb/gh" <<'SH' +#!/usr/bin/env bash +case "${1:-} ${2:-}" in "pr view") echo "error: pull request not found" >&2; exit 1 ;; esac +exit 0 +SH + cat > "$fb/no-mistakes" <<'SH' +#!/usr/bin/env bash +case "${1:-}" in + axi) shift; [ "${1:-}" = status ] && printf ''; exit 0 ;; + runs) exit 0 ;; +esac +exit 0 +SH + chmod +x "$fb"/* + + git init -q --bare "$c/origin.git" + git -C "$c/origin.git" symbolic-ref HEAD refs/heads/main + git clone -q "$c/origin.git" "$c/_seed" 2>/dev/null + git -C "$c/_seed" -c user.email=t@t -c user.name=t commit -q --allow-empty -m baseline + git -C "$c/_seed" push -q origin main + rm -rf "$c/_seed" + git clone -q "$c/origin.git" "$c/project" + git -C "$c/project" remote set-head origin main 2>/dev/null || true + git -C "$c/project" worktree add -q -b fm/task-x1 "$c/wt" main + touch "$c/state/.last-watcher-beat" + cat > "$c/state/task-x1.meta" < "$c/state/task-x1.deliberate-stop" + + out=$(FM_ROOT_OVERRIDE="$ROOT" FM_STATE_OVERRIDE="$c/state" \ + FM_DATA_OVERRIDE="$c/data" FM_CONFIG_OVERRIDE="$c/config" \ + PATH="$SHIM_DIR:$fb:$PATH" "$ROOT/bin/fm-teardown.sh" task-x1 --force 2>&1); rc=$? + expect_code 0 "$rc" "the forced teardown should complete"$'\n'"$out" + [ ! -e "$c/state/task-x1.deliberate-stop" ] \ + || fail "teardown left the deliberate-stop marker behind" + [ ! -e "$c/state/task-x1.meta" ] || fail "teardown left the task record behind" + pass "live: teardown retires the deliberate-stop marker for a real task" +} + +# --------------------------------------------------------------------------- +# Scenario H: a real relaunch replaces the stopped worker and clears the marker; +# a relaunch whose replacement cannot be launched retains the parked stop, so a +# still-stopped task is never misclassified as a running replacement. +# --------------------------------------------------------------------------- +relaunch_case() { # -> echoes on two lines + local id=$1 body=$2 c + c="$TMP/rl-$id" + mkdir -p "$c/home/state" "$c/home/data/$id" "$c/user-home" + git init -q "$c/proj" + git -C "$c/proj" -c user.email=t@t -c user.name=t commit -q --allow-empty -m init + git -C "$c/proj" worktree add -q -b "task-$id" "$c/wt" + printf '%s' "$body" > "$c/home/data/$id/brief.md" + { + echo "window=$SESSION:fm-$id" + echo "endpoint_task_id=$id" + echo "worktree=$c/wt" + echo "project=$c/proj" + echo "harness=pi" + echo "kind=ship" + echo "mode=local-only" + echo "yolo=off" + echo "model=default" + echo "effort=default" + } > "$c/home/state/$id.meta" + printf '%s\n' "$(date +%s)" > "$c/home/state/$id.deliberate-stop" + "$REAL_TMUX" -L "$SOCKET" new-window -d -t "$SESSION:" -c "$c/wt" -n "fm-$id" \ + "bash -c 'exec bash'" || fail "could not create relaunch pane fm-$id" + printf '%s\n%s\n' "$c/home" "$c/wt" +} + +test_relaunch_clears_on_success_and_retains_on_abort() { + command -v pi >/dev/null 2>&1 || return 0 + local home out rc + + # Success: the replacement launches in the same worktree and the marker clears. + home=$(relaunch_case rl1 $'# Task\n## Captain\'s intent\nExercise relaunch live and confirm the replacement is supervised normally.\n\n## Firstmate spec\nReplace the stopped agent in the same endpoint and worktree.\n' | sed -n 1p) + out=$(env -u HERDR_ENV FM_SPAWN_NO_GUARD=1 HOME="$TMP/rl-rl1/user-home" \ + FM_HOME="$home" FM_CONTROL_POLL=0.1 FM_CONTROL_EXIT_WAIT=1 FM_CONTROL_LAUNCH_WAIT=3 \ + timeout 90 "$CONTROL" rl1 relaunch --note "resume live test" 2>&1); rc=$? + expect_code 0 "$rc" "the live relaunch should succeed"$'\n'"$out" + [ ! -e "$home/state/rl1.deliberate-stop" ] \ + || fail "a delivered relaunch did not clear the deliberate-stop marker" + + # Abort: a brief the launch owner refuses stops the worker but delivers no + # replacement, so the parked stop must survive. + home=$(relaunch_case rl2 $'# Task\n## Captain\'s intent\nNo Firstmate spec subsection here.\n' | sed -n 1p) + out=$(env -u HERDR_ENV FM_SPAWN_NO_GUARD=1 HOME="$TMP/rl-rl2/user-home" \ + FM_HOME="$home" FM_CONTROL_POLL=0.1 FM_CONTROL_EXIT_WAIT=1 FM_CONTROL_LAUNCH_WAIT=3 \ + timeout 90 "$CONTROL" rl2 relaunch --note "resume live test" 2>&1); rc=$? + expect_code 1 "$rc" "a refused replacement launch should fail the relaunch"$'\n'"$out" + [ -e "$home/state/rl2.deliberate-stop" ] \ + || fail "an aborted relaunch cleared the parked deliberate-stop marker" + pass "live: a real relaunch clears the marker and an aborted relaunch retains the parked stop" +} + +test_control_exit_records_marker_on_a_verified_stop_and_refuses_without_one +test_watcher_parks_a_deliberately_stopped_idle_pane +test_watcher_parks_a_deliberately_stopped_busy_pane +test_watcher_rechecks_a_deliberately_stopped_churning_pane +test_watcher_still_surfaces_a_markerless_finished_pane +test_teardown_clears_the_deliberate_stop_marker +test_relaunch_clears_on_success_and_retains_on_abort From 8bfac280a1303904e8432fe851d345264ee73d8e Mon Sep 17 00:00:00 2001 From: sdivanl Date: Mon, 21 Sep 2026 19:44:25 +0800 Subject: [PATCH 12/12] no-mistakes(document): Correct deliberate-stop away-mode anchor and recheck wording --- docs/architecture.md | 2 +- docs/configuration.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 72ea2f94b40..d90b4d54d45 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -99,7 +99,7 @@ A secondmate home's terminal child ledger lines, PR registrations, captain holds Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. Each `fm-wake-drain.sh` presentation runs the same liveness guard as the supervision scripts, so a lapsed watcher chain surfaces even on a turn that only handles queued wakes. Routine watcher polling, supervision no-ops, elapsed waiting time, and absorbed benign wakes stay silent. -A declared external wait, an attended verified captain-held transfer, or a deliberately stopped parked task trades that silence for one bounded recheck per pause window, naming which human the wait is on; while the away-posture record exists, captain-held work waits without rechecks and remains visible in the return brief. +A declared external wait or an attended verified captain-held transfer trades that silence for one bounded recheck per pause window, naming which human the wait is on, and a deliberately stopped parked task trades it for the same bounded recheck, asking firstmate to relaunch the worker or clean up the finished task; while the away-posture record exists, captain-held work waits without rechecks and remains visible in the return brief. Crew status files are append-only wake-event logs, not current-state fields. Because of that, a per-wake read of only the latest line can bury an earlier still-open `needs-decision`/`blocked` under later unrelated appends; `fm-wake-drain.sh` prints a separate, fleet-wide OPEN DECISIONS section on every presentation (including the empty-queue path session-start relies on), built through `fm-classify-lib.sh`'s cursor-backed incremental scan using the authoritative `status_open_decisions` fold semantics so the buried decision keeps surfacing until that fold closes it while each presentation folds only new status-log appends. The drain coordinates that fold and its annotations through a locked fleet-wide snapshot whose `.status-presentation-cursor` manifest records each status file's identity plus independent annotation and outcome-backstop byte offsets. diff --git a/docs/configuration.md b/docs/configuration.md index f725c41a92a..7b4942c74f4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1173,7 +1173,7 @@ FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|read FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates, unless that pane's own worker declared a wait that has not elapsed, or, where config/wedge-defer-parked-gate arms it, that pane's crew is parked at a validation gate awaiting the supervisor's decision on it that the crew raised under that run's key and nobody has answered yet, or the task carries the durable deliberate-stop marker (`state/.deliberate-stop`, written by `bin/fm-control.sh exit`), any of which takes the FM_PAUSE_RESURFACE_SECS recheck below instead; stale panes whose crew is not provably working surface immediately unless admitted directly to the declared-wait cadence, while a live idle declared wait still surfaces once before that cadence bounds repeats; at that same escalation moment a recovery-grade agent-state probe (docs/architecture.md owns that dead-record contract) reports a pane whose endpoint is proven `dead` or `missing` once and stops re-escalating it while it stays that way FM_BUSY_TURN_MAX_SECS=3600 # maximum age without a completed turn or explicit native-harness progress (bin/fm-watch.sh owns marker selection), before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a declared external wait, a deliberately stopped parked task (`state/.deliberate-stop`), an attended verified captain-held transfer, or - where config/wedge-defer-parked-gate arms it - a validation gate of the crew's own awaiting the supervisor's still-unanswered decision takes the FM_PAUSE_RESURFACE_SECS recheck below instead -FM_PAUSE_RESURFACE_SECS=14400 # four hours between bounded rechecks of a declared external wait, a verified captain-held transfer, or a deliberately stopped parked task (`state/.deliberate-stop`), and between repeated new-hash stale alarms for an ordinary crew task with an open backlog captain call; a structured until time can make an external-wait recheck occur sooner but cannot extend this bound; this includes a live idle pane after its first inconclusive stale wake, a provably-working pane whose own unelapsed declared wait or, where config/wedge-defer-parked-gate arms it, unanswered supervisor-owed validation gate defers its FM_STALE_ESCALATE_SECS escalation, and a live busy pane past FM_BUSY_TURN_MAX_SECS, while the away-mode daemon uses the same setting and ages its window against the crew's own latest status line rather than pane busy state; a captain-held transfer is never rechecked while the away-posture record exists, while an armed validation gate awaiting the supervisor's decision keeps this recheck in either posture +FM_PAUSE_RESURFACE_SECS=14400 # four hours between bounded rechecks of a declared external wait, a verified captain-held transfer, or a deliberately stopped parked task (`state/.deliberate-stop`), and between repeated new-hash stale alarms for an ordinary crew task with an open backlog captain call; a structured until time can make an external-wait recheck occur sooner but cannot extend this bound; this includes a live idle pane after its first inconclusive stale wake, a provably-working pane whose own unelapsed declared wait or, where config/wedge-defer-parked-gate arms it, unanswered supervisor-owed validation gate defers its FM_STALE_ESCALATE_SECS escalation, and a live busy pane past FM_BUSY_TURN_MAX_SECS, while the away-mode daemon uses the same setting and ages its window against the crew's own latest status line rather than pane busy state, except that a deliberately stopped parked task is anchored on its stop marker's mtime; a captain-held transfer is never rechecked while the away-posture record exists, while an armed validation gate awaiting the supervisor's decision keeps this recheck in either posture FM_SECONDMATE_WAKE_STALL_SECS=180 # minimum interval with no change of the oldest actionable foreign wake-queue row (it advances as the mate drains, and a queue reprovisioned under the same task id starts a fresh interval at whatever sequence it restarts) before an endpoint-recorded local secondmate produces one durable parent wake-loop-stall notification for that no-progress episode; a mate that is provably inside an active turn (an exact busy verdict) does not escalate until that same no-progress interval reaches FM_BUSY_TURN_MAX_SECS above, declared external-wait pause rows are excluded, and zero or invalid values use 180 FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WORKTREE_WRITE_PRUNE='.git node_modules .venv venv __pycache__ .mypy_cache .pytest_cache .ruff_cache .tox target dist build .next .cache vendor' # directory names the wedge detector's task-worktree write probe skips; the default keeps .git out so a supervisor's own read-only git command can never look like crew progress; set it to the empty string to prune nothing, which widens the probe to the whole depth-bounded tree rather than disabling it