fix(daemon): stop away-mode daemon wedging on its own busy pane

.agents/skills/afk/SKILL.md

@@ -92,7 +92,8 @@ The daemon never injects into an in-use pane. Two checks run before every
 injection, dispatched through `bin/fm-backend.sh` for the supervisor's own
 backend (tmux or herdr; see "Auto-discovered supervisor pane" below):
 
-- **Primary-pane busy guard** - `pane_is_busy` trusts Herdr native `busy` when available, otherwise matches rendered output against only the detected primary harness's signature.
+- **Primary-pane busy guard** - `pane_is_busy` requires the rendered output to match the detected primary harness's busy signature; a native `busy` verdict (e.g. Herdr's `agent_status`) is corroborating evidence, not sufficient alone, because a live tracked background shell (such as this daemon's own launch) can pin native state to `working` long after the pane is actually idle.
+  When the pane cannot be captured at all, `pane_is_busy` falls back to the native verdict rather than guessing idle.
   This narrow delivery guard never classifies a recorded worker task and never uses a global union of vendor patterns.
 - **Composer-state guard** - `inject_msg` reads the full `empty`/`pending`/`pending-unproven`/`unknown` verdict from `fm_backend_composer_state` and injects only when it is affirmatively `empty`.
   Every other or future verdict defers, including an unreadable pane, ambiguous geometry, a blank unidentified row, and a bare shell prompt left after the agent exits.
@@ -105,12 +106,17 @@ In afk mode the composer guard is belt-and-suspenders (no human is typing), but
 
 **Max-defer escape (the daemon must never silently wedge).**
 If anything stays buffered past `FM_MAX_DEFER_SECS` (default 300), the daemon
-attempts one normal flush, which still requires an idle pane and an affirmatively empty composer.
+attempts one FORCED flush that drops only the busy guard; the composer-empty guard
+still applies unconditionally, since that is what actually protects against merging
+with a human's half-typed line.
+This bounds worst-case silent non-delivery to `FM_MAX_DEFER_SECS` instead of retrying
+the identical blocked call forever.
 The alarm is defense in depth rather than a substitute for keeping every genuinely idle supported composer injectable.
-If that submit cannot be confirmed, it raises a loud, rate-limited wedge alarm:
+If that forced submit still cannot be confirmed, it raises a loud, rate-limited wedge alarm:
 an ERROR in the daemon log, a durable
 `state/.subsuper-inject-wedged` marker (surface it on the "while you were out"
-catch-up if present), a tmux status-line flash when applicable, and a configurable backend-independent active alert.
+catch-up if present), a tmux status-line flash when applicable, and a configurable backend-independent active alert
+carrying the buffered item count and first line so the banner is actionable rather than a bare age and marker path.
 `docs/wedge-alarm.md` owns the alert channel setup, and `docs/verification/supervision.md` "Wedge-alarm channels" owns active evidence.
 So a guard false-positive becomes a visible stall, never an unbounded silent no-op.
 
@@ -181,10 +187,13 @@ the operational prefix lets firstmate distinguish it from a real captain message
   A blank or otherwise unidentified input row carries no positive container proof and defers injection, so a modal dialog or a mid-redraw pane is never an injection target.
 - **Max-defer escape** - the daemon must never silently wedge. If anything stays
   buffered past `FM_MAX_DEFER_SECS` (default 300s), the daemon attempts one
-  normal flush, which still requires an idle pane and an affirmatively empty composer. If that
+  FORCED flush that drops only the busy guard, keeping the composer-empty guard
+  unconditionally, bounding worst-case silent non-delivery to `FM_MAX_DEFER_SECS`
+  instead of retrying the identical blocked call forever. If that
   cannot confirm a submit, it raises a loud, rate-limited wedge alarm: ERROR log,
   durable `state/.subsuper-inject-wedged` marker, a tmux status-line flash when
-  applicable, and a backend-independent active alert. A
+  applicable, and a backend-independent active alert carrying the buffered item
+  count and first line. A
   composer false-positive surfaces as a visible stall, never an unbounded silent
   no-op.
 - **Verified type-once submit model** - the digest is typed once (`send-keys -l`

bin/fm-afk-launch.sh

@@ -571,7 +571,7 @@ fm_afk_launch_start_native() {
 }
 
 fm_afk_launch_stop() {
-  local pid pid_identity current_identity result=0 read_result
+  local pid pid_identity current_identity result=0 read_result stop_wait_iters
   fm_afk_launch_record_read
   read_result=$?
   if [ "$read_result" -eq 2 ]; then
@@ -592,7 +592,26 @@ fm_afk_launch_stop() {
       fm_afk_launch_log "failed to signal away-mode daemon pid=$pid"
       result=1
     fi
-    for _ in $(seq 1 40); do
+    # fm-afk-inject-wedge Fix 3: the daemon's cleanup trap flushes, then kills
+    # and waits on its watcher child (bin/fm-watch.sh) before exiting. That
+    # watcher traps TERM but spends most of its life in a foreground
+    # `sleep "${FM_POLL:-15}"`, and bash defers a trapped signal until the
+    # foreground command returns, so shutdown can legitimately take up to
+    # FM_POLL seconds plus the flush. This wait must stay strictly greater
+    # than that budget or a watcher caught mid-sleep reports a false "did not
+    # exit after SIGTERM" and leaves lifecycle state preserved for no reason.
+    # FM_POLL is not always an integer (tests use fractional values like 0.1),
+    # and bash arithmetic expansion errors on those, so compute the iteration
+    # count (rounded up, at 0.25s per iteration) with awk instead.
+    stop_wait_iters=$(awk -v poll="${FM_POLL:-15}" 'BEGIN {
+      p = poll + 0
+      if (p < 0) p = 0
+      n = (p + 10) * 4
+      i = int(n)
+      if (i < n) i++
+      print i
+    }')
+    for _ in $(seq 1 "$stop_wait_iters"); do
       fm_pid_alive "$pid" || break
       sleep 0.25
     done

bin/fm-supervise-daemon.sh

@@ -592,14 +592,26 @@ fm_daemon_primary_harness() {
   printf '%s' "$FM_DAEMON_PRIMARY_HARNESS"
 }
 
+# A live tracked background shell (e.g. this daemon's own `run_in_background`
+# launch, per the /afk skill's no-separate-terminal exception) pins Herdr's
+# native agent_status to `working` for as long as the shell is alive, even
+# once the turn has actually ended and the pane sits idle at an empty prompt
+# (fm-afk-inject-wedge: proven live, 34 wedge-free hours with the daemon
+# launched OUTSIDE the pane vs. thousands of false "busy" deferrals with it
+# launched inside). A native "busy" verdict is therefore NOT sufficient proof
+# on its own; require the harness-scoped rendered busy footer (the same
+# fm_busy_lines_match signature fm_backend_herdr_send_text_submit already uses
+# to confirm a queued-while-busy submit) to corroborate before treating the
+# pane as unavailable. A native verdict of idle/unknown was never trusted
+# alone either, so this only changes the busy case. If the pane cannot be
+# captured at all, fall back to the native verdict rather than guessing idle:
+# a false "busy" costs one deferred cycle, a false "idle" risks injecting into
+# an unreadable pane.
 pane_is_busy() {  # <target> [backend]
   local target=$1 backend=${2:-tmux} native tail40 harness
   harness=$(fm_daemon_primary_harness)
   native=$(fm_backend_busy_state "$backend" "$target" 2>/dev/null)
-  case "$native" in
-    busy) return 0 ;;
-  esac
-  tail40=$(fm_backend_capture "$backend" "$target" 40 2>/dev/null) || return 1
+  tail40=$(fm_backend_capture "$backend" "$target" 40 2>/dev/null) || { [ "$native" = busy ]; return $?; }
   printf '%s' "$tail40" | grep -v '^[[:space:]]*$' | tail -12 \
     | fm_busy_lines_match "$harness"
 }
@@ -651,9 +663,10 @@ escalate_add() {  # <state> <distilled-item>
 
 # Flush the escalation buffer as ONE batched, single-line digest to the
 # supervisor pane. Returns 0 on successful inject (or empty buffer), non-zero on
-# inject failure (buffer preserved for retry / catch-up).
-escalate_flush() {  # <state>
-  local state=$1 buf item n msg
+# inject failure (buffer preserved for retry / catch-up). <force>=1 passes
+# through to inject_msg's forced mode (max-defer escape only, see inject_msg).
+escalate_flush() {  # <state> [force]
+  local state=$1 force="${2:-0}" buf item n msg
   buf="$state/.subsuper-escalations"
   [ -s "$buf" ] || return 0
   n=$(wc -l < "$buf" 2>/dev/null || echo 0)
@@ -662,7 +675,7 @@ escalate_flush() {  # <state>
   # Single-line wrapper: no embedded newlines (inject_msg also collapses as a
   # safety net, but keeping the source single-line makes the intent explicit).
   msg=$(printf 'Supervisor escalate (%s event(s)): %s (pre-read; re-arm not needed — watcher daemon-managed)' "$n" "$msg")
-  if inject_msg "$msg" "$state"; then : > "$buf"; rm -f "${buf}.since" "$state/.subsuper-inject-wedged"; return 0; fi
+  if inject_msg "$msg" "$state" "$force"; then : > "$buf"; rm -f "${buf}.since" "$state/.subsuper-inject-wedged"; return 0; fi
   return 1
 }
 
@@ -905,7 +918,7 @@ wedge_alarm_notify() {  # <summary> <marker>
 # is lost - the buffer and the
 # wake-queue both survive - but the stall stops being invisible.
 inject_wedge_alarm() {  # <state> <age-seconds>
-  local state=$1 age=$2 marker target backend max_defer now notify=1
+  local state=$1 age=$2 marker target backend max_defer now notify=1 n first_item summary
   marker="$state/.subsuper-inject-wedged"
   max_defer="${FM_MAX_DEFER_SECS:-$MAX_DEFER_SECS_DEFAULT}"
   # Re-alarm at most once per max-defer window so a long wedge does not spam.
@@ -939,7 +952,14 @@ inject_wedge_alarm() {  # <state> <age-seconds>
   # incident fell through. Configurable and best-effort; the marker above stays
   # the durable record whether or not any channel fires.
   if [ "$notify" -eq 1 ]; then
-    wedge_alarm_notify "away-mode escalations WEDGED ${age}s undelivered - see $marker" "$marker"
+    # fm-afk-inject-wedge Fix 2: a bare age+path told the captain something was
+    # stuck but never what, so 47 identical banners in one 4h episode read as
+    # noise rather than an alert. Carry the item count and first buffered line
+    # so one banner is actually informative.
+    n=$(wc -l < "$state/.subsuper-escalations" 2>/dev/null || echo 0)
+    first_item=$(head -n1 "$state/.subsuper-escalations" 2>/dev/null)
+    summary="away-mode escalations WEDGED ${age}s undelivered (${n} item(s)): ${first_item} - see $marker"
+    wedge_alarm_notify "$summary" "$marker"
   fi
 }
 
@@ -985,8 +1005,12 @@ housekeeping() {  # <state>
   fi
 
   # (1b) max-defer escape. If anything is still buffered past MAX_DEFER_SECS,
-  # retry the normal delivery path. If that still cannot confirm, raise a loud
-  # wedge alarm while preserving the buffer.
+  # FORCE the delivery path (fm-afk-inject-wedge Fix 2): this is the one call
+  # site that drops the busy guard (the composer-empty guard still applies),
+  # so unbounded silent non-delivery becomes a bounded worst case of
+  # MAX_DEFER_SECS instead of retrying the identical blocked call forever. If
+  # that still cannot confirm a submit, raise a loud wedge alarm while
+  # preserving the buffer.
   max_defer=${FM_MAX_DEFER_SECS:-$MAX_DEFER_SECS_DEFAULT}
   if afk_active "$state" && [ "$max_defer" -gt 0 ] && [ -s "$state/.subsuper-escalations" ]; then
     oldest=$(_oldest_line_age "$state/.subsuper-escalations")
@@ -995,8 +1019,8 @@ housekeeping() {  # <state>
     # and waits.
     if [ "$oldest" -ge "$max_defer" ] \
        && [ "$(_file_age "$state/.subsuper-inject-wedged")" -ge "$max_defer" ]; then
-      if escalate_flush "$state"; then
-        log "inject recovered: max-defer flush succeeded after ${oldest}s undelivered"
+      if escalate_flush "$state" 1; then
+        log "inject recovered: max-defer forced flush succeeded after ${oldest}s undelivered"
         rm -f "$state/.subsuper-inject-wedged"
       else
         inject_wedge_alarm "$state" "$oldest"
@@ -1141,9 +1165,10 @@ window_for_task() {  # <task-key> [state]
 #     after dim/faint ghost text and borders are ignored (a human's half-typed
 #     line, or a previous injection's unsent text), defer entirely - injecting
 #     would merge with the human's text.
-inject_msg() {  # <message> [state]
-  local msg=$1 state target backend retries sleep_s verdict composer encoded
+inject_msg() {  # <message> [state] [force]
+  local msg=$1 state force target backend retries sleep_s verdict composer encoded
   state="${2:-$(_state_root)}"
+  force="${3:-0}"
   # (1) Presence-gate: inject ONLY when afk is active. When afk is off, the
   # daemon self-handles and stays quiet; firstmate drives the normal always-on
   # watcher triage. Escalations buffer and survive for the next catch-up flush.
@@ -1163,8 +1188,14 @@ inject_msg() {  # <message> [state]
   # discovery), matching this function's pre-existing default assumption.
   backend="${FM_SUPERVISOR_BACKEND:-tmux}"
   fm_backend_target_exists "$backend" "$target" || return 1
-  # (3) Busy-guard: never inject into an in-use supervisor pane.
-  if pane_is_busy "$target" "$backend"; then
+  # (3) Busy-guard: never inject into an in-use supervisor pane, UNLESS this is
+  # the max-defer forced escape (fm-afk-inject-wedge Fix 2). The composer-empty
+  # guard below still applies unconditionally - it is what actually protects
+  # against merging with a human's half-typed line - so a forced call can still
+  # queue behind a real turn (fm_backend_send_text_submit already confirms a
+  # queued-while-busy submit) without risking a corrupted composer. This turns
+  # unbounded silent non-delivery into a bounded worst case of FM_MAX_DEFER_SECS.
+  if [ "$force" != 1 ] && pane_is_busy "$target" "$backend"; then
     log "inject deferred: supervisor pane busy (agent mid-turn)"
     return 1
   fi

tests/fm-afk-launch.test.sh

@@ -732,6 +732,70 @@ unit_stop_confirms_daemon_exit() {
   rm -rf "$st"
 }
 
+# fm-afk-inject-wedge Fix 3: the watcher child the daemon's cleanup() waits on
+# traps TERM but can sit in a foreground `sleep FM_POLL` (15s default), so
+# shutdown can legitimately take up to FM_POLL seconds plus the flush. The stop
+# wait must stay strictly greater than FM_POLL, not the old fixed 10s budget
+# that reported a false "did not exit after SIGTERM" whenever the watcher
+# happened to be mid-sleep.
+unit_stop_wait_exceeds_configured_poll() {
+  local st daemon_pid seq_arg
+  st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-stop-poll-budget.XXXXXX")
+  mkdir -p "$st/state/.supervise-daemon.lock"
+  : > "$st/state/.afk"
+  printf 'none\t-\tnative\n' > "$st/state/.afk-daemon-terminal"
+  bash -c 'trap "" TERM; while :; do sleep 1; done' &
+  daemon_pid=$!
+  printf '%s' "$daemon_pid" > "$st/state/.supervise-daemon.lock/pid"
+  ( . "$ROOT/bin/fm-wake-lib.sh"; fm_pid_identity "$daemon_pid" > "$st/state/.supervise-daemon.lock/pid-identity" )
+  seq_arg="$st/seq-arg"
+  SEQ_ARG="$seq_arg" FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" FM_POLL=30 bash -c '
+    . "$1"
+    seq() { printf "%s" "$2" > "$SEQ_ARG"; printf "1\n"; }
+    sleep() { :; }
+    kill() { command kill "$@"; }
+    ! fm_afk_launch_stop
+  ' _ "$LAUNCH" >/dev/null 2>&1 || true
+  if [ -e "$seq_arg" ] && [ "$(cat "$seq_arg")" -gt $(( (30 + 10) * 4 - 1 )) ]; then
+    pass "stop wait: iteration budget exceeds FM_POLL plus margin"
+  else
+    fail "stop wait: iteration budget did not scale with FM_POLL (got $(cat "$seq_arg" 2>/dev/null))"
+  fi
+  kill -KILL "$daemon_pid" 2>/dev/null || true
+  wait "$daemon_pid" 2>/dev/null || true
+  rm -rf "$st"
+}
+
+# A fractional FM_POLL (tests use 0.1, 0.02) must not break the stop wait: bash
+# arithmetic expansion errors on a non-integer operand, which would silently
+# skip the wait and falsely report the daemon as not exiting.
+unit_stop_wait_survives_fractional_poll() {
+  local st daemon_pid out status
+  st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-stop-fractional-poll.XXXXXX")
+  mkdir -p "$st/state/.supervise-daemon.lock"
+  : > "$st/state/.afk"
+  printf 'none\t-\tnative\n' > "$st/state/.afk-daemon-terminal"
+  bash -c 'trap "" TERM; while :; do sleep 1; done' &
+  daemon_pid=$!
+  printf '%s' "$daemon_pid" > "$st/state/.supervise-daemon.lock/pid"
+  ( . "$ROOT/bin/fm-wake-lib.sh"; fm_pid_identity "$daemon_pid" > "$st/state/.supervise-daemon.lock/pid-identity" )
+  out=$(FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" FM_POLL=0.1 bash -c '
+    . "$1"
+    sleep() { :; }
+    kill() { command kill "$@"; }
+    fm_afk_launch_stop
+  ' _ "$LAUNCH" 2>&1)
+  status=$?
+  if [ "$status" -eq 1 ] && printf '%s' "$out" | grep -F 'did not exit after SIGTERM' >/dev/null; then
+    pass "stop wait: a fractional FM_POLL still produces a real bounded wait, not an arithmetic crash"
+  else
+    fail "stop wait: fractional FM_POLL broke the wait budget (status=$status out=$out)"
+  fi
+  kill -KILL "$daemon_pid" 2>/dev/null || true
+  wait "$daemon_pid" 2>/dev/null || true
+  rm -rf "$st"
+}
+
 unit_refresh_validates_record() {
   local st daemon_pid
   st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-refresh-record.XXXXXX")
@@ -950,6 +1014,8 @@ unit_stop_validates_before_signal
 unit_lock_requires_complete_metadata
 unit_stop_surfaces_afk_removal_failure
 unit_stop_confirms_daemon_exit
+unit_stop_wait_exceeds_configured_poll
+unit_stop_wait_survives_fractional_poll
 unit_refresh_validates_record
 unit_clear_failure_aborts_entry
 unit_confirmed_absence_succeeds