Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ state/ volatile runtime signals; gitignored
.wake-queue durable queued wakes: epoch<TAB>seq<TAB>kind<TAB>key<TAB>payload
.afk durable away-mode flag; present = sub-supervisor may inject escalations (set by /afk, cleared on user return)
.watch.lock .wake-queue.lock watcher singleton and queue serialization locks
.hash-* .count-* .stale-* .stale-since-* .paused-* .wedge-escalations-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch
.hash-* .count-* .stale-* .stale-since-* .paused-* .pause-handoff-* .wedge-escalations-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch
.watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete
.last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it
.subsuper-* .supervise-daemon.* sub-supervisor internals; never touch
Expand Down
6 changes: 4 additions & 2 deletions bin/backends/herdr.sh
Original file line number Diff line number Diff line change
Expand Up @@ -730,7 +730,9 @@ FM_BACKEND_HERDR_IDLE_RE=${FM_BACKEND_HERDR_IDLE_RE:-'^Type a message\.\.\.$'}
# Known bare (unbordered) prompt glyphs a composer row may start with: ❯
# (claude) and › (codex) only. Generic shell-style glyphs > $ % # are still
# recognized after a bordered composer row has already been structurally found.
FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^[❯›]'}
# Alternation, not a bracket expression, keeps byte-wise C-locale matching
# from accepting a box corner that shares the glyphs' leading byte.
FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^(❯|›)'}

fm_backend_herdr_composer_state() { # <target> -> empty|pending|unknown
local target=$1 cap line trimmed found=0 shape="" raw_match="" bordered=0 stripped
Expand Down Expand Up @@ -782,7 +784,7 @@ fm_backend_herdr_composer_state() { # <target> -> empty|pending|unknown
fi
# Delegate the empty/pending/unknown decision to the shared owner. The bare
# shape only ever starts with an AGENT glyph (FM_BACKEND_HERDR_BARE_PROMPT_RE
# is '^[❯›]'), so a bare shell prompt never reaches here - it stays 'unknown'
# is '^(❯|›)'), so a bare shell prompt never reaches here - it stays 'unknown'
# via the no-composer-row path above, exactly as before.
fm_composer_classify_content "$bordered" "$stripped" "$FM_BACKEND_HERDR_IDLE_RE"
}
Expand Down
39 changes: 34 additions & 5 deletions bin/fm-classify-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
# overlapping triage policy lives in one place instead of two copies that can
# drift apart.
#
# Most functions are pure, side-effect-free reads of status files: each takes
# what it needs as arguments and touches no globals beyond the optional
# FM_CAPTAIN_RE override. Consumers layer their own dedup/marker state on top (the
# daemon keeps its escalation-digest seen-markers; the watcher keeps its .seen-*
# signatures).
# Most functions are pure, side-effect-free reads of status files or the
# watcher-owned pause handoff record: each takes what it needs as arguments and
# touches no globals beyond the optional FM_CAPTAIN_RE override. Consumers layer
# their own dedup/marker state on top (the daemon keeps its escalation-digest
# seen-markers; the watcher keeps its .seen-* signatures).
#
# The one exception is the absorb classification (crew_absorb_class and its
# working/paused wrappers). It is NOT a pure status-file read: it reuses
Expand Down Expand Up @@ -224,6 +224,28 @@ signal_reason_is_actionable() { # <file> ...
return 1
}

# Return the watcher-owned, short-lived coalesced-pause handoff path for one
# task, so successor stale triage can distinguish a pause that was surfaced in
# an exiting signal batch from an arbitrary old paused status event.
crew_pause_handoff_file() { # <id> [state-dir]
local id=$1 state=${2:-${STATE:-${FM_STATE_OVERRIDE:-}}}
[ -n "$id" ] && [ -n "$state" ] || return 1
printf '%s/.pause-handoff-%s' "$state" "$id"
}

# Accept a handoff only when it is a regular file whose recorded final status
# still exactly matches the task's final paused event; this prevents recovery
# from masking a newer status transition or following a symlinked artifact.
crew_pause_handoff_allows_recovery() { # <id> [state-dir]
local id=$1 state=${2:-${STATE:-${FM_STATE_OVERRIDE:-}}} marker last recorded
marker=$(crew_pause_handoff_file "$id" "$state") || return 1
[ -f "$marker" ] && [ ! -L "$marker" ] || return 1
last=$(last_status_line "$state/$id.status")
status_is_paused "$last" || return 1
IFS= read -r recorded < "$marker" || return 1
[ "$recorded" = "$last" ]
}

# Classify WHY an idle/stale crew MIGHT be safely absorbed instead of surfaced,
# from bin/fm-crew-state.sh's one authoritative current-state line
# ("state: <s> · source: <src> · <detail>"). Prints exactly one token:
Expand Down Expand Up @@ -251,6 +273,13 @@ crew_absorb_class() { # <id>
src=${line#*source: }; src=${src%% *}
case "$src" in run-step|pane) printf 'working'; return ;; esac
fi
if [ "$state" = unknown ]; then
src=${line#*source: }; src=${src%% *}
if [ "$src" = none ] && crew_pause_handoff_allows_recovery "$id"; then
printf 'paused'
return
fi
fi
printf 'none'
}

Expand Down
14 changes: 12 additions & 2 deletions bin/fm-composer-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,18 @@ fm_composer_classify_content() { # <bordered> <content> [idle_re] [idle_case] [
fi
# Strip a leading prompt glyph, then re-judge the remainder.
case "$content" in
'❯ '*|'› '*|'> '*|'$ '*|'% '*|'# '*) content=${content#??} ;;
'❯'*|'›'*|'>'*|'$'*|'%'*|'#'*) content=${content#?} ;;
'❯ '*) content=${content#'❯ '} ;;
'› '*) content=${content#'› '} ;;
'> '*) content=${content#'> '} ;;
'$ '*) content=${content#'$ '} ;;
'% '*) content=${content#'% '} ;;
'# '*) content=${content#'# '} ;;
'❯'*) content=${content#'❯'} ;;
'›'*) content=${content#'›'} ;;
'>'*) content=${content#'>'} ;;
'$'*) content=${content#'$'} ;;
'%'*) content=${content#'%'} ;;
'#'*) content=${content#'#'} ;;
esac
content="${content#"${content%%[![:space:]]*}"}"
content="${content%"${content##*[![:space:]]}"}"
Expand Down
28 changes: 27 additions & 1 deletion bin/fm-watch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,9 @@ pause_state_class() { # <window> <task>
fi
class=$(crew_absorb_class "$task")
case "$class" in
paused) date +%s > "$recheck_file" ;;
paused) date +%s > "$recheck_file"
rm -f "$(crew_pause_handoff_file "$task" "$STATE")"
;;
*) rm -f "$recheck_file" ;;
esac
printf '%s' "$class"
Expand Down Expand Up @@ -406,6 +408,26 @@ scan_signals() {
return 0
}

# Persist final paused events from a signal batch that will make this watcher
# exit, allowing its successor to rebuild pause tracking when no live
# current-state source is available.
record_coalesced_pause_handoffs() { # <pending-signal-rows>
local pending=$1 sf sig f last task
while IFS=$(printf '\t') read -r sf sig f; do
[ -n "$f" ] || continue
case "$f" in
*.status)
last=$(last_status_line "$f")
status_is_paused "$last" || continue
task=$(basename "$f"); task=${task%.status}
printf '%s\n' "$last" > "$(crew_pause_handoff_file "$task" "$STATE")"
;;
esac
done <<EOF
$pending
EOF
}

run_check() {
local c=$1
if command -v timeout >/dev/null 2>&1; then
Expand Down Expand Up @@ -692,6 +714,7 @@ EOF
done <<EOF
$pending
EOF
record_coalesced_pause_handoffs "$pending"
wake "$reason"
else
while IFS=$(printf '\t') read -r sf sig f; do
Expand All @@ -718,6 +741,9 @@ EOF
if ! status_is_paused "$last" && [ -e "$STATE/.paused-$key" ]; then
clear_pause_tracking "$w"
fi
if ! status_is_paused "$last"; then
rm -f "$(crew_pause_handoff_file "$task" "$STATE")"
fi
if [ "$kind" = secondmate ] && ! status_is_paused "$last"; then
continue
fi
Expand Down
3 changes: 3 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ Those actionable wakes are written to a durable local queue (`state/.wake-queue`
No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step for that crew's branch or a backend busy signature.
A crew that declares `paused:` for a known external wait is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge.
Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck.
When a `paused:` event is coalesced with another actionable signal, the outgoing watcher writes `state/.pause-handoff-<id>` with that task's final non-blank status line before it exits.
A successor may rebuild pause tracking from that handoff only when `bin/fm-crew-state.sh` reports `unknown` with source `none` and the current final status line exactly matches the recorded line.
After rebuilding the pause marker, stale triage consumes the handoff, and any later non-paused final status event clears it.
Fresh stale panes use the same current-state read before trusting the status log, so an active run or busy pane outranks an old captain-relevant status-log line left behind before validation.
No-change heartbeats are also benign.
Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn.
Expand Down
2 changes: 1 addition & 1 deletion docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ FM_BACKEND= # optional runtime backend override for new spawns; tmux
HERDR_SESSION=default # herdr-only: named session for normal backend ops; not enough for destructive cleanup (docs/herdr-backend.md)
FM_BACKEND_HERDR_COMPOSER_LINES=20 # herdr-only: tail lines scanned by composer-state guard/fallback paths; idle-baseline submit confirmation uses agent-state
FM_BACKEND_HERDR_IDLE_RE='^Type a message\.\.\.$' # herdr-only: empty-composer placeholder regex after shared ghost extraction plus border and prompt stripping
FM_BACKEND_HERDR_BARE_PROMPT_RE='^[❯›]' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)")
FM_BACKEND_HERDR_BARE_PROMPT_RE='^(❯|›)' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; alternation prevents a non-UTF-8 locale from byte-wise matching a box corner with the same leading byte; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)")
FM_BACKEND_HERDR_SUBMIT_POLLS=6 # herdr-only: agent-state samples spread across each Enter attempt's budget when confirming a submit (docs/herdr-backend.md "Native agent-state submit confirmation")
FM_BACKEND_HERDR_SUBMIT_MIN_SLEEP=0.6 # herdr-only: minimum per-Enter confirmation budget before polling agent-state after an idle baseline
FM_BACKEND_ORCA_COMPOSER_LINES=200 # orca-only: terminal-read lines scanned to locate the composer row for submit verification
Expand Down
2 changes: 1 addition & 1 deletion docs/herdr-backend.md
Original file line number Diff line number Diff line change
Expand Up @@ -809,7 +809,7 @@ Covered by the unit cases in `tests/fm-afk-launch.test.sh` (clear-on-fresh-entry
If a future herdr build strips ANSI style from `--format ansi`, the classifier loses its ghost signal and falls back to reading the suggestion text as `pending` - the fail-safe direction (it defers rather than risks overwriting a human draft), which the max-defer alarm then surfaces.
- **RESOLVED: a "paused / awaiting-external" crew state for the stale-wedge escalation.** Raised alongside the 2026-07-07 incident: an in-flight crew intentionally idling on a known external wait (a vendor rate limit, say) still tripped `bin/fm-supervise-daemon.sh`'s "stale persisted ... (possible wedge)" escalation exactly like a genuinely wedged crew, with no way to mark the wait as expected.
Fixed by the `paused:` external-wait verb: a crew declares a deliberate wait, and both `bin/fm-watch.sh` and `bin/fm-supervise-daemon.sh` absorb its idle pane through the shared `bin/fm-classify-lib.sh` vocabulary (`status_is_paused`, `crew_absorb_class`, `FM_PAUSE_RESURFACE_SECS`), re-surfacing it for a recheck on a long cadence instead of a wedge escalation.
See `AGENTS.md` section 8 and the crew-facing brief contract in `bin/fm-brief.sh`.
See `AGENTS.md` section 8, the [event-driven supervision contract](architecture.md#event-driven-supervision) for coalesced-signal pause recovery, and the crew-facing brief contract in `bin/fm-brief.sh`.
- **Not implemented: mid-session secondmate liveness.** The `fm_backend_agent_alive`-driven respawn sweep (`bin/fm-bootstrap.sh`, see "Agent liveness probe reuses the husk classifier" above) only runs at session start.
A secondmate dying mid-session is a harder follow-on: the watcher deliberately exempts secondmates from stale-pane detection (an idle secondmate pane is healthy by design), so catching a mid-session death would need a periodic liveness beacon distinct from that exemption, not implemented here.
Deferred as a separate item - it changes the stale-classification/status vocabulary shared with `bin/fm-watch.sh` and `bin/fm-classify-lib.sh`, which is a bigger surface than this redelivery-loop fix should carry.
2 changes: 1 addition & 1 deletion tests/fm-backend-herdr.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ test_create_task_refuses_duplicate_label() {
dir="$TMP_ROOT/dup-task"; mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log"
printf '{"result":{"tabs":[{"tab_id":"w1:t2","label":"fm-dup1","workspace_id":"w1"}]}}\n' > "$resp/1.out"
fb=$(make_herdr_fakebin "$dir")
out=$( PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \
out=$( LC_ALL=C PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \
bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_create_task fmtest:w1 fm-dup1 /tmp/proj' "$ROOT" 2>&1 )
status=$?
[ "$status" -ne 0 ] || fail "create_task should refuse an existing tab label (herdr itself does not enforce uniqueness)"
Expand Down
2 changes: 1 addition & 1 deletion tests/fm-composer-lib.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ test_idle_placeholder_is_empty() {
out=$(classify 1 'Type a message...' "$idle")
[ "$out" = empty ] || fail "the grok idle placeholder should read empty, got '$out'"
# Placeholder after an agent glyph (post-strip match).
out=$(classify 0 '❯ Type a message...' "$idle")
out=$(LC_ALL=C classify 0 '❯ Type a message...' "$idle")
[ "$out" = empty ] || fail "the idle placeholder after a glyph should read empty, got '$out'"
# Without the idle regex it is just text -> pending.
out=$(classify 1 'Type a message...')
Expand Down
Loading
Loading