Skip to content

fix(bin): name the task in watcher wedge and dead-endpoint stale reasons #6761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dsdevq wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(bin): name the task in watcher wedge and dead-endpoint stale reasons #6761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5396db7
fix(watch): name the task in wedge and dead-endpoint stale reasons
dsdevq Oct 7, 2026
e126a11
no-mistakes(ci): Fixed the Greptile finding (ci-1). Invariant: each e…
dsdevq Oct 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions bin/fm-watch.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@
# agent, for human inspection only - never an automatic
# interrupt, signal, or restart of the worker or its
# tool process.
# The possible-wedge escalation and the dead-endpoint report end
# with " task=<id>" after the parenthetical, so the supervisor
# reads the task without looking the pane up; consumers read
# the window up to the first " (" and ignore the suffix.
# stale: <window> (unread firstmate instruction: ...)
# stale: <window> (steering-inbox ladder bookkeeping unwritable: ...)
# stale: <window> (steering-inbox busy bookkeeping unwritable: ...)
Expand Down Expand Up @@ -1484,7 +1488,7 @@ wedge_dead_record() { # <window> <since-file> <triage-label> <idle-age> <pane-h
triage_log "absorbed $label (agent $agent_state, already reported once, idle ${age}s): $win"
return 0
fi
reason="stale: $win (idle ${age}s, agent $agent_state - $detail, so this is not a wedge; reported once and not re-escalated while it stays that way - reconcile this record, and check for unlanded work before any cleanup)"
reason="stale: $win (idle ${age}s, agent $agent_state - $detail, so this is not a wedge; reported once and not re-escalated while it stays that way - reconcile this record, and check for unlanded work before any cleanup) task=$task"
# Append before the marker, for the reason stale_wait_record gives: a marker
# written ahead of a failed append outlives it, and the next sighting would then
# absorb the retry - the one way this bound could swallow the report outright
Expand Down Expand Up @@ -1542,9 +1546,9 @@ wedge_timer_check() { # <window> <since-file> <triage-label> <escalation-count-
fi
n=$(( $(cat "$escalation_file" 2>/dev/null || echo 0) + 1 ))
echo "$n" > "$escalation_file"
reason="stale: $win (idle ${age}s, possible wedge, escalation $n)"
reason="stale: $win (idle ${age}s, possible wedge, escalation $n) task=$task"
if [ "$n" -ge "$FM_WEDGE_DEMAND_INSPECT_COUNT" ]; then
reason="stale: $win (idle ${age}s, possible wedge, escalation $n, demand-deep-inspection: same pane has wedge-escalated $n times in a row - do not re-absorb on the run-step/pane state alone)"
reason="stale: $win (idle ${age}s, possible wedge, escalation $n, demand-deep-inspection: same pane has wedge-escalated $n times in a row - do not re-absorb on the run-step/pane state alone) task=$task"
fi
fm_wake_append stale "$win" "$reason" || exit 1
rm -f "$since_file"
Expand Down
6 changes: 3 additions & 3 deletions tests/fm-daemon.test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -893,9 +893,9 @@ test_enriched_wedge_under_declared_wait_uses_pause_cadence() {
if [ "$i" -ge 3 ]; then
# Past FM_WEDGE_DEMAND_INSPECT_COUNT the watcher enriches the same reason with
# its demand-deep-inspection marker; a declaration outranks both forms.
reason="stale: $win (idle 250s, possible wedge, escalation $i, demand-deep-inspection: same pane has wedge-escalated $i times in a row - do not re-absorb on the run-step/pane state alone)"
reason="stale: $win (idle 250s, possible wedge, escalation $i, demand-deep-inspection: same pane has wedge-escalated $i times in a row - do not re-absorb on the run-step/pane state alone) task=$task"
else
reason="stale: $win (idle 250s, possible wedge, escalation $i)"
reason="stale: $win (idle 250s, possible wedge, escalation $i) task=$task"
fi
LOG="$dir/daemon.log" FM_STATE_OVERRIDE="$state" handle_wake "$reason" "$state"
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \
Expand Down Expand Up @@ -929,7 +929,7 @@ test_enriched_wedge_under_declared_wait_uses_pause_cadence() {
# enriched wedge escalates again, unchanged.
: > "$state/.subsuper-escalations"
printf 'working: the audit finished, resuming\n' >> "$state/$task.status"
reason="stale: $win (idle 250s, possible wedge, escalation 6)"
reason="stale: $win (idle 250s, possible wedge, escalation 6) task=$task"
LOG="$dir/daemon.log" FM_STATE_OVERRIDE="$state" handle_wake "$reason" "$state"
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$win" FM_FAKE_TMUX_CAPTURE="$pane" \
FM_STATE_OVERRIDE="$state" FM_ESCALATE_BATCH_SECS=999999 FM_PAUSE_RESURFACE_SECS=3600 \
Expand Down
5 changes: 5 additions & 0 deletions tests/fm-watch-triage.test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2341,6 +2341,7 @@ test_stale_terminal_status_overridden_by_active_run() {
wait_for_exit "$pid" 100 || fail "watcher did not escalate an overridden stale terminal status past the threshold"
grep -F "stale: $window" "$out" >/dev/null || fail "escalation did not print a stale wake"
grep -F "possible wedge" "$out" >/dev/null || fail "escalation did not flag a possible wedge"
grep -F ' task=validating' "$out" >/dev/null || fail "the wedge escalation did not name its task: $(cat "$out")"
unset FM_FAKE_CREW_STATE
pass "a stale terminal-looking status is overridden and absorbed while a run is actively working, then wedge-escalated"
}
Expand Down Expand Up @@ -3122,6 +3123,8 @@ test_wedge_threshold_defers_to_a_declared_wait_under_a_working_verdict() {
ack_stopped_cycle "$state" || fail "could not acknowledge undeclared escalation $n"
grep -F "possible wedge, escalation $n" "$out" >/dev/null \
|| fail "an undeclared working lane did not reach escalation $n: $(cat "$out")"
grep -F "possible wedge, escalation $n" "$out" | grep -F ' task=wedge' >/dev/null \
|| fail "escalation $n did not name its task: $(cat "$out")"
n=$((n + 1))
done
grep -F 'demand-deep-inspection: same pane has wedge-escalated 3 times in a row' "$out" >/dev/null \
Expand Down Expand Up @@ -3781,6 +3784,8 @@ test_gone_report_rearms_when_the_endpoint_comes_back() {
|| fail "a second death in the same window was never reported: $(cat "$out")"
grep -F 'agent dead' "$out" >/dev/null \
|| fail "a second death was not reported as a gone endpoint: $(cat "$out")"
grep -F ' task=wedge' "$out" >/dev/null \
|| fail "the gone-endpoint report did not name its task: $(cat "$out")"
ack_stopped_cycle "$state" || fail "could not acknowledge the second gone report"
unset FM_TEST_PANE_COMMAND FM_TEST_TMUX_WINDOWS
pass "the once-only gone report re-arms when the endpoint comes back, and reports a later death again"
Expand Down
Loading