Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions bin/fm-watch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2573,12 +2573,25 @@ EOF
# On the first changed signal, linger one grace period and re-scan before
# classifying: a crewmate's final status write and the same turn's turn-end
# hook land seconds apart, and reporting them as separate actionable wakes
# costs a full firstmate turn each. The re-scan also picks up a newer
# signature for an already-pending file (last write wins below).
# costs a full firstmate turn each. Re-scan status files only: an answerer
# may finish self-announcing its append during grace, while a first-scan
# turn-end must still surface if its marker changes again or disappears.
pending=$(scan_signals)
if [ -n "$pending" ]; then
sleep "$SIGNAL_GRACE"
pending=$(printf '%s\n%s' "$pending" "$(scan_signals)")
first_turnends=$(printf '%s\n' "$pending" | while IFS=$(printf '\t') read -r sf sig f; do
if [[ $f == *.turn-ended ]]; then
printf '%s\t%s\t%s\n' "$sf" "$sig" "$f"
fi
done)
rescanned=$(scan_signals)
if [ -n "$first_turnends" ]; then
pending=$(printf '%s\n%s' "$first_turnends" "$rescanned")
else
pending=$rescanned
fi
fi
if [ -n "$pending" ]; then
# The final coalesced signal set is the watcher-carried status-change
# trigger for this home's published summary. Start it before either
# surfacing or absorbing the signal, but never wait on it: see
Expand Down
3 changes: 3 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,9 @@ Terminal verbs remain captain-relevant, while a nonterminal progress verb cannot
The shared latest-event read takes the most recent line that leads with a recognized verb or legacy token, so continuation prose and trailing blank lines after a multi-line record cannot hide a declared wait.
Both supervisors classify the status bytes appended since they last classified that log, never its last line alone, and report every actionable event through the captured endpoint before committing that position.
The watcher's `.seen-*` and `.hb-surfaced-<task>` markers and the daemon's `.subsuper-seen-status-<task>` marker independently track reported file state and successfully classified position, so an unchanged unreadable state reports once without advancing past unread content, while a changed state retries and an unusable position re-reads the whole log.
After the watcher's first signal scan starts the `FM_SIGNAL_GRACE` coalescing window, its final classification uses only a fresh post-grace scan for status files, so an answerer's `--resolve-key` close whose seen marker commits during that window does not re-wake its own home while a later worker append still surfaces.
First-scan turn-end markers remain in the coalesced set even if their file disappears or returns to its reported signature during grace.
This guarantee depends on the status-log append-only contract; an in-place rewrite that preserves the same inode and size can hide a genuine signal and is outside that contract.
A keyed `needs-decision` or `blocked` transition accepted by the whole-file decision fold is retired only when that fold retires it - an explicit close for its exact key, or a terminal declaration by the ship or scout that owns the log - while a reserved-key transition the fold rejects surfaces as a reconciliation signal without becoming an open decision.
The fold remains the sole owner of open/closed semantics, including same-key reopening and reserved-key handling, shared with the durable OPEN DECISIONS surface.
The always-on watcher also uses that library's absorb classification on no-verb signals and first-sighting stale panes before status-log terminality is trusted, while the daemon maintains distinct wedge and declared-wait recheck cadences.
Expand Down
91 changes: 91 additions & 0 deletions tests/fm-send-resolve-key.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,96 @@ test_separate_resolve_key_answers_do_not_rewake() {
fi
pass "fm-send --resolve-key: separate answers do not each re-wake; later lines still do"
}
# Freeze the real sender after its close append but before its seen-marker
# commit, then let the real watcher capture that transient state. The grace
# rescan must drop it once the sender commits; a worker blocker still wakes.
test_answer_close_during_signal_grace() (
local dir fb home log watcher sender i out
dir="$TMP_ROOT/self-announced-race"; mkdir -p "$dir/watchbin"
fb=$(make_stubs "$dir"); log="$dir/send.log"
home=$(setup_home self-announced-race)
fm_write_meta "$home/state/t9.meta" "window=sess:fm-t9" "kind=ship"
printf 'needs-decision [key=port-choice]: 8080 or 9090\n' > "$home/state/t9.status"
FM_STATE_OVERRIDE="$home/state" bash -c '
. "$1"; fm_wake_status_mark_current "$2" "$3"
' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t9.status" \
|| fail "could not prime the announced baseline"
drain_out "$home" >/dev/null
# Keep transport and timing fakes outside the append/scan/queue code.
cat > "$dir/size-reader" <<'SH'
#!/usr/bin/env bash
if grep -q '^resolved ' "$1" && [ ! -e "$FM_RACE_DIR/release" ]; then
touch "$FM_RACE_DIR/appended"
for ((i=0; i<300; i++)); do
[ ! -e "$FM_RACE_DIR/release" ] || break
/bin/sleep 0.1
done
[ -e "$FM_RACE_DIR/release" ] || exit 1
fi
if [ "$(uname)" = Darwin ]; then /usr/bin/stat -f '%z' "$1"; else stat -c '%s' "$1"; fi
SH
cat > "$dir/watchbin/sleep" <<'SH'
#!/usr/bin/env bash
if [ "$1" = 7 ]; then
touch "$FM_RACE_DIR/grace"
for ((i=0; i<300; i++)); do
[ ! -e "$FM_RACE_DIR/sent" ] || exit 0
/bin/sleep 0.1
done
exit 1
fi
[ "$1" != 1 ] || touch "$FM_RACE_DIR/poll"
exec /bin/sleep "$@"
SH
printf '#!/usr/bin/env bash\nexit 0\n' > "$dir/watchbin/tmux"
chmod +x "$dir/size-reader" "$dir/watchbin/sleep" "$dir/watchbin/tmux"
sender='' watcher=''
trap '[ -z "$sender" ] || kill "$sender" 2>/dev/null; [ -z "$watcher" ] || kill "$watcher" 2>/dev/null; wait 2>/dev/null || true' EXIT
FM_RACE_DIR="$dir" FM_STATUS_SIZE_READER="$dir/size-reader" \
run_send "$fb" "$home" "$log" t9 --resolve-key port-choice "use 9090" > "$dir/send.out" &
sender=$!
for ((i=0; i<300; i++)); do
[ ! -e "$dir/appended" ] || break
/bin/sleep 0.1
done
[ -e "$dir/appended" ] || fail "sender never reached the post-append barrier"
FM_RACE_DIR="$dir" PATH="$dir/watchbin:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$home" \
FM_STATE_OVERRIDE="$home/state" FM_POLL=1 FM_SIGNAL_GRACE=7 \
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \
"$ROOT/bin/fm-watch.sh" > "$dir/watch.out" 2> "$dir/watch.err" &
watcher=$!
for ((i=0; i<300; i++)); do
[ ! -e "$dir/grace" ] || break
/bin/sleep 0.1
done
[ -e "$dir/grace" ] || fail "watcher never scanned the uncommitted close"
touch "$dir/release"
wait "$sender" || fail "answer send failed"; sender=
rm -f "$dir/poll"
touch "$dir/sent"
for ((i=0; i<300; i++)); do
[ ! -e "$dir/poll" ] || break
[ ! -s "$dir/watch.out" ] || break
/bin/sleep 0.1
done
if [ -s "$dir/watch.out" ] || [ -s "$home/state/.wake-queue" ]; then
out=$(drain_out "$home")
fail "the committed close survived the grace rescan: $(cat "$dir/watch.out")\n$out"
fi
[ -e "$dir/poll" ] || fail "watcher did not complete a quiet poll"
printf 'blocked [key=worker-access]: worker needs release access\n' >> "$home/state/t9.status"
for ((i=0; i<300; i++)); do
[ ! -s "$dir/watch.out" ] || break
/bin/sleep 0.1
done
grep -F "signal: $home/state/t9.status" "$dir/watch.out" >/dev/null \
|| fail "worker blocker after the self-close did not signal"
wait "$watcher" || fail "watcher failed"; watcher=
out=$(drain_out "$home")
assert_contains "$out" 'blocked [key=worker-access]: worker needs release access' \
"worker blocker must appear in the wake annotation"
pass "fm-send --resolve-key: grace rescan drops a self-close but preserves the next worker blocker"
)

# The reported failure behind issue #2109: a worker that put the colon first
# (needs-decision: [key=X] ...) had its key silently folded to "default", so
Expand Down Expand Up @@ -905,6 +995,7 @@ test_decision_answer_partition_relocates_under_the_record() {
pass "fm-send --resolve-key: a decision answer refuses the attended branch before sending, a blocked: key stays steering, and the away-posture record relocates the answer"
}

test_answer_close_during_signal_grace || exit 1
test_answer_send_closes_open_decision
test_answer_close_is_self_announced
test_separate_resolve_key_answers_do_not_rewake
Expand Down
60 changes: 60 additions & 0 deletions tests/fm-watch-triage.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -737,6 +737,64 @@ test_turn_ended_provably_working_absorbed() {
pass "a bare turn-end whose crew is provably working (busy pane) is absorbed"
}

# A changed turn-end is observed first, then deleted or restored to its
# already-reported timestamp during the real 30-second grace.
# Both first-scan signals must still wake.
run_turnend_grace_case() (
local name=$1 dir state marker watcher i trace out
dir="$TMP_ROOT/grace-turnend-$name"; state="$dir/state"; marker="$state/unit.turn-ended"
mkdir -p "$state" "$dir/config" "$dir/fakebin"
printf '#!/bin/sh\nexit 0\n' > "$dir/fakebin/tmux"
printf '#!/bin/sh\nprintf "state: unknown · source: none · no current-state source available\\n"\n' \
> "$dir/fakebin/fm-crew-state.sh"
chmod +x "$dir/fakebin/tmux" "$dir/fakebin/fm-crew-state.sh"
printf 'working: %s\n' "$(printf '%055d' 0)" > "$state/unit.status"
FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_wake_status_mark_current "$2" "$3"' _ \
"$ROOT/bin/fm-wake-lib.sh" "$state" "$state/unit.status" \
|| fail "$name: could not prime the status baseline"
: > "$marker"
touch -t 200109090146.40 "$marker"
FM_STATE_OVERRIDE="$state" bash -c '
. "$1"
seen=$(fm_wake_signal_seen_path "$2" "$3")
fm_wake_signal_sig "$3" > "$seen"
' _ "$ROOT/bin/fm-wake-lib.sh" "$state" "$marker" \
|| fail "$name: could not prime the turn-end baseline"
touch "$marker"
trace="$dir/watch.trace"; out="$dir/watch.out"
PATH="$dir/fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$dir" \
FM_STATE_OVERRIDE="$state" FM_CONFIG_OVERRIDE="$dir/config" \
FM_CREW_STATE_BIN="$dir/fakebin/fm-crew-state.sh" \
FM_HEARTBEAT=999999 FM_CHECK_INTERVAL=999999 \
/bin/bash -x "$ROOT/bin/fm-watch.sh" > "$out" 2> "$trace" &
watcher=$!
trap 'kill "$watcher" 2>/dev/null || true; wait "$watcher" 2>/dev/null || true' EXIT
for ((i=0; i<300; i++)); do
grep -F '+ sleep 30' "$trace" >/dev/null 2>&1 && break
kill -0 "$watcher" 2>/dev/null || fail "$name: watcher exited before grace: $(cat "$out")"
/bin/sleep 0.1
done
grep -F '+ sleep 30' "$trace" >/dev/null \
|| fail "$name: watcher never entered signal grace"
case "$name" in
delete) rm "$marker" ;;
restore) touch -t 200109090146.40 "$marker" ;;
esac
for ((i=0; i<650; i++)); do
[ -s "$out" ] && break
[ "$(grep -Fc '+ scan_signals' "$trace" 2>/dev/null || true)" -lt 3 ] || break
/bin/sleep 0.1
done
grep -F "signal: $marker" "$out" >/dev/null \
|| fail "$name: first-scan turn-end was lost during grace (scans=$(grep -Fc '+ scan_signals' "$trace"))"
grep -F "$marker" "$state/.wake-queue" >/dev/null \
|| fail "$name: turn-end wake was not queued"
pass "first-scan turn-end still wakes after grace when $name"
)

test_turnend_grace_delete() { run_turnend_grace_case delete || fail "deleted turn-end grace case failed"; }
test_turnend_grace_restore() { run_turnend_grace_case restore || fail "restored turn-end grace case failed"; }

# --- a no-verb signal whose crew is NOT provably working SURFACES -------------
# This is the swallowed-finish fix: a crew that finished (or stopped and waits)
# reports its final turn-end with no captain-relevant status and no running
Expand Down Expand Up @@ -6056,6 +6114,8 @@ test_folded_worker_decision_without_home_append_still_wakes
test_separate_self_announced_answers_after_fold_wake_once
test_self_announced_close_after_fold_still_surfaces_folded_worker_failure
test_self_announced_close_after_fold_still_surfaces_folded_secondmate_lines
test_turnend_grace_delete || exit 1
test_turnend_grace_restore || exit 1
test_actionable_signal_surfaced
test_needs_decision_signal_payload_marked_for_branch_exclusion
test_needs_decision_reconciliation_required_still_marked
Expand Down