From f951ec3164107eb6394eca9cb742d11fec8a0621 Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sat, 19 Sep 2026 04:34:02 +0000 Subject: [PATCH 1/9] fix(bin): treat home-owned status closes as already read Self-announced bookkeeping appends now record their exact byte ranges. Later drains and signal scans skip those ranges, so two distinct --resolve-key answers after an OPEN DECISIONS fold do not each wake the supervisor. Worker-authored lines outside that ledger still signal. --- AGENTS.md | 1 + bin/fm-classify-lib.sh | 203 ++++++++++++++++++++++++++++-- bin/fm-wake-lib.sh | 127 ++++++++++++++----- docs/architecture.md | 4 +- docs/scripts.md | 2 +- tests/fm-send-resolve-key.test.sh | 41 ++++++ tests/fm-wake-queue.test.sh | 114 +++++++++++++++++ tests/fm-watch-triage.test.sh | 33 +++++ 8 files changed, 483 insertions(+), 42 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c4f62af7dbc..330db08bdf2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,6 +142,7 @@ state/ runtime records and signals; gitignored .wake-queue durable queued wakes retained until post-handling acknowledgement: epochseqkindkeypayload .watcher-down private generation-bound recovery state coupling watcher downtime, durable wake presentation, and post-handling acknowledgement; never touch ..open-decisions-cursor per-task byte cursor and folded open-decision set bounding the OPEN DECISIONS scan's cost to new status-log appends; written only by fm-classify-lib.sh's status_open_decisions_incremental, removed by teardown, safe to delete (forces one full re-fold) + ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so later drains and signal scans treat those ranges as already owned; written only by fm-classify-lib.sh's status_home_appends_record, removed by teardown, safe to delete .status-presentation-cursor .status-presentation-lock fleet-wide per-task status identity plus independent annotation and outcome-backstop byte offsets, with a serialization lock preventing already-presented lines from replaying while preserving delayed signal annotations; owned by fm-classify-lib.sh, with each task's row retired by teardown .afk-contract the away-posture record: the captain's verbatim away words, expected return, reach profile, spend cap, and structured mandate clauses; written only by bin/fm-afk-contract.sh after the captain confirms the read-back, archived under afk-contracts/ at return; its presence IS the away posture in every harness; its sibling .afk-contract.lock serializes actions authorized by the live record (contract: bin/fm-afk-contract.sh) afk-contracts/ archived away-posture records: one final record per away window keyed by entry time, plus any superseded mandates from that window diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 0752e52f370..8b4dcbefca8 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -27,7 +27,7 @@ # A missing, malformed, identity-mismatched, or past-end classified position reads # from byte 0, preferring a bounded duplicate over a lost event. # -# There are three documented exceptions. The absorb classification +# There are four documented exceptions. The absorb classification # (crew_absorb_class and its working/paused wrappers) is NOT a pure status-file # read: it reuses bin/fm-crew-state.sh, which may make a bounded no-mistakes call, # to decide whether a crew that just stopped its turn or went stale is working, @@ -37,9 +37,12 @@ # open-decisions fold" below) also writes: it persists a per-status-file byte # cursor and folded open-set as a side effect, so a per-drain fleet-wide scan # stays bounded by new appends instead of re-reading each task's whole lifetime -# log every time. crew_worktree_written_since reads the task's meta file and walks -# a bounded slice of its worktree instead of a status file, so callers run it only -# at the moment they would otherwise escalate. +# log every time. status_home_appends_record writes the per-task home-owned +# append ledger (see "home-owned status-append ledger" below) so later drains +# and signal scans can treat this home's own bookkeeping bytes as already owned. +# crew_worktree_written_since reads the task's meta file and walks a bounded slice +# of its worktree instead of a status file, so callers run it only at the moment +# they would otherwise escalate. # Directory of this library, used to locate the sibling fm-crew-state.sh reader. # Resolved at source time from BASH_SOURCE so it works whether sourced by a @@ -1319,13 +1322,14 @@ status_presentation_marker_commit() { status_retire_presentation_task() { # local state=$1 task=$2 lock manifest tmp data row_task ident offset backstop extra rc=0 found=0 - local signal_marker heartbeat_marker daemon_marker + local signal_marker heartbeat_marker daemon_marker home_appends lock="$state/.status-presentation-lock" manifest="$state/.status-presentation-cursor" tmp="$manifest.tmp.$$" signal_marker=$(status_signal_seen_marker_path "$state" "$task") heartbeat_marker=$(status_heartbeat_seen_marker_path "$state" "$task") daemon_marker=$(status_daemon_seen_marker_path "$state" "$task") + home_appends="$state/.$task.home-appends" # A remote-home teardown can legitimately retire an endpoint ID that has no # status log in that home. Do not contend with that home's unrelated status @@ -1335,6 +1339,7 @@ status_retire_presentation_task() { # if [ ! -e "$state/$task.status" ] && [ ! -L "$state/$task.status" ] \ && [ ! -e "$state/.$task.open-decisions-cursor" ] \ && [ ! -L "$state/.$task.open-decisions-cursor" ] \ + && [ ! -e "$home_appends" ] && [ ! -L "$home_appends" ] \ && [ ! -e "$signal_marker" ] && [ ! -L "$signal_marker" ] \ && [ ! -e "$heartbeat_marker" ] && [ ! -L "$heartbeat_marker" ] \ && [ ! -e "$daemon_marker" ] && [ ! -L "$daemon_marker" ]; then @@ -1384,7 +1389,7 @@ EOF fi if [ "$rc" -eq 0 ]; then rm -f -- "$state/$task.status" "$state/.$task.open-decisions-cursor" \ - "$signal_marker" "$heartbeat_marker" "$daemon_marker" || rc=1 + "$home_appends" "$signal_marker" "$heartbeat_marker" "$daemon_marker" || rc=1 fi fm_lock_release "$lock" || rc=1 return "$rc" @@ -1571,6 +1576,8 @@ status_open_decisions_cursor_offset() { # # print nothing. status_new_lines_since_cursor() { # [] local f=$1 captured_end=${2:-} cf offset size actual_size chunk_file line rc=0 + local pos line_start owned_ranges + local LC_ALL=C [ -f "$f" ] && [ -r "$f" ] && [ ! -L "$f" ] || return 0 cf=$(_fm_open_decisions_cursor_path "$f") chunk_file="$cf.unread.$$" @@ -1589,9 +1596,16 @@ status_new_lines_since_cursor() { # [] [ "$offset" -lt "$size" ] || return 0 _fm_status_read_span "$f" "$offset" "$((size - offset))" > "$chunk_file" 2>/dev/null \ || { rm -f "$chunk_file"; return 1; } + owned_ranges=$(status_home_appends_ranges "$f") + pos=$offset while IFS= read -r line || [ -n "$line" ]; do + line_start=$pos + pos=$((pos + ${#line} + 1)) case "$line" in - *[![:space:]]*) printf '%s\n' "$line" || { rc=1; break; } ;; + *[![:space:]]*) + _fm_offset_in_home_append_ranges "$owned_ranges" "$line_start" && continue + printf '%s\n' "$line" || { rc=1; break; } + ;; esac done < "$chunk_file" rm -f "$chunk_file" @@ -1733,6 +1747,172 @@ window_to_task() { t="${w##*:}"; t="${t#fm-}"; printf '%s' "$t" } +# --- home-owned status-append ledger ---------------------------------------- +# +# This home's bookkeeping closes (fm_wake_status_append_self_announced) record +# the exact byte range they appended so later drains and signal scans can treat +# those bytes as already owned. That is the multi-answer path: two distinct +# --resolve-key closes must not each force a captain-facing wake solely because +# each one appended a status line, while a worker-authored line that is not in +# this ledger still signals. +# The ledger does not use lag verbs to hide a worker `resolved` line. Only +# bytes this home itself recorded as owned are skipped. +# +# Path: state/..home-appends +# Format: +# v1 +# ident= +# +# Ranges are half-open [start, end), merged when adjacent or overlapping. +# An identity mismatch (file rotated) discards the ledger. Teardown deletes it. +# Not a pure status-file read: status_home_appends_record writes this sidecar. + +status_home_appends_path() { # + local f=$1 dir base + dir=$(dirname "$f") + base=$(basename "$f") + printf '%s/.%s.home-appends' "$dir" "${base%.status}" +} + +status_home_appends_ranges() { # -> startend lines + local f=$1 path ident data first rest line start end extra + path=$(status_home_appends_path "$f") + [ -f "$path" ] && [ -r "$path" ] && [ ! -L "$path" ] || return 0 + ident=$(_fm_open_decisions_file_ident "$f") || return 0 + data=$(LC_ALL=C command cat "$path" 2>/dev/null) || return 0 + first=${data%%$'\n'*} + [ "$first" = v1 ] || return 0 + rest=${data#*$'\n'} + [ "$rest" != "$data" ] || return 0 + line=${rest%%$'\n'*} + case "$line" in ident=*) ;; *) return 0 ;; esac + [ "${line#ident=}" = "$ident" ] || return 0 + case "$rest" in + *$'\n'*) rest=${rest#*$'\n'} ;; + *) return 0 ;; + esac + while IFS=$(printf '\t') read -r start end extra || [ -n "$start" ]; do + [ -n "$start" ] || continue + [ -z "$extra" ] || continue + case "$start:$end" in *[!0-9:]*) continue ;; esac + [ "$end" -gt "$start" ] || continue + printf '%s\t%s\n' "$start" "$end" || return 1 + done < + local ranges=$1 offset=$2 start end + case "$offset" in ''|*[!0-9]*) return 1 ;; esac + while IFS=$(printf '\t') read -r start end; do + [ -n "$start" ] || continue + case "$start:$end" in *[!0-9:]*) continue ;; esac + if [ "$offset" -ge "$start" ] && [ "$offset" -lt "$end" ]; then + return 0 + fi + done < + _fm_offset_in_home_append_ranges "$(status_home_appends_ranges "$1")" "$2" +} + +status_home_appends_covers() { # + local start=$2 end=$3 range_start range_end + case "$start:$end" in *[!0-9:]*) return 1 ;; esac + [ "$end" -ge "$start" ] || return 1 + [ "$end" -eq "$start" ] && return 0 + while IFS=$(printf '\t') read -r range_start range_end; do + [ -n "$range_start" ] || continue + case "$range_start:$range_end" in *[!0-9:]*) continue ;; esac + [ "$range_start" -le "$start" ] || continue + if [ "$range_end" -gt "$start" ]; then + start=$range_end + fi + if [ "$start" -ge "$end" ]; then + return 0 + fi + done < + local f=$1 start=$2 end=$3 path ident tmp merged + local LC_ALL=C + case "$start:$end" in *[!0-9:]*) return 1 ;; esac + [ "$end" -gt "$start" ] || return 1 + ident=$(_fm_open_decisions_file_ident "$f") || return 1 + path=$(status_home_appends_path "$f") + merged=$(printf '%s\n%s\t%s\n' "$(status_home_appends_ranges "$f")" "$start" "$end" | awk ' + NF == 2 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ && $2+0 > $1+0 { + n++ + s[n] = $1 + 0 + e[n] = $2 + 0 + } + END { + for (i = 1; i <= n; i++) { + for (j = i + 1; j <= n; j++) { + if (s[j] < s[i] || (s[j] == s[i] && e[j] < e[i])) { + t = s[i]; s[i] = s[j]; s[j] = t + t = e[i]; e[i] = e[j]; e[j] = t + } + } + } + m = 0 + for (i = 1; i <= n; i++) { + if (m == 0 || s[i] > me[m]) { + m++ + ms[m] = s[i] + me[m] = e[i] + } else if (e[i] > me[m]) { + me[m] = e[i] + } + } + for (i = 1; i <= m; i++) printf "%s\t%s\n", ms[i], me[i] + } + ') || return 1 + tmp="$path.tmp.$$" + { + printf 'v1\nident=%s\n' "$ident" + if [ -n "$merged" ]; then + printf '%s' "$merged" + case "$merged" in *$'\n') ;; *) printf '\n' ;; esac + fi + } > "$tmp" || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$path" || { rm -f "$tmp"; return 1; } +} + +# 0 when every non-blank line in [start, end) is a keyed needs-decision or +# blocked line. Empty spans succeed. A worker `resolved`, `failed`, `paused`, +# `working`, or unkeyed line fails, so lag never hides those. +_fm_status_span_is_decision_lag() { # + local f=$1 start=$2 end=$3 chunk line verb + local LC_ALL=C + case "$start:$end" in *[!0-9:]*) return 1 ;; esac + [ "$end" -ge "$start" ] || return 1 + [ "$end" -eq "$start" ] && return 0 + chunk=$(_fm_status_read_span "$f" "$start" "$((end - start))") || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in *[![:space:]]*) ;; *) continue ;; esac + verb=$(status_line_verb "$line") + case "$verb" in + needs-decision|blocked) ;; + *) return 1 ;; + esac + _fm_key_before_colon "$line" || _fm_key_at_note_head "$line" >/dev/null || return 1 + _fm_decision_key "$line" >/dev/null || return 1 + done < under # one size-and-identity snapshot. # The record form produces `\t\t` and returns 0 when @@ -1756,6 +1936,8 @@ window_to_task() { # reconciliation signal and never treated here as an open decision. # status_open_decisions remains the single owner of open/closed semantics, # including same-key reopening and reserved-key handling. +# Bytes recorded in this home's owned-append ledger are not classified as +# events: this home already wrote them as bookkeeping. # Every other captain-relevant event is terminal and always actionable. _fm_decision_origin_drop() { # local origin @@ -1804,6 +1986,8 @@ _fm_status_open_decision_origins() { # [] status_span_first_actionable_record() { # [record-var] [needs-decision-var] local f=$1 start=${2:-0} output_var=${3-} needs_var=${4-} size ident cur_ident scratch chunk_file full_file prefix_file result local line verb key origins='' folded=0 rc=1 failed=0 prefix_lines=0 line_number=0 live_line='' events='' _line _key _fm_span_needs_decision=0 + local pos line_start owned_ranges + local LC_ALL=C [ -e "$f" ] || { [ -L "$f" ] && return 2; return 1; } [ -f "$f" ] && [ -r "$f" ] && [ ! -L "$f" ] || return 2 ident=$(_fm_open_decisions_file_ident "$f") || return 2 @@ -1830,9 +2014,14 @@ status_span_first_actionable_record() { # [record- rm -f "$chunk_file" "$full_file" "$prefix_file"; return 2; } [ "$cur_ident" = "$ident" ] || { rm -f "$chunk_file" "$full_file" "$prefix_file"; return 2; } + owned_ranges=$(status_home_appends_ranges "$f") + pos=$start while IFS= read -r line || [ -n "$line" ]; do + line_start=$pos + pos=$((pos + ${#line} + 1)) line_number=$((line_number + 1)) case "$line" in *[![:space:]]*) ;; *) continue ;; esac + _fm_offset_in_home_append_ranges "$owned_ranges" "$line_start" && continue if status_is_captain_held "$line"; then # A transfer closes the status-log decision and remains non-actionable to # stale classification. The side-band marker lets signal routing surface diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index e3582cbc4da..5ace31df768 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -2117,21 +2117,36 @@ fm_wake_signal_seen_size() { # esac } -# 0 when 's current signature matches its recorded reported state. -# For a status file this means the current state was already reported, not that -# every byte was successfully classified; the separate classified position owns -# that fact. -# A missing marker or unreadable signature is not a match, so uncertainty reads -# as an unreported state. +# 0 when 's current signature matches its recorded reported state, or +# when every byte past the already-read bound is in this home's owned-append +# ledger (optionally after OPEN DECISIONS fold lag of keyed needs-decision or +# blocked lines). For a status file a reported match means the current state +# was already reported, not that every byte was successfully classified; the +# separate classified position owns that fact. Owned-only growth after a fold +# or classified bound is this home's own bookkeeping and is not a new signal. +# A missing marker or unreadable signature is not a match unless the owned +# ledger plus fold lag covers the unread suffix, so uncertainty still reads as +# an unreported state. fm_wake_signal_seen_current() { # - local sig marker + local sig marker classified size folded sig=$(fm_wake_signal_sig "$2") || return 1 [ -n "$sig" ] || return 1 marker=$(fm_wake_signal_seen_path "$1" "$2") case "$2" in *.status) _fm_wake_require_classify || return 1 - status_presentation_marker_reported_matches "$marker" "$sig" + status_presentation_marker_reported_matches "$marker" "$sig" && return 0 + classified=$(fm_wake_signal_seen_size "$1" "$2") + size=$(_fm_status_file_size "$2") || return 1 + size=${size//[[:space:]]/} + case "$classified:$size" in *[!0-9:]*) return 1 ;; esac + [ "$classified" -le "$size" ] || return 1 + status_home_appends_covers "$2" "$classified" "$size" && return 0 + folded=$(status_open_decisions_cursor_offset "$2" 2>/dev/null) || folded=0 + case "$folded" in ''|*[!0-9]*) folded=0 ;; esac + [ "$folded" -ge "$classified" ] && [ "$folded" -le "$size" ] || return 1 + _fm_status_span_is_decision_lag "$2" "$classified" "$folded" || return 1 + status_home_appends_covers "$2" "$folded" "$size" ;; *) [ "$(cat "$marker" 2>/dev/null)" = "$sig" ] ;; esac @@ -2161,39 +2176,65 @@ fm_wake_status_mark_current() { # # line THIS home's own machinery writes as bookkeeping it has already presented # in the very turn or tick that writes it (an answerer-closes resolved line, a # pending-reply escalation close, a captain-held transfer). Such a close must -# not wake the session that wrote it, so this appends the line and then -# advances the watcher's seen marker to cover exactly the appended bytes and -# nothing else. The advance is provenance-gated and fails toward waking: -# - the marker advances ONLY when the file's pre-append signature matched the -# recorded seen marker (every earlier byte was already announced or -# deliberately absorbed), AND the post-append size equals the pre-append -# size plus exactly the appended bytes (no foreign write interleaved); -# - on ANY other condition - missing marker, pending foreign bytes, an -# interleaved writer, an unreadable signature - the line is still appended -# but the marker is left alone, so the watcher surfaces the file normally. -# A later, different line from any other writer grows the size past the marker -# and wakes as before: task identity alone can never suppress new content. +# not wake the session that wrote it. This appends the line, records the exact +# appended byte range in the home-owned append ledger (bin/fm-classify-lib.sh), +# and then advances the watcher's seen marker across those bytes when this home +# already read every earlier byte. Already-read means: +# - the watcher's classified offset equals the pre-append size, or +# - the OPEN DECISIONS fold cursor reaches the pre-append size (or owned +# ranges cover the gap from that cursor to the pre-append size) and every +# non-blank line the watcher has not classified yet is a keyed +# needs-decision or blocked line, which OPEN DECISIONS listed as open. +# The fold reads bytes it never prints, so a worker's `failed:`, `paused:`, +# `working:`, `resolved`, or verb-less line there must still wake. The post- +# append size must equal the pre-append size plus exactly the appended bytes +# (no foreign write interleaved), and the span classifier from the already-read +# bound, skipping owned ranges, must find no actionable event. +# On ANY other condition - missing file, pending foreign bytes that are not +# fold lag, an interleaved writer, an unreadable size or identity - the line is +# still appended and the owned range is still recorded when growth is proven, +# but the marker is left alone so the watcher surfaces unowned foreign bytes. +# Later drains treat owned ranges as already owned even when the prefix cursor +# has not caught up, so separate --resolve-key answers do not each force a +# captain-facing wake. A later different line from any other writer grows the +# size past the owned ranges and wakes as before: task identity alone can never +# suppress new content. # Returns 0 appended and self-announced, 1 appended but left for the watcher # (the safe direction), 2 the append itself failed. fm_wake_status_append_self_announced() { # - local state=$1 file=$2 line=$3 marker pre_sig='' pre_size='' pre_ident='' post_size post_ident + local state=$1 file=$2 line=$3 pre_size='' pre_ident='' post_size post_ident + local classified folded bound span_rc=0 local LC_ALL=C _fm_wake_require_classify || return 1 - marker=$(fm_wake_signal_seen_path "$state" "$file") if [ -e "$file" ]; then - pre_sig=$(fm_wake_signal_sig "$file") || pre_sig='' pre_size=$(_fm_status_file_size "$file") || pre_size='' pre_ident=$(_fm_open_decisions_file_ident "$file") || pre_ident='' fi printf '%s\n' "$line" >> "$file" || return 2 - [ -n "$pre_sig" ] || return 1 - status_presentation_marker_reported_matches "$marker" "$pre_sig" || return 1 - [ "$(status_presentation_marker_offset "$marker" "$file")" = "$pre_size" ] || return 1 + case "$pre_size" in ''|*[!0-9]*) return 1 ;; esac post_size=$(_fm_status_file_size "$file") || return 1 post_ident=$(_fm_open_decisions_file_ident "$file") || return 1 - case "$pre_size$post_size" in ''|*[!0-9]*) return 1 ;; esac + case "$post_size" in ''|*[!0-9]*) return 1 ;; esac [ -n "$pre_ident" ] && [ "$post_ident" = "$pre_ident" ] || return 1 [ "$post_size" -eq $((pre_size + ${#line} + 1)) ] || return 1 + status_home_appends_record "$file" "$pre_size" "$post_size" || return 1 + classified=$(fm_wake_signal_seen_size "$state" "$file") + case "$classified" in ''|*[!0-9]*) classified=0 ;; esac + bound= + if [ "$classified" = "$pre_size" ]; then + bound=$classified + else + folded=$(status_open_decisions_cursor_offset "$file" 2>/dev/null) || folded=0 + case "$folded" in ''|*[!0-9]*) folded=0 ;; esac + if [ "$folded" -le "$pre_size" ] && [ "$classified" -le "$folded" ] \ + && _fm_status_span_is_decision_lag "$file" "$classified" "$folded" \ + && status_home_appends_covers "$file" "$folded" "$pre_size"; then + bound=$folded + fi + fi + [ -n "$bound" ] || return 1 + status_span_first_actionable_record "$file" "$bound" >/dev/null || span_rc=$? + [ "$span_rc" -eq 1 ] || return 1 fm_wake_status_seen_commit "$state" "$file" "$post_size" "$post_ident" || return 1 return 0 } @@ -2254,10 +2295,11 @@ fm_wake_status_cursor_offset() { # -> already-presented # O_NOFOLLOW read of every still-unread status byte. min-offset is the # already-presented cursor from classify-lib. Lines whose bytes begin before -# that offset are not replayed. Prints nothing and returns 1 when no unread -# non-blank line exists. +# that offset are not replayed, nor are lines this home recorded as its own +# bookkeeping appends. Prints nothing and returns 1 when no unread non-blank +# line exists. fm_wake_unread_events() { # [] - local path=$1 min_offset=$3 end_offset=${4:-} result size chunk chunk_start + local path=$1 min_offset=$3 end_offset=${4:-} result size chunk chunk_start owned local LC_ALL=C FM_WAKE_EVENT_LINE= FM_WAKE_UNREAD_LINES= @@ -2288,12 +2330,31 @@ fm_wake_unread_events() { # /dev/null 2>&1; then + owned=$(status_home_appends_ranges "$path") || owned= + fi + FM_WAKE_UNREAD_LINES=$(printf '%s' "$chunk" | LC_ALL=C awk -v start="$chunk_start" -v min="$min_offset" -v owned="$owned" ' + BEGIN { + pos = start + 0 + n = split(owned, rows, "\n") + for (i = 1; i <= n; i++) { + split(rows[i], p, "\t") + if (p[1] ~ /^[0-9]+$/ && p[2] ~ /^[0-9]+$/) { + nr++ + os[nr] = p[1] + 0 + oe[nr] = p[2] + 0 + } + } + } + function is_owned(off, i) { + for (i = 1; i <= nr; i++) if (off >= os[i] && off < oe[i]) return 1 + return 0 + } { line_start = pos pos += length($0) + 1 - if ($0 ~ /[^[:space:]]/ && line_start >= min) print $0 + if ($0 ~ /[^[:space:]]/ && line_start >= min && !is_owned(line_start)) print $0 } ') || return 1 [ -n "$FM_WAKE_UNREAD_LINES" ] || return 1 diff --git a/docs/architecture.md b/docs/architecture.md index 7cdc3ff6d5e..d08192cdb1e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,9 +89,11 @@ A queued signal annotation prints every status line still unread at that cursor, A third bounded section, RECORD DIVERGENCE, prints on the same drains for the opposite failure: the status fold went quiet on a key that the durable captain-held task still shows as open, so the status side reads as complete while the two records contradict each other; `bin/fm-captain-hold.sh diverged` decides what counts and closes nothing, and `docs/captain-hold-lifecycle.md` owns the mechanism. A failed read, output, or concurrent-replacement check prevents the snapshot cursor from advancing across uncertain bytes, and teardown retires a task's manifest row before that task ID can be reused. The explicit resolution is written by the actor that answers, not the busy worker: `fm-send`'s `--resolve-key` appends the closing `resolved` line to this home's own copy of the ledger at answer time, which covers crewmates, local secondmates, and remote secondmates identically because a remote mate's escalations reach that local copy through the parent-replies ingest and only the answer message itself crosses the transport. -This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, so they advance the watcher marker only across their own bytes when all earlier bytes were already announced; pending or interleaved foreign bytes fail toward an ordinary wake. +This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so later drains and signal scans treat those bytes as already owned. +The watcher marker still advances only when earlier bytes were already announced or were OPEN DECISIONS fold lag (keyed `needs-decision` or `blocked` lines the drain listed); a worker `resolved`, `failed`, `paused`, or `working` line in that span, or any interleaved foreign write, still wakes. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. Any direct or remaining historical annotation prints every status line unread at the presentation cursor instead of replaying only the latest line. +Lines whose bytes this home recorded as its own bookkeeping appends are treated as already owned and are not replayed. `bin/fm-crew-state.sh ` is the cheap current-state read for an actionable heartbeat review: it attributes an active or terminal no-mistakes run under the shared run-attribution contract, then keeps that run-step authoritative even if the pane has closed, except that a `blocked:` event reporting a refused or missing daemon socket outranks a potentially stale active run record only while that socket-down declaration is itself the log's latest recognized event, since any later event, including another `blocked:` one, means the crew moved on. For other daemon, timeout, or unreachability claims, a running or fixing run with recent pipeline-reported activity supersedes the event and names reattachment as the recovery instead of surfacing a false block. [`bin/fm-nm-run-lib.sh`](../bin/fm-nm-run-lib.sh) owns branch, head, and pipeline-custody attribution, plus complete same-branch run selection, optional inventory lookup, and ambiguity reporting. diff --git a/docs/scripts.md b/docs/scripts.md index 5b8ceb56d3e..093ddf20722 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -111,7 +111,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-wake-drain.sh` | Present and acknowledge the current actor's claimed wake rows alongside status, outcome-backstop, decision, divergence, recovery, and supervision checks | | `fm-wake-grant.sh` | Serialize Pi supervision-branch wake-row claim activation, publication, release, and deactivation | | `fm-wake-lib.sh` | Shared durable wake queue, recovery generations, portable locks, and watcher identity/health helpers | -| `fm-classify-lib.sh` | Shared wake classification, durable keyed-decision folds and scans, unread status selection, and bounded latest-event snapshots | +| `fm-classify-lib.sh` | Shared wake classification, durable keyed-decision folds and scans, unread status selection, home-owned status-append ranges, and bounded latest-event snapshots | | `fm-send.sh` | Steer a task via a durable inbox record plus doorbell, or send a supported key or typed harness invocation through the recorded backend | | `fm-branch-prompt.sh` | Emit the Pi supervision branch's byte-stable system prompt ([pi-supervision-branch.md](pi-supervision-branch.md)) | | `fm-branch-outcome.sh` | Own the supervision branch's append-only outcome store, cursors, bounded status-coverage indexes, and session-start replay | diff --git a/tests/fm-send-resolve-key.test.sh b/tests/fm-send-resolve-key.test.sh index 62a8f05d2d5..001ab63ea2d 100755 --- a/tests/fm-send-resolve-key.test.sh +++ b/tests/fm-send-resolve-key.test.sh @@ -180,6 +180,46 @@ test_answer_close_is_self_announced() { pass "fm-send --resolve-key: the close never re-wakes its own home, later lines still do" } +# Two distinct --resolve-key answers after an OPEN DECISIONS fold (no watcher +# seen marker) must each stay quiet. This is the multi-decision residual: lag +# verbs on unread status lines are not enough, because each answer is its own +# event. A later worker line on the same task still wakes. +test_separate_resolve_key_answers_after_fold_do_not_rewake() { + local dir fb log home rc + dir="$TMP_ROOT/separate-answers"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log" + home=$(setup_home separate-answers) + fm_write_meta "$home/state/t7.meta" "window=sess:fm-t7" "kind=ship" + { + printf 'needs-decision [key=budget]: approve spend?\n' + printf 'needs-decision [key=vendor]: pick a vendor\n' + } > "$home/state/t7.status" + + drain_out "$home" >/dev/null + + run_send "$fb" "$home" "$log" t7 --resolve-key budget "approved"; rc=$? + expect_code 0 "$rc" "the first folded answer should succeed" + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_signal_seen_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ + || fail "the first --resolve-key answer after a fold was left to re-wake this home" + + run_send "$fb" "$home" "$log" t7 --resolve-key vendor "acme"; rc=$? + expect_code 0 "$rc" "the second folded answer should succeed" + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_signal_seen_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ + || fail "the second --resolve-key answer after a fold was left to re-wake this home" + + printf 'blocked: need staging credentials\n' >> "$home/state/t7.status" + if FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_signal_seen_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status"; then + fail "a later worker line after two folded answers was swallowed" + fi + pass "fm-send --resolve-key: separate answers after a fold do not each re-wake; later lines still do" +} + # The reported failure behind issue #2109: a worker that put the colon first # (needs-decision: [key=X] ...) had its key silently folded to "default", so # the answer's --resolve-key X refused with "no open decision or blocker with @@ -724,6 +764,7 @@ test_remote_reserved_pending_reply_key_closes_locally() { test_answer_send_closes_open_decision test_answer_close_is_self_announced +test_separate_resolve_key_answers_after_fold_do_not_rewake test_colon_first_key_position_is_answerable test_answer_starts_work_never_orphans test_routine_steer_never_closes diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 92f46266f02..bf4ab81404e 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -1628,6 +1628,117 @@ test_self_announced_append_guards() { pass "self-announced appends suppress only their own bytes and fail toward waking" } +# Two distinct --resolve-key closes after an OPEN DECISIONS fold must not each +# leave the file wake-worthy. The fold listed both keys; each answer records its +# own byte range so later scans treat those ranges as already owned. A worker +# line that is not in that ledger still wakes, including a worker `resolved` +# sitting in the folded span (lag verbs are only keyed needs-decision/blocked). +test_separate_self_announced_answers_after_fold_are_owned() { + local dir state status rc + dir=$(make_case multi-answer-owned) + state="$dir/state" + status="$state/t.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + { + printf 'needs-decision [key=k1]: pick REST or RPC\n' + printf 'needs-decision [key=k2]: pick us-east or eu-west\n' + } > "$status" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k1]: answered: REST' \ + || fail "the first folded answer was not self-announced (rc=$?)" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + || fail "the first folded answer was left to re-wake this home" + + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k2]: answered: eu-west' \ + || fail "the second folded answer was not self-announced (rc=$?)" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + || fail "the second folded answer was left to re-wake this home" + + printf 'blocked [key=creds]: need staging credentials\n' >> "$status" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + && fail "a later worker line after two owned answers was swallowed" + + pass "separate self-announced answers after a fold stay owned; a later worker line still wakes" +} + +test_folded_worker_resolved_is_not_owned_lag() { + local dir state status rc + dir=$(make_case folded-worker-resolved) + state="$dir/state" + status="$state/t.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + { + printf 'needs-decision [key=budget]: approve spend?\n' + printf 'needs-decision [key=vendor]: vendor A or B?\n' + printf 'resolved [key=vendor]: picked vendor B myself, cheaper\n' + } > "$status" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + + rc=0 + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=budget]: answered: approved' || rc=$? + [ "$rc" -eq 1 ] || fail "a close over a folded worker resolved did not fail toward waking (rc=$rc)" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + && fail "a worker resolved in the folded span was treated as already owned" + + pass "a worker resolved in fold lag still wakes after this home's close" +} + +test_owned_appends_are_not_replayed_as_unread() { + local dir state status out + dir=$(make_case owned-unread) + state="$dir/state" + status="$state/t.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + { + printf 'needs-decision [key=k1]: first\n' + printf 'needs-decision [key=k2]: second\n' + } > "$status" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k1]: answered: first' \ + || fail "the first owned close failed" + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k2]: answered: second' \ + || fail "the second owned close failed" + printf 'blocked: need staging credentials\n' >> "$status" + append_wake "$state" signal t.status "signal: $status" \ + || fail "could not queue the later worker signal" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2>"$dir/drain.err" \ + || fail "drain after the worker line failed" + out=$(cat "$dir/drain.out") + printf '%s' "$out" | grep -F 'blocked: need staging credentials' >/dev/null \ + || fail "the later worker line was not annotated: $out" + if printf '%s' "$out" | grep -E 'resolved \[key=k[12]\]' >/dev/null; then + fail "an owned bookkeeping close was replayed as unread: $out" + fi + pass "owned append ranges are not replayed as unread on a later real wake" +} + # A trap that fires inside a lock's critical section abandons the holding # frame, and the exit path then re-acquires the same lock (a TERM inside a # recovery-marker section is the reproduced case: the watcher's reap wedged @@ -1994,6 +2105,9 @@ test_secondmate_stall_marker_rejects_symlink test_acknowledged_stall_publication_survives_pre_marker_crash test_empty_prefix_mate_preserves_other_mate_receipt test_self_announced_append_guards +test_separate_self_announced_answers_after_fold_are_owned +test_folded_worker_resolved_is_not_owned_lag +test_owned_appends_are_not_replayed_as_unread test_historical_annotation_skips_announced_status test_concurrent_append_and_drain test_signal_catchup_without_running_watcher diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 29d6633cf12..0a07dbcfc40 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1573,6 +1573,38 @@ test_self_announced_close_does_not_rewake_but_next_note_does() { pass "a self-announced close never wakes its own home, and the next real note still does" } +test_separate_self_announced_answers_after_fold_do_not_rewake() { + local dir state fakebin out status_file pid rc + dir=$(make_case multi-answer-fold); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + status_file="$state/task.status" + { + printf 'needs-decision [key=k1]: pick REST or RPC\n' + printf 'needs-decision [key=k2]: pick us-east or eu-west\n' + } > "$status_file" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + rc=0 + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_wake_status_append_self_announced "$2" "$3" "resolved [key=k1]: answered: REST" || exit $? + fm_wake_status_append_self_announced "$2" "$3" "resolved [key=k2]: answered: eu-west" || exit $? + ' _ "$ROOT/bin/fm-wake-lib.sh" "$state" "$status_file" || rc=$? + [ "$rc" -eq 0 ] || fail "the folded answers were not self-announced (rc=$rc)" + export FM_FAKE_CREW_STATE='state: unknown · source: none · idle worker' + watch_bg "$state" "$fakebin" "$out" + pid=$! + if ! wait_poll_cycle "$state" "$pid"; then + reap "$pid"; fail "separate folded answers re-woke the watcher: $(cat "$out")" + fi + [ ! -s "$out" ] || { reap "$pid"; fail "separate folded answers printed a wake reason: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "separate folded answers enqueued a durable wake"; } + printf 'blocked: need staging credentials\n' >> "$status_file" + wait_for_exit "$pid" 100 || fail "a later worker line after two folded answers was swallowed" + grep -F "signal: $status_file" "$out" >/dev/null \ + || fail "the later worker line did not surface as a signal" + pass "separate self-announced answers after a fold never wake, and the next real note still does" +} + # --- actionable wakes are surfaced (queue + exit) --------------------------- test_actionable_signal_surfaced() { @@ -5473,6 +5505,7 @@ test_working_note_not_working_surfaced test_secondmate_status_note_surfaced_despite_busy_agent test_secondmate_buried_block_wakes_despite_busy_agent test_self_announced_close_does_not_rewake_but_next_note_does +test_separate_self_announced_answers_after_fold_do_not_rewake test_actionable_signal_surfaced test_needs_decision_signal_payload_marked_for_branch_exclusion test_needs_decision_reconciliation_required_still_marked From fa6e22a2431ba8372804cba07a11b01d541b13cd Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sat, 19 Sep 2026 04:53:45 +0000 Subject: [PATCH 2/9] no-mistakes(review): Keep owned closes in unread status; lock ledger writes --- bin/fm-classify-lib.sh | 28 +++++------ docs/architecture.md | 2 +- tests/fm-wake-drain-unread-status.test.sh | 60 +++++++++++++++++++++++ 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 8b4dcbefca8..c22af25907c 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -1576,8 +1576,6 @@ status_open_decisions_cursor_offset() { # # print nothing. status_new_lines_since_cursor() { # [] local f=$1 captured_end=${2:-} cf offset size actual_size chunk_file line rc=0 - local pos line_start owned_ranges - local LC_ALL=C [ -f "$f" ] && [ -r "$f" ] && [ ! -L "$f" ] || return 0 cf=$(_fm_open_decisions_cursor_path "$f") chunk_file="$cf.unread.$$" @@ -1596,16 +1594,9 @@ status_new_lines_since_cursor() { # [] [ "$offset" -lt "$size" ] || return 0 _fm_status_read_span "$f" "$offset" "$((size - offset))" > "$chunk_file" 2>/dev/null \ || { rm -f "$chunk_file"; return 1; } - owned_ranges=$(status_home_appends_ranges "$f") - pos=$offset while IFS= read -r line || [ -n "$line" ]; do - line_start=$pos - pos=$((pos + ${#line} + 1)) case "$line" in - *[![:space:]]*) - _fm_offset_in_home_append_ranges "$owned_ranges" "$line_start" && continue - printf '%s\n' "$line" || { rc=1; break; } - ;; + *[![:space:]]*) printf '%s\n' "$line" || { rc=1; break; } ;; esac done < "$chunk_file" rm -f "$chunk_file" @@ -1843,12 +1834,21 @@ EOF } status_home_appends_record() { # - local f=$1 start=$2 end=$3 path ident tmp merged - local LC_ALL=C + local f=$1 start=$2 end=$3 path lock rc=0 case "$start:$end" in *[!0-9:]*) return 1 ;; esac [ "$end" -gt "$start" ] || return 1 - ident=$(_fm_open_decisions_file_ident "$f") || return 1 path=$(status_home_appends_path "$f") + lock="$path.lock" + fm_lock_acquire_wait "$lock" || return 1 + _fm_status_home_appends_merge_locked "$f" "$path" "$start" "$end" || rc=1 + fm_lock_release "$lock" || rc=1 + return "$rc" +} + +_fm_status_home_appends_merge_locked() { # + local f=$1 path=$2 start=$3 end=$4 ident tmp merged + local LC_ALL=C + ident=$(_fm_open_decisions_file_ident "$f") || return 1 merged=$(printf '%s\n%s\t%s\n' "$(status_home_appends_ranges "$f")" "$start" "$end" | awk ' NF == 2 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ && $2+0 > $1+0 { n++ @@ -2021,7 +2021,7 @@ status_span_first_actionable_record() { # [record- pos=$((pos + ${#line} + 1)) line_number=$((line_number + 1)) case "$line" in *[![:space:]]*) ;; *) continue ;; esac - _fm_offset_in_home_append_ranges "$owned_ranges" "$line_start" && continue + [ -z "$owned_ranges" ] || ! _fm_offset_in_home_append_ranges "$owned_ranges" "$line_start" || continue if status_is_captain_held "$line"; then # A transfer closes the status-log decision and remains non-actionable to # stale classification. The side-band marker lets signal routing surface diff --git a/docs/architecture.md b/docs/architecture.md index d08192cdb1e..ef55b061dc0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,7 +93,7 @@ This home's answerer close, pending-reply escalation close, and captain-held tra The watcher marker still advances only when earlier bytes were already announced or were OPEN DECISIONS fold lag (keyed `needs-decision` or `blocked` lines the drain listed); a worker `resolved`, `failed`, `paused`, or `working` line in that span, or any interleaved foreign write, still wakes. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. Any direct or remaining historical annotation prints every status line unread at the presentation cursor instead of replaying only the latest line. -Lines whose bytes this home recorded as its own bookkeeping appends are treated as already owned and are not replayed. +Lines whose bytes this home recorded as its own bookkeeping appends are treated as already owned and are not replayed in that annotation, but the UNREAD STATUS section still presents them, since it is the only guaranteed presentation of a reserved-key resolution. `bin/fm-crew-state.sh ` is the cheap current-state read for an actionable heartbeat review: it attributes an active or terminal no-mistakes run under the shared run-attribution contract, then keeps that run-step authoritative even if the pane has closed, except that a `blocked:` event reporting a refused or missing daemon socket outranks a potentially stale active run record only while that socket-down declaration is itself the log's latest recognized event, since any later event, including another `blocked:` one, means the crew moved on. For other daemon, timeout, or unreachability claims, a running or fixing run with recent pipeline-reported activity supersedes the event and names reattachment as the recovery instead of surfacing a false block. [`bin/fm-nm-run-lib.sh`](../bin/fm-nm-run-lib.sh) owns branch, head, and pipeline-custody attribution, plus complete same-branch run selection, optional inventory lookup, and ambiguity reporting. diff --git a/tests/fm-wake-drain-unread-status.test.sh b/tests/fm-wake-drain-unread-status.test.sh index ccf8bb96abd..c12285ebb08 100755 --- a/tests/fm-wake-drain-unread-status.test.sh +++ b/tests/fm-wake-drain-unread-status.test.sh @@ -158,6 +158,65 @@ test_pending_reply_resolution_surfaces_once() { pass "a pending-reply resolution buried under a later note surfaces once and closes OPEN DECISIONS" } +# The watcher's pending-reply close goes through the self-announced append, so +# it records its bytes as this home's own and never wakes. The drain must still +# present that reserved-key resolution in UNREAD STATUS, its only guaranteed +# presentation. +test_self_announced_pending_reply_close_still_surfaces() { + local dir state out status corr + dir=$(make_case self-announced-pending-reply) + state="$dir/state" + out="$dir/drain.out" + status="$state/task6.status" + + run_pending_reply() { + FM_STATE_OVERRIDE="$state" FM_PENDING_REPLY_NOW=5000 bash -c ' + . "$1"; . "$2"; shift 2; "$@" + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + corr=$(run_pending_reply fm_pending_reply_create "$dir" "$state" task6 "ship it") \ + || fail "could not create the pending-reply record" + run_pending_reply fm_pending_reply_mark_delivered "$state" "$corr" \ + || fail "could not mark the pending-reply request delivered" + FM_STATE_OVERRIDE="$state" FM_PENDING_REPLY_NOW=5000 bash -c ' + . "$1"; rec=$(fm_pending_reply_path "$2" "$3") + fm_pending_reply_set "$rec" phase escalated && fm_pending_reply_set "$rec" escalated_epoch 4950 + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$state" "$corr" \ + || fail "could not mark the pending-reply request escalated" + + printf 'blocked [key=pending-reply-%s]: pending-reply-missed: task=task6 pending-reply-id=%s request=ship it\n' \ + "$corr" "$corr" > "$status" + prime_status_seen "$state" "$status" || fail "could not mark the status file surfaced" + append_wake "$state" signal task6.status "signal: task6.status" \ + || fail "queueing the pending-reply escalation signal failed" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null || fail "drain of the escalation failed" + printf 'done [corr=%s]: shipped after all\n' "$corr" >> "$status" + prime_status_seen "$state" "$status" || fail "could not mark the status file surfaced" + append_wake "$state" signal task6.status "signal: task6.status" \ + || fail "queueing the delayed reply signal failed" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null || fail "drain of the delayed reply failed" + + run_pending_reply fm_pending_reply_try_resolve "$state" "$corr" \ + || fail "the delayed reply did not resolve the pending-reply record" + grep -F "resolved [key=pending-reply-$corr]: pending-reply-resolved:" "$status" >/dev/null \ + || fail "the resolve did not append the escalation close: $(cat "$status")" + [ -s "$state/.task6.home-appends" ] \ + || fail "the escalation close did not go through the self-announced append" + run_pending_reply fm_wake_signal_seen_current "$state" "$status" \ + || fail "the self-announced escalation close was left to re-wake this home" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain after the escalation close failed" + grep -F "task6 resolved [key=pending-reply-$corr]: pending-reply-resolved: task=task6 pending-reply-id=$corr" "$out" >/dev/null \ + || fail "the self-announced pending-reply resolution was hidden from UNREAD STATUS: $(cat "$out")" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "second drain after the escalation close failed" + if grep -F 'pending-reply-resolved:' "$out" >/dev/null; then + fail "an already-presented self-announced resolution was replayed: $(cat "$out")" + fi + pass "a self-announced pending-reply close does not wake yet still surfaces once in UNREAD STATUS" +} + test_unread_output_over_cap_remains_recoverable() { local dir state out status i payload dir=$(make_case unread-over-cap) @@ -379,6 +438,7 @@ test_already_presented_notes_are_not_replayed test_brand_new_note_after_presentation_is_surfaced test_signal_annotation_surfaces_every_unread_note_not_only_the_newest test_pending_reply_resolution_surfaces_once +test_self_announced_pending_reply_close_still_surfaces test_unread_output_over_cap_remains_recoverable test_snapshot_does_not_ack_a_later_append test_retired_task_id_starts_new_status_unread From 57ec75d90a7a204f3ff41c92a3df05fe19953d02 Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sat, 19 Sep 2026 05:57:21 +0000 Subject: [PATCH 3/9] no-mistakes(review): Drop fold-lag wake suppression so folded worker decisions still wake --- bin/fm-classify-lib.sh | 29 ------------- bin/fm-wake-lib.sh | 71 ++++++++++--------------------- docs/architecture.md | 2 +- tests/fm-send-resolve-key.test.sh | 30 +++++++------ tests/fm-wake-queue.test.sh | 53 ++++++++++++++--------- tests/fm-watch-triage.test.sh | 64 +++++++++++++++++++++------- 6 files changed, 122 insertions(+), 127 deletions(-) diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index c22af25907c..9a42dc1364a 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -1808,10 +1808,6 @@ EOF return 1 } -status_home_appends_contains_offset() { # - _fm_offset_in_home_append_ranges "$(status_home_appends_ranges "$1")" "$2" -} - status_home_appends_covers() { # local start=$2 end=$3 range_start range_end case "$start:$end" in *[!0-9:]*) return 1 ;; esac @@ -1888,31 +1884,6 @@ _fm_status_home_appends_merge_locked() { # mv -f "$tmp" "$path" || { rm -f "$tmp"; return 1; } } -# 0 when every non-blank line in [start, end) is a keyed needs-decision or -# blocked line. Empty spans succeed. A worker `resolved`, `failed`, `paused`, -# `working`, or unkeyed line fails, so lag never hides those. -_fm_status_span_is_decision_lag() { # - local f=$1 start=$2 end=$3 chunk line verb - local LC_ALL=C - case "$start:$end" in *[!0-9:]*) return 1 ;; esac - [ "$end" -ge "$start" ] || return 1 - [ "$end" -eq "$start" ] && return 0 - chunk=$(_fm_status_read_span "$f" "$start" "$((end - start))") || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in *[![:space:]]*) ;; *) continue ;; esac - verb=$(status_line_verb "$line") - case "$verb" in - needs-decision|blocked) ;; - *) return 1 ;; - esac - _fm_key_before_colon "$line" || _fm_key_at_note_head "$line" >/dev/null || return 1 - _fm_decision_key "$line" >/dev/null || return 1 - done < under # one size-and-identity snapshot. # The record form produces `\t\t` and returns 0 when diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 5ace31df768..78dc964b95f 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -2118,17 +2118,16 @@ fm_wake_signal_seen_size() { # } # 0 when 's current signature matches its recorded reported state, or -# when every byte past the already-read bound is in this home's owned-append -# ledger (optionally after OPEN DECISIONS fold lag of keyed needs-decision or -# blocked lines). For a status file a reported match means the current state -# was already reported, not that every byte was successfully classified; the -# separate classified position owns that fact. Owned-only growth after a fold -# or classified bound is this home's own bookkeeping and is not a new signal. +# when every byte past the watcher's classified offset is in this home's +# owned-append ledger. For a status file a reported match means the current +# state was already reported, not that every byte was successfully classified; +# the separate classified position owns that fact. Owned-only growth past the +# classified offset is this home's own bookkeeping and is not a new signal. # A missing marker or unreadable signature is not a match unless the owned -# ledger plus fold lag covers the unread suffix, so uncertainty still reads as -# an unreported state. +# ledger covers the unread suffix, so uncertainty still reads as an unreported +# state. fm_wake_signal_seen_current() { # - local sig marker classified size folded + local sig marker classified size sig=$(fm_wake_signal_sig "$2") || return 1 [ -n "$sig" ] || return 1 marker=$(fm_wake_signal_seen_path "$1" "$2") @@ -2141,12 +2140,7 @@ fm_wake_signal_seen_current() { # size=${size//[[:space:]]/} case "$classified:$size" in *[!0-9:]*) return 1 ;; esac [ "$classified" -le "$size" ] || return 1 - status_home_appends_covers "$2" "$classified" "$size" && return 0 - folded=$(status_open_decisions_cursor_offset "$2" 2>/dev/null) || folded=0 - case "$folded" in ''|*[!0-9]*) folded=0 ;; esac - [ "$folded" -ge "$classified" ] && [ "$folded" -le "$size" ] || return 1 - _fm_status_span_is_decision_lag "$2" "$classified" "$folded" || return 1 - status_home_appends_covers "$2" "$folded" "$size" + status_home_appends_covers "$2" "$classified" "$size" ;; *) [ "$(cat "$marker" 2>/dev/null)" = "$sig" ] ;; esac @@ -2178,23 +2172,17 @@ fm_wake_status_mark_current() { # # pending-reply escalation close, a captain-held transfer). Such a close must # not wake the session that wrote it. This appends the line, records the exact # appended byte range in the home-owned append ledger (bin/fm-classify-lib.sh), -# and then advances the watcher's seen marker across those bytes when this home -# already read every earlier byte. Already-read means: -# - the watcher's classified offset equals the pre-append size, or -# - the OPEN DECISIONS fold cursor reaches the pre-append size (or owned -# ranges cover the gap from that cursor to the pre-append size) and every -# non-blank line the watcher has not classified yet is a keyed -# needs-decision or blocked line, which OPEN DECISIONS listed as open. -# The fold reads bytes it never prints, so a worker's `failed:`, `paused:`, -# `working:`, `resolved`, or verb-less line there must still wake. The post- -# append size must equal the pre-append size plus exactly the appended bytes -# (no foreign write interleaved), and the span classifier from the already-read -# bound, skipping owned ranges, must find no actionable event. -# On ANY other condition - missing file, pending foreign bytes that are not -# fold lag, an interleaved writer, an unreadable size or identity - the line is -# still appended and the owned range is still recorded when growth is proven, -# but the marker is left alone so the watcher surfaces unowned foreign bytes. -# Later drains treat owned ranges as already owned even when the prefix cursor +# and then advances the watcher's seen marker across those bytes only when the +# watcher's classified offset equals the pre-append size (every earlier byte was +# already classified) and the post-append size equals the pre-append size plus +# exactly the appended bytes (no foreign write interleaved). +# On ANY other condition - missing file, pending foreign bytes, an interleaved +# writer, an unreadable size or identity - the line is still appended and the +# owned range is still recorded when growth is proven, but the marker is left +# alone so the watcher surfaces the unclassified foreign bytes. An OPEN +# DECISIONS fold is no substitute: any actor's drain folds, so a worker line +# only the fold read must still wake. +# Later signal scans treat owned ranges as already owned even when the watcher # has not caught up, so separate --resolve-key answers do not each force a # captain-facing wake. A later different line from any other writer grows the # size past the owned ranges and wakes as before: task identity alone can never @@ -2203,7 +2191,7 @@ fm_wake_status_mark_current() { # # (the safe direction), 2 the append itself failed. fm_wake_status_append_self_announced() { # local state=$1 file=$2 line=$3 pre_size='' pre_ident='' post_size post_ident - local classified folded bound span_rc=0 + local classified local LC_ALL=C _fm_wake_require_classify || return 1 if [ -e "$file" ]; then @@ -2219,22 +2207,7 @@ fm_wake_status_append_self_announced() { # [ "$post_size" -eq $((pre_size + ${#line} + 1)) ] || return 1 status_home_appends_record "$file" "$pre_size" "$post_size" || return 1 classified=$(fm_wake_signal_seen_size "$state" "$file") - case "$classified" in ''|*[!0-9]*) classified=0 ;; esac - bound= - if [ "$classified" = "$pre_size" ]; then - bound=$classified - else - folded=$(status_open_decisions_cursor_offset "$file" 2>/dev/null) || folded=0 - case "$folded" in ''|*[!0-9]*) folded=0 ;; esac - if [ "$folded" -le "$pre_size" ] && [ "$classified" -le "$folded" ] \ - && _fm_status_span_is_decision_lag "$file" "$classified" "$folded" \ - && status_home_appends_covers "$file" "$folded" "$pre_size"; then - bound=$folded - fi - fi - [ -n "$bound" ] || return 1 - status_span_first_actionable_record "$file" "$bound" >/dev/null || span_rc=$? - [ "$span_rc" -eq 1 ] || return 1 + [ "$classified" = "$pre_size" ] || return 1 fm_wake_status_seen_commit "$state" "$file" "$post_size" "$post_ident" || return 1 return 0 } diff --git a/docs/architecture.md b/docs/architecture.md index ef55b061dc0..d523d43f37a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,7 +90,7 @@ A third bounded section, RECORD DIVERGENCE, prints on the same drains for the op A failed read, output, or concurrent-replacement check prevents the snapshot cursor from advancing across uncertain bytes, and teardown retires a task's manifest row before that task ID can be reused. The explicit resolution is written by the actor that answers, not the busy worker: `fm-send`'s `--resolve-key` appends the closing `resolved` line to this home's own copy of the ledger at answer time, which covers crewmates, local secondmates, and remote secondmates identically because a remote mate's escalations reach that local copy through the parent-replies ingest and only the answer message itself crosses the transport. This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so later drains and signal scans treat those bytes as already owned. -The watcher marker still advances only when earlier bytes were already announced or were OPEN DECISIONS fold lag (keyed `needs-decision` or `blocked` lines the drain listed); a worker `resolved`, `failed`, `paused`, or `working` line in that span, or any interleaved foreign write, still wakes. +The watcher marker still advances only when the watcher had already classified every earlier byte; a worker line the watcher has not classified yet, even one an OPEN DECISIONS fold already listed, or any interleaved foreign write, still wakes. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. Any direct or remaining historical annotation prints every status line unread at the presentation cursor instead of replaying only the latest line. Lines whose bytes this home recorded as its own bookkeeping appends are treated as already owned and are not replayed in that annotation, but the UNREAD STATUS section still presents them, since it is the only guaranteed presentation of a reserved-key resolution. diff --git a/tests/fm-send-resolve-key.test.sh b/tests/fm-send-resolve-key.test.sh index 001ab63ea2d..07b99604730 100755 --- a/tests/fm-send-resolve-key.test.sh +++ b/tests/fm-send-resolve-key.test.sh @@ -180,11 +180,11 @@ test_answer_close_is_self_announced() { pass "fm-send --resolve-key: the close never re-wakes its own home, later lines still do" } -# Two distinct --resolve-key answers after an OPEN DECISIONS fold (no watcher -# seen marker) must each stay quiet. This is the multi-decision residual: lag -# verbs on unread status lines are not enough, because each answer is its own -# event. A later worker line on the same task still wakes. -test_separate_resolve_key_answers_after_fold_do_not_rewake() { +# Two distinct --resolve-key answers to decisions the watcher already +# classified must each stay quiet, even when neither answer advances the seen +# marker: the answers are this home's owned ranges. A later worker line on the +# same task still wakes. +test_separate_resolve_key_answers_do_not_rewake() { local dir fb log home rc dir="$TMP_ROOT/separate-answers"; mkdir -p "$dir" fb=$(make_stubs "$dir"); log="$dir/send.log" @@ -194,30 +194,32 @@ test_separate_resolve_key_answers_after_fold_do_not_rewake() { printf 'needs-decision [key=budget]: approve spend?\n' printf 'needs-decision [key=vendor]: pick a vendor\n' } > "$home/state/t7.status" - - drain_out "$home" >/dev/null + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_status_mark_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ + || fail "could not prime the announced baseline" run_send "$fb" "$home" "$log" t7 --resolve-key budget "approved"; rc=$? - expect_code 0 "$rc" "the first folded answer should succeed" + expect_code 0 "$rc" "the first answer should succeed" FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_signal_seen_current "$2" "$3" ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ - || fail "the first --resolve-key answer after a fold was left to re-wake this home" + || fail "the first --resolve-key answer was left to re-wake this home" run_send "$fb" "$home" "$log" t7 --resolve-key vendor "acme"; rc=$? - expect_code 0 "$rc" "the second folded answer should succeed" + expect_code 0 "$rc" "the second answer should succeed" FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_signal_seen_current "$2" "$3" ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ - || fail "the second --resolve-key answer after a fold was left to re-wake this home" + || fail "the second --resolve-key answer was left to re-wake this home" printf 'blocked: need staging credentials\n' >> "$home/state/t7.status" if FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_signal_seen_current "$2" "$3" ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status"; then - fail "a later worker line after two folded answers was swallowed" + fail "a later worker line after two answers was swallowed" fi - pass "fm-send --resolve-key: separate answers after a fold do not each re-wake; later lines still do" + pass "fm-send --resolve-key: separate answers do not each re-wake; later lines still do" } # The reported failure behind issue #2109: a worker that put the colon first @@ -764,7 +766,7 @@ test_remote_reserved_pending_reply_key_closes_locally() { test_answer_send_closes_open_decision test_answer_close_is_self_announced -test_separate_resolve_key_answers_after_fold_do_not_rewake +test_separate_resolve_key_answers_do_not_rewake test_colon_first_key_position_is_answerable test_answer_starts_work_never_orphans test_routine_steer_never_closes diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index bf4ab81404e..fee08e1978c 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -1628,13 +1628,14 @@ test_self_announced_append_guards() { pass "self-announced appends suppress only their own bytes and fail toward waking" } -# Two distinct --resolve-key closes after an OPEN DECISIONS fold must not each -# leave the file wake-worthy. The fold listed both keys; each answer records its -# own byte range so later scans treat those ranges as already owned. A worker -# line that is not in that ledger still wakes, including a worker `resolved` -# sitting in the folded span (lag verbs are only keyed needs-decision/blocked). +# Two distinct --resolve-key closes after an OPEN DECISIONS fold record their +# own byte ranges, so the watcher's span classification never reports the +# answers. The fold alone does not mark the worker's decisions seen, because +# any actor's drain folds: a folded decision this home has not answered still +# classifies as a new signal. Once the watcher has classified the span, the +# owned answers stay quiet and a later worker line wakes. test_separate_self_announced_answers_after_fold_are_owned() { - local dir state status rc + local dir state status rc events dir=$(make_case multi-answer-owned) state="$dir/state" status="$state/t.status" @@ -1648,27 +1649,39 @@ test_separate_self_announced_answers_after_fold_are_owned() { { printf 'needs-decision [key=k1]: pick REST or RPC\n' printf 'needs-decision [key=k2]: pick us-east or eu-west\n' + printf 'needs-decision [key=k3]: pick a database\n' } > "$status" FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ || fail "the OPEN DECISIONS fold drain failed" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + && fail "a fold alone marked unclassified worker decisions as seen" + rc=0 + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k1]: answered: REST' || rc=$? + [ "$rc" -eq 1 ] || fail "the first answer over unclassified decisions did not fail toward waking (rc=$rc)" + rc=0 run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ - 'resolved [key=k1]: answered: REST' \ - || fail "the first folded answer was not self-announced (rc=$?)" + 'resolved [key=k2]: answered: eu-west' || rc=$? + [ "$rc" -eq 1 ] || fail "the second answer over unclassified decisions did not fail toward waking (rc=$rc)" run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ - || fail "the first folded answer was left to re-wake this home" + && fail "unclassified worker decisions were hidden behind this home's answers" - run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ - 'resolved [key=k2]: answered: eu-west' \ - || fail "the second folded answer was not self-announced (rc=$?)" + events=$(FM_STATE_OVERRIDE="$state" bash -c '. "$1"; status_span_first_actionable "$2" 0' _ "$ROOT/bin/fm-classify-lib.sh" "$status") \ + || fail "the unanswered folded decision was not classified as actionable" + [ "$events" = 'needs-decision [key=k3]: pick a database' ] \ + || fail "the span classification reported more than the unanswered decision: $events" + + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not record the watcher classifying the decisions" run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ - || fail "the second folded answer was left to re-wake this home" + || fail "the owned answers were left to re-wake this home" printf 'blocked [key=creds]: need staging credentials\n' >> "$status" run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ && fail "a later worker line after two owned answers was swallowed" - pass "separate self-announced answers after a fold stay owned; a later worker line still wakes" + pass "separate self-announced answers after a fold stay owned; worker decisions and later lines still wake" } test_folded_worker_resolved_is_not_owned_lag() { @@ -1702,7 +1715,7 @@ test_folded_worker_resolved_is_not_owned_lag() { } test_owned_appends_are_not_replayed_as_unread() { - local dir state status out + local dir state status out rc dir=$(make_case owned-unread) state="$dir/state" status="$state/t.status" @@ -1719,12 +1732,14 @@ test_owned_appends_are_not_replayed_as_unread() { } > "$status" FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ || fail "the OPEN DECISIONS fold drain failed" + rc=0 run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ - 'resolved [key=k1]: answered: first' \ - || fail "the first owned close failed" + 'resolved [key=k1]: answered: first' || rc=$? + [ "$rc" -le 1 ] || fail "the first owned close was not appended (rc=$rc)" + rc=0 run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ - 'resolved [key=k2]: answered: second' \ - || fail "the second owned close failed" + 'resolved [key=k2]: answered: second' || rc=$? + [ "$rc" -le 1 ] || fail "the second owned close was not appended (rc=$rc)" printf 'blocked: need staging credentials\n' >> "$status" append_wake "$state" signal t.status "signal: $status" \ || fail "could not queue the later worker signal" diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 0a07dbcfc40..21c15e3f37f 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1573,8 +1573,34 @@ test_self_announced_close_does_not_rewake_but_next_note_does() { pass "a self-announced close never wakes its own home, and the next real note still does" } -test_separate_self_announced_answers_after_fold_do_not_rewake() { - local dir state fakebin out status_file pid rc +# Any actor's drain folds OPEN DECISIONS, including a Pi branch drain, so a +# fold is no proof the watcher's owner saw the line. A fresh worker decision the +# fold already read must still wake when this home appended nothing. +test_folded_worker_decision_without_home_append_still_wakes() { + local dir state fakebin out status_file pid + dir=$(make_case folded-decision-wakes); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + status_file="$state/task.status" + printf 'working: building\n' > "$status_file" + prime_status_seen "$state" "$status_file" || fail "could not prime the announced baseline" + printf 'needs-decision [key=k3]: pick a region\n' >> "$status_file" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + export FM_FAKE_CREW_STATE='state: unknown · source: none · idle worker' + watch_bg "$state" "$fakebin" "$out" + pid=$! + wait_for_exit "$pid" 100 || fail "a folded worker decision with no home append was swallowed" + grep -F "signal: $status_file" "$out" >/dev/null \ + || fail "the folded worker decision did not surface as a signal: $(cat "$out")" + pass "a folded worker decision with no home append still wakes" +} + +# Two distinct --resolve-key answers to decisions the watcher never classified +# leave the marker alone, since a fold is no proof the watcher's owner saw them. +# That costs one wake for the worker's decisions, not one per answer: the +# answers are this home's owned ranges. Once the watcher surfaced the +# decisions, the owned answers stay quiet and the next real note still wakes. +test_separate_self_announced_answers_after_fold_wake_once() { + local dir state fakebin out status_file pid rc answer dir=$(make_case multi-answer-fold); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" status_file="$state/task.status" { @@ -1583,26 +1609,33 @@ test_separate_self_announced_answers_after_fold_do_not_rewake() { } > "$status_file" FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ || fail "the OPEN DECISIONS fold drain failed" - rc=0 - FM_STATE_OVERRIDE="$state" bash -c ' - . "$1" - fm_wake_status_append_self_announced "$2" "$3" "resolved [key=k1]: answered: REST" || exit $? - fm_wake_status_append_self_announced "$2" "$3" "resolved [key=k2]: answered: eu-west" || exit $? - ' _ "$ROOT/bin/fm-wake-lib.sh" "$state" "$status_file" || rc=$? - [ "$rc" -eq 0 ] || fail "the folded answers were not self-announced (rc=$rc)" + for answer in 'resolved [key=k1]: answered: REST' 'resolved [key=k2]: answered: eu-west'; do + rc=0 + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; fm_wake_status_append_self_announced "$2" "$3" "$4" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$state" "$status_file" "$answer" || rc=$? + [ "$rc" -eq 1 ] || fail "an answer over unclassified worker decisions did not fail toward waking (rc=$rc)" + done export FM_FAKE_CREW_STATE='state: unknown · source: none · idle worker' watch_bg "$state" "$fakebin" "$out" pid=$! + wait_for_exit "$pid" 100 || fail "the unclassified worker decisions were swallowed" + grep -F "signal: $status_file" "$out" >/dev/null \ + || fail "the worker decisions did not surface as a signal: $(cat "$out")" + ack_stopped_cycle "$state" || fail "could not handle the worker decisions' wake" + : > "$out" + watch_bg "$state" "$fakebin" "$out" + pid=$! if ! wait_poll_cycle "$state" "$pid"; then - reap "$pid"; fail "separate folded answers re-woke the watcher: $(cat "$out")" + reap "$pid"; fail "the owned answers re-woke the watcher: $(cat "$out")" fi - [ ! -s "$out" ] || { reap "$pid"; fail "separate folded answers printed a wake reason: $(cat "$out")"; } - [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "separate folded answers enqueued a durable wake"; } + [ ! -s "$out" ] || { reap "$pid"; fail "the owned answers printed a wake reason: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the owned answers enqueued another durable wake"; } printf 'blocked: need staging credentials\n' >> "$status_file" - wait_for_exit "$pid" 100 || fail "a later worker line after two folded answers was swallowed" + wait_for_exit "$pid" 100 || fail "a later worker line after two owned answers was swallowed" grep -F "signal: $status_file" "$out" >/dev/null \ || fail "the later worker line did not surface as a signal" - pass "separate self-announced answers after a fold never wake, and the next real note still does" + pass "separate answers over unclassified decisions wake once, and the next real note still does" } # --- actionable wakes are surfaced (queue + exit) --------------------------- @@ -5505,7 +5538,8 @@ test_working_note_not_working_surfaced test_secondmate_status_note_surfaced_despite_busy_agent test_secondmate_buried_block_wakes_despite_busy_agent test_self_announced_close_does_not_rewake_but_next_note_does -test_separate_self_announced_answers_after_fold_do_not_rewake +test_folded_worker_decision_without_home_append_still_wakes +test_separate_self_announced_answers_after_fold_wake_once test_actionable_signal_surfaced test_needs_decision_signal_payload_marked_for_branch_exclusion test_needs_decision_reconciliation_required_still_marked From 007c321429e0b7b6e479cb3daef4c42eb6c39f56 Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sat, 19 Sep 2026 06:26:40 +0000 Subject: [PATCH 4/9] no-mistakes(review): Require real owned growth before ledger marks status seen --- bin/fm-wake-lib.sh | 13 +++++----- tests/fm-wake-queue.test.sh | 50 +++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 78dc964b95f..6f2162efb5a 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -2118,14 +2118,15 @@ fm_wake_signal_seen_size() { # } # 0 when 's current signature matches its recorded reported state, or -# when every byte past the watcher's classified offset is in this home's -# owned-append ledger. For a status file a reported match means the current +# when the file is a readable regular file that grew past the watcher's +# classified offset and every grown byte is in this home's owned-append ledger. +# For a status file a reported match means the current # state was already reported, not that every byte was successfully classified; # the separate classified position owns that fact. Owned-only growth past the # classified offset is this home's own bookkeeping and is not a new signal. -# A missing marker or unreadable signature is not a match unless the owned -# ledger covers the unread suffix, so uncertainty still reads as an unreported -# state. +# A missing marker, an unreadable signature, or any other signature change +# without owned growth is not a match, so uncertainty still reads as an +# unreported state. fm_wake_signal_seen_current() { # local sig marker classified size sig=$(fm_wake_signal_sig "$2") || return 1 @@ -2139,7 +2140,7 @@ fm_wake_signal_seen_current() { # size=$(_fm_status_file_size "$2") || return 1 size=${size//[[:space:]]/} case "$classified:$size" in *[!0-9:]*) return 1 ;; esac - [ "$classified" -le "$size" ] || return 1 + [ "$classified" -lt "$size" ] && [ -f "$2" ] && [ -r "$2" ] && [ ! -L "$2" ] || return 1 status_home_appends_covers "$2" "$classified" "$size" ;; *) [ "$(cat "$marker" 2>/dev/null)" = "$sig" ] ;; diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index fee08e1978c..1800d46ea17 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -1684,6 +1684,55 @@ test_separate_self_announced_answers_after_fold_are_owned() { pass "separate self-announced answers after a fold stay owned; worker decisions and later lines still wake" } +# The owned ledger only vouches for growth it recorded. A signature change +# with no growth past the classified offset, such as the log turning +# unreadable, must still read as unreported, before and after owned growth. +test_unreadable_status_is_not_owned() { + local dir state status + dir=$(make_case owned-unreadable) + state="$dir/state" + status="$state/t.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + if [ "$(id -u)" -eq 0 ]; then + pass "unreadable status check skipped: root reads mode-000 files" + return 0 + fi + printf 'needs-decision [key=k1]: pick one\n' > "$status" + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not prime the announced baseline" + chmod 000 "$status" + if run_wake_lib fm_wake_signal_seen_current "$state" "$status"; then + chmod 600 "$status" + fail "an unreadable fully classified status read as already seen" + fi + chmod 600 "$status" + + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not re-prime the announced baseline" + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k1]: answered: one' \ + || fail "the owned close was not self-announced" + printf 'needs-decision [key=k2]: pick two\n' >> "$status" + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not record the watcher classifying the worker line" + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k2]: answered: two' \ + || fail "the second owned close was not self-announced" + chmod 000 "$status" + if run_wake_lib fm_wake_signal_seen_current "$state" "$status"; then + chmod 600 "$status" + fail "an unreadable status after owned growth read as already seen" + fi + chmod 600 "$status" + pass "an unreadable status still reads as unreported, with or without owned growth" +} + test_folded_worker_resolved_is_not_owned_lag() { local dir state status rc dir=$(make_case folded-worker-resolved) @@ -2121,6 +2170,7 @@ test_acknowledged_stall_publication_survives_pre_marker_crash test_empty_prefix_mate_preserves_other_mate_receipt test_self_announced_append_guards test_separate_self_announced_answers_after_fold_are_owned +test_unreadable_status_is_not_owned test_folded_worker_resolved_is_not_owned_lag test_owned_appends_are_not_replayed_as_unread test_historical_annotation_skips_announced_status From 0a21cd050b118e2b0785edf82a556548aaed1ad3 Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sat, 19 Sep 2026 06:51:14 +0000 Subject: [PATCH 5/9] no-mistakes(document): Clarify home-appends ledger scope versus UNREAD STATUS --- AGENTS.md | 2 +- bin/fm-classify-lib.sh | 9 +++++---- docs/architecture.md | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 330db08bdf2..9b42c781dc4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,7 +142,7 @@ state/ runtime records and signals; gitignored .wake-queue durable queued wakes retained until post-handling acknowledgement: epochseqkindkeypayload .watcher-down private generation-bound recovery state coupling watcher downtime, durable wake presentation, and post-handling acknowledgement; never touch ..open-decisions-cursor per-task byte cursor and folded open-decision set bounding the OPEN DECISIONS scan's cost to new status-log appends; written only by fm-classify-lib.sh's status_open_decisions_incremental, removed by teardown, safe to delete (forces one full re-fold) - ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so later drains and signal scans treat those ranges as already owned; written only by fm-classify-lib.sh's status_home_appends_record, removed by teardown, safe to delete + ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so signal scans and wake annotations treat those ranges as already owned (UNREAD STATUS still presents them); written only by fm-classify-lib.sh's status_home_appends_record, removed by teardown, safe to delete .status-presentation-cursor .status-presentation-lock fleet-wide per-task status identity plus independent annotation and outcome-backstop byte offsets, with a serialization lock preventing already-presented lines from replaying while preserving delayed signal annotations; owned by fm-classify-lib.sh, with each task's row retired by teardown .afk-contract the away-posture record: the captain's verbatim away words, expected return, reach profile, spend cap, and structured mandate clauses; written only by bin/fm-afk-contract.sh after the captain confirms the read-back, archived under afk-contracts/ at return; its presence IS the away posture in every harness; its sibling .afk-contract.lock serializes actions authorized by the live record (contract: bin/fm-afk-contract.sh) afk-contracts/ archived away-posture records: one final record per away window keyed by entry time, plus any superseded mandates from that window diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 9a42dc1364a..748e98bacad 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -38,8 +38,8 @@ # cursor and folded open-set as a side effect, so a per-drain fleet-wide scan # stays bounded by new appends instead of re-reading each task's whole lifetime # log every time. status_home_appends_record writes the per-task home-owned -# append ledger (see "home-owned status-append ledger" below) so later drains -# and signal scans can treat this home's own bookkeeping bytes as already owned. +# append ledger (see "home-owned status-append ledger" below) so signal scans +# and wake annotations can treat this home's own bookkeeping bytes as already owned. # crew_worktree_written_since reads the task's meta file and walks a bounded slice # of its worktree instead of a status file, so callers run it only at the moment # they would otherwise escalate. @@ -1741,8 +1741,9 @@ window_to_task() { # --- home-owned status-append ledger ---------------------------------------- # # This home's bookkeeping closes (fm_wake_status_append_self_announced) record -# the exact byte range they appended so later drains and signal scans can treat -# those bytes as already owned. That is the multi-answer path: two distinct +# the exact byte range they appended so signal scans and wake annotations treat +# those bytes as already owned, while the drain's UNREAD STATUS section still +# presents them. That is the multi-answer path: two distinct # --resolve-key closes must not each force a captain-facing wake solely because # each one appended a status line, while a worker-authored line that is not in # this ledger still signals. diff --git a/docs/architecture.md b/docs/architecture.md index d523d43f37a..6491946085a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,7 +89,7 @@ A queued signal annotation prints every status line still unread at that cursor, A third bounded section, RECORD DIVERGENCE, prints on the same drains for the opposite failure: the status fold went quiet on a key that the durable captain-held task still shows as open, so the status side reads as complete while the two records contradict each other; `bin/fm-captain-hold.sh diverged` decides what counts and closes nothing, and `docs/captain-hold-lifecycle.md` owns the mechanism. A failed read, output, or concurrent-replacement check prevents the snapshot cursor from advancing across uncertain bytes, and teardown retires a task's manifest row before that task ID can be reused. The explicit resolution is written by the actor that answers, not the busy worker: `fm-send`'s `--resolve-key` appends the closing `resolved` line to this home's own copy of the ledger at answer time, which covers crewmates, local secondmates, and remote secondmates identically because a remote mate's escalations reach that local copy through the parent-replies ingest and only the answer message itself crosses the transport. -This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so later drains and signal scans treat those bytes as already owned. +This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so signal scans and wake annotations treat those bytes as already owned. The watcher marker still advances only when the watcher had already classified every earlier byte; a worker line the watcher has not classified yet, even one an OPEN DECISIONS fold already listed, or any interleaved foreign write, still wakes. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. Any direct or remaining historical annotation prints every status line unread at the presentation cursor instead of replaying only the latest line. From 13e1bad038e7d3185690e99f59ffcfb1d555c204 Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sun, 20 Sep 2026 00:10:45 +0000 Subject: [PATCH 6/9] no-mistakes(review): Restore fold-lag path, drop owned-range filters, fix test --- AGENTS.md | 2 +- bin/fm-classify-lib.sh | 21 -------- bin/fm-wake-lib.sh | 87 +++++++++++++++++-------------- docs/architecture.md | 4 +- tests/fm-send-resolve-key.test.sh | 55 +++++++++++-------- tests/fm-wake-queue.test.sh | 70 +++++++++---------------- tests/fm-watch-triage.test.sh | 35 +++++++++++++ 7 files changed, 143 insertions(+), 131 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c0e281e4ac1..e26099e7b57 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,7 +142,7 @@ state/ runtime records and signals; gitignored .wake-queue durable queued wakes retained until post-handling acknowledgement: epochseqkindkeypayload .watcher-down private generation-bound recovery state coupling watcher downtime, durable wake presentation, and post-handling acknowledgement; never touch ..open-decisions-cursor per-task byte cursor and folded open-decision set bounding the OPEN DECISIONS scan's cost to new status-log appends; written only by fm-classify-lib.sh's status_open_decisions_incremental, removed by teardown, safe to delete (forces one full re-fold) - ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so signal scans and wake annotations treat those ranges as already owned (UNREAD STATUS still presents them); written only by fm-classify-lib.sh's status_home_appends_record, removed by teardown, safe to delete + ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so a wake scan can tell its own growth from a foreign write instead of waking on it; presentation is unaffected, so both the signal annotation and UNREAD STATUS still print those lines; written only by fm-classify-lib.sh's status_home_appends_record, removed by teardown, safe to delete .status-presentation-cursor .status-presentation-lock fleet-wide per-task status identity plus independent annotation and outcome-backstop byte offsets, with a serialization lock preventing already-presented lines from replaying while preserving delayed signal annotations; owned by fm-classify-lib.sh, with each task's row retired by teardown .afk-contract the away-posture record: the captain's verbatim away words, expected return, reach profile, spend cap, and structured mandate clauses; written only by bin/fm-afk-contract.sh after the captain confirms the read-back, archived under afk-contracts/ at return; its presence IS the away posture in every harness; its sibling .afk-contract.lock serializes actions authorized by the live record (contract: bin/fm-afk-contract.sh) afk-contracts/ archived away-posture records: one final record per away window keyed by entry time, plus any superseded mandates from that window diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 748e98bacad..8631556b780 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -1794,21 +1794,6 @@ $rest EOF } -_fm_offset_in_home_append_ranges() { # - local ranges=$1 offset=$2 start end - case "$offset" in ''|*[!0-9]*) return 1 ;; esac - while IFS=$(printf '\t') read -r start end; do - [ -n "$start" ] || continue - case "$start:$end" in *[!0-9:]*) continue ;; esac - if [ "$offset" -ge "$start" ] && [ "$offset" -lt "$end" ]; then - return 0 - fi - done < local start=$2 end=$3 range_start range_end case "$start:$end" in *[!0-9:]*) return 1 ;; esac @@ -1958,7 +1943,6 @@ _fm_status_open_decision_origins() { # [] status_span_first_actionable_record() { # [record-var] [needs-decision-var] local f=$1 start=${2:-0} output_var=${3-} needs_var=${4-} size ident cur_ident scratch chunk_file full_file prefix_file result local line verb key origins='' folded=0 rc=1 failed=0 prefix_lines=0 line_number=0 live_line='' events='' _line _key _fm_span_needs_decision=0 - local pos line_start owned_ranges local LC_ALL=C [ -e "$f" ] || { [ -L "$f" ] && return 2; return 1; } [ -f "$f" ] && [ -r "$f" ] && [ ! -L "$f" ] || return 2 @@ -1986,14 +1970,9 @@ status_span_first_actionable_record() { # [record- rm -f "$chunk_file" "$full_file" "$prefix_file"; return 2; } [ "$cur_ident" = "$ident" ] || { rm -f "$chunk_file" "$full_file" "$prefix_file"; return 2; } - owned_ranges=$(status_home_appends_ranges "$f") - pos=$start while IFS= read -r line || [ -n "$line" ]; do - line_start=$pos - pos=$((pos + ${#line} + 1)) line_number=$((line_number + 1)) case "$line" in *[![:space:]]*) ;; *) continue ;; esac - [ -z "$owned_ranges" ] || ! _fm_offset_in_home_append_ranges "$owned_ranges" "$line_start" || continue if status_is_captain_held "$line"; then # A transfer closes the status-log decision and remains non-actionable to # stale classification. The side-band marker lets signal routing surface diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 75292670a1b..879ad42f1f2 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -2171,29 +2171,39 @@ fm_wake_status_mark_current() { # # lines THIS home's own machinery writes as bookkeeping it has already presented # in the very turn or tick that writes them (answerer-closes resolved lines, a # pending-reply escalation close, captain-held transfers). Such a close must -# not wake the session that wrote it. This appends one command's lines +# not wake the session that wrote it, so this appends one command's lines # together, records the exact appended byte range in the home-owned append # ledger (bin/fm-classify-lib.sh), and then advances the watcher's seen marker -# across those bytes only when the watcher's classified offset equals the -# pre-append size (every earlier byte was already classified) and the -# post-append size equals the pre-append size plus exactly the appended bytes -# (no foreign write interleaved). -# On ANY other condition - missing file, pending foreign bytes, an interleaved -# writer, an unreadable size or identity - the lines are still appended and the -# owned range is still recorded when growth is proven, but the marker is left -# alone so the watcher surfaces the unclassified foreign bytes. An OPEN -# DECISIONS fold is no substitute: any actor's drain folds, so a worker line -# only the fold read must still wake. +# across the appended bytes and no byte this home has not already read. The +# advance is provenance-gated and fails toward waking: +# - the marker advances only when this home already read every pre-append +# byte, the post-append size equals that size plus exactly the appended +# bytes (no foreign write interleaved), AND the watcher's own span +# classifier finds no actionable event from its classified offset through +# the post-append end (classifying after the append keeps the just-closed +# decisions from counting as live); +# - "already read" means the watcher's classified seen offset equals the +# pre-append size, or the OPEN DECISIONS fold cursor does and every +# non-blank line the watcher has not classified yet is a keyed +# needs-decision or blocked line, which OPEN DECISIONS listed as open. The +# fold reads bytes it never prints, so a worker's `failed:`, `paused:`, +# `working:`, `resolved` or verb-less line there must still wake, and so +# must a captain-held line, which raises the watcher's needs-decision +# side-band; +# - on ANY other condition - a missing file, pending foreign bytes, an +# interleaved writer, an unreadable size or identity - the lines are still +# appended and the owned range is still recorded when growth is proven, but +# the marker is left alone, so the watcher surfaces the file normally. # Later signal scans treat owned ranges as already owned even when the watcher # has not caught up, so separate --resolve-key answers do not each force a -# captain-facing wake. A later different line from any other writer grows the +# captain-facing wake. A later, different line from any other writer grows the # size past the owned ranges and wakes as before: task identity alone can never # suppress new content. # Returns 0 appended and self-announced, 1 appended but left for the watcher # (the safe direction), 2 the append itself failed. fm_wake_status_append_self_announced() { # ... local state=$1 file=$2 line appended=0 pre_size='' pre_ident='' post_size post_ident - local classified + local classified folded lag span_rc=0 local LC_ALL=C shift 2 _fm_wake_require_classify || return 1 @@ -2211,7 +2221,24 @@ fm_wake_status_append_self_announced() { # ... [ "$post_size" -eq $((pre_size + appended)) ] || return 1 status_home_appends_record "$file" "$pre_size" "$post_size" || return 1 classified=$(fm_wake_signal_seen_size "$state" "$file") - [ "$classified" = "$pre_size" ] || return 1 + if [ "$classified" != "$pre_size" ]; then + folded=$(status_open_decisions_cursor_offset "$file") || folded=0 + [ "$folded" = "$pre_size" ] && [ "$classified" -lt "$pre_size" ] || return 1 + lag=$(_fm_status_read_span "$file" "$classified" "$((pre_size - classified))") || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in *[![:space:]]*) ;; *) continue ;; esac + case "$(status_line_verb "$line")" in + needs-decision|blocked) ;; + *) return 1 ;; + esac + _fm_key_before_colon "$line" || _fm_key_at_note_head "$line" >/dev/null || return 1 + _fm_decision_key "$line" >/dev/null || return 1 + done </dev/null || span_rc=$? + [ "$span_rc" -eq 1 ] || return 1 fm_wake_status_seen_commit "$state" "$file" "$post_size" "$post_ident" || return 1 return 0 } @@ -2272,11 +2299,10 @@ fm_wake_status_cursor_offset() { # -> already-presented # O_NOFOLLOW read of every still-unread status byte. min-offset is the # already-presented cursor from classify-lib. Lines whose bytes begin before -# that offset are not replayed, nor are lines this home recorded as its own -# bookkeeping appends. Prints nothing and returns 1 when no unread non-blank -# line exists. +# that offset are not replayed. Prints nothing and returns 1 when no unread +# non-blank line exists. fm_wake_unread_events() { # [] - local path=$1 min_offset=$3 end_offset=${4:-} result size chunk chunk_start owned + local path=$1 min_offset=$3 end_offset=${4:-} result size chunk chunk_start local LC_ALL=C FM_WAKE_EVENT_LINE= FM_WAKE_UNREAD_LINES= @@ -2307,31 +2333,12 @@ fm_wake_unread_events() { # /dev/null 2>&1; then - owned=$(status_home_appends_ranges "$path") || owned= - fi - FM_WAKE_UNREAD_LINES=$(printf '%s' "$chunk" | LC_ALL=C awk -v start="$chunk_start" -v min="$min_offset" -v owned="$owned" ' - BEGIN { - pos = start + 0 - n = split(owned, rows, "\n") - for (i = 1; i <= n; i++) { - split(rows[i], p, "\t") - if (p[1] ~ /^[0-9]+$/ && p[2] ~ /^[0-9]+$/) { - nr++ - os[nr] = p[1] + 0 - oe[nr] = p[2] + 0 - } - } - } - function is_owned(off, i) { - for (i = 1; i <= nr; i++) if (off >= os[i] && off < oe[i]) return 1 - return 0 - } + FM_WAKE_UNREAD_LINES=$(printf '%s' "$chunk" | LC_ALL=C awk -v start="$chunk_start" -v min="$min_offset" ' + BEGIN { pos = start + 0 } { line_start = pos pos += length($0) + 1 - if ($0 ~ /[^[:space:]]/ && line_start >= min && !is_owned(line_start)) print $0 + if ($0 ~ /[^[:space:]]/ && line_start >= min) print $0 } ') || return 1 [ -n "$FM_WAKE_UNREAD_LINES" ] || return 1 diff --git a/docs/architecture.md b/docs/architecture.md index e7d83dfe06c..0c76c516044 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,11 +89,11 @@ A queued signal annotation prints every status line still unread at that cursor, A third bounded section, RECORD DIVERGENCE, prints on the same drains for the opposite failure: the status fold went quiet on a key that the durable captain-held task still shows as open, so the status side reads as complete while the two records contradict each other; `bin/fm-captain-hold.sh diverged` decides what counts and closes nothing, and `docs/captain-hold-lifecycle.md` owns the mechanism. A failed read, output, or concurrent-replacement check prevents the snapshot cursor from advancing across uncertain bytes, and teardown retires a task's manifest row before that task ID can be reused. The explicit resolution is written by the actor that answers, not the busy worker: `fm-send`'s `--resolve-key` appends the closing `resolved` line to this home's own copy of the ledger at answer time, which covers crewmates, local secondmates, and remote secondmates identically because a remote mate's escalations reach that local copy through the parent-replies ingest and only the answer message itself crosses the transport. -This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so signal scans and wake annotations treat those bytes as already owned. +This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so a later wake scan can tell this home's own growth from a foreign write instead of waking on it. The watcher marker still advances only when the watcher had already classified every earlier byte; a worker line the watcher has not classified yet, even one an OPEN DECISIONS fold already listed, or any interleaved foreign write, still wakes. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. Any direct or remaining historical annotation prints every status line unread at the presentation cursor instead of replaying only the latest line. -Lines whose bytes this home recorded as its own bookkeeping appends are treated as already owned and are not replayed in that annotation, but the UNREAD STATUS section still presents them, since it is the only guaranteed presentation of a reserved-key resolution. +The owned-append ledger only decides whether growth wakes this home; it never removes a line from presentation, so both that annotation and the UNREAD STATUS section still print this home's own bookkeeping closes. `bin/fm-crew-state.sh ` is the cheap current-state read for an actionable heartbeat review: it attributes an active or terminal no-mistakes run under the shared run-attribution contract, then keeps that run-step authoritative even if the pane has closed, except that a `blocked:` event reporting a refused or missing daemon socket outranks a potentially stale active run record only while that socket-down declaration is itself the log's latest recognized event, since any later event, including another `blocked:` one, means the crew moved on. For other daemon, timeout, or unreachability claims, a running or fixing run with recent pipeline-reported activity supersedes the event and names reattachment as the recovery instead of surfacing a false block. [`bin/fm-nm-run-lib.sh`](../bin/fm-nm-run-lib.sh) owns branch, head, and pipeline-custody attribution, plus complete same-branch run selection, optional inventory lookup, and ambiguity reporting. diff --git a/tests/fm-send-resolve-key.test.sh b/tests/fm-send-resolve-key.test.sh index 8b87b023527..8e11e12bf3e 100755 --- a/tests/fm-send-resolve-key.test.sh +++ b/tests/fm-send-resolve-key.test.sh @@ -187,43 +187,58 @@ test_answer_close_is_self_announced() { pass "fm-send --resolve-key: the close never re-wakes its own home, later lines still do" } -# Two distinct --resolve-key answers to decisions the watcher already -# classified must each stay quiet, even when neither answer advances the seen -# marker: the answers are this home's owned ranges. A later worker line on the -# same task still wakes. +# Two distinct --resolve-key answers must each stay quiet even when the seen +# marker does NOT cover them. An in-flight watcher classification that lands +# after the first answer regresses the classified offset behind that answer's +# bytes, so the marker no longer vouches for them; only the home-appends ledger +# does. Without the ledger the second scan re-wakes this home over its own +# close. A later worker line on the same task still wakes. test_separate_resolve_key_answers_do_not_rewake() { - local dir fb log home rc + local dir fb log home rc status pre_answer ident dir="$TMP_ROOT/separate-answers"; mkdir -p "$dir" fb=$(make_stubs "$dir"); log="$dir/send.log" home=$(setup_home separate-answers) + status="$home/state/t7.status" fm_write_meta "$home/state/t7.meta" "window=sess:fm-t7" "kind=ship" { printf 'needs-decision [key=budget]: approve spend?\n' printf 'needs-decision [key=vendor]: pick a vendor\n' - } > "$home/state/t7.status" + } > "$status" FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_status_mark_current "$2" "$3" - ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" \ || fail "could not prime the announced baseline" + pre_answer=$(wc -c < "$status" | tr -d '[:space:]') run_send "$fb" "$home" "$log" t7 --resolve-key budget "approved"; rc=$? expect_code 0 "$rc" "the first answer should succeed" + + # A watcher classification captured before the answer commits afterwards and + # rewinds the classified offset behind the answer's bytes. + ident=$(FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; _fm_open_decisions_file_ident "$2" + ' _ "$ROOT/bin/fm-classify-lib.sh" "$status") \ + || fail "could not read the status identity" + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_status_seen_commit "$2" "$3" "$4" "$5" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" "$pre_answer" "$ident" \ + || fail "could not replay the stale watcher classification" FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_signal_seen_current "$2" "$3" - ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" \ || fail "the first --resolve-key answer was left to re-wake this home" run_send "$fb" "$home" "$log" t7 --resolve-key vendor "acme"; rc=$? expect_code 0 "$rc" "the second answer should succeed" FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_signal_seen_current "$2" "$3" - ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" \ || fail "the second --resolve-key answer was left to re-wake this home" - printf 'blocked: need staging credentials\n' >> "$home/state/t7.status" + printf 'blocked: need staging credentials\n' >> "$status" if FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_signal_seen_current "$2" "$3" - ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status"; then + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status"; then fail "a later worker line after two answers was swallowed" fi pass "fm-send --resolve-key: separate answers do not each re-wake; later lines still do" @@ -409,12 +424,9 @@ test_multiple_keys_close_together() { pass "fm-send --resolve-key: one answer closes each named key and only those" } -# A session-start drain may list both decisions (folding them without a -# watcher seen marker) and one answer still closes both. The fold is no -# substitute for classification, so the worker's decision bytes stay -# wake-worthy; the closes themselves succeed and drop those keys from OPEN -# DECISIONS. Owned answer ranges keep later scans from treating the closes as -# extra signals once the watcher has classified the span. +# Issue 4767: the session-start drain listed both decisions (folding them +# without a watcher seen marker), and one answer closes both. The closes are +# this home's own bookkeeping, so the watcher must not wake it to reread them. test_multiple_keys_close_after_fold_is_self_announced() { local dir fb log home rc out dir="$TMP_ROOT/multi-fold"; mkdir -p "$dir" @@ -432,16 +444,15 @@ test_multiple_keys_close_after_fold_is_self_announced() { run_send "$fb" "$home" "$log" t7 --resolve-key budget --resolve-key vendor \ "approve spend, pick acme"; rc=$? expect_code 0 "$rc" "an answer resolving two folded keys should succeed" - if FM_STATE_OVERRIDE="$home/state" bash -c ' + FM_STATE_OVERRIDE="$home/state" bash -c ' . "$1"; fm_wake_signal_seen_current "$2" "$3" - ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status"; then - fail "folding then answering swallowed the unclassified worker decisions" - fi + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$home/state/t7.status" \ + || fail "one answer's two closes after an OPEN DECISIONS drain were left to re-wake this home" out=$(drain_out "$home") if printf '%s' "$out" | grep -F 'OPEN DECISIONS' >/dev/null; then fail "an answered folded key is still open: $out" fi - pass "fm-send --resolve-key: one answer closes folded keys without swallowing the worker decisions" + pass "fm-send --resolve-key: one answer's closes after a drain fold never wake this home" } test_local_secondmate_answer_marked_and_closed() { diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 61f73c9bc37..32557f6f640 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -1569,15 +1569,16 @@ test_interruption_before_and_after_raw_commit() { # The guarded self-announced status append (fm_wake_status_append_self_announced) # and the seen-signature gate it shares with the watcher's signal scan. Both # directions of the dedup contract are pinned through the real library -# functions: a fully announced file plus the home's own bookkeeping close stays +# functions: a file this home already knows (seen marker or OPEN DECISIONS +# fold) plus the home's own bookkeeping close stays # announced (no wake), while ANY unannounced byte - a pending foreign line, a -# missing marker, a later different note - reads as wake-worthy. An OPEN -# DECISIONS fold is no substitute for the watcher's classified offset. +# missing cursor, a later different note - reads as wake-worthy. test_self_announced_append_guards() { - local dir state status rc=0 + local dir state status folded rc=0 dir=$(make_case self-announced-append) state="$dir/state" status="$state/t.status" + folded="$state/folded.status" run_wake_lib() { FM_STATE_OVERRIDE="$state" bash -c ' @@ -1633,6 +1634,26 @@ test_self_announced_append_guards() { run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ || fail "multibyte byte accounting broke the self-announce guard" + # Issue 4767: a drain that folded OPEN DECISIONS has already presented those + # bytes to this home even when the watcher has not written a matching seen + # marker. The bookkeeping close must stay quiet; a later worker line must not. + printf 'needs-decision [key=k3]: pick one\n' > "$folded" + run_wake_lib fm_wake_signal_seen_current "$state" "$folded" \ + && fail "an unfolded file without a seen marker read as announced" + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + status_open_decisions_incremental "$2" >/dev/null + ' _ "$ROOT/bin/fm-classify-lib.sh" "$folded" \ + || fail "could not fold the open decision" + run_wake_lib fm_wake_status_append_self_announced "$state" "$folded" \ + 'resolved [key=k3]: answered: folded close' \ + || fail "a close after an OPEN DECISIONS fold was not self-announced (rc=$?)" + run_wake_lib fm_wake_signal_seen_current "$state" "$folded" \ + || fail "the folded close left unannounced bytes behind" + printf 'blocked: worker still needs help\n' >> "$folded" + run_wake_lib fm_wake_signal_seen_current "$state" "$folded" \ + && fail "a later worker line after a folded close was swallowed" + pass "self-announced appends suppress only their own bytes and fail toward waking" } @@ -1771,46 +1792,6 @@ test_folded_worker_resolved_is_not_owned_lag() { pass "a worker resolved in fold lag still wakes after this home's close" } -test_owned_appends_are_not_replayed_as_unread() { - local dir state status out rc - dir=$(make_case owned-unread) - state="$dir/state" - status="$state/t.status" - - run_wake_lib() { - FM_STATE_OVERRIDE="$state" bash -c ' - . "$1"; shift; "$@" - ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" - } - - { - printf 'needs-decision [key=k1]: first\n' - printf 'needs-decision [key=k2]: second\n' - } > "$status" - FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ - || fail "the OPEN DECISIONS fold drain failed" - rc=0 - run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ - 'resolved [key=k1]: answered: first' || rc=$? - [ "$rc" -le 1 ] || fail "the first owned close was not appended (rc=$rc)" - rc=0 - run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ - 'resolved [key=k2]: answered: second' || rc=$? - [ "$rc" -le 1 ] || fail "the second owned close was not appended (rc=$rc)" - printf 'blocked: need staging credentials\n' >> "$status" - append_wake "$state" signal t.status "signal: $status" \ - || fail "could not queue the later worker signal" - FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2>"$dir/drain.err" \ - || fail "drain after the worker line failed" - out=$(cat "$dir/drain.out") - printf '%s' "$out" | grep -F 'blocked: need staging credentials' >/dev/null \ - || fail "the later worker line was not annotated: $out" - if printf '%s' "$out" | grep -E 'resolved \[key=k[12]\]' >/dev/null; then - fail "an owned bookkeeping close was replayed as unread: $out" - fi - pass "owned append ranges are not replayed as unread on a later real wake" -} - # A trap that fires inside a lock's critical section abandons the holding # frame, and the exit path then re-acquires the same lock (a TERM inside a # recovery-marker section is the reproduced case: the watcher's reap wedged @@ -2180,7 +2161,6 @@ test_self_announced_append_guards test_separate_self_announced_answers_after_fold_are_owned test_unreadable_status_is_not_owned test_folded_worker_resolved_is_not_owned_lag -test_owned_appends_are_not_replayed_as_unread test_historical_annotation_skips_announced_status test_concurrent_append_and_drain test_signal_catchup_without_running_watcher diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 44d1a46dd7d..3df6899e818 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1573,6 +1573,40 @@ test_self_announced_close_does_not_rewake_but_next_note_does() { pass "a self-announced close never wakes its own home, and the next real note still does" } +test_self_announced_close_after_open_decisions_fold_does_not_rewake() { + local dir state fakebin out status_file pid rc + dir=$(make_case self-close-after-fold); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + status_file="$state/task.status" + printf 'needs-decision [key=k1]: pick one\n' > "$status_file" + # Session-start drain folds OPEN DECISIONS without writing a watcher seen + # marker. That is the issue 4767 path: the supervisor then closes the listed + # decision and must not get a signal wake of its own resolved line. + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + status_open_decisions_incremental "$2" >/dev/null + ' _ "$ROOT/bin/fm-classify-lib.sh" "$status_file" \ + || fail "could not fold the open decision" + rc=0 + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_wake_status_append_self_announced "$2" "$3" "resolved [key=k1]: answered: closed after fold" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$state" "$status_file" || rc=$? + [ "$rc" -eq 0 ] || fail "the bookkeeping close after OPEN DECISIONS fold was not self-announced (rc=$rc)" + export FM_FAKE_CREW_STATE='state: unknown · source: none · idle worker' + watch_bg "$state" "$fakebin" "$out" + pid=$! + if ! wait_poll_cycle "$state" "$pid"; then + reap "$pid"; fail "a close after OPEN DECISIONS fold re-woke its own watcher: $(cat "$out")" + fi + [ ! -s "$out" ] || { reap "$pid"; fail "folded close printed a wake reason: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "folded close enqueued a durable wake"; } + printf 'blocked: worker still needs help\n' >> "$status_file" + wait_for_exit "$pid" 100 || fail "a later worker line after a folded close was swallowed" + grep -F "signal: $status_file" "$out" >/dev/null \ + || fail "the later worker line did not surface as a signal" + pass "a close after OPEN DECISIONS fold never wakes its own home, and the next real note still does" +} + # Any actor's drain folds OPEN DECISIONS, including a Pi branch drain, so a # fold is no proof the watcher's owner saw the line. A fresh worker decision the # fold already read must still wake when this home appended nothing. @@ -5604,6 +5638,7 @@ test_working_note_not_working_surfaced test_secondmate_status_note_surfaced_despite_busy_agent test_secondmate_buried_block_wakes_despite_busy_agent test_self_announced_close_does_not_rewake_but_next_note_does +test_self_announced_close_after_open_decisions_fold_does_not_rewake test_folded_worker_decision_without_home_append_still_wakes test_separate_self_announced_answers_after_fold_wake_once test_self_announced_close_after_fold_still_surfaces_folded_worker_failure From 01dcdb84557a7b350648976b51946879a7ac149f Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sun, 20 Sep 2026 00:28:36 +0000 Subject: [PATCH 7/9] no-mistakes(review): Align ledger docs and scope ledger to wake path only --- bin/fm-classify-lib.sh | 20 +++++++------- bin/fm-send.sh | 6 ++-- bin/fm-wake-lib.sh | 55 +++++++++++++++++++++++-------------- docs/architecture.md | 2 +- tests/fm-wake-queue.test.sh | 44 +++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 33 deletions(-) diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 8631556b780..feffdb6c477 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -38,8 +38,8 @@ # cursor and folded open-set as a side effect, so a per-drain fleet-wide scan # stays bounded by new appends instead of re-reading each task's whole lifetime # log every time. status_home_appends_record writes the per-task home-owned -# append ledger (see "home-owned status-append ledger" below) so signal scans -# and wake annotations can treat this home's own bookkeeping bytes as already owned. +# append ledger (see "home-owned status-append ledger" below) so the wake scan +# can treat this home's own bookkeeping bytes as already owned. # crew_worktree_written_since reads the task's meta file and walks a bounded slice # of its worktree instead of a status file, so callers run it only at the moment # they would otherwise escalate. @@ -1741,14 +1741,17 @@ window_to_task() { # --- home-owned status-append ledger ---------------------------------------- # # This home's bookkeeping closes (fm_wake_status_append_self_announced) record -# the exact byte range they appended so signal scans and wake annotations treat -# those bytes as already owned, while the drain's UNREAD STATUS section still -# presents them. That is the multi-answer path: two distinct +# the exact byte range they appended so the wake scan can tell this home's own +# growth from a foreign write. That is the multi-answer path: two distinct # --resolve-key closes must not each force a captain-facing wake solely because # each one appended a status line, while a worker-authored line that is not in # this ledger still signals. -# The ledger does not use lag verbs to hide a worker `resolved` line. Only -# bytes this home itself recorded as owned are skipped. +# fm_wake_signal_seen_current (bin/fm-wake-lib.sh) is the ONLY consumer. The +# ledger decides whether growth wakes this home and nothing else: it never +# removes a line from presentation, so the drain's signal annotation and its +# UNREAD STATUS section both still print these bytes. +# The ledger does not use lag verbs to hide a worker `resolved` line; only +# bytes this home itself recorded as owned are ever treated as owned. # # Path: state/..home-appends # Format: @@ -1893,8 +1896,6 @@ _fm_status_home_appends_merge_locked() { # # reconciliation signal and never treated here as an open decision. # status_open_decisions remains the single owner of open/closed semantics, # including same-key reopening and reserved-key handling. -# Bytes recorded in this home's owned-append ledger are not classified as -# events: this home already wrote them as bookkeeping. # Every other captain-relevant event is terminal and always actionable. _fm_decision_origin_drop() { # local origin @@ -1943,7 +1944,6 @@ _fm_status_open_decision_origins() { # [] status_span_first_actionable_record() { # [record-var] [needs-decision-var] local f=$1 start=${2:-0} output_var=${3-} needs_var=${4-} size ident cur_ident scratch chunk_file full_file prefix_file result local line verb key origins='' folded=0 rc=1 failed=0 prefix_lines=0 line_number=0 live_line='' events='' _line _key _fm_span_needs_decision=0 - local LC_ALL=C [ -e "$f" ] || { [ -L "$f" ] && return 2; return 1; } [ -f "$f" ] && [ -r "$f" ] && [ ! -L "$f" ] || return 2 ident=$(_fm_open_decisions_file_ident "$f") || return 2 diff --git a/bin/fm-send.sh b/bin/fm-send.sh index ad74ea55308..fd3cc0adf3a 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -686,8 +686,10 @@ fi # very turn that answered the decisions, so they go through ONE guarded # self-announced append (bin/fm-wake-lib.sh). That records the appended byte # range so separate --resolve-key answers do not each wake this same session, -# while any concurrent foreign status bytes, or a worker line an OPEN -# DECISIONS fold read, still leave the watcher's wake path untouched. +# including when this home already folded those bytes through OPEN DECISIONS +# without a matching watcher seen marker; any concurrent foreign status bytes, +# or a worker line the fold read but never listed, leave the watcher's wake +# path untouched. fm_send_close_resolved_keys() { # local note=$1 k close_note append_rc still manual_close_cmd close_lines=() i=0 note=$(printf '%s' "$note" | tr '\n\r\t' ' ' | LC_ALL=C tr -d '\000-\037\177') diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 879ad42f1f2..6a6fa4e031f 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -2117,36 +2117,51 @@ fm_wake_signal_seen_size() { # esac } -# 0 when 's current signature matches its recorded reported state, or -# when the file is a readable regular file that grew past the watcher's -# classified offset and every grown byte is in this home's owned-append ledger. -# For a status file a reported match means the current -# state was already reported, not that every byte was successfully classified; -# the separate classified position owns that fact. Owned-only growth past the -# classified offset is this home's own bookkeeping and is not a new signal. -# A missing marker, an unreadable signature, or any other signature change -# without owned growth is not a match, so uncertainty still reads as an -# unreported state. -fm_wake_signal_seen_current() { # - local sig marker classified size +# 0 when 's current signature matches its recorded reported state. +# For a status file this means the current state was already reported, not that +# every byte was successfully classified; the separate classified position owns +# that fact. +# A missing marker or unreadable signature is not a match, so uncertainty reads +# as an unreported state. +# This predicate never consults the owned-append ledger, which is what makes it +# the safe gate for a captain-facing surface: a line must never be withheld from +# presentation merely because this home is the writer that appended it. +fm_wake_signal_reported_current() { # + local sig marker sig=$(fm_wake_signal_sig "$2") || return 1 [ -n "$sig" ] || return 1 marker=$(fm_wake_signal_seen_path "$1" "$2") case "$2" in *.status) _fm_wake_require_classify || return 1 - status_presentation_marker_reported_matches "$marker" "$sig" && return 0 - classified=$(fm_wake_signal_seen_size "$1" "$2") - size=$(_fm_status_file_size "$2") || return 1 - size=${size//[[:space:]]/} - case "$classified:$size" in *[!0-9:]*) return 1 ;; esac - [ "$classified" -lt "$size" ] && [ -f "$2" ] && [ -r "$2" ] && [ ! -L "$2" ] || return 1 - status_home_appends_covers "$2" "$classified" "$size" + status_presentation_marker_reported_matches "$marker" "$sig" ;; *) [ "$(cat "$marker" 2>/dev/null)" = "$sig" ] ;; esac } +# 0 when the state was already reported, or when the file is a readable regular +# file that grew past the watcher's classified offset and every grown byte is in +# this home's owned-append ledger. Owned-only growth past the classified offset +# is this home's own bookkeeping and is not a new signal, so separate +# --resolve-key answers do not each force a wake. Any other signature change +# without owned growth is not a match, so uncertainty still reads as unreported. +# This is the wake-scan predicate and answers only "should this wake the home?". +# Presentation asks the different question and uses +# fm_wake_signal_reported_current. +fm_wake_signal_seen_current() { # + local classified size + fm_wake_signal_reported_current "$1" "$2" && return 0 + case "$2" in *.status) ;; *) return 1 ;; esac + _fm_wake_require_classify || return 1 + classified=$(fm_wake_signal_seen_size "$1" "$2") + size=$(_fm_status_file_size "$2") || return 1 + size=${size//[[:space:]]/} + case "$classified:$size" in *[!0-9:]*) return 1 ;; esac + [ "$classified" -lt "$size" ] && [ -f "$2" ] && [ -r "$2" ] && [ ! -L "$2" ] || return 1 + status_home_appends_covers "$2" "$classified" "$size" +} + fm_wake_status_reported_commit() { # _fm_wake_require_classify || return 1 status_presentation_marker_report "$(fm_wake_signal_seen_path "$1" "$2")" "$3" @@ -2394,7 +2409,7 @@ fm_wake_print_annotations() { # [] # existing historical caveat. A direct status row is annotated for every # still-unread line since the last drain presentation; already-presented # bytes are not replayed. - if [ "$mode" = historical ] && fm_wake_signal_seen_current "$STATE" "$path"; then + if [ "$mode" = historical ] && fm_wake_signal_reported_current "$STATE" "$path"; then continue fi offset=$(fm_wake_status_cursor_offset "$path") || return 1 diff --git a/docs/architecture.md b/docs/architecture.md index 0c76c516044..9f70482bead 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,7 +90,7 @@ A third bounded section, RECORD DIVERGENCE, prints on the same drains for the op A failed read, output, or concurrent-replacement check prevents the snapshot cursor from advancing across uncertain bytes, and teardown retires a task's manifest row before that task ID can be reused. The explicit resolution is written by the actor that answers, not the busy worker: `fm-send`'s `--resolve-key` appends the closing `resolved` line to this home's own copy of the ledger at answer time, which covers crewmates, local secondmates, and remote secondmates identically because a remote mate's escalations reach that local copy through the parent-replies ingest and only the answer message itself crosses the transport. This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so a later wake scan can tell this home's own growth from a foreign write instead of waking on it. -The watcher marker still advances only when the watcher had already classified every earlier byte; a worker line the watcher has not classified yet, even one an OPEN DECISIONS fold already listed, or any interleaved foreign write, still wakes. +The watcher marker advances past those bytes only when every earlier byte was already classified by the watcher or listed as an open decision by the OPEN DECISIONS fold; any other earlier line, including a worker line the fold read but never listed, and any interleaved foreign write, fails toward an ordinary wake. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. Any direct or remaining historical annotation prints every status line unread at the presentation cursor instead of replaying only the latest line. The owned-append ledger only decides whether growth wakes this home; it never removes a line from presentation, so both that annotation and the UNREAD STATUS section still print this home's own bookkeeping closes. diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 32557f6f640..adf7129df74 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -2099,6 +2099,49 @@ test_malformed_presentation_lock_reports_acquire_failure() { # not present an already-announced status line as a new update, while a status # file with unannounced bytes keeps its annotation and a direct status row is # always annotated. Driven through the real drain executable. +# The owned-append ledger is wake-only: it must never withhold a captain-facing +# turn-ended annotation. An in-flight watcher classification that commits after +# this home's own close regresses the classified offset behind the owned bytes - +# exactly the state the wake scan treats as already owned - so the wake stays +# suppressed while the historical annotation must still present the line. +test_owned_growth_still_annotates_turn_ended() { + local dir state out err status pre_close ident + dir=$(make_case owned-historical) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + status="$state/scout.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + printf 'needs-decision [key=budget]: approve spend?\n' > "$status" + prime_status_seen "$state" "$status" || fail "could not prime the scout seen marker" + pre_close=$(wc -c < "$status" | tr -d '[:space:]') + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=budget]: answered: approved' \ + || fail "the answerer close was not self-announced" + ident=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; _fm_open_decisions_file_ident "$2" + ' _ "$ROOT/bin/fm-classify-lib.sh" "$status") \ + || fail "could not read the status identity" + run_wake_lib fm_wake_status_seen_commit "$state" "$status" "$pre_close" "$ident" \ + || fail "could not replay the stale watcher classification" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + || fail "owned-only growth did not suppress the wake" + + : > "$state/scout.turn-ended" + append_wake "$state" signal scout.turn-ended "signal: $state/scout.turn-ended" \ + || fail "turn-ended wake append failed" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" || fail "drain failed" + grep -F 'scout.status: resolved [key=budget]: answered: approved' "$out" >/dev/null \ + || fail "owned growth hid this home's own close from the turn-ended annotation: $(cat "$out")" + pass "owned growth suppresses the wake without hiding the turn-ended annotation" +} + test_historical_annotation_skips_announced_status() { local dir state out err dir=$(make_case historical-annotation) @@ -2161,6 +2204,7 @@ test_self_announced_append_guards test_separate_self_announced_answers_after_fold_are_owned test_unreadable_status_is_not_owned test_folded_worker_resolved_is_not_owned_lag +test_owned_growth_still_annotates_turn_ended test_historical_annotation_skips_announced_status test_concurrent_append_and_drain test_signal_catchup_without_running_watcher From 205c84f34571c8de16da1c93c12a2e88ac1f2fe6 Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sun, 20 Sep 2026 01:33:08 +0000 Subject: [PATCH 8/9] no-mistakes(review): Restore stranded historical-annotation test comment to its function --- tests/fm-wake-queue.test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index adf7129df74..5dc70897e31 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -2095,10 +2095,6 @@ test_malformed_presentation_lock_reports_acquire_failure() { pass "malformed presentation locks report acquire failure instead of contention" } -# Drain-time historical annotation staleness: a turn-ended-only wake row must -# not present an already-announced status line as a new update, while a status -# file with unannounced bytes keeps its annotation and a direct status row is -# always annotated. Driven through the real drain executable. # The owned-append ledger is wake-only: it must never withhold a captain-facing # turn-ended annotation. An in-flight watcher classification that commits after # this home's own close regresses the classified offset behind the owned bytes - @@ -2142,6 +2138,10 @@ test_owned_growth_still_annotates_turn_ended() { pass "owned growth suppresses the wake without hiding the turn-ended annotation" } +# Drain-time historical annotation staleness: a turn-ended-only wake row must +# not present an already-announced status line as a new update, while a status +# file with unannounced bytes keeps its annotation and a direct status row is +# always annotated. Driven through the real drain executable. test_historical_annotation_skips_announced_status() { local dir state out err dir=$(make_case historical-annotation) From a3624c944a4fc98c06a2eae0342868ad01486b7e Mon Sep 17 00:00:00 2001 From: Tiago Peixoto Date: Sun, 20 Sep 2026 02:05:45 +0000 Subject: [PATCH 9/9] no-mistakes(review): Retire the home-appends lock alongside its ledger --- AGENTS.md | 2 +- bin/fm-classify-lib.sh | 5 ++++- tests/fm-wake-drain-unread-status.test.sh | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e26099e7b57..ab9ec009776 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,7 +142,7 @@ state/ runtime records and signals; gitignored .wake-queue durable queued wakes retained until post-handling acknowledgement: epochseqkindkeypayload .watcher-down private generation-bound recovery state coupling watcher downtime, durable wake presentation, and post-handling acknowledgement; never touch ..open-decisions-cursor per-task byte cursor and folded open-decision set bounding the OPEN DECISIONS scan's cost to new status-log appends; written only by fm-classify-lib.sh's status_open_decisions_incremental, removed by teardown, safe to delete (forces one full re-fold) - ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so a wake scan can tell its own growth from a foreign write instead of waking on it; presentation is unaffected, so both the signal annotation and UNREAD STATUS still print those lines; written only by fm-classify-lib.sh's status_home_appends_record, removed by teardown, safe to delete + ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so a wake scan can tell its own growth from a foreign write instead of waking on it; presentation is unaffected, so both the signal annotation and UNREAD STATUS still print those lines; written only by fm-classify-lib.sh's status_home_appends_record; its sibling ..home-appends.lock serializes that ledger's read-merge-write; both removed by teardown, safe to delete .status-presentation-cursor .status-presentation-lock fleet-wide per-task status identity plus independent annotation and outcome-backstop byte offsets, with a serialization lock preventing already-presented lines from replaying while preserving delayed signal annotations; owned by fm-classify-lib.sh, with each task's row retired by teardown .afk-contract the away-posture record: the captain's verbatim away words, expected return, reach profile, spend cap, and structured mandate clauses; written only by bin/fm-afk-contract.sh after the captain confirms the read-back, archived under afk-contracts/ at return; its presence IS the away posture in every harness; its sibling .afk-contract.lock serializes actions authorized by the live record (contract: bin/fm-afk-contract.sh) afk-contracts/ archived away-posture records: one final record per away window keyed by entry time, plus any superseded mandates from that window diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index feffdb6c477..4c4737aaf4e 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -1322,7 +1322,7 @@ status_presentation_marker_commit() { status_retire_presentation_task() { # local state=$1 task=$2 lock manifest tmp data row_task ident offset backstop extra rc=0 found=0 - local signal_marker heartbeat_marker daemon_marker home_appends + local signal_marker heartbeat_marker daemon_marker home_appends home_appends_lock lock="$state/.status-presentation-lock" manifest="$state/.status-presentation-cursor" tmp="$manifest.tmp.$$" @@ -1330,6 +1330,7 @@ status_retire_presentation_task() { # heartbeat_marker=$(status_heartbeat_seen_marker_path "$state" "$task") daemon_marker=$(status_daemon_seen_marker_path "$state" "$task") home_appends="$state/.$task.home-appends" + home_appends_lock="$home_appends.lock" # A remote-home teardown can legitimately retire an endpoint ID that has no # status log in that home. Do not contend with that home's unrelated status @@ -1340,6 +1341,7 @@ status_retire_presentation_task() { # && [ ! -e "$state/.$task.open-decisions-cursor" ] \ && [ ! -L "$state/.$task.open-decisions-cursor" ] \ && [ ! -e "$home_appends" ] && [ ! -L "$home_appends" ] \ + && [ ! -e "$home_appends_lock" ] && [ ! -L "$home_appends_lock" ] \ && [ ! -e "$signal_marker" ] && [ ! -L "$signal_marker" ] \ && [ ! -e "$heartbeat_marker" ] && [ ! -L "$heartbeat_marker" ] \ && [ ! -e "$daemon_marker" ] && [ ! -L "$daemon_marker" ]; then @@ -1390,6 +1392,7 @@ EOF if [ "$rc" -eq 0 ]; then rm -f -- "$state/$task.status" "$state/.$task.open-decisions-cursor" \ "$home_appends" "$signal_marker" "$heartbeat_marker" "$daemon_marker" || rc=1 + fm_lock_remove_path "$home_appends_lock" 2>/dev/null || true fi fm_lock_release "$lock" || rc=1 return "$rc" diff --git a/tests/fm-wake-drain-unread-status.test.sh b/tests/fm-wake-drain-unread-status.test.sh index c12285ebb08..8bb19013ee4 100755 --- a/tests/fm-wake-drain-unread-status.test.sh +++ b/tests/fm-wake-drain-unread-status.test.sh @@ -287,11 +287,17 @@ test_retired_task_id_starts_new_status_unread() { printf "40@$(cat "$2")" > "$(status_signal_seen_marker_path "$STATE" reused)" printf "40@$(cat "$2")" > "$(status_heartbeat_seen_marker_path "$STATE" reused)" printf "40@$(cat "$2")" > "$(status_daemon_seen_marker_path "$STATE" reused)" + ledger=$(status_home_appends_path "$STATE/reused.status") + status_home_appends_record "$STATE/reused.status" 0 12 || exit 1 + [ -f "$ledger" ] || exit 1 + mkdir -p "$ledger.lock" || exit 1 + printf "%s\n" 2147483646 > "$ledger.lock/pid" || exit 1 status_retire_presentation_task "$STATE" reused || exit 1 for marker in \ "$(status_signal_seen_marker_path "$STATE" reused)" \ "$(status_heartbeat_seen_marker_path "$STATE" reused)" \ - "$(status_daemon_seen_marker_path "$STATE" reused)"; do + "$(status_daemon_seen_marker_path "$STATE" reused)" \ + "$ledger" "$ledger.lock"; do [ ! -e "$marker" ] && [ ! -L "$marker" ] || exit 1 done ' _ "$ROOT" "$dir/old-ident" || fail "retiring the reused task presentation state failed"