diff --git a/AGENTS.md b/AGENTS.md index c3634216137..ab9ec009776 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,6 +142,7 @@ state/ runtime records and signals; gitignored .wake-queue durable queued wakes retained until post-handling acknowledgement: epochseqkindkeypayload .watcher-down private generation-bound recovery state coupling watcher downtime, durable wake presentation, and post-handling acknowledgement; never touch ..open-decisions-cursor per-task byte cursor and folded open-decision set bounding the OPEN DECISIONS scan's cost to new status-log appends; written only by fm-classify-lib.sh's status_open_decisions_incremental, removed by teardown, safe to delete (forces one full re-fold) + ..home-appends per-task ledger of byte ranges this home itself appended as bookkeeping closes, so a wake scan can tell its own growth from a foreign write instead of waking on it; presentation is unaffected, so both the signal annotation and UNREAD STATUS still print those lines; written only by fm-classify-lib.sh's status_home_appends_record; its sibling ..home-appends.lock serializes that ledger's read-merge-write; both removed by teardown, safe to delete .status-presentation-cursor .status-presentation-lock fleet-wide per-task status identity plus independent annotation and outcome-backstop byte offsets, with a serialization lock preventing already-presented lines from replaying while preserving delayed signal annotations; owned by fm-classify-lib.sh, with each task's row retired by teardown .afk-contract the away-posture record: the captain's verbatim away words, expected return, reach profile, spend cap, and structured mandate clauses; written only by bin/fm-afk-contract.sh after the captain confirms the read-back, archived under afk-contracts/ at return; its presence IS the away posture in every harness; its sibling .afk-contract.lock serializes actions authorized by the live record (contract: bin/fm-afk-contract.sh) afk-contracts/ archived away-posture records: one final record per away window keyed by entry time, plus any superseded mandates from that window diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 0752e52f370..4c4737aaf4e 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -27,7 +27,7 @@ # A missing, malformed, identity-mismatched, or past-end classified position reads # from byte 0, preferring a bounded duplicate over a lost event. # -# There are three documented exceptions. The absorb classification +# There are four documented exceptions. The absorb classification # (crew_absorb_class and its working/paused wrappers) is NOT a pure status-file # read: it reuses bin/fm-crew-state.sh, which may make a bounded no-mistakes call, # to decide whether a crew that just stopped its turn or went stale is working, @@ -37,9 +37,12 @@ # open-decisions fold" below) also writes: it persists a per-status-file byte # cursor and folded open-set as a side effect, so a per-drain fleet-wide scan # stays bounded by new appends instead of re-reading each task's whole lifetime -# log every time. crew_worktree_written_since reads the task's meta file and walks -# a bounded slice of its worktree instead of a status file, so callers run it only -# at the moment they would otherwise escalate. +# log every time. status_home_appends_record writes the per-task home-owned +# append ledger (see "home-owned status-append ledger" below) so the wake scan +# can treat this home's own bookkeeping bytes as already owned. +# crew_worktree_written_since reads the task's meta file and walks a bounded slice +# of its worktree instead of a status file, so callers run it only at the moment +# they would otherwise escalate. # Directory of this library, used to locate the sibling fm-crew-state.sh reader. # Resolved at source time from BASH_SOURCE so it works whether sourced by a @@ -1319,13 +1322,15 @@ status_presentation_marker_commit() { status_retire_presentation_task() { # local state=$1 task=$2 lock manifest tmp data row_task ident offset backstop extra rc=0 found=0 - local signal_marker heartbeat_marker daemon_marker + local signal_marker heartbeat_marker daemon_marker home_appends home_appends_lock lock="$state/.status-presentation-lock" manifest="$state/.status-presentation-cursor" tmp="$manifest.tmp.$$" signal_marker=$(status_signal_seen_marker_path "$state" "$task") heartbeat_marker=$(status_heartbeat_seen_marker_path "$state" "$task") daemon_marker=$(status_daemon_seen_marker_path "$state" "$task") + home_appends="$state/.$task.home-appends" + home_appends_lock="$home_appends.lock" # A remote-home teardown can legitimately retire an endpoint ID that has no # status log in that home. Do not contend with that home's unrelated status @@ -1335,6 +1340,8 @@ status_retire_presentation_task() { # if [ ! -e "$state/$task.status" ] && [ ! -L "$state/$task.status" ] \ && [ ! -e "$state/.$task.open-decisions-cursor" ] \ && [ ! -L "$state/.$task.open-decisions-cursor" ] \ + && [ ! -e "$home_appends" ] && [ ! -L "$home_appends" ] \ + && [ ! -e "$home_appends_lock" ] && [ ! -L "$home_appends_lock" ] \ && [ ! -e "$signal_marker" ] && [ ! -L "$signal_marker" ] \ && [ ! -e "$heartbeat_marker" ] && [ ! -L "$heartbeat_marker" ] \ && [ ! -e "$daemon_marker" ] && [ ! -L "$daemon_marker" ]; then @@ -1384,7 +1391,8 @@ EOF fi if [ "$rc" -eq 0 ]; then rm -f -- "$state/$task.status" "$state/.$task.open-decisions-cursor" \ - "$signal_marker" "$heartbeat_marker" "$daemon_marker" || rc=1 + "$home_appends" "$signal_marker" "$heartbeat_marker" "$daemon_marker" || rc=1 + fm_lock_remove_path "$home_appends_lock" 2>/dev/null || true fi fm_lock_release "$lock" || rc=1 return "$rc" @@ -1733,6 +1741,141 @@ window_to_task() { t="${w##*:}"; t="${t#fm-}"; printf '%s' "$t" } +# --- home-owned status-append ledger ---------------------------------------- +# +# This home's bookkeeping closes (fm_wake_status_append_self_announced) record +# the exact byte range they appended so the wake scan can tell this home's own +# growth from a foreign write. That is the multi-answer path: two distinct +# --resolve-key closes must not each force a captain-facing wake solely because +# each one appended a status line, while a worker-authored line that is not in +# this ledger still signals. +# fm_wake_signal_seen_current (bin/fm-wake-lib.sh) is the ONLY consumer. The +# ledger decides whether growth wakes this home and nothing else: it never +# removes a line from presentation, so the drain's signal annotation and its +# UNREAD STATUS section both still print these bytes. +# The ledger does not use lag verbs to hide a worker `resolved` line; only +# bytes this home itself recorded as owned are ever treated as owned. +# +# Path: state/..home-appends +# Format: +# v1 +# ident= +# +# Ranges are half-open [start, end), merged when adjacent or overlapping. +# An identity mismatch (file rotated) discards the ledger. Teardown deletes it. +# Not a pure status-file read: status_home_appends_record writes this sidecar. + +status_home_appends_path() { # + local f=$1 dir base + dir=$(dirname "$f") + base=$(basename "$f") + printf '%s/.%s.home-appends' "$dir" "${base%.status}" +} + +status_home_appends_ranges() { # -> startend lines + local f=$1 path ident data first rest line start end extra + path=$(status_home_appends_path "$f") + [ -f "$path" ] && [ -r "$path" ] && [ ! -L "$path" ] || return 0 + ident=$(_fm_open_decisions_file_ident "$f") || return 0 + data=$(LC_ALL=C command cat "$path" 2>/dev/null) || return 0 + first=${data%%$'\n'*} + [ "$first" = v1 ] || return 0 + rest=${data#*$'\n'} + [ "$rest" != "$data" ] || return 0 + line=${rest%%$'\n'*} + case "$line" in ident=*) ;; *) return 0 ;; esac + [ "${line#ident=}" = "$ident" ] || return 0 + case "$rest" in + *$'\n'*) rest=${rest#*$'\n'} ;; + *) return 0 ;; + esac + while IFS=$(printf '\t') read -r start end extra || [ -n "$start" ]; do + [ -n "$start" ] || continue + [ -z "$extra" ] || continue + case "$start:$end" in *[!0-9:]*) continue ;; esac + [ "$end" -gt "$start" ] || continue + printf '%s\t%s\n' "$start" "$end" || return 1 + done < + local start=$2 end=$3 range_start range_end + case "$start:$end" in *[!0-9:]*) return 1 ;; esac + [ "$end" -ge "$start" ] || return 1 + [ "$end" -eq "$start" ] && return 0 + while IFS=$(printf '\t') read -r range_start range_end; do + [ -n "$range_start" ] || continue + case "$range_start:$range_end" in *[!0-9:]*) continue ;; esac + [ "$range_start" -le "$start" ] || continue + if [ "$range_end" -gt "$start" ]; then + start=$range_end + fi + if [ "$start" -ge "$end" ]; then + return 0 + fi + done < + local f=$1 start=$2 end=$3 path lock rc=0 + case "$start:$end" in *[!0-9:]*) return 1 ;; esac + [ "$end" -gt "$start" ] || return 1 + path=$(status_home_appends_path "$f") + lock="$path.lock" + fm_lock_acquire_wait "$lock" || return 1 + _fm_status_home_appends_merge_locked "$f" "$path" "$start" "$end" || rc=1 + fm_lock_release "$lock" || rc=1 + return "$rc" +} + +_fm_status_home_appends_merge_locked() { # + local f=$1 path=$2 start=$3 end=$4 ident tmp merged + local LC_ALL=C + ident=$(_fm_open_decisions_file_ident "$f") || return 1 + merged=$(printf '%s\n%s\t%s\n' "$(status_home_appends_ranges "$f")" "$start" "$end" | awk ' + NF == 2 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ && $2+0 > $1+0 { + n++ + s[n] = $1 + 0 + e[n] = $2 + 0 + } + END { + for (i = 1; i <= n; i++) { + for (j = i + 1; j <= n; j++) { + if (s[j] < s[i] || (s[j] == s[i] && e[j] < e[i])) { + t = s[i]; s[i] = s[j]; s[j] = t + t = e[i]; e[i] = e[j]; e[j] = t + } + } + } + m = 0 + for (i = 1; i <= n; i++) { + if (m == 0 || s[i] > me[m]) { + m++ + ms[m] = s[i] + me[m] = e[i] + } else if (e[i] > me[m]) { + me[m] = e[i] + } + } + for (i = 1; i <= m; i++) printf "%s\t%s\n", ms[i], me[i] + } + ') || return 1 + tmp="$path.tmp.$$" + { + printf 'v1\nident=%s\n' "$ident" + if [ -n "$merged" ]; then + printf '%s' "$merged" + case "$merged" in *$'\n') ;; *) printf '\n' ;; esac + fi + } > "$tmp" || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$path" || { rm -f "$tmp"; return 1; } +} + # Capture the bytes of an append-only status log at or after under # one size-and-identity snapshot. # The record form produces `\t\t` and returns 0 when diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 5e42f354213..fd3cc0adf3a 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -684,11 +684,12 @@ fi # command; the decision then stays open and re-surfaces, never silently lost. # All of one answer's closes are this home's own bookkeeping, written by the # very turn that answered the decisions, so they go through ONE guarded -# self-announced append (bin/fm-wake-lib.sh) and do not wake this same session -# again, including when this home already folded those bytes through OPEN -# DECISIONS without a matching watcher seen marker; any concurrent foreign -# status bytes, or a worker line the fold read but never listed, leave the -# watcher's wake path untouched. +# self-announced append (bin/fm-wake-lib.sh). That records the appended byte +# range so separate --resolve-key answers do not each wake this same session, +# including when this home already folded those bytes through OPEN DECISIONS +# without a matching watcher seen marker; any concurrent foreign status bytes, +# or a worker line the fold read but never listed, leave the watcher's wake +# path untouched. fm_send_close_resolved_keys() { # local note=$1 k close_note append_rc still manual_close_cmd close_lines=() i=0 note=$(printf '%s' "$note" | tr '\n\r\t' ' ' | LC_ALL=C tr -d '\000-\037\177') diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index d59672d9da4..6a6fa4e031f 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -2123,7 +2123,10 @@ fm_wake_signal_seen_size() { # # that fact. # A missing marker or unreadable signature is not a match, so uncertainty reads # as an unreported state. -fm_wake_signal_seen_current() { # +# This predicate never consults the owned-append ledger, which is what makes it +# the safe gate for a captain-facing surface: a line must never be withheld from +# presentation merely because this home is the writer that appended it. +fm_wake_signal_reported_current() { # local sig marker sig=$(fm_wake_signal_sig "$2") || return 1 [ -n "$sig" ] || return 1 @@ -2137,6 +2140,28 @@ fm_wake_signal_seen_current() { # esac } +# 0 when the state was already reported, or when the file is a readable regular +# file that grew past the watcher's classified offset and every grown byte is in +# this home's owned-append ledger. Owned-only growth past the classified offset +# is this home's own bookkeeping and is not a new signal, so separate +# --resolve-key answers do not each force a wake. Any other signature change +# without owned growth is not a match, so uncertainty still reads as unreported. +# This is the wake-scan predicate and answers only "should this wake the home?". +# Presentation asks the different question and uses +# fm_wake_signal_reported_current. +fm_wake_signal_seen_current() { # + local classified size + fm_wake_signal_reported_current "$1" "$2" && return 0 + case "$2" in *.status) ;; *) return 1 ;; esac + _fm_wake_require_classify || return 1 + classified=$(fm_wake_signal_seen_size "$1" "$2") + size=$(_fm_status_file_size "$2") || return 1 + size=${size//[[:space:]]/} + case "$classified:$size" in *[!0-9:]*) return 1 ;; esac + [ "$classified" -lt "$size" ] && [ -f "$2" ] && [ -r "$2" ] && [ ! -L "$2" ] || return 1 + status_home_appends_covers "$2" "$classified" "$size" +} + fm_wake_status_reported_commit() { # _fm_wake_require_classify || return 1 status_presentation_marker_report "$(fm_wake_signal_seen_path "$1" "$2")" "$3" @@ -2162,9 +2187,10 @@ fm_wake_status_mark_current() { # # in the very turn or tick that writes them (answerer-closes resolved lines, a # pending-reply escalation close, captain-held transfers). Such a close must # not wake the session that wrote it, so this appends one command's lines -# together and then advances the watcher's seen marker across the appended -# bytes and no byte this home has not already read. The advance is -# provenance-gated and fails toward waking: +# together, records the exact appended byte range in the home-owned append +# ledger (bin/fm-classify-lib.sh), and then advances the watcher's seen marker +# across the appended bytes and no byte this home has not already read. The +# advance is provenance-gated and fails toward waking: # - the marker advances only when this home already read every pre-append # byte, the post-append size equals that size plus exactly the appended # bytes (no foreign write interleaved), AND the watcher's own span @@ -2181,14 +2207,18 @@ fm_wake_status_mark_current() { # # side-band; # - on ANY other condition - a missing file, pending foreign bytes, an # interleaved writer, an unreadable size or identity - the lines are still -# appended but the marker is left alone, so the watcher surfaces the file -# normally. -# A later, different line from any other writer grows the size past the marker -# and wakes as before: task identity alone can never suppress new content. +# appended and the owned range is still recorded when growth is proven, but +# the marker is left alone, so the watcher surfaces the file normally. +# Later signal scans treat owned ranges as already owned even when the watcher +# has not caught up, so separate --resolve-key answers do not each force a +# captain-facing wake. A later, different line from any other writer grows the +# size past the owned ranges and wakes as before: task identity alone can never +# suppress new content. # Returns 0 appended and self-announced, 1 appended but left for the watcher # (the safe direction), 2 the append itself failed. fm_wake_status_append_self_announced() { # ... - local state=$1 file=$2 line appended=0 pre_size='' pre_ident='' post_size post_ident classified folded lag span_rc=0 + local state=$1 file=$2 line appended=0 pre_size='' pre_ident='' post_size post_ident + local classified folded lag span_rc=0 local LC_ALL=C shift 2 _fm_wake_require_classify || return 1 @@ -2197,12 +2227,14 @@ fm_wake_status_append_self_announced() { # ... pre_ident=$(_fm_open_decisions_file_ident "$file") || pre_ident='' fi printf '%s\n' "$@" >> "$file" || return 2 + case "$pre_size" in ''|*[!0-9]*) return 1 ;; esac post_size=$(_fm_status_file_size "$file") || return 1 post_ident=$(_fm_open_decisions_file_ident "$file") || return 1 - case "$pre_size$post_size" in ''|*[!0-9]*) return 1 ;; esac + case "$post_size" in ''|*[!0-9]*) return 1 ;; esac [ -n "$pre_ident" ] && [ "$post_ident" = "$pre_ident" ] || return 1 for line in "$@"; do appended=$((appended + ${#line} + 1)); done [ "$post_size" -eq $((pre_size + appended)) ] || return 1 + status_home_appends_record "$file" "$pre_size" "$post_size" || return 1 classified=$(fm_wake_signal_seen_size "$state" "$file") if [ "$classified" != "$pre_size" ]; then folded=$(status_open_decisions_cursor_offset "$file") || folded=0 @@ -2377,7 +2409,7 @@ fm_wake_print_annotations() { # [] # existing historical caveat. A direct status row is annotated for every # still-unread line since the last drain presentation; already-presented # bytes are not replayed. - if [ "$mode" = historical ] && fm_wake_signal_seen_current "$STATE" "$path"; then + if [ "$mode" = historical ] && fm_wake_signal_reported_current "$STATE" "$path"; then continue fi offset=$(fm_wake_status_cursor_offset "$path") || return 1 diff --git a/docs/architecture.md b/docs/architecture.md index 57c2d05af76..9f70482bead 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,9 +89,11 @@ A queued signal annotation prints every status line still unread at that cursor, A third bounded section, RECORD DIVERGENCE, prints on the same drains for the opposite failure: the status fold went quiet on a key that the durable captain-held task still shows as open, so the status side reads as complete while the two records contradict each other; `bin/fm-captain-hold.sh diverged` decides what counts and closes nothing, and `docs/captain-hold-lifecycle.md` owns the mechanism. A failed read, output, or concurrent-replacement check prevents the snapshot cursor from advancing across uncertain bytes, and teardown retires a task's manifest row before that task ID can be reused. The explicit resolution is written by the actor that answers, not the busy worker: `fm-send`'s `--resolve-key` appends the closing `resolved` line to this home's own copy of the ledger at answer time, which covers crewmates, local secondmates, and remote secondmates identically because a remote mate's escalations reach that local copy through the parent-replies ingest and only the answer message itself crosses the transport. -This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, so they advance the watcher marker past their own bytes only when every earlier byte was already classified by the watcher or listed as an open decision; any other earlier line, and any interleaved foreign write, fails toward an ordinary wake. +This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, which records the exact byte range it appended so a later wake scan can tell this home's own growth from a foreign write instead of waking on it. +The watcher marker advances past those bytes only when every earlier byte was already classified by the watcher or listed as an open decision by the OPEN DECISIONS fold; any other earlier line, including a worker line the fold read but never listed, and any interleaved foreign write, fails toward an ordinary wake. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. Any direct or remaining historical annotation prints every status line unread at the presentation cursor instead of replaying only the latest line. +The owned-append ledger only decides whether growth wakes this home; it never removes a line from presentation, so both that annotation and the UNREAD STATUS section still print this home's own bookkeeping closes. `bin/fm-crew-state.sh ` is the cheap current-state read for an actionable heartbeat review: it attributes an active or terminal no-mistakes run under the shared run-attribution contract, then keeps that run-step authoritative even if the pane has closed, except that a `blocked:` event reporting a refused or missing daemon socket outranks a potentially stale active run record only while that socket-down declaration is itself the log's latest recognized event, since any later event, including another `blocked:` one, means the crew moved on. For other daemon, timeout, or unreachability claims, a running or fixing run with recent pipeline-reported activity supersedes the event and names reattachment as the recovery instead of surfacing a false block. [`bin/fm-nm-run-lib.sh`](../bin/fm-nm-run-lib.sh) owns branch, head, and pipeline-custody attribution, plus complete same-branch run selection, optional inventory lookup, and ambiguity reporting. diff --git a/docs/scripts.md b/docs/scripts.md index ef45d68dfa2..6d7e415c437 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -111,7 +111,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-wake-drain.sh` | Present and acknowledge the current actor's claimed wake rows alongside status, outcome-backstop, decision, divergence, recovery, and supervision checks | | `fm-wake-grant.sh` | Serialize Pi supervision-branch wake-row claim activation, publication, release, and deactivation | | `fm-wake-lib.sh` | Shared durable wake queue, recovery generations, portable locks, and watcher identity/health helpers | -| `fm-classify-lib.sh` | Shared wake classification, durable keyed-decision folds and scans, unread status selection, and bounded latest-event snapshots | +| `fm-classify-lib.sh` | Shared wake classification, durable keyed-decision folds and scans, unread status selection, home-owned status-append ranges, and bounded latest-event snapshots | | `fm-send.sh` | Steer a task via a durable inbox record plus doorbell, or send a supported key or typed harness invocation through the recorded backend | | `fm-branch-prompt.sh` | Emit the Pi supervision branch's byte-stable system prompt ([pi-supervision-branch.md](pi-supervision-branch.md)) | | `fm-branch-outcome.sh` | Own the supervision branch's append-only outcome store, cursors, bounded status-coverage indexes, and session-start replay | diff --git a/tests/fm-send-resolve-key.test.sh b/tests/fm-send-resolve-key.test.sh index 9c57b0ea1a3..8e11e12bf3e 100755 --- a/tests/fm-send-resolve-key.test.sh +++ b/tests/fm-send-resolve-key.test.sh @@ -187,6 +187,63 @@ test_answer_close_is_self_announced() { pass "fm-send --resolve-key: the close never re-wakes its own home, later lines still do" } +# Two distinct --resolve-key answers must each stay quiet even when the seen +# marker does NOT cover them. An in-flight watcher classification that lands +# after the first answer regresses the classified offset behind that answer's +# bytes, so the marker no longer vouches for them; only the home-appends ledger +# does. Without the ledger the second scan re-wakes this home over its own +# close. A later worker line on the same task still wakes. +test_separate_resolve_key_answers_do_not_rewake() { + local dir fb log home rc status pre_answer ident + dir="$TMP_ROOT/separate-answers"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log" + home=$(setup_home separate-answers) + status="$home/state/t7.status" + fm_write_meta "$home/state/t7.meta" "window=sess:fm-t7" "kind=ship" + { + printf 'needs-decision [key=budget]: approve spend?\n' + printf 'needs-decision [key=vendor]: pick a vendor\n' + } > "$status" + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_status_mark_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" \ + || fail "could not prime the announced baseline" + pre_answer=$(wc -c < "$status" | tr -d '[:space:]') + + run_send "$fb" "$home" "$log" t7 --resolve-key budget "approved"; rc=$? + expect_code 0 "$rc" "the first answer should succeed" + + # A watcher classification captured before the answer commits afterwards and + # rewinds the classified offset behind the answer's bytes. + ident=$(FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; _fm_open_decisions_file_ident "$2" + ' _ "$ROOT/bin/fm-classify-lib.sh" "$status") \ + || fail "could not read the status identity" + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_status_seen_commit "$2" "$3" "$4" "$5" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" "$pre_answer" "$ident" \ + || fail "could not replay the stale watcher classification" + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_signal_seen_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" \ + || fail "the first --resolve-key answer was left to re-wake this home" + + run_send "$fb" "$home" "$log" t7 --resolve-key vendor "acme"; rc=$? + expect_code 0 "$rc" "the second answer should succeed" + FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_signal_seen_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status" \ + || fail "the second --resolve-key answer was left to re-wake this home" + + printf 'blocked: need staging credentials\n' >> "$status" + if FM_STATE_OVERRIDE="$home/state" bash -c ' + . "$1"; fm_wake_signal_seen_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$home/state" "$status"; then + fail "a later worker line after two answers was swallowed" + fi + pass "fm-send --resolve-key: separate answers do not each re-wake; later lines still do" +} + # The reported failure behind issue #2109: a worker that put the colon first # (needs-decision: [key=X] ...) had its key silently folded to "default", so # the answer's --resolve-key X refused with "no open decision or blocker with @@ -825,6 +882,7 @@ test_decision_answer_partition_relocates_under_the_record() { test_answer_send_closes_open_decision test_answer_close_is_self_announced +test_separate_resolve_key_answers_do_not_rewake test_colon_first_key_position_is_answerable test_answer_starts_work_never_orphans test_routine_steer_never_closes diff --git a/tests/fm-wake-drain-unread-status.test.sh b/tests/fm-wake-drain-unread-status.test.sh index ccf8bb96abd..8bb19013ee4 100755 --- a/tests/fm-wake-drain-unread-status.test.sh +++ b/tests/fm-wake-drain-unread-status.test.sh @@ -158,6 +158,65 @@ test_pending_reply_resolution_surfaces_once() { pass "a pending-reply resolution buried under a later note surfaces once and closes OPEN DECISIONS" } +# The watcher's pending-reply close goes through the self-announced append, so +# it records its bytes as this home's own and never wakes. The drain must still +# present that reserved-key resolution in UNREAD STATUS, its only guaranteed +# presentation. +test_self_announced_pending_reply_close_still_surfaces() { + local dir state out status corr + dir=$(make_case self-announced-pending-reply) + state="$dir/state" + out="$dir/drain.out" + status="$state/task6.status" + + run_pending_reply() { + FM_STATE_OVERRIDE="$state" FM_PENDING_REPLY_NOW=5000 bash -c ' + . "$1"; . "$2"; shift 2; "$@" + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + corr=$(run_pending_reply fm_pending_reply_create "$dir" "$state" task6 "ship it") \ + || fail "could not create the pending-reply record" + run_pending_reply fm_pending_reply_mark_delivered "$state" "$corr" \ + || fail "could not mark the pending-reply request delivered" + FM_STATE_OVERRIDE="$state" FM_PENDING_REPLY_NOW=5000 bash -c ' + . "$1"; rec=$(fm_pending_reply_path "$2" "$3") + fm_pending_reply_set "$rec" phase escalated && fm_pending_reply_set "$rec" escalated_epoch 4950 + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$state" "$corr" \ + || fail "could not mark the pending-reply request escalated" + + printf 'blocked [key=pending-reply-%s]: pending-reply-missed: task=task6 pending-reply-id=%s request=ship it\n' \ + "$corr" "$corr" > "$status" + prime_status_seen "$state" "$status" || fail "could not mark the status file surfaced" + append_wake "$state" signal task6.status "signal: task6.status" \ + || fail "queueing the pending-reply escalation signal failed" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null || fail "drain of the escalation failed" + printf 'done [corr=%s]: shipped after all\n' "$corr" >> "$status" + prime_status_seen "$state" "$status" || fail "could not mark the status file surfaced" + append_wake "$state" signal task6.status "signal: task6.status" \ + || fail "queueing the delayed reply signal failed" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null || fail "drain of the delayed reply failed" + + run_pending_reply fm_pending_reply_try_resolve "$state" "$corr" \ + || fail "the delayed reply did not resolve the pending-reply record" + grep -F "resolved [key=pending-reply-$corr]: pending-reply-resolved:" "$status" >/dev/null \ + || fail "the resolve did not append the escalation close: $(cat "$status")" + [ -s "$state/.task6.home-appends" ] \ + || fail "the escalation close did not go through the self-announced append" + run_pending_reply fm_wake_signal_seen_current "$state" "$status" \ + || fail "the self-announced escalation close was left to re-wake this home" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "drain after the escalation close failed" + grep -F "task6 resolved [key=pending-reply-$corr]: pending-reply-resolved: task=task6 pending-reply-id=$corr" "$out" >/dev/null \ + || fail "the self-announced pending-reply resolution was hidden from UNREAD STATUS: $(cat "$out")" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" || fail "second drain after the escalation close failed" + if grep -F 'pending-reply-resolved:' "$out" >/dev/null; then + fail "an already-presented self-announced resolution was replayed: $(cat "$out")" + fi + pass "a self-announced pending-reply close does not wake yet still surfaces once in UNREAD STATUS" +} + test_unread_output_over_cap_remains_recoverable() { local dir state out status i payload dir=$(make_case unread-over-cap) @@ -228,11 +287,17 @@ test_retired_task_id_starts_new_status_unread() { printf "40@$(cat "$2")" > "$(status_signal_seen_marker_path "$STATE" reused)" printf "40@$(cat "$2")" > "$(status_heartbeat_seen_marker_path "$STATE" reused)" printf "40@$(cat "$2")" > "$(status_daemon_seen_marker_path "$STATE" reused)" + ledger=$(status_home_appends_path "$STATE/reused.status") + status_home_appends_record "$STATE/reused.status" 0 12 || exit 1 + [ -f "$ledger" ] || exit 1 + mkdir -p "$ledger.lock" || exit 1 + printf "%s\n" 2147483646 > "$ledger.lock/pid" || exit 1 status_retire_presentation_task "$STATE" reused || exit 1 for marker in \ "$(status_signal_seen_marker_path "$STATE" reused)" \ "$(status_heartbeat_seen_marker_path "$STATE" reused)" \ - "$(status_daemon_seen_marker_path "$STATE" reused)"; do + "$(status_daemon_seen_marker_path "$STATE" reused)" \ + "$ledger" "$ledger.lock"; do [ ! -e "$marker" ] && [ ! -L "$marker" ] || exit 1 done ' _ "$ROOT" "$dir/old-ident" || fail "retiring the reused task presentation state failed" @@ -379,6 +444,7 @@ test_already_presented_notes_are_not_replayed test_brand_new_note_after_presentation_is_surfaced test_signal_annotation_surfaces_every_unread_note_not_only_the_newest test_pending_reply_resolution_surfaces_once +test_self_announced_pending_reply_close_still_surfaces test_unread_output_over_cap_remains_recoverable test_snapshot_does_not_ack_a_later_append test_retired_task_id_starts_new_status_unread diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 14cb8cc8d89..5dc70897e31 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -1657,6 +1657,141 @@ test_self_announced_append_guards() { pass "self-announced appends suppress only their own bytes and fail toward waking" } +# Two distinct --resolve-key closes after an OPEN DECISIONS fold record their +# own byte ranges, so the watcher's span classification never reports the +# answers. The fold alone does not mark the worker's decisions seen, because +# any actor's drain folds: a folded decision this home has not answered still +# classifies as a new signal. Once the watcher has classified the span, the +# owned answers stay quiet and a later worker line wakes. +test_separate_self_announced_answers_after_fold_are_owned() { + local dir state status rc events + dir=$(make_case multi-answer-owned) + state="$dir/state" + status="$state/t.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + { + printf 'needs-decision [key=k1]: pick REST or RPC\n' + printf 'needs-decision [key=k2]: pick us-east or eu-west\n' + printf 'needs-decision [key=k3]: pick a database\n' + } > "$status" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + && fail "a fold alone marked unclassified worker decisions as seen" + + rc=0 + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k1]: answered: REST' || rc=$? + [ "$rc" -eq 1 ] || fail "the first answer over unclassified decisions did not fail toward waking (rc=$rc)" + rc=0 + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k2]: answered: eu-west' || rc=$? + [ "$rc" -eq 1 ] || fail "the second answer over unclassified decisions did not fail toward waking (rc=$rc)" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + && fail "unclassified worker decisions were hidden behind this home's answers" + + events=$(FM_STATE_OVERRIDE="$state" bash -c '. "$1"; status_span_first_actionable "$2" 0' _ "$ROOT/bin/fm-classify-lib.sh" "$status") \ + || fail "the unanswered folded decision was not classified as actionable" + [ "$events" = 'needs-decision [key=k3]: pick a database' ] \ + || fail "the span classification reported more than the unanswered decision: $events" + + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not record the watcher classifying the decisions" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + || fail "the owned answers were left to re-wake this home" + + printf 'blocked [key=creds]: need staging credentials\n' >> "$status" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + && fail "a later worker line after two owned answers was swallowed" + + pass "separate self-announced answers after a fold stay owned; worker decisions and later lines still wake" +} + +# The owned ledger only vouches for growth it recorded. A signature change +# with no growth past the classified offset, such as the log turning +# unreadable, must still read as unreported, before and after owned growth. +test_unreadable_status_is_not_owned() { + local dir state status + dir=$(make_case owned-unreadable) + state="$dir/state" + status="$state/t.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + if [ "$(id -u)" -eq 0 ]; then + pass "unreadable status check skipped: root reads mode-000 files" + return 0 + fi + printf 'needs-decision [key=k1]: pick one\n' > "$status" + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not prime the announced baseline" + chmod 000 "$status" + if run_wake_lib fm_wake_signal_seen_current "$state" "$status"; then + chmod 600 "$status" + fail "an unreadable fully classified status read as already seen" + fi + chmod 600 "$status" + + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not re-prime the announced baseline" + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k1]: answered: one' \ + || fail "the owned close was not self-announced" + printf 'needs-decision [key=k2]: pick two\n' >> "$status" + run_wake_lib fm_wake_status_mark_current "$state" "$status" \ + || fail "could not record the watcher classifying the worker line" + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=k2]: answered: two' \ + || fail "the second owned close was not self-announced" + chmod 000 "$status" + if run_wake_lib fm_wake_signal_seen_current "$state" "$status"; then + chmod 600 "$status" + fail "an unreadable status after owned growth read as already seen" + fi + chmod 600 "$status" + pass "an unreadable status still reads as unreported, with or without owned growth" +} + +test_folded_worker_resolved_is_not_owned_lag() { + local dir state status rc + dir=$(make_case folded-worker-resolved) + state="$dir/state" + status="$state/t.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + { + printf 'needs-decision [key=budget]: approve spend?\n' + printf 'needs-decision [key=vendor]: vendor A or B?\n' + printf 'resolved [key=vendor]: picked vendor B myself, cheaper\n' + } > "$status" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + + rc=0 + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=budget]: answered: approved' || rc=$? + [ "$rc" -eq 1 ] || fail "a close over a folded worker resolved did not fail toward waking (rc=$rc)" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + && fail "a worker resolved in the folded span was treated as already owned" + + pass "a worker resolved in fold lag still wakes after this home's close" +} + # A trap that fires inside a lock's critical section abandons the holding # frame, and the exit path then re-acquires the same lock (a TERM inside a # recovery-marker section is the reproduced case: the watcher's reap wedged @@ -1960,6 +2095,49 @@ test_malformed_presentation_lock_reports_acquire_failure() { pass "malformed presentation locks report acquire failure instead of contention" } +# The owned-append ledger is wake-only: it must never withhold a captain-facing +# turn-ended annotation. An in-flight watcher classification that commits after +# this home's own close regresses the classified offset behind the owned bytes - +# exactly the state the wake scan treats as already owned - so the wake stays +# suppressed while the historical annotation must still present the line. +test_owned_growth_still_annotates_turn_ended() { + local dir state out err status pre_close ident + dir=$(make_case owned-historical) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + status="$state/scout.status" + + run_wake_lib() { + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; shift; "$@" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$@" + } + + printf 'needs-decision [key=budget]: approve spend?\n' > "$status" + prime_status_seen "$state" "$status" || fail "could not prime the scout seen marker" + pre_close=$(wc -c < "$status" | tr -d '[:space:]') + run_wake_lib fm_wake_status_append_self_announced "$state" "$status" \ + 'resolved [key=budget]: answered: approved' \ + || fail "the answerer close was not self-announced" + ident=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; _fm_open_decisions_file_ident "$2" + ' _ "$ROOT/bin/fm-classify-lib.sh" "$status") \ + || fail "could not read the status identity" + run_wake_lib fm_wake_status_seen_commit "$state" "$status" "$pre_close" "$ident" \ + || fail "could not replay the stale watcher classification" + run_wake_lib fm_wake_signal_seen_current "$state" "$status" \ + || fail "owned-only growth did not suppress the wake" + + : > "$state/scout.turn-ended" + append_wake "$state" signal scout.turn-ended "signal: $state/scout.turn-ended" \ + || fail "turn-ended wake append failed" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" || fail "drain failed" + grep -F 'scout.status: resolved [key=budget]: answered: approved' "$out" >/dev/null \ + || fail "owned growth hid this home's own close from the turn-ended annotation: $(cat "$out")" + pass "owned growth suppresses the wake without hiding the turn-ended annotation" +} + # Drain-time historical annotation staleness: a turn-ended-only wake row must # not present an already-announced status line as a new update, while a status # file with unannounced bytes keeps its annotation and a direct status row is @@ -2023,6 +2201,10 @@ test_secondmate_stall_marker_rejects_symlink test_acknowledged_stall_publication_survives_pre_marker_crash test_empty_prefix_mate_preserves_other_mate_receipt test_self_announced_append_guards +test_separate_self_announced_answers_after_fold_are_owned +test_unreadable_status_is_not_owned +test_folded_worker_resolved_is_not_owned_lag +test_owned_growth_still_annotates_turn_ended test_historical_annotation_skips_announced_status test_concurrent_append_and_drain test_signal_catchup_without_running_watcher diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 13c8648e53a..3df6899e818 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1607,6 +1607,71 @@ test_self_announced_close_after_open_decisions_fold_does_not_rewake() { pass "a close after OPEN DECISIONS fold never wakes its own home, and the next real note still does" } +# Any actor's drain folds OPEN DECISIONS, including a Pi branch drain, so a +# fold is no proof the watcher's owner saw the line. A fresh worker decision the +# fold already read must still wake when this home appended nothing. +test_folded_worker_decision_without_home_append_still_wakes() { + local dir state fakebin out status_file pid + dir=$(make_case folded-decision-wakes); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + status_file="$state/task.status" + printf 'working: building\n' > "$status_file" + prime_status_seen "$state" "$status_file" || fail "could not prime the announced baseline" + printf 'needs-decision [key=k3]: pick a region\n' >> "$status_file" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + export FM_FAKE_CREW_STATE='state: unknown · source: none · idle worker' + watch_bg "$state" "$fakebin" "$out" + pid=$! + wait_for_exit "$pid" 100 || fail "a folded worker decision with no home append was swallowed" + grep -F "signal: $status_file" "$out" >/dev/null \ + || fail "the folded worker decision did not surface as a signal: $(cat "$out")" + pass "a folded worker decision with no home append still wakes" +} + +# Two distinct --resolve-key answers to decisions the watcher never classified +# leave the marker alone, since a fold is no proof the watcher's owner saw them. +# That costs one wake for the worker's decisions, not one per answer: the +# answers are this home's owned ranges. Once the watcher surfaced the +# decisions, the owned answers stay quiet and the next real note still wakes. +test_separate_self_announced_answers_after_fold_wake_once() { + local dir state fakebin out status_file pid rc answer + dir=$(make_case multi-answer-fold); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + status_file="$state/task.status" + { + printf 'needs-decision [key=k1]: pick REST or RPC\n' + printf 'needs-decision [key=k2]: pick us-east or eu-west\n' + } > "$status_file" + FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>"$dir/fold.err" \ + || fail "the OPEN DECISIONS fold drain failed" + for answer in 'resolved [key=k1]: answered: REST' 'resolved [key=k2]: answered: eu-west'; do + rc=0 + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; fm_wake_status_append_self_announced "$2" "$3" "$4" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$state" "$status_file" "$answer" || rc=$? + [ "$rc" -eq 1 ] || fail "an answer over unclassified worker decisions did not fail toward waking (rc=$rc)" + done + export FM_FAKE_CREW_STATE='state: unknown · source: none · idle worker' + watch_bg "$state" "$fakebin" "$out" + pid=$! + wait_for_exit "$pid" 100 || fail "the unclassified worker decisions were swallowed" + grep -F "signal: $status_file" "$out" >/dev/null \ + || fail "the worker decisions did not surface as a signal: $(cat "$out")" + ack_stopped_cycle "$state" || fail "could not handle the worker decisions' wake" + : > "$out" + watch_bg "$state" "$fakebin" "$out" + pid=$! + if ! wait_poll_cycle "$state" "$pid"; then + reap "$pid"; fail "the owned answers re-woke the watcher: $(cat "$out")" + fi + [ ! -s "$out" ] || { reap "$pid"; fail "the owned answers printed a wake reason: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the owned answers enqueued another durable wake"; } + printf 'blocked: need staging credentials\n' >> "$status_file" + wait_for_exit "$pid" 100 || fail "a later worker line after two owned answers was swallowed" + grep -F "signal: $status_file" "$out" >/dev/null \ + || fail "the later worker line did not surface as a signal" + pass "separate answers over unclassified decisions wake once, and the next real note still does" +} + test_self_announced_close_after_fold_still_surfaces_folded_worker_failure() { local dir state fakebin out status_file pid rc dir=$(make_case self-close-folded-failure); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" @@ -5574,6 +5639,8 @@ test_secondmate_status_note_surfaced_despite_busy_agent test_secondmate_buried_block_wakes_despite_busy_agent test_self_announced_close_does_not_rewake_but_next_note_does test_self_announced_close_after_open_decisions_fold_does_not_rewake +test_folded_worker_decision_without_home_append_still_wakes +test_separate_self_announced_answers_after_fold_wake_once test_self_announced_close_after_fold_still_surfaces_folded_worker_failure test_self_announced_close_after_fold_still_surfaces_folded_secondmate_lines test_actionable_signal_surfaced