diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 26a2fe61ff2..cde6ba85fcb 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -804,12 +804,11 @@ EOF # because no code path in this repo ever does that to a status file. # # The other real failure mode is OUR OWN read failing (a stat/wc/tail I/O -# error), not a malformed writer: every such read here is checked, and on -# failure this reports the already-trusted persisted set unchanged rather than -# risking a silent invalidation that would wipe it - never a bare "empty" as if -# nothing were open. Such a fallback call succeeds only when the set it reports -# is non-empty, so a caller can never read one as a computed "nothing open"; -# the fallback block inside status_open_decisions_incremental owns why. +# error), not a malformed writer: every such read here is checked, leaves the +# already-trusted persisted set untouched on disk, and returns failure without +# printing that set. Even a non-empty persisted set is stale when unread bytes +# could have resolved one decision or opened another, so a caller must never +# present it as this call's authoritative result. # # Not a pure status-file read: this writes/rewrites the sibling cursor file as a # side effect (state/..open-decisions-cursor), the library's second @@ -915,7 +914,7 @@ _fm_status_read_span() { # } status_open_decisions_incremental() { # [] - local f=$1 captured_end=${2:-} cf offset ident open='' trusted_open='' cursor_data first rest offset_line ident_line + local f=$1 captured_end=${2:-} cf offset ident open='' cursor_data first rest offset_line ident_line local version='' size actual_size cur_ident resolve held chunk_file chunk_size line cursor_dirty=0 local target_cursor [ -f "$f" ] && [ -r "$f" ] && [ ! -L "$f" ] || return 0 @@ -950,7 +949,6 @@ status_open_decisions_incremental() { # [] case "$rest" in *$'\n'*) open=${rest#*$'\n'} ;; esac - if [ -n "$version" ] && [ -n "$ident" ]; then trusted_open=$open; fi ;; *) offset=0; version='' ;; esac @@ -963,26 +961,20 @@ status_open_decisions_incremental() { # [] esac fi - # A stat/size-read failure is a genuine I/O error, not "the file is empty" - - # report the already-trusted persisted set unchanged rather than risking a - # silent invalidation that would wipe it. That fallback is only honest while - # the set it reports has something IN it: an EMPTY set returned with rc 0 - # asserts "computed, nothing open" about bytes this call never read - just as - # wrong when the cursor was trusted but lagging a log that has since grown as - # when it was untrusted outright. So every fallback below reports the set it - # has and succeeds only when that set is non-empty; nothing to report means - # an incomplete fold, which the drain's existing notice already surfaces. - cur_ident=$(_fm_open_decisions_file_ident "$f") || { printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return; } - [ -n "$cur_ident" ] || { printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return; } - actual_size=$(_fm_status_file_size "$f") \ - || { printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return; } +# A stat, size, or span-read failure is a genuine I/O error, not "the file is +# empty" and not permission to replay the persisted open set. Leave that set +# untouched for a later recovery call, but return failure without printing it: +# unread bytes can make any cached set stale in either direction. + cur_ident=$(_fm_open_decisions_file_ident "$f") || return 1 + [ -n "$cur_ident" ] || return 1 + actual_size=$(_fm_status_file_size "$f") || return 1 actual_size=${actual_size//[[:space:]]/} - case "$actual_size" in ''|*[!0-9]*) printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return ;; esac + case "$actual_size" in ''|*[!0-9]*) return 1 ;; esac if [ -n "$captured_end" ]; then case "$captured_end" in - ''|*[!0-9]*) printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return ;; + ''|*[!0-9]*) return 1 ;; esac - [ "$captured_end" -le "$actual_size" ] || { printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return; } + [ "$captured_end" -le "$actual_size" ] || return 1 size=$captured_end else size=$actual_size @@ -991,19 +983,18 @@ status_open_decisions_incremental() { # [] if [ -z "$version" ] || [ -z "$ident" ] || [ "$ident" != "$cur_ident" ] || [ "$offset" -gt "$actual_size" ]; then offset=0 open='' - trusted_open='' cursor_dirty=1 fi if [ "$offset" -lt "$size" ]; then chunk_file="$cf.read.$$" _fm_status_read_span "$f" "$offset" "$((size - offset))" > "$chunk_file" 2>/dev/null \ - || { rm -f "$chunk_file"; printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return; } + || { rm -f "$chunk_file"; return 1; } chunk_size=$(LC_ALL=C wc -c < "$chunk_file" 2>/dev/null) \ - || { rm -f "$chunk_file"; printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return; } + || { rm -f "$chunk_file"; return 1; } chunk_size=${chunk_size//[[:space:]]/} case "$chunk_size" in - ''|*[!0-9]*) rm -f "$chunk_file"; printf '%s' "$trusted_open"; [ -n "$trusted_open" ]; return ;; + ''|*[!0-9]*) rm -f "$chunk_file"; return 1 ;; esac # Test-only observability seam (off by default, no production behavior # change): when set, records exactly how many bytes THIS call folded, so a @@ -1403,6 +1394,9 @@ EOF return "$rc" } +# Only a drain that computed its whole presentation reaches this pass at all, so +# every task here was presented in full: an annotation or snapshot failure +# anywhere skips the acknowledge and commit entirely and no cursor moves. status_acknowledge_presented_snapshot() { # [] local state=$1 snapshot=$2 fully_presented=${3:-} task endpoint ident f offset lines line safe while IFS=$(printf '\t') read -r task endpoint ident; do diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 41fe28b17e0..c766a7062e6 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -399,7 +399,9 @@ EOF # cursor-backed unread span as the annotation path, and runs on every drain - # including the empty-queue fast path - so a buried answer cannot be swallowed # when the fold later advances the cursor. Prints nothing when nothing is -# unread, which is the common case. +# unread, which is the common case. The header's "not re-printed" promise holds +# because this section runs only on a drain that goes on to commit its +# presentation receipt: any earlier failure skips the sections outright. print_unread_status_section() { # local snapshot=$1 unread task line shown=0 @@ -543,28 +545,41 @@ EOF printf 'RECORD DIVERGENCE: reconcile each one - record the captain'"'"'s own words with bin/fm-captain-hold.sh answer --decision-file , or re-open the status decision when that resolution was not the captain'"'"'s word.\n' || return 1 } -# A read or write hiccup anywhere in the drain's fleet-wide section passes (one -# task's status log, one task's open-decisions cursor, the scratch file the -# sections are prepared into) must never be indistinguishable from "computed, -# and genuinely nothing is open or unread": that silence is exactly what let a -# captain-facing OPEN DECISIONS section vanish for a drain even though several -# tasks' needs-decision/blocked lines were still open and unresolved in their -# own durable status logs: a failure on any ONE task aborts the whole -# acknowledge pass via its `|| return 1` before a single section is prepared, -# and the preparation that follows is itself all-or-nothing. -# Print this notice on every such failure instead of returning silently, so an -# empty presentation can only ever mean the passes ran to completion and found -# nothing - never that they could not be computed. print_status_presentation -# emits it from one place for every failure it observes, so no new failure -# path inside those passes can return without it. -print_status_sections_incomplete_notice() { - printf 'STATUS PRESENTATION INCOMPLETE: unread status, outcome backstop, OPEN DECISIONS, and record divergence could not be fully computed this drain (a status log or cursor read/write failed); do not read this drain'"'"'s silence as nothing open or unread - retry on the next drain.\n' +# A computation failure before the prepared section bytes reach stdout must +# never be indistinguishable from "computed, and genuinely nothing is open or +# unread". The caller supplies the failed operation so the notice says both +# what is missing and why. Receipt persistence is deliberately separate: once +# every prepared byte reached stdout, only the receipt can be incomplete. +print_status_sections_incomplete_notice() { # + local reason=$1 + printf 'STATUS PRESENTATION INCOMPLETE: unread status, outcome backstop, OPEN DECISIONS, and record divergence could not be fully computed this drain (%s); do not read this drain'"'"'s silence as nothing open or unread - retry on the next drain.\n' "$reason" +} + +print_status_receipt_failure_notice() { + printf 'STATUS PRESENTATION RECEIPT FAILED: the status sections above were fully computed and printed, but their presentation receipt could not be committed; they may repeat on the next drain.\n' +} + +# One notice for every failure that leaves this drain with no presentation to +# make at all - the lock, the fleet snapshot, or the annotation pass, whether or +# not the failure can name a task. Such a drain acknowledges nothing and commits +# nothing, so "nothing was marked as seen" needs no per-task bookkeeping to be +# true: every presentation cursor is still where the previous drain left it. +print_status_presentation_incomplete_notice() { # + local reason=$1 + printf 'STATUS PRESENTATION INCOMPLETE: %s; no status annotations or fleet-wide status sections were computed this drain, nothing was marked as seen and no presentation cursor advanced, so every unread status line is still unread.\n' \ + "$reason" } print_status_sections() { # [] local snapshot=$1 fully_presented=${2:-} acknowledged prepared - acknowledged=$(status_acknowledge_presented_snapshot "$STATE" "$snapshot" "$fully_presented") || return 1 - prepared=$(mktemp "$STATE/.status-presentation.prepared.XXXXXX") || return 1 + acknowledged=$(status_acknowledge_presented_snapshot "$STATE" "$snapshot" "$fully_presented") || { + print_status_sections_incomplete_notice 'a status log or presentation cursor could not be read' + return 1 + } + prepared=$(mktemp "$STATE/.status-presentation.prepared.XXXXXX") || { + print_status_sections_incomplete_notice 'the prepared-output file could not be created' + return 1 + } if ! { print_unread_status_section "$snapshot" \ && print_status_outcome_backstop_section "$snapshot" \ @@ -572,6 +587,7 @@ print_status_sections() { # [ "$prepared"; then rm -f -- "$prepared" + print_status_sections_incomplete_notice 'a status log, cursor, or prepared-output write could not be completed' return 1 fi # Prepare every section before presentation, but do not commit its receipt @@ -579,18 +595,20 @@ print_status_sections() { # [] - local rows=${1:-} lock="$STATE/.status-presentation-lock" snapshot annotation_manifest fully_presented='' rc=0 - local lock_rc holder_pid + local rows=${1:-} lock="$STATE/.status-presentation-lock" snapshot='' annotation_manifest fully_presented='' rc=0 + local lock_rc holder_pid incomplete_reason='' if fm_lock_acquire_wait_bounded "$lock" "$PRESENTATION_LOCK_TIMEOUT"; then : else @@ -600,7 +618,7 @@ print_status_presentation() { # [] printf 'STATUS PRESENTATION SKIPPED: lock remains held by live pid %s after %ss; retry on the next drain.\n' \ "$holder_pid" "$PRESENTATION_LOCK_TIMEOUT" else - printf 'wake drain: status presentation lock could not be acquired safely\n' >&2 + print_status_presentation_incomplete_notice 'status presentation lock could not be acquired safely' fi return 1 fi @@ -608,21 +626,30 @@ print_status_presentation() { # [] # in the captured value. Acknowledging and committing that truncated fleet view # would rewrite the shared presentation-cursor manifest without the tasks the # read never reached, resetting their unread and outcome-backstop cursors for - # good, so rc=1 here keeps the annotation, acknowledge and commit passes below - # from running at all rather than presenting a partial fleet view. + # good, so the partial capture is discarded here and nothing below runs on it. snapshot=$(status_presentation_snapshot "$STATE") || { - printf 'STATUS PRESENTATION INCOMPLETE: status snapshot could not be read.\n' + snapshot= + incomplete_reason='status snapshot could not be read' rc=1 } if [ "$rc" -eq 0 ] && [ -n "$rows" ]; then - fm_wake_print_annotations "$rows" "$snapshot" || rc=1 - if [ "$rc" -eq 0 ]; then - annotation_manifest=$(fm_wake_annotation_manifest "$rows") || rc=1 - fully_presented=$(printf '%s\n' "$annotation_manifest" | awk -F '\t' '$2 == "direct" { sub(/\.status$/, "", $1); print $1 }') || rc=1 + if fm_wake_print_annotations "$rows" "$snapshot"; then + annotation_manifest=$(fm_wake_annotation_manifest "$rows") + fully_presented=$(printf '%s\n' "$annotation_manifest" | awk -F '\t' '$2 == "direct" { sub(/\.status$/, "", $1); print $1 }') + else + incomplete_reason='a supplemental status annotation could not be computed' + rc=1 fi fi - if [ "$rc" -eq 0 ] && [ -n "$snapshot" ]; then - print_status_sections "$snapshot" "$fully_presented" || { rc=1; print_status_sections_incomplete_notice; } + # One rule for the snapshot and the annotation pass alike, whether or not the + # failure can name a task: a drain that could not compute either presents + # nothing as seen. It says so once and skips the acknowledge and commit passes + # entirely, so no task's presentation cursor moves and the next drain still + # owes every unread line. + if [ -n "$incomplete_reason" ]; then + print_status_presentation_incomplete_notice "$incomplete_reason" + elif [ -n "$snapshot" ]; then + print_status_sections "$snapshot" "$fully_presented" || rc=1 fi fm_lock_release "$lock" return "$rc" diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 5f359524087..f0492058f61 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -2254,8 +2254,9 @@ fm_wake_status_cursor_offset() { # -> already-presented # O_NOFOLLOW read of every still-unread status byte. min-offset is the # already-presented cursor from classify-lib. Lines whose bytes begin before -# that offset are not replayed. Prints nothing and returns 1 when no unread -# non-blank line exists. +# that offset are not replayed. Prints nothing and returns 1 when the span holds +# no unread non-blank line, and 2 when the span itself could not be read: the +# caller must not treat an unread span it never read as one with nothing in it. fm_wake_unread_events() { # [] local path=$1 min_offset=$3 end_offset=${4:-} result size chunk chunk_start local LC_ALL=C @@ -2281,10 +2282,10 @@ fm_wake_unread_events() { # /dev/null) || return 1 + ' "$path" "$min_offset" "$end_offset" 2>/dev/null) || return 2 size=${result%%$'\t'*} chunk=${result#*$'\t'} - case "$size" in ''|*[!0-9]*) return 1 ;; esac + case "$size" in ''|*[!0-9]*) return 2 ;; esac [ -n "$chunk" ] || return 1 [ "$min_offset" -lt "$size" ] || return 1 chunk_start=$min_offset @@ -2295,7 +2296,7 @@ fm_wake_unread_events() { # = min) print $0 } - ') || return 1 + ') || return 2 [ -n "$FM_WAKE_UNREAD_LINES" ] || return 1 FM_WAKE_EVENT_LINE=$(printf '%s\n' "$FM_WAKE_UNREAD_LINES" | tail -1) FM_WAKE_EVENT_LINE=$(printf '%s' "$FM_WAKE_EVENT_LINE" | LC_ALL=C tr '\t\r' ' ') @@ -2306,10 +2307,13 @@ fm_wake_latest_event() { # } # Print supplemental drain-time context only after the caller has committed the -# raw queue consumption and released the append lock. +# raw queue consumption and released the append lock. Returns 1 when any part of +# the pass could not be computed, whatever the cause: the caller's contract is +# that such a drain presents nothing as seen at all. fm_wake_print_annotations() { # [] local rows=$1 snapshot=${2:-} manifest status_key mode path prefix line task endpoint - local snapshot_task snapshot_endpoint _snapshot_ident offset last_event event_line + local snapshot_task snapshot_endpoint _snapshot_ident offset last_event event_line events_rc + local incomplete=0 local LC_ALL=C manifest=$(fm_wake_annotation_manifest "$rows" | awk -F '\t' ' @@ -2326,7 +2330,7 @@ fm_wake_print_annotations() { # [] END { for (i = 1; i <= count; i++) print order[i] "\t" mode[order[i]] } - ') || return 0 + ') || return 1 # Test-only latency seam for proving that queue appends remain independent of # a slow best-effort annotation phase. @@ -2352,7 +2356,20 @@ fm_wake_print_annotations() { # [] if [ "$mode" = historical ] && fm_wake_signal_seen_current "$STATE" "$path"; then continue fi - offset=$(fm_wake_status_cursor_offset "$path") || continue + # A row can outlive normal task teardown. With no safe status file there is + # no live annotation to compute, so keep the durable row without calling a + # missing file a cursor failure. A cursor failure for a still-readable live + # file is different and must be reported below. + [ -f "$path" ] && [ -r "$path" ] && [ ! -L "$path" ] || continue + # A live row whose presentation cursor cannot be read has not been + # annotated. Report the failure so the drain can say that plainly on stdout + # and mark nothing as seen, leaving these bytes for the next drain to + # annotate; silently continuing makes the durable row look fully enriched + # when it is not. + offset=$(fm_wake_status_cursor_offset "$path") || { + incomplete=1 + continue + } endpoint= if [ -n "$snapshot" ]; then task=${status_key%.status} @@ -2364,11 +2381,16 @@ EOF [ -n "$endpoint" ] || continue fi if [ -n "$endpoint" ] && [ "$offset" -ge "$endpoint" ]; then continue; fi - if ! fm_wake_unread_events "$path" 0 "$offset" "$endpoint"; then + events_rc=0 + fm_wake_unread_events "$path" 0 "$offset" "$endpoint" || events_rc=$? + if [ "$events_rc" -ne 0 ]; then # Annotation enrichment is supplemental to the already-printed durable - # wake rows. A file that disappears, rotates, or becomes unreadable after - # the snapshot must not suppress annotations for other status files; the - # presentation commit will reject a changed snapshot identity. + # wake rows, so a file that disappears, rotates, or becomes unreadable + # after the snapshot must not suppress annotations for other status files. + # A span that could not be read (rc 2) is not a span with nothing in it: + # report it so the drain marks nothing as seen and the unread bytes + # survive to the next drain. + [ "$events_rc" -ne 2 ] || incomplete=1 continue fi last_event=$FM_WAKE_EVENT_LINE @@ -2391,5 +2413,6 @@ EOF $manifest EOF + [ "$incomplete" -eq 0 ] || return 1 return 0 } diff --git a/docs/architecture.md b/docs/architecture.md index 74fac0c21ea..a644137ba5d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,7 +74,10 @@ The drain coordinates that fold and its annotations through a locked fleet-wide [`pi-supervision-branch.md`](pi-supervision-branch.md#lost-wake-outcome-backstop) owns the bounded lost-wake backstop that uses the latter offset. A queued signal annotation prints every status line still unread at that cursor, while the fleet-wide UNREAD STATUS section prints `note:` lines and reserved-key pending-reply resolutions once even on an empty-queue drain because those verbs never enter the OPEN DECISIONS fold. A third bounded section, RECORD DIVERGENCE, prints on the same drains for the opposite failure: the status fold went quiet on a key that the durable captain-held task still shows as open, so the status side reads as complete while the two records contradict each other; `bin/fm-captain-hold.sh diverged` decides what counts and closes nothing, and `docs/captain-hold-lifecycle.md` owns the mechanism. -A failed read, output, or concurrent-replacement check prevents the snapshot cursor from advancing across uncertain bytes, and teardown retires a task's manifest row before that task ID can be reused. +Any failure before those prepared section bytes are fully delivered to stdout - the presentation lock, the fleet snapshot read, a supplemental annotation, a concurrent-replacement check, or a status log, cursor, or prepared-output read or write - leaves the drain with nothing it may present as seen: it states on stdout what it could not compute and why, and no task's annotation, unread, or outcome-backstop offset advances, whether or not the failure could name a task. +A cursor-backed fold whose own read failed reports that failure instead of replaying its persisted open set, because the bytes it never read could have resolved one decision or opened another. +Once every prepared byte has reached stdout only the receipt can still fail, and that case is reported as a receipt failure rather than relabeling a complete presentation incomplete, so those sections may repeat on the next drain. +Teardown retires a task's manifest row before that task ID can be reused. The explicit resolution is written by the actor that answers, not the busy worker: `fm-send`'s `--resolve-key` appends the closing `resolved` line to this home's own copy of the ledger at answer time, which covers crewmates, local secondmates, and remote secondmates identically because a remote mate's escalations reach that local copy through the parent-replies ingest and only the answer message itself crosses the transport. This home's answerer close, pending-reply escalation close, and captain-held transfer use the provenance-guarded append owned by `bin/fm-wake-lib.sh`, so they advance the watcher marker only across their own bytes when all earlier bytes were already announced; pending or interleaved foreign bytes fail toward an ordinary wake. A turn-ended-only queue row omits its historical status annotation when that status file exactly matches the same seen marker. diff --git a/tests/fm-wake-drain-open-decisions.test.sh b/tests/fm-wake-drain-open-decisions.test.sh index da6529630ae..40a163ed033 100755 --- a/tests/fm-wake-drain-open-decisions.test.sh +++ b/tests/fm-wake-drain-open-decisions.test.sh @@ -306,8 +306,10 @@ IDENT : > "$dir/fail-ident" FM_STATE_OVERRIDE="$state" FM_STATUS_IDENTITY_READER="$ident" "$DRAIN" > "$out" \ || fail "wake drain failed instead of reporting an unreadable snapshot" - grep -F 'STATUS PRESENTATION INCOMPLETE: status snapshot could not be read.' "$out" >/dev/null \ + grep -F 'STATUS PRESENTATION INCOMPLETE: status snapshot could not be read' "$out" >/dev/null \ || fail "a failed snapshot read went unreported: $(command cat "$out")" + grep -F 'nothing was marked as seen and no presentation cursor advanced' "$out" >/dev/null \ + || fail "a failed snapshot read did not state that nothing was marked as seen: $(command cat "$out")" rm -f "$dir/fail-ident" FM_STATE_OVERRIDE="$state" FM_STATUS_IDENTITY_READER="$ident" "$DRAIN" > "$out" \ @@ -390,6 +392,107 @@ test_trusted_empty_fold_cursor_read_failure_is_not_a_silent_empty() { pass "a trusted fold cursor's empty persisted set is not replayed as a computed empty section" } +# A trusted NON-empty fold is still stale when this call cannot read the bytes +# after its cursor. Replaying that set as authoritative can show a decision that +# the unread span resolved and hide a new decision from the same span. Stage the +# fold cursor at the first drain while leaving the presentation cursor at the +# second drain's EOF, so the injected span-read failure reaches only the fold. +test_trusted_nonempty_fold_cursor_read_failure_is_not_authoritative() { + local dir state out err reader cursor saved_cursor task log + dir=$(make_case trusted-nonempty-fold-cursor) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + reader="$dir/fail-reader" + task=firstmate-reconcile-fork-with-upstream + log="$state/$task.status" + cursor="$state/.$task.open-decisions-cursor" + saved_cursor="$dir/stale-open-set.cursor" + + printf 'needs-decision [key=old-choice]: choose the old route\n' > "$log" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" \ + || fail "bootstrap drain before the trusted nonempty-cursor failure failed" + grep -F "$task [key=old-choice] needs-decision: choose the old route" "$out" >/dev/null \ + || fail "the original decision did not surface before staging the stale fold" + cp "$cursor" "$saved_cursor" || fail "could not save the trusted nonempty fold cursor" + + { + printf 'resolved [key=old-choice]: the old route is closed\n' + printf 'needs-decision [key=new-choice]: choose the new route\n' + } >> "$log" + append_wake "$state" signal "$task.status" "signal: $log" \ + || fail "could not seed the direct row that advances the presentation cursor" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "clean drain before staging the trusted nonempty cursor failed" + grep -F "$task [key=new-choice] needs-decision: choose the new route" "$out" >/dev/null \ + || fail "the replacement decision did not surface before the injected failure" + if grep -F "$task [key=old-choice]" "$out" >/dev/null; then + fail "the clean drain still showed the resolved decision: $(command cat "$out")" + fi + ack_drain_err "$state" "$err" \ + || fail "could not acknowledge the cursor-staging wake" + cp "$saved_cursor" "$cursor" || fail "could not restore the stale trusted open set" + + printf '#!/usr/bin/env bash\nexit 1\n' > "$reader" + chmod +x "$reader" + FM_STATE_OVERRIDE="$state" FM_STATUS_SPAN_READER="$reader" "$DRAIN" > "$out" \ + || fail "wake drain failed instead of reporting the stale fold as incomplete" + if grep -F 'OPEN DECISIONS (still open' "$out" >/dev/null \ + || grep -F "$task [key=old-choice]" "$out" >/dev/null \ + || grep -F "$task [key=new-choice]" "$out" >/dev/null; then + fail "a failed fold read printed a stale nonempty set as authoritative: $(command cat "$out")" + fi + grep -F 'STATUS PRESENTATION INCOMPLETE: unread status, outcome backstop, OPEN DECISIONS' "$out" >/dev/null \ + || fail "the stale nonempty fold read failure was not reported: $(command cat "$out")" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" \ + || fail "recovery drain after the stale nonempty fold failure cleared failed" + grep -F "$task [key=new-choice] needs-decision: choose the new route" "$out" >/dev/null \ + || fail "the new decision did not appear after the fold read recovered: $(command cat "$out")" + if grep -F "$task [key=old-choice]" "$out" >/dev/null; then + fail "the resolved decision reappeared after the fold read recovered: $(command cat "$out")" + fi + + pass "a trusted nonempty fold is never authoritative when its unread span cannot be read" +} + +# Once every section's bytes have reached stdout, a failure to persist the +# trailing receipt means only that the sections can repeat. It must not relabel +# those fully computed and printed sections as an incomplete presentation. +test_receipt_failure_does_not_relabel_printed_sections_incomplete() { + local dir state out fakebin real_mv + dir=$(make_case receipt-write-failure) + state="$dir/state" + out="$dir/drain.out" + fakebin="$dir/fakebin" + real_mv=$(command -v mv) + + printf 'needs-decision [key=route]: choose the presentation route\n' > "$state/task.status" + cat > "$fakebin/mv" <<'SH' +#!/usr/bin/env bash +set -u +last= +for arg in "$@"; do last=$arg; done +case "$last" in + */.status-presentation-cursor) exit 1 ;; +esac +exec "$FM_TEST_REAL_MV" "$@" +SH + chmod +x "$fakebin/mv" + + PATH="$fakebin:$PATH" FM_TEST_REAL_MV="$real_mv" FM_STATE_OVERRIDE="$state" \ + "$DRAIN" > "$out" || fail "drain failed after the injected receipt write failure" + grep -F 'task [key=route] needs-decision: choose the presentation route' "$out" >/dev/null \ + || fail "the receipt failure hid the already-computed sections: $(command cat "$out")" + grep -F 'STATUS PRESENTATION RECEIPT FAILED:' "$out" >/dev/null \ + || fail "the receipt write failure was not reported distinctly: $(command cat "$out")" + if grep -F 'STATUS PRESENTATION INCOMPLETE:' "$out" >/dev/null; then + fail "a receipt-only failure mislabeled fully printed sections as incomplete: $(command cat "$out")" + fi + + pass "a failed presentation receipt never contradicts the complete sections already printed" +} + # An untrusted per-task fold cursor has no persisted open set to fall back on, # so a span-read failure there cannot honestly report "nothing open". The # acknowledge and unread-status passes both short-circuit at EOF here, so this @@ -504,4 +607,6 @@ test_torn_down_task_wake_row_does_not_blank_the_sections test_partial_snapshot_does_not_truncate_the_cursor_manifest test_untrusted_fold_cursor_read_failure_is_not_a_silent_empty test_trusted_empty_fold_cursor_read_failure_is_not_a_silent_empty +test_trusted_nonempty_fold_cursor_read_failure_is_not_authoritative +test_receipt_failure_does_not_relabel_printed_sections_incomplete test_status_symlink_is_not_followed diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 9e7faea0335..401039c22b6 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -1848,8 +1848,11 @@ test_malformed_presentation_lock_reports_acquire_failure() { FM_STATE_OVERRIDE="$state" FM_STATUS_PRESENTATION_LOCK_TIMEOUT=1 \ "$DRAIN" > "$out" 2> "$err" || fail "malformed-lock drain failed" - grep -F 'wake drain: status presentation lock could not be acquired safely' "$err" >/dev/null \ - || fail "malformed presentation lock did not report an acquire failure" + grep -F 'STATUS PRESENTATION INCOMPLETE: status presentation lock could not be acquired safely' "$out" >/dev/null \ + || fail "malformed presentation lock did not report its acquire failure on stdout" + if grep -F 'wake drain: status presentation lock could not be acquired safely' "$err" >/dev/null; then + fail "malformed presentation lock still reported only through the diagnostic channel" + fi if grep -F 'STATUS PRESENTATION SKIPPED: lock remains held by live pid' "$out" >/dev/null; then fail "malformed presentation lock was reported as live-holder contention" fi @@ -1858,6 +1861,325 @@ test_malformed_presentation_lock_reports_acquire_failure() { pass "malformed presentation locks report acquire failure instead of contention" } +# The annotation pass and fleet-wide sections share the presentation cursor, +# but the former used to treat a cursor-read failure as "skip this live row". +# Fail exactly the annotation's first cursor read, then let every later read +# recover. One rule governs what the drain owes: it says ONCE that it computed +# nothing and marked nothing as seen, and no presentation cursor moves - not the +# failing task's, and not an untouched bystander's - so every unread line, +# including the `working:` line whose only surface is that annotation, is still +# owed on the next drain. +test_annotation_cursor_failure_is_reported_on_stdout() { + local dir state out err fakebin real_cat manifest status bystander notices + dir=$(make_case annotation-cursor-failure) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + fakebin="$dir/fakebin" + manifest="$state/.status-presentation-cursor" + status="$state/task.status" + bystander="$state/bystander.status" + real_cat=$(command -v cat) + + printf 'note: prime the presentation cursor\n' > "$status" + printf 'note: prime the bystander cursor\n' > "$bystander" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" \ + || fail "could not prime the annotation cursor fixture" + [ -s "$manifest" ] || fail "the annotation cursor fixture wrote no presentation manifest" + + printf 'working: live row must not disappear silently\n' >> "$status" + printf 'note: the captain is still owed this one\n' >> "$status" + printf 'note: an untouched task keeps its unread line too\n' >> "$bystander" + append_wake "$state" signal task.status "signal: $status" \ + || fail "could not seed the annotation cursor wake" + cat > "$fakebin/cat" <<'SH' +#!/usr/bin/env bash +set -u +if [ "$#" -eq 1 ] && [ "$1" = "$FM_TEST_CURSOR_MANIFEST" ] \ + && [ ! -e "$FM_TEST_CURSOR_FAILURE_USED" ]; then + : > "$FM_TEST_CURSOR_FAILURE_USED" + exit 1 +fi +exec "$FM_TEST_REAL_CAT" "$@" +SH + chmod +x "$fakebin/cat" + + PATH="$fakebin:$PATH" FM_TEST_REAL_CAT="$real_cat" \ + FM_TEST_CURSOR_MANIFEST="$manifest" \ + FM_TEST_CURSOR_FAILURE_USED="$dir/cursor-failure-used" \ + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "drain failed after the injected annotation cursor failure" + grep "$(printf '\tsignal\t')" "$out" >/dev/null \ + || fail "the annotation cursor failure dropped the durable wake row" + if grep -F 'wake annotation:' "$out" >/dev/null; then + fail "the failed annotation cursor read still printed an authoritative annotation" + fi + notices=$(grep -c 'STATUS PRESENTATION INCOMPLETE' "$out") + [ "$notices" -eq 1 ] \ + || fail "the annotation cursor failure owed exactly one notice, got $notices: $(command cat "$out")" + grep -F 'nothing was marked as seen and no presentation cursor advanced' "$out" >/dev/null \ + || fail "the annotation cursor failure did not state that nothing was marked as seen: $(command cat "$out")" + if grep -F 'UNREAD STATUS' "$out" >/dev/null; then + fail "a drain that marked nothing as seen still printed the one-shot unread section: $(command cat "$out")" + fi + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "the drain after the injected annotation cursor failure cleared failed" + grep -F 'wake annotation:' "$out" | grep -F 'working: live row must not disappear silently' >/dev/null \ + || fail "the unannotated line was acknowledged instead of retried: $(command cat "$out")" + grep -F 'note: the captain is still owed this one' "$out" >/dev/null \ + || fail "the failing task's unread line never came back: $(command cat "$out")" + grep -F 'note: an untouched task keeps its unread line too' "$out" >/dev/null \ + || fail "an untouched task's cursor advanced on a drain that computed nothing: $(command cat "$out")" + + pass "a live row's annotation cursor failure is explicit on stdout and moves no cursor" +} + +# A span read that FAILS and a span with nothing unread in it used to be the same +# return, so a transient read failure dropped a live task's annotation with no +# notice at all and the presentation cursor still advanced past the bytes the +# drain never printed. Fail exactly the annotation's first span read. +test_annotation_span_read_failure_is_reported_and_retried() { + local dir state out err fakebin real_perl status + dir=$(make_case annotation-span-read-failure) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + fakebin="$dir/fakebin" + status="$state/task.status" + real_perl=$(command -v perl) + + printf 'note: prime the presentation cursor\n' > "$status" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" \ + || fail "could not prime the annotation span fixture" + + printf 'working: span read must not vanish silently\n' >> "$status" + printf 'note: an unread surface the fleet-wide section acknowledges\n' >> "$status" + append_wake "$state" signal task.status "signal: $status" \ + || fail "could not seed the annotation span wake" + cat > "$fakebin/perl" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + if [ "$arg" = "$FM_TEST_SPAN_PATH" ] && [ ! -e "$FM_TEST_SPAN_FAILURE_USED" ]; then + : > "$FM_TEST_SPAN_FAILURE_USED" + exit 1 + fi +done +exec "$FM_TEST_REAL_PERL" "$@" +SH + chmod +x "$fakebin/perl" + + PATH="$fakebin:$PATH" FM_TEST_REAL_PERL="$real_perl" \ + FM_TEST_SPAN_PATH="$status" \ + FM_TEST_SPAN_FAILURE_USED="$dir/span-failure-used" \ + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "drain failed after the injected annotation span read failure" + if grep -F 'wake annotation:' "$out" >/dev/null; then + fail "the failed span read still printed an authoritative annotation" + fi + if grep -F 'STATUS PRESENTATION INCOMPLETE: unread status, outcome backstop, OPEN DECISIONS' "$out" >/dev/null; then + fail "the injected read failure reached the fleet-wide sections instead of the annotation span" + fi + grep -F 'STATUS PRESENTATION INCOMPLETE: a supplemental status annotation could not be computed' "$out" >/dev/null \ + || fail "the failed annotation span read remained silent on stdout: $(command cat "$out")" + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "the drain after the injected annotation span failure cleared failed" + grep -F 'wake annotation:' "$out" | grep -F 'working: span read must not vanish silently' >/dev/null \ + || fail "the span nobody read was acknowledged instead of retried: $(command cat "$out")" + grep -F 'note: an unread surface the fleet-wide section acknowledges' "$out" >/dev/null \ + || fail "the unread surface beside the failed span was marked as seen anyway: $(command cat "$out")" + + pass "a failed annotation span read is reported and its unread bytes stay unread" +} + +# An annotation failure that names no task still let the acknowledge pass advance +# cursors past bytes the annotation never printed. Fail the annotation pass's own +# manifest build - the one failure path that can name nothing - over a MIXED +# span: the trailing `note:` is an unread surface, so the fleet rule alone would +# acknowledge the whole span and drop the `working:` line whose only surface is +# the annotation that never ran. +test_unattributed_annotation_failure_holds_every_cursor() { + local dir state out err fakebin real_awk status + dir=$(make_case unattributed-annotation-failure) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + fakebin="$dir/fakebin" + status="$state/task.status" + real_awk=$(command -v awk) + + printf 'note: prime the presentation cursor\n' > "$status" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" \ + || fail "could not prime the unattributed annotation fixture" + + # A `working:` line has no fleet-wide surface: the annotation is its only + # presentation, so acknowledging it after an uncomputed annotation loses it. + printf 'working: only the annotation can carry this one\n' >> "$status" + printf 'note: an unread surface follows it in the same span\n' >> "$status" + append_wake "$state" signal task.status "signal: $status" \ + || fail "could not seed the unattributed annotation wake" + cat > "$fakebin/awk" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + case "$arg" in + *"if (!(key in seen))"*) + if [ ! -e "$FM_TEST_AWK_FAILURE_USED" ]; then + : > "$FM_TEST_AWK_FAILURE_USED" + exit 1 + fi + ;; + esac +done +exec "$FM_TEST_REAL_AWK" "$@" +SH + chmod +x "$fakebin/awk" + + PATH="$fakebin:$PATH" FM_TEST_REAL_AWK="$real_awk" \ + FM_TEST_AWK_FAILURE_USED="$dir/awk-failure-used" \ + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "drain failed after the injected annotation manifest failure" + [ -e "$dir/awk-failure-used" ] \ + || fail "the injected annotation manifest failure never fired" + grep -F 'STATUS PRESENTATION INCOMPLETE: a supplemental status annotation could not be computed' "$out" >/dev/null \ + || fail "the unattributed annotation failure remained silent on stdout: $(command cat "$out")" + if grep -F 'wake annotation:' "$out" >/dev/null; then + fail "the uncomputed annotation manifest still printed an annotation" + fi + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "the drain after the injected annotation manifest failure cleared failed" + grep -F 'wake annotation:' "$out" | grep -F 'working: only the annotation can carry this one' >/dev/null \ + || fail "an annotation failure that named no task acknowledged the span anyway: $(command cat "$out")" + grep -F 'note: an unread surface follows it in the same span' "$out" >/dev/null \ + || fail "the unread surface in the same span was marked as seen by a drain that computed nothing: $(command cat "$out")" + + pass "an annotation failure that names no task holds every presentation cursor" +} + +# A per-task hold used to still let the drain claim every other direct row was +# fully presented, so a sibling task's cursor advanced on the strength of a pass +# that had already failed. Fail one task's annotation cursor read with two direct +# rows queued and prove the drain names no task at all and BOTH `working:` spans - +# the failing one and its sibling's, whose only surface is that annotation - +# survive to the next drain. +test_annotation_failure_holds_sibling_cursors_too() { + local dir state out err fakebin real_cat manifest first second notices + dir=$(make_case annotation-sibling-hold) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + fakebin="$dir/fakebin" + manifest="$state/.status-presentation-cursor" + first="$state/alpha.status" + second="$state/bravo.status" + real_cat=$(command -v cat) + + printf 'note: prime alpha\n' > "$first" + printf 'note: prime bravo\n' > "$second" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" \ + || fail "could not prime the sibling annotation fixture" + [ -s "$manifest" ] || fail "the sibling annotation fixture wrote no presentation manifest" + + # Neither `working:` line has a fleet-wide surface: the annotation is the only + # presentation either one gets. + printf 'working: alpha needs its annotation\n' >> "$first" + printf 'working: bravo needs its annotation\n' >> "$second" + append_wake "$state" signal alpha.status "signal: $first" \ + || fail "could not seed the alpha wake" + append_wake "$state" signal bravo.status "signal: $second" \ + || fail "could not seed the bravo wake" + cat > "$fakebin/cat" <<'SH' +#!/usr/bin/env bash +set -u +if [ "$#" -eq 1 ] && [ "$1" = "$FM_TEST_CURSOR_MANIFEST" ] \ + && [ ! -e "$FM_TEST_CURSOR_FAILURE_USED" ]; then + : > "$FM_TEST_CURSOR_FAILURE_USED" + exit 1 +fi +exec "$FM_TEST_REAL_CAT" "$@" +SH + chmod +x "$fakebin/cat" + + PATH="$fakebin:$PATH" FM_TEST_REAL_CAT="$real_cat" \ + FM_TEST_CURSOR_MANIFEST="$manifest" \ + FM_TEST_CURSOR_FAILURE_USED="$dir/cursor-failure-used" \ + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "drain failed after the injected sibling cursor failure" + [ -e "$dir/cursor-failure-used" ] \ + || fail "the injected sibling cursor failure never fired" + notices=$(grep -c 'STATUS PRESENTATION INCOMPLETE' "$out") + [ "$notices" -eq 1 ] \ + || fail "the sibling cursor failure owed exactly one notice, got $notices: $(command cat "$out")" + if grep 'STATUS PRESENTATION INCOMPLETE' "$out" | grep -F "$state/" >/dev/null; then + fail "the one notice named a per-task status log instead of staying generic: $(command cat "$out")" + fi + + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out" 2> "$err" \ + || fail "the drain after the injected sibling cursor failure cleared failed" + grep -F 'wake annotation:' "$out" | grep -F 'working: alpha needs its annotation' >/dev/null \ + || fail "alpha's unannotated span was acknowledged instead of retried: $(command cat "$out")" + grep -F 'wake annotation:' "$out" | grep -F 'working: bravo needs its annotation' >/dev/null \ + || fail "bravo's unannotated span was acknowledged instead of retried: $(command cat "$out")" + + pass "an annotation failure holds the sibling cursors it never proved it presented" +} + +# The fleet snapshot is the first step of the same pass. Its failure used to +# print a notice about the sections alone and skip the annotation pass in +# silence, so a live row's unread `working:` line - whose only surface is that +# annotation - read as nothing unread. Fail the snapshot's identity read with a +# direct row queued: the same one notice is owed, and nothing may be marked as +# seen. +test_snapshot_failure_reports_the_uncomputed_annotations() { + local dir state out err ident status notices + dir=$(make_case snapshot-failure-annotations) + state="$dir/state" + out="$dir/drain.out" + err="$dir/drain.err" + ident="$dir/ident-reader" + status="$state/task.status" + + cat > "$ident" < "$status" + FM_STATE_OVERRIDE="$state" FM_STATUS_IDENTITY_READER="$ident" "$DRAIN" > "$out" \ + || fail "could not prime the snapshot-failure fixture" + + printf 'working: only the annotation can carry this one\n' >> "$status" + append_wake "$state" signal task.status "signal: $status" \ + || fail "could not seed the snapshot-failure wake" + + : > "$dir/fail-ident" + FM_STATE_OVERRIDE="$state" FM_STATUS_IDENTITY_READER="$ident" "$DRAIN" > "$out" 2> "$err" \ + || fail "drain failed instead of reporting an unreadable snapshot" + grep "$(printf '\tsignal\t')" "$out" >/dev/null \ + || fail "the snapshot failure dropped the durable wake row" + notices=$(grep -c 'STATUS PRESENTATION INCOMPLETE' "$out") + [ "$notices" -eq 1 ] \ + || fail "the snapshot failure owed exactly one notice, got $notices: $(command cat "$out")" + if grep -F 'wake annotation:' "$out" >/dev/null; then + fail "an unreadable snapshot still printed an authoritative annotation: $(command cat "$out")" + fi + grep -F 'nothing was marked as seen and no presentation cursor advanced' "$out" >/dev/null \ + || fail "the snapshot failure said nothing about the annotations it skipped: $(command cat "$out")" + + rm -f "$dir/fail-ident" + FM_STATE_OVERRIDE="$state" FM_STATUS_IDENTITY_READER="$ident" "$DRAIN" > "$out" 2> "$err" \ + || fail "the drain after the snapshot failure cleared failed" + grep -F 'wake annotation:' "$out" | grep -F 'working: only the annotation can carry this one' >/dev/null \ + || fail "the span the snapshot failure never annotated was acknowledged anyway: $(command cat "$out")" + + pass "an unreadable fleet snapshot reports its uncomputed annotations and moves no cursor" +} + # Drain-time historical annotation staleness: a turn-ended-only wake row must # not present an already-announced status line as a new update, while a status # file with unannounced bytes keeps its annotation and a direct status row is @@ -1912,6 +2234,11 @@ test_subshell_lock_ownership_without_bashpid test_bounded_lock_handoff_after_contention test_live_presentation_holder_is_deadlined_without_weakening_ack test_malformed_presentation_lock_reports_acquire_failure +test_annotation_cursor_failure_is_reported_on_stdout +test_annotation_span_read_failure_is_reported_and_retried +test_unattributed_annotation_failure_holds_every_cursor +test_annotation_failure_holds_sibling_cursors_too +test_snapshot_failure_reports_the_uncomputed_annotations test_secondmate_foreign_queue_stall_tracks_progress_and_alerts_once test_secondmate_declared_pause_rows_do_not_feed_stall_escalation test_secondmate_reprovisioned_queue_starts_a_fresh_interval