From 6cd030e94bcba42dab0e3040facd6cbb52e6bb86 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sat, 22 Aug 2026 23:51:49 -0700 Subject: [PATCH 01/24] fix: surface stalled secondmate queues and wake handoffs --- bin/fm-backlog-handoff.sh | 36 +++++++++++++- bin/fm-watch.sh | 78 ++++++++++++++++++++++++++++++ tests/fm-backlog-handoff.test.sh | 41 ++++++++++++++++ tests/fm-wake-queue.test.sh | 83 ++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 1 deletion(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 97bda75c331..5ef8714be0f 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -51,6 +51,9 @@ # selected set from the dispatchable backlog into data/handoff/.outbox.md, # then an idempotent confined transfer and fm-backlog-receive.sh deliver it. # A present outbox is the whole recovery record. No two-phase journal exists. +# Every successful delivery also sends one marked wake to the receiving endpoint. +# A missing endpoint is reported as an observable warning, while a live endpoint +# that rejects the wake makes the handoff fail with the delivered backlog intact. # Usage: fm-backlog-handoff.sh ... # fm-backlog-handoff.sh --resume-pending set -eu @@ -300,6 +303,32 @@ warn_stale_public_commitments() { # ... return 0 } +# Wake a live receiver after its backlog has become durable. The marked message +# uses the normal endpoint route, so local and remote secondmates share the same +# verified submit and failure semantics. A seeded but not-yet-spawned home is a +# valid handoff destination, but its missing endpoint is reported rather than +# pretending the task was started. +wake_secondmate_receiver() { # + local id=$1 meta="$STATE/$1.meta" out rc=0 + if [ ! -f "$meta" ] || [ -L "$meta" ]; then + printf 'warning: handed off work to secondmate %s, but no live receiver endpoint is recorded; the destination backlog is durable and needs a later wake\n' "$id" >&2 + return 0 + fi + [ "$(grep '^kind=' "$meta" | cut -d= -f2-)" = secondmate ] || { + printf 'error: secondmate %s has non-secondmate endpoint metadata; backlog is durable but the receiver was not woken\n' "$id" >&2 + return 1 + } + out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_ROOT_OVERRIDE="$FM_ROOT" \ + "$SCRIPT_DIR/fm-send.sh" "$id" \ + 'New routed work is in your backlog. Run bin/fm-session-start.sh now, then act on the routed task.' 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + [ -z "$out" ] || printf '%s\n' "$out" >&2 + printf 'error: backlog delivery to secondmate %s succeeded, but its receiver wake failed; rerun this handoff to retry the wake\n' "$id" >&2 + return 1 + fi + [ -z "$out" ] || printf '%s\n' "$out" +} + outbox_item_count() { # awk '/^- \[[ x]\] / { count++ } END { print count + 0 }' "$1" } @@ -348,8 +377,12 @@ remote_deliver_outbox() { # echo "error: handoff receipt by $id was unavailable or completion is unknown; outbox preserved at $outbox" >&2 return 1 fi + if ! wake_secondmate_receiver "$id"; then + echo "error: remote backlog is durable at $id; outbox preserved at $outbox for wake retry" >&2 + return 1 + fi rm -f -- "$outbox" || { - echo "error: remote receipt was confirmed but local outbox cleanup failed: $outbox" >&2 + echo "error: receiver wake was confirmed but local outbox cleanup failed: $outbox" >&2 return 1 } printf '%s\n' "$receive_out" @@ -608,6 +641,7 @@ fi echo "handed off ${#TO_MOVE[@]} item(s) to $ID: ${TO_MOVE[*]}" echo " into $SUB_BACKLOG" +wake_secondmate_receiver "$ID" || exit 1 if [ "${#ALREADY[@]}" -gt 0 ]; then echo " already present (skipped): ${ALREADY[*]}" fi diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index d1d59d3ceb5..d4b51116800 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -174,6 +174,9 @@ STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provabl # turn-ended and resets the age. Set generously above any legitimate interval # between completed turns, including long tool calls, builds, or test runs. BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600} +# A local secondmate's foreign queue is checked on every poll, but only after this +# bounded age can it produce a parent notification. +SECONDMATE_WAKE_STALL_SECS=${FM_SECONDMATE_WAKE_STALL_SECS:-60} # A crew that declared a pause is idling on a known external wait, so its stale # pane is absorbed rather than wedge-escalated. # A captain-held or paused crew whose agent has confidently exited uses the same @@ -294,6 +297,73 @@ recorded_windows() { done } +# Print the oldest structurally valid row in a local secondmate's foreign queue. +# This is a read-only observation: the receiving home owns acknowledgement and +# this parent never changes the row or the foreign queue. +secondmate_oldest_queue_row() { # + local queue=$1 + [ -f "$queue" ] && [ ! -L "$queue" ] || return 0 + awk -F '\t' ' + NF >= 5 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ { + if (!found || $2 < seq) { + found = 1 + seq = $2 + row = $0 + } + } + END { if (found) print row } + ' "$queue" 2>/dev/null || true +} + +# Surface one durable parent check for one unchanged foreign row after its +# bounded age. The primary marker and queued-key check make repeated watcher +# cycles converge without a notification storm, while an empty queue removes +# only this home's marker so a later row can be observed. +secondmate_wake_stall_tick() { + local now=$(( $(date +%s) )) threshold=$SECONDMATE_WAKE_STALL_SECS + local meta task kind remote_host home queue row epoch seq row_key marker notify_key queued age reason + case "$threshold" in ''|*[!0-9]*|0) threshold=60 ;; esac + for meta in "$STATE"/*.meta; do + [ -e "$meta" ] || continue + kind=$(fm_meta_get "$meta" kind) + [ "$kind" = secondmate ] || continue + remote_host=$(fm_meta_get "$meta" remote_host) + [ -z "$remote_host" ] || continue + task=${meta##*/} + task=${task%.meta} + case "$task" in ''|*[!A-Za-z0-9._-]*) continue ;; esac + home=$(fm_meta_get "$meta" home) + [ -n "$home" ] || continue + [ -f "$home/.fm-secondmate-home" ] && [ ! -L "$home/.fm-secondmate-home" ] || continue + [ "$(cat "$home/.fm-secondmate-home" 2>/dev/null || true)" = "$task" ] || continue + queue="$home/state/.wake-queue" + row=$(secondmate_oldest_queue_row "$queue") + marker="$STATE/.secondmate-wake-stall-$task" + if [ -z "$row" ]; then + rm -f "$marker" + continue + fi + IFS=$(printf '\t') read -r epoch seq _row_kind _row_key _row_payload </dev/null || true)" = "$row_key" ] && continue + notify_key="secondmate-wake-loop-$task-$row_key" + reason="check: secondmate wake-loop stalled: mate=$task row=$seq age=${age}s" + queued=$(fm_wake_queued_keys check) + if ! printf '%s\n' "$queued" | grep -Fx "$notify_key" >/dev/null 2>&1; then + fm_wake_append check "$notify_key" "$reason" || return 1 + fi + printf '%s\n' "$row_key" > "$marker" + wake "$reason" + done + return 0 +} + # Consecutive wedge-escalation count for a window past FM_WEDGE_DEMAND_INSPECT_COUNT # (default 3): a pane that keeps re-wedging on the SAME stale hash - each # escalation gets absorbed again as "still validating" one poll later, since the @@ -971,6 +1041,14 @@ while :; do # No conversation scraping; unresolved records are never silently expired. fm_pending_reply_tick "$STATE" || true + # A live secondmate endpoint does not prove that its own wake loop is alive. + # Observe the foreign queue before the rest of this cycle so an aged row wakes + # the parent without consuming or rewriting the receiving home's record. + secondmate_wake_stall_tick || { + echo "watcher: secondmate wake-loop observation failed" >&2 + exit 1 + } + # Process-to-event liveness repair. This never discovers a result by polling: # each registered source has its own child blocking on that source, and this # only republishes results already captured durably and restarts a source diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 94b50f8a647..37314ffeaef 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -25,6 +25,46 @@ setup_homes() { "$id" "$sub_abs" > "$home/data/secondmates.md" } +# A live local receiver must get the same marked endpoint wake that direct routed +# requests use. This test drives the real handoff and fm-send path through the +# fake tmux adapter, then asserts the adapter received a submission. +test_handoff_wakes_live_local_receiver() { + local home="$TMP_ROOT/live-wake-main" sub="$TMP_ROOT/live-wake-sub" fakebin out + setup_homes "$home" "$sub" + mkdir -p "$sub/state" "$sub/data" + cat > "$home/state/design.meta" < "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] wake-item - routed to a live receiver (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + fakebin=$(make_fake_tmux "$TMP_ROOT/live-wake-fake") + out="$TMP_ROOT/live-wake.out" + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" PATH="$fakebin:$PATH" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/live-wake-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/live-wake-fake/pane.txt" \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 \ + "$ROOT/bin/fm-backlog-handoff.sh" design wake-item > "$out" 2>&1 \ + || fail "handoff to a live receiver failed: $(cat "$out")" + grep -F 'wake-item' "$sub/data/backlog.md" >/dev/null \ + || fail "live receiver did not receive the routed backlog item" + grep -F 'send-keys' "$TMP_ROOT/live-wake-tmux.log" >/dev/null \ + || fail "handoff did not wake the live receiver endpoint" + grep -F 'New routed work is in your backlog.' "$TMP_ROOT/live-wake-tmux.log" >/dev/null \ + || fail "receiver wake did not carry the routed-work instruction" + pass "a routed handoff wakes the live local receiver through its verified endpoint" +} + # Exact multi-line block extract: header matching key plus following body lines # (indented lines and blank separators between paragraphs), stopping at the next # item header or unindented section heading (column-0 ##). @@ -632,6 +672,7 @@ EOF pass "registry entry without (home: ...) fails cleanly with has no home" } +test_handoff_wakes_live_local_receiver test_body_moves_when_followed_by_another_item test_body_moves_when_followed_by_section_heading test_multi_paragraph_body_with_internal_blanks_moves_whole diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 0a3619ce0ed..cccf386d829 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -235,6 +235,88 @@ test_drain_dedupes_obvious_duplicates() { # plain drain-and-handle turn that runs no other supervision script. It must warn # when work is in flight with no live watcher, and stay silent right after a # normal fire from a live watcher with a fresh beacon, so it never false-alarms. +test_secondmate_foreign_queue_stall_is_one_shot_and_read_only() { + local dir state sub fakebin out row_before row_after stall_count + dir=$(make_case secondmate-foreign-stall) + state="$dir/state" + sub="$dir/secondmate" + mkdir -p "$sub/state" "$sub/data" "$sub/bin" + printf '# Firstmate\n' > "$sub/AGENTS.md" + printf 'mate\n' > "$sub/.fm-secondmate-home" + printf 'window=firstmate:fm-mate\nkind=secondmate\nharness=claude\nbackend=tmux\nhome=%s\n' \ + "$sub" > "$state/mate.meta" + printf '%s\t7\tcheck\trouted\tcheck: routed row\n' "$(( $(date +%s) - 10 ))" > "$sub/state/.wake-queue" + row_before="$dir/foreign-before" + row_after="$dir/foreign-after" + cp "$sub/state/.wake-queue" "$row_before" + fakebin="$dir/fakebin" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +case "${1:-}" in + list-windows) printf '%s\n' "${FM_FAKE_TMUX_WINDOW:-}" ;; + capture-pane) cat "${FM_FAKE_TMUX_CAPTURE:-/dev/null}" ;; + display-message) printf '0\n' ;; + *) exit 0 ;; +esac +SH + chmod +x "$fakebin/tmux" + out="$dir/watch.out" + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 3 > "$out" 2> "$dir/watch.err" || true + grep -F 'check: secondmate wake-loop stalled: mate=mate row=7' "$out" >/dev/null \ + || fail "an aged foreign row did not wake the parent checkpoint: $(cat "$out"); err=$(cat "$dir/watch.err"); meta=$(cat "$state/mate.meta"); foreign=$(cat "$sub/state/.wake-queue")" + [ -s "$state/.wake-queue" ] || fail "the parent notification was not durable" + stall_count=$(grep -c 'secondmate-wake-loop-mate-' "$state/.wake-queue" || true) + [ "$stall_count" -eq 1 ] || fail "the first parent checkpoint did not publish exactly one stall notification" + + cmp -s "$row_before" "$sub/state/.wake-queue" \ + || fail "foreign queue row changed during read-only stall detection" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2> "$dir/drain.err" \ + || fail "parent drain failed after the stall notification" + ack_drain_err "$state" "$dir/drain.err" \ + || fail "parent stall notification could not be acknowledged" + + sleep 1 + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$dir/watch-second.out" 2> "$dir/watch-second.err" || true + [ ! -s "$state/.wake-queue" ] || { + stall_count=$(grep -c 'secondmate-wake-loop-mate-' "$state/.wake-queue" || true) + [ "$stall_count" -eq 0 ] || fail "repeated checkpoint re-published the same stall notification" + } + cp "$sub/state/.wake-queue" "$row_after" + cmp -s "$row_before" "$row_after" || fail "foreign queue changed after idempotent re-check" + + : > "$sub/state/.wake-queue" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$dir/watch-empty.out" 2> "$dir/watch-empty.err" || true + ! grep -F 'secondmate wake-loop stalled' "$dir/watch-empty.out" >/dev/null \ + || fail "an empty foreign queue produced a stall notification" + + printf '%s\t8\tcheck\thealthy\tcheck: healthy row\n' "$(date +%s)" > "$sub/state/.wake-queue" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=60 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$dir/watch-healthy.out" 2> "$dir/watch-healthy.err" || true + ! grep -F 'secondmate wake-loop stalled' "$dir/watch-healthy.out" >/dev/null \ + || fail "a healthy foreign queue produced a stall notification" + pass "foreign secondmate queue stalls notify once, remain byte-stable, and stay quiet when empty or healthy" +} + test_drain_asserts_watcher_liveness() { local dir state err identity dir=$(make_case drain-liveness) @@ -793,6 +875,7 @@ test_historical_annotation_skips_announced_status() { } test_self_held_lock_reclaims_instead_of_deadlocking +test_secondmate_foreign_queue_stall_is_one_shot_and_read_only test_self_announced_append_guards test_historical_annotation_skips_announced_status test_concurrent_append_and_drain From 8135c9cfa1cee4c1c9d310a12d23b27a75584581 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sat, 22 Aug 2026 23:59:09 -0700 Subject: [PATCH 02/24] no-mistakes(review): Make handoff wakes retryable and stall alerts crash-safe --- bin/fm-backlog-handoff.sh | 5 ++-- bin/fm-wake-drain.sh | 4 +++ bin/fm-wake-lib.sh | 33 +++++++++++++++++++++ bin/fm-watch.sh | 7 +++-- tests/fm-backlog-handoff.test.sh | 49 ++++++++++++++++++++++++++++++++ tests/fm-wake-queue.test.sh | 39 +++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 4 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 5ef8714be0f..5f40b5d7cee 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -311,8 +311,8 @@ warn_stale_public_commitments() { # ... wake_secondmate_receiver() { # local id=$1 meta="$STATE/$1.meta" out rc=0 if [ ! -f "$meta" ] || [ -L "$meta" ]; then - printf 'warning: handed off work to secondmate %s, but no live receiver endpoint is recorded; the destination backlog is durable and needs a later wake\n' "$id" >&2 - return 0 + printf 'error: handed off work to secondmate %s, but no live receiver endpoint is recorded; the destination backlog is durable and the receiver was not woken\n' "$id" >&2 + return 1 fi [ "$(grep '^kind=' "$meta" | cut -d= -f2-)" = secondmate ] || { printf 'error: secondmate %s has non-secondmate endpoint metadata; backlog is durable but the receiver was not woken\n' "$id" >&2 @@ -591,6 +591,7 @@ fi if [ "${#TO_MOVE[@]}" -eq 0 ]; then echo "nothing to move: ${ALREADY[*]:-no keys} already present in $SUB_BACKLOG" + wake_secondmate_receiver "$ID" || exit 1 exit 0 fi diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 203765be80f..14599aaf8da 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -298,6 +298,10 @@ if [ -n "$ACK_THROUGH" ]; then awk -F '\t' -v cutoff="$ACK_THROUGH" ' NF < 5 || $2 !~ /^[0-9]+$/ || $2 > cutoff { print } ' "$FM_WAKE_QUEUE" > "$DRAIN_TMP" || exit 1 + fm_wake_commit_secondmate_stall_receipts_through "$ACK_THROUGH" || { + echo "wake drain: secondmate stall receipt could not be recorded safely" >&2 + exit 1 + } if [ ! -s "$DRAIN_TMP" ]; then fm_recovery_marker_ack "$RECOVERY_MARKER" "$ACK_GENERATION" RECOVERY_ACK_STATUS=$? diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 28249b661f3..ecfa7d85ee4 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -1176,6 +1176,39 @@ fm_wake_queued_keys_locked() { "$FM_WAKE_QUEUE" 2>/dev/null || true } +fm_wake_secondmate_stall_receipt_write() { # + local task=$1 row_key=$2 receipt tmp + case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$row_key" in ''|*[!0-9-]*) return 1 ;; esac + receipt="$STATE/.secondmate-wake-stall-receipt-$task-$row_key" + [ "$(cat "$receipt" 2>/dev/null || true)" != "$row_key" ] || return 0 + tmp=$(mktemp "$STATE/.secondmate-wake-stall-receipt.XXXXXX") || return 1 + if ! printf '%s\n' "$row_key" > "$tmp" || ! chmod 0600 "$tmp" \ + || ! _fm_atomic_replace "$tmp" "$receipt"; then + rm -f -- "$tmp" + return 1 + fi +} + +fm_wake_commit_secondmate_stall_receipts_through() { # + local cutoff=$1 key seq rest epoch task row_key + while IFS= read -r key; do + seq=${key##*-} + rest=${key%-*} + epoch=${rest##*-} + task=${rest#secondmate-wake-loop-} + task=${task%-$epoch} + case "$seq" in ''|*[!0-9]*) return 1 ;; esac + case "$epoch" in ''|*[!0-9]*) return 1 ;; esac + case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + row_key="$epoch-$seq" + fm_wake_secondmate_stall_receipt_write "$task" "$row_key" || return 1 + done < <(awk -F '\t' -v cutoff="$cutoff" ' + NF >= 5 && $2 ~ /^[0-9]+$/ && $2 <= cutoff && $3 == "check" \ + && $4 ~ /^secondmate-wake-loop-[A-Za-z0-9._-]+-[0-9]+-[0-9]+$/ { print $4 } + ' "$FM_WAKE_QUEUE" 2>/dev/null) +} + fm_wake_restore_queue() { local drained=$1 restore restore="$STATE/.wake-queue.restore.$(fm_current_pid)" diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index d4b51116800..27f2507acdb 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -321,7 +321,7 @@ secondmate_oldest_queue_row() { # # only this home's marker so a later row can be observed. secondmate_wake_stall_tick() { local now=$(( $(date +%s) )) threshold=$SECONDMATE_WAKE_STALL_SECS - local meta task kind remote_host home queue row epoch seq row_key marker notify_key queued age reason + local meta task kind remote_host home queue row epoch seq row_key marker receipt notify_key queued age reason case "$threshold" in ''|*[!0-9]*|0) threshold=60 ;; esac for meta in "$STATE"/*.meta; do [ -e "$meta" ] || continue @@ -340,7 +340,7 @@ secondmate_wake_stall_tick() { row=$(secondmate_oldest_queue_row "$queue") marker="$STATE/.secondmate-wake-stall-$task" if [ -z "$row" ]; then - rm -f "$marker" + rm -f "$marker" "$STATE"/.secondmate-wake-stall-receipt-"$task"-* continue fi IFS=$(printf '\t') read -r epoch seq _row_kind _row_key _row_payload </dev/null || true)" = "$row_key" ] && continue + [ "$(cat "$receipt" 2>/dev/null || true)" = "$row_key" ] && continue notify_key="secondmate-wake-loop-$task-$row_key" reason="check: secondmate wake-loop stalled: mate=$task row=$seq age=${age}s" queued=$(fm_wake_queued_keys check) if ! printf '%s\n' "$queued" | grep -Fx "$notify_key" >/dev/null 2>&1; then fm_wake_append check "$notify_key" "$reason" || return 1 fi + fm_wake_secondmate_stall_receipt_write "$task" "$row_key" || return 1 printf '%s\n' "$row_key" > "$marker" wake "$reason" done diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 37314ffeaef..124615f0c81 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -14,6 +14,12 @@ set -u command -v tasks-axi >/dev/null 2>&1 || { echo "skip: tasks-axi not found (required by the delegated handoff path)"; exit 0; } TMP_ROOT=$(fm_test_tmproot fm-backlog-handoff) +HANDOFF_FAKEBIN=$(make_fake_tmux "$TMP_ROOT/default-fake") +export PATH="$HANDOFF_FAKEBIN:$PATH" +export FM_FAKE_TMUX_WINDOW='firstmate:fm-design' +export FM_FAKE_TMUX_LOG="$TMP_ROOT/default-tmux.log" +export FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/default-fake/pane.txt" +export FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 setup_homes() { local home=$1 subhome=$2 id=${3:-design} @@ -23,6 +29,14 @@ setup_homes() { sub_abs=$(cd "$subhome" && pwd -P) printf -- '- %s - feature work (home: %s; scope: feature work; projects: alpha; added 2026-07-09)\n' \ "$id" "$sub_abs" > "$home/data/secondmates.md" + cat > "$home/state/$id.meta" < "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] retry-item - wake must be retried (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + + out=$(FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design retry-item 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "handoff without a receiver endpoint reported success" + assert_contains "$out" "receiver was not woken" "missing receiver failure was not observable" + assert_grep 'retry-item' "$sub/data/backlog.md" "failed wake lost the durably handed-off item" + + cat > "$home/state/design.meta" < "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design retry-item > "$TMP_ROOT/retry-wake.out" 2>&1 \ + || fail "an already-present handoff did not retry its receiver wake: $(cat "$TMP_ROOT/retry-wake.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "the recovery handoff did not retry delivery through the receiver endpoint" + pass "a failed receiver wake is loud and retries from an already-present handoff" +} + # Exact multi-line block extract: header matching key plus following body lines # (indented lines and blank separators between paragraphs), stopping at the next # item header or unindented section heading (column-0 ##). @@ -673,6 +721,7 @@ EOF } test_handoff_wakes_live_local_receiver +test_failed_wake_retries_when_the_item_is_already_present test_body_moves_when_followed_by_another_item test_body_moves_when_followed_by_section_heading test_multi_paragraph_body_with_internal_blanks_moves_whole diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index cccf386d829..d9a3d15a2fc 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -317,6 +317,44 @@ SH pass "foreign secondmate queue stalls notify once, remain byte-stable, and stay quiet when empty or healthy" } +test_acknowledged_stall_publication_survives_pre_marker_crash() { + local dir state sub fakebin out epoch row_before + dir=$(make_case secondmate-stall-crash) + state="$dir/state" + sub="$dir/secondmate" + mkdir -p "$sub/state" "$sub/data" + printf 'mate\n' > "$sub/.fm-secondmate-home" + printf 'window=firstmate:fm-mate\nkind=secondmate\nharness=claude\nbackend=tmux\nhome=%s\n' \ + "$sub" > "$state/mate.meta" + epoch=$(( $(date +%s) - 10 )) + printf '%s\t7\tcheck\trouted\tcheck: routed row\n' "$epoch" > "$sub/state/.wake-queue" + row_before="$dir/foreign-before" + cp "$sub/state/.wake-queue" "$row_before" + append_wake "$state" check "secondmate-wake-loop-mate-$epoch-7" \ + "check: secondmate wake-loop stalled: mate=mate row=7 age=10s" \ + || fail "could not seed the pre-marker crash publication" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2> "$dir/drain.err" \ + || fail "pre-marker crash publication could not be drained" + ack_drain_err "$state" "$dir/drain.err" \ + || fail "pre-marker crash publication could not be acknowledged" + + fakebin="$dir/fakebin" + out="$dir/watch.out" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$out" 2> "$dir/watch.err" || true + ! grep -F 'secondmate wake-loop stalled' "$out" >/dev/null \ + || fail "an acknowledged publication was duplicated after the pre-marker crash state" + [ ! -s "$state/.wake-queue" ] \ + || fail "the replacement watcher re-published an acknowledged stall notification" + cmp -s "$row_before" "$sub/state/.wake-queue" \ + || fail "pre-marker crash recovery changed the foreign queue row" + pass "stall publication acknowledgement closes the pre-marker crash window" +} + test_drain_asserts_watcher_liveness() { local dir state err identity dir=$(make_case drain-liveness) @@ -876,6 +914,7 @@ test_historical_annotation_skips_announced_status() { test_self_held_lock_reclaims_instead_of_deadlocking test_secondmate_foreign_queue_stall_is_one_shot_and_read_only +test_acknowledged_stall_publication_survives_pre_marker_crash test_self_announced_append_guards test_historical_annotation_skips_announced_status test_concurrent_append_and_drain From 111aca497a42126be93b92207d38b9424b097c89 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:04:07 -0700 Subject: [PATCH 03/24] no-mistakes(review): Prevent duplicate handoff wakes and cover remote delivery --- bin/fm-backlog-handoff.sh | 50 ++++++++++++++++++++++--- tests/fm-backlog-handoff.test.sh | 14 ++++++- tests/fm-remote-backlog-handoff.test.sh | 40 ++++++++++++++++++++ 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 5f40b5d7cee..9491064e804 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -52,8 +52,8 @@ # then an idempotent confined transfer and fm-backlog-receive.sh deliver it. # A present outbox is the whole recovery record. No two-phase journal exists. # Every successful delivery also sends one marked wake to the receiving endpoint. -# A missing endpoint is reported as an observable warning, while a live endpoint -# that rejects the wake makes the handoff fail with the delivered backlog intact. +# A missing endpoint or a live endpoint that rejects the wake makes the handoff +# fail with the delivered backlog intact. # Usage: fm-backlog-handoff.sh ... # fm-backlog-handoff.sh --resume-pending set -eu @@ -308,6 +308,23 @@ warn_stale_public_commitments() { # ... # verified submit and failure semantics. A seeded but not-yet-spawned home is a # valid handoff destination, but its missing endpoint is reported rather than # pretending the task was started. +receiver_wake_mark_pending() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" tmp + case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + if [ -e "$marker" ] || [ -L "$marker" ]; then + if [ -f "$marker" ] && [ ! -L "$marker" ] \ + && [ "$(cat "$marker" 2>/dev/null || true)" = pending ]; then + return 0 + fi + return 1 + fi + tmp=$(umask 077; mktemp "$STATE/.backlog-handoff-wake.XXXXXX") || return 1 + if ! printf 'pending\n' > "$tmp" || ! chmod 600 "$tmp" || ! mv -f -- "$tmp" "$marker"; then + rm -f -- "$tmp" + return 1 + fi +} + wake_secondmate_receiver() { # local id=$1 meta="$STATE/$1.meta" out rc=0 if [ ! -f "$meta" ] || [ -L "$meta" ]; then @@ -329,6 +346,21 @@ wake_secondmate_receiver() { # [ -z "$out" ] || printf '%s\n' "$out" } +wake_pending_secondmate_receiver() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + if [ ! -f "$marker" ] || [ -L "$marker" ] \ + || [ "$(cat "$marker" 2>/dev/null || true)" != pending ]; then + printf 'error: receiver wake state for secondmate %s is unsafe or invalid\n' "$id" >&2 + return 1 + fi + wake_secondmate_receiver "$id" || return 1 + rm -f -- "$marker" || { + printf 'error: receiver wake for secondmate %s was confirmed, but pending state could not be cleared\n' "$id" >&2 + return 1 + } +} + outbox_item_count() { # awk '/^- \[[ x]\] / { count++ } END { print count + 0 }' "$1" } @@ -377,7 +409,11 @@ remote_deliver_outbox() { # echo "error: handoff receipt by $id was unavailable or completion is unknown; outbox preserved at $outbox" >&2 return 1 fi - if ! wake_secondmate_receiver "$id"; then + receiver_wake_mark_pending "$id" || { + echo "error: remote backlog is durable at $id, but receiver wake state could not be recorded; outbox preserved at $outbox" >&2 + return 1 + } + if ! wake_pending_secondmate_receiver "$id"; then echo "error: remote backlog is durable at $id; outbox preserved at $outbox for wake retry" >&2 return 1 fi @@ -591,7 +627,7 @@ fi if [ "${#TO_MOVE[@]}" -eq 0 ]; then echo "nothing to move: ${ALREADY[*]:-no keys} already present in $SUB_BACKLOG" - wake_secondmate_receiver "$ID" || exit 1 + wake_pending_secondmate_receiver "$ID" || exit 1 exit 0 fi @@ -642,7 +678,11 @@ fi echo "handed off ${#TO_MOVE[@]} item(s) to $ID: ${TO_MOVE[*]}" echo " into $SUB_BACKLOG" -wake_secondmate_receiver "$ID" || exit 1 +receiver_wake_mark_pending "$ID" || { + echo "error: backlog delivery to secondmate $ID succeeded, but receiver wake state could not be recorded" >&2 + exit 1 +} +wake_pending_secondmate_receiver "$ID" || exit 1 if [ "${#ALREADY[@]}" -gt 0 ]; then echo " already present (skipped): ${ALREADY[*]}" fi diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 124615f0c81..8a7a1353706 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -43,7 +43,7 @@ EOF # requests use. This test drives the real handoff and fm-send path through the # fake tmux adapter, then asserts the adapter received a submission. test_handoff_wakes_live_local_receiver() { - local home="$TMP_ROOT/live-wake-main" sub="$TMP_ROOT/live-wake-sub" fakebin out + local home="$TMP_ROOT/live-wake-main" sub="$TMP_ROOT/live-wake-sub" fakebin out wake_count setup_homes "$home" "$sub" mkdir -p "$sub/state" "$sub/data" cat > "$home/state/design.meta" </dev/null \ || fail "receiver wake did not carry the routed-work instruction" - pass "a routed handoff wakes the live local receiver through its verified endpoint" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/live-wake-tmux.log") + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" PATH="$fakebin:$PATH" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/live-wake-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/live-wake-fake/pane.txt" \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 \ + "$ROOT/bin/fm-backlog-handoff.sh" design wake-item > "$TMP_ROOT/live-wake-rerun.out" 2>&1 \ + || fail "idempotent successful handoff rerun failed: $(cat "$TMP_ROOT/live-wake-rerun.out")" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/live-wake-tmux.log")" -eq "$wake_count" ] \ + || fail "idempotent successful handoff rerun duplicated the receiver wake" + pass "a routed handoff wakes once and a successful rerun stays idempotent" } test_failed_wake_retries_when_the_item_is_already_present() { diff --git a/tests/fm-remote-backlog-handoff.test.sh b/tests/fm-remote-backlog-handoff.test.sh index bcfcd7dd7f0..eaf7c47238b 100755 --- a/tests/fm-remote-backlog-handoff.test.sh +++ b/tests/fm-remote-backlog-handoff.test.sh @@ -15,6 +15,7 @@ REMOTE_ROOT="$TMP_ROOT/remote-root" REMOTE="$TMP_ROOT/remote" FAKEBIN=$(fm_fakebin "$TMP_ROOT/fake") SSH_COUNT="$TMP_ROOT/ssh.count" +WAKE_LOG="$TMP_ROOT/wake.log" mkdir -p "$PARENT/data" "$PARENT/state" "$REMOTE_ROOT/bin" \ "$REMOTE/data" "$REMOTE/state" "$REMOTE/config" "$REMOTE/projects" "$REMOTE/bin" # Tear down deterministically. Releasing the blocked stages and killing the @@ -66,6 +67,19 @@ printf 'ios\n' > "$REMOTE/.fm-secondmate-home" cat > "$PARENT/data/secondmates.md" < "$PARENT/state/ios.meta" < "$WAKE_LOG" cat > "$FAKEBIN/fake-ssh" <<'SH' #!/usr/bin/env bash @@ -86,6 +100,11 @@ shift 2 argv_b64=$4 command_name=$(perl -MMIME::Base64=decode_base64 -e '$d=decode_base64($ARGV[0]); ($c)=split(/\0/, $d); print $c' "$argv_b64") case "${FM_FAKE_SSH_MODE:-normal}:$command_name" in + *:fm-remote-secondmate-control.sh) + printf '%s\n' "$command_name" >> "$FM_FAKE_REMOTE_WAKE_LOG" + [ "${FM_FAKE_REMOTE_WAKE_RC:-0}" -eq 0 ] || printf 'remote receiver wake failed\n' >&2 + exit "${FM_FAKE_REMOTE_WAKE_RC:-0}" + ;; unreachable:*) exit 255 ;; serialize:fm-backlog-receive.sh) if mkdir "$FM_FAKE_SERIALIZE_ONCE" 2>/dev/null; then @@ -112,6 +131,8 @@ handoff_env() { FM_ROOT_OVERRIDE="$ROOT" \ FM_SSH_BIN="$FAKEBIN/fake-ssh" \ FM_FAKE_SSH_COUNT="$SSH_COUNT" \ + FM_FAKE_REMOTE_WAKE_LOG="$WAKE_LOG" \ + FM_FAKE_REMOTE_WAKE_RC="${FM_FAKE_REMOTE_WAKE_RC:-0}" \ FM_FAKE_SERIALIZE_ONCE="$TMP_ROOT/serialize.once" \ FM_FAKE_SERIALIZE_ENTERED="$TMP_ROOT/serialize.entered" \ FM_FAKE_SERIALIZE_RELEASE="$TMP_ROOT/serialize.release" \ @@ -222,6 +243,8 @@ pass "ambiguous receipt leaves one durable outbox and no duplicate dispatchable out=$(handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending) assert_contains "$out" 'received: ios moved=0 already=2' "retry did not classify already-delivered keys idempotently" +[ "$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG")" -eq 1 ] \ + || fail "confirmed remote receipt did not wake its supported receiver endpoint exactly once" assert_absent "$PARENT/data/handoff/ios.outbox.md" "confirmed retry did not clean the local outbox" [ "$(grep -cF -- '- [ ] ios-a - first iOS task' "$REMOTE/data/backlog.md")" -eq 1 ] \ || fail "receipt retry duplicated ios-a" @@ -315,6 +338,23 @@ handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending >/dev/null \ || fail "pending bootstrap-visible outbox did not later converge" pass "bootstrap detects pending outbox handoffs without a journal" +write_backlog '- [ ] remote-wake-fail - receiver failure stays recoverable (repo: alpha)' +set +e +FM_FAKE_REMOTE_WAKE_RC=1 handoff_env "$ROOT/bin/fm-backlog-handoff.sh" ios remote-wake-fail \ + > "$TMP_ROOT/remote-wake-fail.out" 2>&1 +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "remote handoff claimed success after its receiver wake failed" +assert_contains "$(cat "$TMP_ROOT/remote-wake-fail.out")" 'receiver wake failed' \ + "remote receiver wake failure was not surfaced" +assert_present "$PARENT/data/handoff/ios.outbox.md" \ + "remote receiver wake failure discarded the recoverable outbox" +handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending >/dev/null \ + || fail "remote receiver wake failure did not recover through resume-pending" +assert_absent "$PARENT/data/handoff/ios.outbox.md" \ + "remote receiver wake recovery left its outbox pending" +pass "remote handoff wakes its supported endpoint or remains loudly recoverable" + write_backlog '- [ ] route-race - remains dispatchable through retirement (repo: alpha)' registry_lock="$PARENT/state/.secondmate-registry.lock" handoff_lock="$PARENT/state/.backlog-handoff-ios.lock" From e357cf1756cd11825ac064311595943da2140bf1 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:08:59 -0700 Subject: [PATCH 04/24] no-mistakes(review): Serialize local handoffs and preserve pre-move wake intent --- bin/fm-backlog-handoff.sh | 26 +++++-- tests/fm-backlog-handoff.test.sh | 124 +++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 5 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 9491064e804..d4fbd7d90fc 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -102,6 +102,7 @@ if [ "${1:-}" = --resume-pending ]; then else [ "$#" -ge 2 ] || { echo "usage: fm-backlog-handoff.sh ..." >&2; exit 1; } ID=$1 + case "$ID" in ''|*[!A-Za-z0-9._-]*) echo "error: unsafe secondmate id: $ID" >&2; exit 1 ;; esac shift fi @@ -571,7 +572,10 @@ if [ "$REMOTE" = 1 ]; then release_remote_locks exit "$rc" fi -release_remote_locks +ACTIVE_HANDOFF_LOCK="$STATE/.backlog-handoff-$ID.lock" +fm_lock_acquire_wait "$ACTIVE_HANDOFF_LOCK" +fm_lock_release "$ACTIVE_REGISTRY_LOCK" +ACTIVE_REGISTRY_LOCK= RAW_HOME=$(secondmate_home "$ID") || exit 1 [ -n "$RAW_HOME" ] || { echo "error: secondmate $ID has no home in $REG" >&2; exit 1; } @@ -649,6 +653,16 @@ if ! fm_tasks_axi_compatible; then exit 1 fi +WAKE_PENDING_BEFORE=0 +WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" +if [ -e "$WAKE_PENDING_MARKER" ] || [ -L "$WAKE_PENDING_MARKER" ]; then + WAKE_PENDING_BEFORE=1 +fi +receiver_wake_mark_pending "$ID" || { + echo "error: receiver wake state for secondmate $ID could not be recorded; nothing was moved" >&2 + exit 1 +} + # Seed the destination with firstmate's standard three-section scaffold when it # does not exist yet, so the moved item lands under the right section. (Left to # create the file itself, tasks-axi mv writes its own `# Backlog` title format, @@ -669,6 +683,12 @@ if ! MV_OUT=$(tasks-axi mv "${TO_MOVE[@]}" --file "$MAIN_BACKLOG" --to "$SUB_BAC if [ "$SUB_CREATED" -eq 1 ]; then rm -f "$SUB_BACKLOG" fi + if [ "$WAKE_PENDING_BEFORE" -eq 0 ]; then + rm -f -- "$WAKE_PENDING_MARKER" || { + echo "error: tasks-axi mv failed and receiver wake state could not be cleared" >&2 + exit 1 + } + fi if [ -n "$MV_OUT" ]; then printf '%s\n' "$MV_OUT" >&2 fi @@ -678,10 +698,6 @@ fi echo "handed off ${#TO_MOVE[@]} item(s) to $ID: ${TO_MOVE[*]}" echo " into $SUB_BACKLOG" -receiver_wake_mark_pending "$ID" || { - echo "error: backlog delivery to secondmate $ID succeeded, but receiver wake state could not be recorded" >&2 - exit 1 -} wake_pending_secondmate_receiver "$ID" || exit 1 if [ "${#ALREADY[@]}" -gt 0 ]; then echo " already present (skipped): ${ALREADY[*]}" diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 8a7a1353706..b3dac516fde 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -123,6 +123,128 @@ EOF pass "a failed receiver wake is loud and retries from an already-present handoff" } +test_move_crash_keeps_wake_pending_for_recovery() { + local home="$TMP_ROOT/move-crash-main" sub="$TMP_ROOT/move-crash-sub" + local fakebin="$TMP_ROOT/move-crash-fakebin" real_tasks rc=0 + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] crash-item - survive the post-move crash (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_tasks=$(command -v tasks-axi) + cat > "$fakebin/tasks-axi" <<'SH' +#!/usr/bin/env bash +"$FM_REAL_TASKS_AXI" "$@" +rc=$? +case " $* " in + *" --file "*" --to "*) + if [ "$rc" -eq 0 ] && [ "${1:-}" = mv ]; then + handoff_pid=$(ps -o ppid= -p "$PPID" | tr -d '[:space:]') + kill -KILL "$handoff_pid" + sleep 1 + fi + ;; +esac +exit "$rc" +SH + chmod +x "$fakebin/tasks-axi" + + set +e + FM_REAL_TASKS_AXI="$real_tasks" PATH="$fakebin:$PATH" FM_HOME="$home" \ + "$ROOT/bin/fm-backlog-handoff.sh" design crash-item > "$TMP_ROOT/move-crash.out" 2>&1 + rc=$? + set +e + [ "$rc" -ne 0 ] || fail "post-move crash fixture unexpectedly reported success" + assert_grep 'crash-item' "$sub/data/backlog.md" "post-move crash did not leave the item durable" + assert_present "$home/state/.backlog-handoff-design.wake-pending" \ + "post-move crash lost receiver wake intent" + + : > "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design crash-item \ + > "$TMP_ROOT/move-crash-retry.out" 2>&1 \ + || fail "post-move crash recovery failed: $(cat "$TMP_ROOT/move-crash-retry.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "post-move crash recovery did not wake the receiver" + assert_absent "$home/state/.backlog-handoff-design.wake-pending" \ + "confirmed crash recovery left receiver wake pending" + pass "a post-move crash preserves wake intent for an idempotent retry" +} + +test_concurrent_local_handoffs_serialize_move_and_wake() { + local home="$TMP_ROOT/concurrent-main" sub="$TMP_ROOT/concurrent-sub" + local basebin blockbin="$TMP_ROOT/concurrent-blockbin" first second i wake_count + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$blockbin" + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] concurrent-a - first routed item (repo: alpha) + +## Done +EOF + basebin=$(make_fake_tmux "$TMP_ROOT/concurrent-fake") + cat > "$blockbin/tmux" <<'SH' +#!/usr/bin/env bash +case "$*" in + *"New routed work is in your backlog."*) + if mkdir "$FM_BLOCK_WAKE_ONCE" 2>/dev/null; then + touch "$FM_BLOCK_WAKE_ENTERED" + while [ ! -f "$FM_BLOCK_WAKE_RELEASE" ]; do sleep 0.02; done + fi + ;; +esac +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$blockbin/tmux" + + PATH="$blockbin:$basebin:$PATH" FM_HOME="$home" FM_BASE_TMUX="$basebin/tmux" \ + FM_BLOCK_WAKE_ONCE="$TMP_ROOT/concurrent.once" \ + FM_BLOCK_WAKE_ENTERED="$TMP_ROOT/concurrent.entered" \ + FM_BLOCK_WAKE_RELEASE="$TMP_ROOT/concurrent.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/concurrent-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/concurrent-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design concurrent-a > "$TMP_ROOT/concurrent-a.out" 2>&1 & + first=$! + i=0 + while [ ! -f "$TMP_ROOT/concurrent.entered" ]; do + kill -0 "$first" 2>/dev/null || fail "first concurrent handoff exited before its blocked wake" + i=$((i + 1)) + [ "$i" -le 250 ] || fail "first concurrent handoff never reached its receiver wake" + sleep 0.02 + done + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] concurrent-b - second routed item (repo: alpha) + +## Done +EOF + PATH="$blockbin:$basebin:$PATH" FM_HOME="$home" FM_BASE_TMUX="$basebin/tmux" \ + FM_BLOCK_WAKE_ONCE="$TMP_ROOT/concurrent.once" \ + FM_BLOCK_WAKE_ENTERED="$TMP_ROOT/concurrent.entered" \ + FM_BLOCK_WAKE_RELEASE="$TMP_ROOT/concurrent.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/concurrent-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/concurrent-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design concurrent-b > "$TMP_ROOT/concurrent-b.out" 2>&1 & + second=$! + sleep 0.2 + assert_grep 'concurrent-b' "$home/data/backlog.md" \ + "second local handoff moved while the first still owned its wake" + touch "$TMP_ROOT/concurrent.release" + wait "$first" || fail "first serialized local handoff failed" + wait "$second" || fail "second serialized local handoff failed" + assert_grep 'concurrent-a' "$sub/data/backlog.md" "first serialized item was lost" + assert_grep 'concurrent-b' "$sub/data/backlog.md" "second serialized item was lost" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/concurrent-tmux.log") + [ "$wake_count" -eq 2 ] || fail "serialized local handoffs produced $wake_count receiver wakes" + pass "concurrent local handoffs serialize each durable move with its wake" +} + # Exact multi-line block extract: header matching key plus following body lines # (indented lines and blank separators between paragraphs), stopping at the next # item header or unindented section heading (column-0 ##). @@ -732,6 +854,8 @@ EOF test_handoff_wakes_live_local_receiver test_failed_wake_retries_when_the_item_is_already_present +test_move_crash_keeps_wake_pending_for_recovery +test_concurrent_local_handoffs_serialize_move_and_wake test_body_moves_when_followed_by_another_item test_body_moves_when_followed_by_section_heading test_multi_paragraph_body_with_internal_blanks_moves_whole From 2f23f26bd3484f3e0de13524398f5a23e3e74f44 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:16:56 -0700 Subject: [PATCH 05/24] no-mistakes(review): Serialize teardown with handoffs and retain remote wake confirmation --- bin/fm-backlog-handoff.sh | 85 +++++++++++++++++++------ bin/fm-teardown.sh | 23 ++++++- tests/fm-backlog-handoff.test.sh | 59 +++++++++++++++++ tests/fm-remote-backlog-handoff.test.sh | 37 +++++++++++ 4 files changed, 180 insertions(+), 24 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index d4fbd7d90fc..61090d38244 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -309,9 +309,19 @@ warn_stale_public_commitments() { # ... # verified submit and failure semantics. A seeded but not-yet-spawned home is a # valid handoff destination, but its missing endpoint is reported rather than # pretending the task was started. -receiver_wake_mark_pending() { # - local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" tmp +receiver_wake_state_write() { # + local id=$1 value=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" tmp case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$value" in pending|confirmed) ;; *) return 1 ;; esac + tmp=$(umask 077; mktemp "$STATE/.backlog-handoff-wake.XXXXXX") || return 1 + if ! printf '%s\n' "$value" > "$tmp" || ! chmod 600 "$tmp" || ! mv -f -- "$tmp" "$marker"; then + rm -f -- "$tmp" + return 1 + fi +} + +receiver_wake_mark_pending() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" if [ -e "$marker" ] || [ -L "$marker" ]; then if [ -f "$marker" ] && [ ! -L "$marker" ] \ && [ "$(cat "$marker" 2>/dev/null || true)" = pending ]; then @@ -319,11 +329,19 @@ receiver_wake_mark_pending() { # fi return 1 fi - tmp=$(umask 077; mktemp "$STATE/.backlog-handoff-wake.XXXXXX") || return 1 - if ! printf 'pending\n' > "$tmp" || ! chmod 600 "$tmp" || ! mv -f -- "$tmp" "$marker"; then - rm -f -- "$tmp" - return 1 - fi + receiver_wake_state_write "$id" pending +} + +receiver_wake_clear_confirmed() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + pending) return 0 ;; + confirmed) rm -f -- "$marker" ;; + *) return 1 ;; + esac } wake_secondmate_receiver() { # @@ -347,19 +365,31 @@ wake_secondmate_receiver() { # [ -z "$out" ] || printf '%s\n' "$out" } -wake_pending_secondmate_receiver() { # - local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" +wake_pending_secondmate_receiver() { # [retain-confirmed] + local id=$1 retain=${2:-0} marker="$STATE/.backlog-handoff-$1.wake-pending" value [ -e "$marker" ] || [ -L "$marker" ] || return 0 - if [ ! -f "$marker" ] || [ -L "$marker" ] \ - || [ "$(cat "$marker" 2>/dev/null || true)" != pending ]; then + if [ ! -f "$marker" ] || [ -L "$marker" ]; then printf 'error: receiver wake state for secondmate %s is unsafe or invalid\n' "$id" >&2 return 1 fi - wake_secondmate_receiver "$id" || return 1 - rm -f -- "$marker" || { - printf 'error: receiver wake for secondmate %s was confirmed, but pending state could not be cleared\n' "$id" >&2 + value=$(cat "$marker" 2>/dev/null || true) + [ "$value" != confirmed ] || return 0 + if [ "$value" != pending ]; then + printf 'error: receiver wake state for secondmate %s is unsafe or invalid\n' "$id" >&2 return 1 - } + fi + wake_secondmate_receiver "$id" || return 1 + if [ "$retain" = 1 ]; then + receiver_wake_state_write "$id" confirmed || { + printf 'error: receiver wake for secondmate %s was confirmed, but confirmed state could not be recorded\n' "$id" >&2 + return 1 + } + else + rm -f -- "$marker" || { + printf 'error: receiver wake for secondmate %s was confirmed, but pending state could not be cleared\n' "$id" >&2 + return 1 + } + fi } outbox_item_count() { # @@ -367,7 +397,7 @@ outbox_item_count() { # } remote_deliver_outbox() { # - local id=$1 outbox=$2 remote_rel receive_out snapshot bytes hash generation counter counter_tmp current + local id=$1 outbox=$2 remote_rel receive_out snapshot bytes hash generation counter counter_tmp current marker [ -f "$outbox" ] && [ ! -L "$outbox" ] || { echo "error: pending outbox is unavailable or unsafe: $outbox" >&2 return 1 @@ -410,11 +440,14 @@ remote_deliver_outbox() { # echo "error: handoff receipt by $id was unavailable or completion is unknown; outbox preserved at $outbox" >&2 return 1 fi - receiver_wake_mark_pending "$id" || { - echo "error: remote backlog is durable at $id, but receiver wake state could not be recorded; outbox preserved at $outbox" >&2 - return 1 - } - if ! wake_pending_secondmate_receiver "$id"; then + marker="$STATE/.backlog-handoff-$id.wake-pending" + if [ "$(cat "$marker" 2>/dev/null || true)" != confirmed ]; then + receiver_wake_mark_pending "$id" || { + echo "error: remote backlog is durable at $id, but receiver wake state could not be recorded; outbox preserved at $outbox" >&2 + return 1 + } + fi + if ! wake_pending_secondmate_receiver "$id" 1; then echo "error: remote backlog is durable at $id; outbox preserved at $outbox for wake retry" >&2 return 1 fi @@ -422,6 +455,10 @@ remote_deliver_outbox() { # echo "error: receiver wake was confirmed but local outbox cleanup failed: $outbox" >&2 return 1 } + rm -f -- "$marker" || { + echo "error: remote outbox cleanup succeeded but confirmed receiver wake state could not be cleared: $marker" >&2 + return 1 + } printf '%s\n' "$receive_out" } @@ -458,6 +495,12 @@ remote_handoff() { # outbox="$DATA/handoff/$id.outbox.md" validate_backlog_file "main backlog" "$MAIN_BACKLOG" || return 1 validate_backlog_file "remote handoff outbox" "$outbox" || return 1 + if [ ! -e "$outbox" ] && [ ! -L "$outbox" ]; then + receiver_wake_clear_confirmed "$id" || { + echo "error: stale receiver wake state for secondmate $id could not be cleared" >&2 + return 1 + } + fi fm_tasks_axi_compatible || { echo "error: a compatible tasks-axi with atomic multi-ID mv support is required to stage remote handoffs; run bin/fm-bootstrap.sh for the required version" >&2 return 1 diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index c84669d3632..54a29c7b3c3 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -194,6 +194,14 @@ teardown_release_locks() { fm_lock_release "${DESCENDANT_LOCK_PATHS[$i]}" || true done DESCENDANT_LOCK_PATHS=() + if [ -n "${LOCAL_HANDOFF_LOCK:-}" ]; then + fm_lock_release "$LOCAL_HANDOFF_LOCK" || true + LOCAL_HANDOFF_LOCK= + fi + if [ -n "${LOCAL_REGISTRY_LOCK:-}" ]; then + fm_lock_release "$LOCAL_REGISTRY_LOCK" || true + LOCAL_REGISTRY_LOCK= + fi if [ "$META_LOCK_HELD" = 1 ]; then fm_lock_release "$META_LOCK" || true META_LOCK_HELD=0 @@ -230,6 +238,8 @@ REMOTE_PENDING_DIR_REAL= REMOTE_HANDOFF_LOCK= REMOTE_REGISTRY_LOCK= REMOTE_REPLY_LIFECYCLE_LOCK= +LOCAL_HANDOFF_LOCK= +LOCAL_REGISTRY_LOCK= remote_teardown_locks_release() { if [ -n "$REMOTE_REPLY_LIFECYCLE_LOCK" ]; then @@ -2267,20 +2277,27 @@ cleanup_firstmate_home_children() { } remove_secondmate_registry_entry() { - local id=$1 tmp lock rc=0 + local id=$1 tmp lock rc=0 acquired=0 [ -f "$SECONDMATE_REG" ] || return 0 lock=$(secondmate_registry_lock_path "$STATE") - fm_lock_acquire_wait "$lock" || return 1 + if [ "$LOCAL_REGISTRY_LOCK" != "$lock" ]; then + fm_lock_acquire_wait "$lock" || return 1 + acquired=1 + fi tmp="$SECONDMATE_REG.tmp.$$" grep -vE "^- $id( |$)" "$SECONDMATE_REG" > "$tmp" || true mv "$tmp" "$SECONDMATE_REG" || rc=$? - fm_lock_release "$lock" + [ "$acquired" -eq 0 ] || fm_lock_release "$lock" return "$rc" } validate_pr_poll_cleanup "$STATE" "$ID" || exit 1 if [ "$KIND" = secondmate ]; then + LOCAL_REGISTRY_LOCK=$(secondmate_registry_lock_path "$STATE") + fm_lock_acquire_wait "$LOCAL_REGISTRY_LOCK" || exit 1 + LOCAL_HANDOFF_LOCK="$STATE/.backlog-handoff-$ID.lock" + fm_lock_acquire_wait "$LOCAL_HANDOFF_LOCK" || exit 1 [ -n "$HOME_PATH" ] || HOME_PATH=$WT validate_firstmate_home_for_removal "$HOME_PATH" "secondmate home" "$ID" >/dev/null || exit 1 if [ "$FORCE" = "--force" ]; then diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index b3dac516fde..15042aca326 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -245,6 +245,64 @@ EOF pass "concurrent local handoffs serialize each durable move with its wake" } +test_local_teardown_waits_for_handoff_wake() { + local home="$TMP_ROOT/teardown-race-main" sub="$TMP_ROOT/teardown-race-sub" + local basebin blockbin="$TMP_ROOT/teardown-race-blockbin" handoff teardown i + setup_homes "$home" "$sub" + printf 'project=%s\n' "$ROOT" >> "$home/state/design.meta" + mkdir -p "$sub/data" "$blockbin" + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] teardown-race - routed while teardown starts (repo: alpha) + +## Done +EOF + basebin=$(make_fake_tmux "$TMP_ROOT/teardown-race-fake") + cat > "$blockbin/tmux" <<'SH' +#!/usr/bin/env bash +case "$*" in + *"New routed work is in your backlog."*) + touch "$FM_BLOCK_WAKE_ENTERED" + while [ ! -f "$FM_BLOCK_WAKE_RELEASE" ]; do sleep 0.02; done + ;; +esac +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$blockbin/tmux" + PATH="$blockbin:$basebin:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_BASE_TMUX="$basebin/tmux" FM_BLOCK_WAKE_ENTERED="$TMP_ROOT/teardown-race.entered" \ + FM_BLOCK_WAKE_RELEASE="$TMP_ROOT/teardown-race.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-race-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-race-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design teardown-race > "$TMP_ROOT/teardown-race-handoff.out" 2>&1 & + handoff=$! + i=0 + while [ ! -f "$TMP_ROOT/teardown-race.entered" ]; do + kill -0 "$handoff" 2>/dev/null || fail "teardown-race handoff exited before its blocked wake" + i=$((i + 1)) + [ "$i" -le 250 ] || fail "teardown-race handoff never reached its receiver wake" + sleep 0.02 + done + PATH="$basebin:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-race-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-race-fake/pane.txt" \ + "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-race-teardown.out" 2>&1 & + teardown=$! + sleep 0.3 + kill -0 "$teardown" 2>/dev/null \ + || fail "local teardown bypassed the in-flight handoff lock: $(cat "$TMP_ROOT/teardown-race-teardown.out")" + [ -d "$sub" ] || fail "local teardown removed the receiver home before handoff wake completed" + assert_grep 'teardown-race' "$sub/data/backlog.md" \ + "local teardown removed routed work before handoff wake completed" + touch "$TMP_ROOT/teardown-race.release" + wait "$handoff" || fail "teardown-race handoff failed after releasing its wake" + wait "$teardown" 2>/dev/null || true + pass "local teardown waits for the routed move and receiver wake" +} + # Exact multi-line block extract: header matching key plus following body lines # (indented lines and blank separators between paragraphs), stopping at the next # item header or unindented section heading (column-0 ##). @@ -856,6 +914,7 @@ test_handoff_wakes_live_local_receiver test_failed_wake_retries_when_the_item_is_already_present test_move_crash_keeps_wake_pending_for_recovery test_concurrent_local_handoffs_serialize_move_and_wake +test_local_teardown_waits_for_handoff_wake test_body_moves_when_followed_by_another_item test_body_moves_when_followed_by_section_heading test_multi_paragraph_body_with_internal_blanks_moves_whole diff --git a/tests/fm-remote-backlog-handoff.test.sh b/tests/fm-remote-backlog-handoff.test.sh index eaf7c47238b..87f0ee18d60 100755 --- a/tests/fm-remote-backlog-handoff.test.sh +++ b/tests/fm-remote-backlog-handoff.test.sh @@ -355,6 +355,43 @@ assert_absent "$PARENT/data/handoff/ios.outbox.md" \ "remote receiver wake recovery left its outbox pending" pass "remote handoff wakes its supported endpoint or remains loudly recoverable" +RM_FAKEBIN="$TMP_ROOT/rm-fakebin" +mkdir -p "$RM_FAKEBIN" +REAL_RM=$(command -v rm) +cat > "$RM_FAKEBIN/rm" <<'SH' +#!/usr/bin/env bash +last=${!#} +if [ "$last" = "$FM_FAIL_RM_PATH" ]; then + exit 1 +fi +exec "$FM_REAL_RM" "$@" +SH +chmod +x "$RM_FAKEBIN/rm" +write_backlog '- [ ] cleanup-retry - confirmed wake survives cleanup retry (repo: alpha)' +wakes_before=$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG") +set +e +PATH="$RM_FAKEBIN:$PATH" FM_REAL_RM="$REAL_RM" \ + FM_FAIL_RM_PATH="$PARENT/data/handoff/ios.outbox.md" \ + handoff_env "$ROOT/bin/fm-backlog-handoff.sh" ios cleanup-retry \ + > "$TMP_ROOT/cleanup-retry.out" 2>&1 +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "remote handoff ignored local outbox cleanup failure" +assert_present "$PARENT/data/handoff/ios.outbox.md" \ + "remote cleanup failure did not preserve the outbox" +[ "$(cat "$PARENT/state/.backlog-handoff-ios.wake-pending")" = confirmed ] \ + || fail "remote cleanup failure did not preserve confirmed wake state" +wakes_after=$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG") +[ "$wakes_after" -eq $((wakes_before + 1)) ] \ + || fail "remote cleanup failure did not perform exactly one receiver wake" +handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending >/dev/null \ + || fail "remote cleanup retry did not converge" +[ "$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG")" -eq "$wakes_after" ] \ + || fail "remote cleanup retry duplicated a confirmed receiver wake" +assert_absent "$PARENT/state/.backlog-handoff-ios.wake-pending" \ + "remote cleanup retry left confirmed wake state behind" +pass "remote cleanup recovery does not duplicate a confirmed receiver wake" + write_backlog '- [ ] route-race - remains dispatchable through retirement (repo: alpha)' registry_lock="$PARENT/state/.secondmate-registry.lock" handoff_lock="$PARENT/state/.backlog-handoff-ios.lock" From bf0ebabea53f9cf5846197157d747142a0bd3ee1 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:23:53 -0700 Subject: [PATCH 06/24] no-mistakes(review): Reconcile correlated handoff wake delivery after crashes --- bin/fm-backlog-handoff.sh | 102 ++++++++++++++++++------ tests/fm-backlog-handoff.test.sh | 51 ++++++++++++ tests/fm-remote-backlog-handoff.test.sh | 6 +- 3 files changed, 134 insertions(+), 25 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 61090d38244..3500a45589e 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -73,6 +73,10 @@ MAIN_BACKLOG="$DATA/backlog.md" . "$SCRIPT_DIR/fm-wake-lib.sh" # shellcheck source=bin/fm-public-followup-lib.sh . "$SCRIPT_DIR/fm-public-followup-lib.sh" +# shellcheck source=bin/fm-pending-reply-lib.sh +. "$SCRIPT_DIR/fm-pending-reply-lib.sh" + +RECEIVER_WAKE_MESSAGE='New routed work is in your backlog. Run bin/fm-session-start.sh now, then act on the routed task.' ACTIVE_HANDOFF_LOCK= ACTIVE_REGISTRY_LOCK= @@ -312,7 +316,12 @@ warn_stale_public_commitments() { # ... receiver_wake_state_write() { # local id=$1 value=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" tmp case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac - case "$value" in pending|confirmed) ;; *) return 1 ;; esac + case "$value" in + pending|confirmed) ;; + pending:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; + confirmed:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; + *) return 1 ;; + esac tmp=$(umask 077; mktemp "$STATE/.backlog-handoff-wake.XXXXXX") || return 1 if ! printf '%s\n' "$value" > "$tmp" || ! chmod 600 "$tmp" || ! mv -f -- "$tmp" "$marker"; then rm -f -- "$tmp" @@ -321,15 +330,43 @@ receiver_wake_state_write() { # } receiver_wake_mark_pending() { # - local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec if [ -e "$marker" ] || [ -L "$marker" ]; then - if [ -f "$marker" ] && [ ! -L "$marker" ] \ - && [ "$(cat "$marker" 2>/dev/null || true)" = pending ]; then - return 0 - fi + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + pending:*) + corr=${value#pending:} + rec=$(fm_pending_reply_path "$STATE" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$id" ] + return $? + ;; + pending) ;; + *) return 1 ;; + esac + fi + corr=$(fm_pending_reply_create "$FM_HOME" "$STATE" "$id" "$RECEIVER_WAKE_MESSAGE") || return 1 + if ! fm_pending_reply_prepare_delivery "$STATE" "$corr" \ + || ! receiver_wake_state_write "$id" "pending:$corr"; then + fm_pending_reply_discard_undelivered "$STATE" "$corr" || true return 1 fi - receiver_wake_state_write "$id" pending +} + +receiver_wake_discard_pending() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + pending:*) + corr=${value#pending:} + fm_pending_reply_discard_undelivered "$STATE" "$corr" || return 1 + ;; + pending) ;; + *) return 1 ;; + esac + rm -f -- "$marker" } receiver_wake_clear_confirmed() { # @@ -338,14 +375,14 @@ receiver_wake_clear_confirmed() { # [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 value=$(cat "$marker" 2>/dev/null || true) case "$value" in - pending) return 0 ;; - confirmed) rm -f -- "$marker" ;; + pending|pending:*) return 0 ;; + confirmed|confirmed:*) rm -f -- "$marker" ;; *) return 1 ;; esac } -wake_secondmate_receiver() { # - local id=$1 meta="$STATE/$1.meta" out rc=0 +wake_secondmate_receiver() { # + local id=$1 corr=$2 meta="$STATE/$1.meta" out rc=0 if [ ! -f "$meta" ] || [ -L "$meta" ]; then printf 'error: handed off work to secondmate %s, but no live receiver endpoint is recorded; the destination backlog is durable and the receiver was not woken\n' "$id" >&2 return 1 @@ -355,8 +392,8 @@ wake_secondmate_receiver() { # return 1 } out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_ROOT_OVERRIDE="$FM_ROOT" \ - "$SCRIPT_DIR/fm-send.sh" "$id" \ - 'New routed work is in your backlog. Run bin/fm-session-start.sh now, then act on the routed task.' 2>&1) || rc=$? + FM_PENDING_REPLY_EXISTING_CORR="$corr" \ + "$SCRIPT_DIR/fm-send.sh" "$id" "$RECEIVER_WAKE_MESSAGE" 2>&1) || rc=$? if [ "$rc" -ne 0 ]; then [ -z "$out" ] || printf '%s\n' "$out" >&2 printf 'error: backlog delivery to secondmate %s succeeded, but its receiver wake failed; rerun this handoff to retry the wake\n' "$id" >&2 @@ -366,21 +403,39 @@ wake_secondmate_receiver() { # } wake_pending_secondmate_receiver() { # [retain-confirmed] - local id=$1 retain=${2:-0} marker="$STATE/.backlog-handoff-$1.wake-pending" value + local id=$1 retain=${2:-0} marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec delivered [ -e "$marker" ] || [ -L "$marker" ] || return 0 if [ ! -f "$marker" ] || [ -L "$marker" ]; then printf 'error: receiver wake state for secondmate %s is unsafe or invalid\n' "$id" >&2 return 1 fi value=$(cat "$marker" 2>/dev/null || true) - [ "$value" != confirmed ] || return 0 - if [ "$value" != pending ]; then + case "$value" in + confirmed|confirmed:*) return 0 ;; + pending) + receiver_wake_mark_pending "$id" || return 1 + value=$(cat "$marker" 2>/dev/null || true) + ;; + esac + case "$value" in pending:*) corr=${value#pending:} ;; *) printf 'error: receiver wake state for secondmate %s is unsafe or invalid\n' "$id" >&2 return 1 + ;; + esac + rec=$(fm_pending_reply_path "$STATE" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$id" ] || return 1 + fm_pending_reply_reconcile_delivery "$STATE" "$corr" >/dev/null 2>&1 || true + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + if [ -z "$delivered" ]; then + fm_pending_reply_corr_reusable "$STATE" "$corr" "$id" || { + printf 'error: receiver wake delivery for secondmate %s is unresolved; refusing to resend correlation %s\n' "$id" "$corr" >&2 + return 1 + } + wake_secondmate_receiver "$id" "$corr" || return 1 fi - wake_secondmate_receiver "$id" || return 1 if [ "$retain" = 1 ]; then - receiver_wake_state_write "$id" confirmed || { + receiver_wake_state_write "$id" "confirmed:$corr" || { printf 'error: receiver wake for secondmate %s was confirmed, but confirmed state could not be recorded\n' "$id" >&2 return 1 } @@ -441,12 +496,13 @@ remote_deliver_outbox() { # return 1 fi marker="$STATE/.backlog-handoff-$id.wake-pending" - if [ "$(cat "$marker" 2>/dev/null || true)" != confirmed ]; then - receiver_wake_mark_pending "$id" || { + case "$(cat "$marker" 2>/dev/null || true)" in + pending:*|confirmed|confirmed:*) ;; + *) receiver_wake_mark_pending "$id" || { echo "error: remote backlog is durable at $id, but receiver wake state could not be recorded; outbox preserved at $outbox" >&2 return 1 - } - fi + } ;; + esac if ! wake_pending_secondmate_receiver "$id" 1; then echo "error: remote backlog is durable at $id; outbox preserved at $outbox for wake retry" >&2 return 1 @@ -727,7 +783,7 @@ if ! MV_OUT=$(tasks-axi mv "${TO_MOVE[@]}" --file "$MAIN_BACKLOG" --to "$SUB_BAC rm -f "$SUB_BACKLOG" fi if [ "$WAKE_PENDING_BEFORE" -eq 0 ]; then - rm -f -- "$WAKE_PENDING_MARKER" || { + receiver_wake_discard_pending "$ID" || { echo "error: tasks-axi mv failed and receiver wake state could not be cleared" >&2 exit 1 } diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 15042aca326..9822a070d39 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -174,6 +174,56 @@ SH pass "a post-move crash preserves wake intent for an idempotent retry" } +test_delivery_confirmation_crash_does_not_resend() { + local home="$TMP_ROOT/confirm-crash-main" sub="$TMP_ROOT/confirm-crash-sub" + local fakebin="$TMP_ROOT/confirm-crash-fakebin" real_sleep rc=0 wake_count + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] confirm-crash - preserve confirmed delivery (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_sleep=$(command -v sleep) + cat > "$fakebin/sleep" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = 1 ] && mkdir "$FM_CONFIRM_CRASH_ONCE" 2>/dev/null; then + handoff_pid=$(ps -o ppid= -p "$PPID" | tr -d '[:space:]') + kill -KILL "$handoff_pid" + exit 0 +fi +exec "$FM_REAL_SLEEP" "$@" +SH + chmod +x "$fakebin/sleep" + : > "$TMP_ROOT/default-tmux.log" + + set +e + PATH="$fakebin:$PATH" FM_REAL_SLEEP="$real_sleep" \ + FM_CONFIRM_CRASH_ONCE="$TMP_ROOT/confirm-crash.once" FM_SEND_SETTLE=1 \ + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design confirm-crash \ + > "$TMP_ROOT/confirm-crash.out" 2>&1 + rc=$? + set +e + [ "$rc" -ne 0 ] || fail "post-confirmation crash fixture unexpectedly reported success" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log") + [ "$wake_count" -eq 1 ] || fail "post-confirmation crash did not deliver exactly one receiver wake" + case "$(cat "$home/state/.backlog-handoff-design.wake-pending")" in + pending:*) ;; + *) fail "post-confirmation crash lost its stable delivery correlation" ;; + esac + + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design confirm-crash \ + > "$TMP_ROOT/confirm-crash-retry.out" 2>&1 \ + || fail "post-confirmation crash recovery failed: $(cat "$TMP_ROOT/confirm-crash-retry.out")" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$wake_count" ] \ + || fail "post-confirmation crash recovery duplicated the receiver wake" + assert_absent "$home/state/.backlog-handoff-design.wake-pending" \ + "post-confirmation crash recovery left wake state pending" + pass "a post-confirmation crash reconciles delivery without resending" +} + test_concurrent_local_handoffs_serialize_move_and_wake() { local home="$TMP_ROOT/concurrent-main" sub="$TMP_ROOT/concurrent-sub" local basebin blockbin="$TMP_ROOT/concurrent-blockbin" first second i wake_count @@ -913,6 +963,7 @@ EOF test_handoff_wakes_live_local_receiver test_failed_wake_retries_when_the_item_is_already_present test_move_crash_keeps_wake_pending_for_recovery +test_delivery_confirmation_crash_does_not_resend test_concurrent_local_handoffs_serialize_move_and_wake test_local_teardown_waits_for_handoff_wake test_body_moves_when_followed_by_another_item diff --git a/tests/fm-remote-backlog-handoff.test.sh b/tests/fm-remote-backlog-handoff.test.sh index 87f0ee18d60..203e5a303c7 100755 --- a/tests/fm-remote-backlog-handoff.test.sh +++ b/tests/fm-remote-backlog-handoff.test.sh @@ -379,8 +379,10 @@ set -e [ "$rc" -ne 0 ] || fail "remote handoff ignored local outbox cleanup failure" assert_present "$PARENT/data/handoff/ios.outbox.md" \ "remote cleanup failure did not preserve the outbox" -[ "$(cat "$PARENT/state/.backlog-handoff-ios.wake-pending")" = confirmed ] \ - || fail "remote cleanup failure did not preserve confirmed wake state" +case "$(cat "$PARENT/state/.backlog-handoff-ios.wake-pending")" in + confirmed:*) ;; + *) fail "remote cleanup failure did not preserve confirmed wake state" ;; +esac wakes_after=$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG") [ "$wakes_after" -eq $((wakes_before + 1)) ] \ || fail "remote cleanup failure did not perform exactly one receiver wake" From 24b3c1522918ab676dd8cfe35769177a4f671c75 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:30:05 -0700 Subject: [PATCH 07/24] no-mistakes(review): Keep failed wakes retryable and isolate stall receipts --- bin/fm-backlog-handoff.sh | 7 +++-- bin/fm-wake-lib.sh | 20 +++++++++++--- bin/fm-watch.sh | 11 +++++--- tests/fm-backlog-handoff.test.sh | 5 +++- tests/fm-wake-queue.test.sh | 46 ++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 9 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 3500a45589e..bc4fd9fc2ce 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -347,8 +347,7 @@ receiver_wake_mark_pending() { # esac fi corr=$(fm_pending_reply_create "$FM_HOME" "$STATE" "$id" "$RECEIVER_WAKE_MESSAGE") || return 1 - if ! fm_pending_reply_prepare_delivery "$STATE" "$corr" \ - || ! receiver_wake_state_write "$id" "pending:$corr"; then + if ! receiver_wake_state_write "$id" "pending:$corr"; then fm_pending_reply_discard_undelivered "$STATE" "$corr" || true return 1 fi @@ -391,6 +390,10 @@ wake_secondmate_receiver() { # printf 'error: secondmate %s has non-secondmate endpoint metadata; backlog is durable but the receiver was not woken\n' "$id" >&2 return 1 } + fm_pending_reply_prepare_delivery "$STATE" "$corr" || { + printf 'error: receiver wake delivery for secondmate %s could not be prepared\n' "$id" >&2 + return 1 + } out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_ROOT_OVERRIDE="$FM_ROOT" \ FM_PENDING_REPLY_EXISTING_CORR="$corr" \ "$SCRIPT_DIR/fm-send.sh" "$id" "$RECEIVER_WAKE_MESSAGE" 2>&1) || rc=$? diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index ecfa7d85ee4..9035108b7d1 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -1177,12 +1177,26 @@ fm_wake_queued_keys_locked() { } fm_wake_secondmate_stall_receipt_write() { # - local task=$1 row_key=$2 receipt tmp + local task=$1 row_key=$2 root task_dir receipt tmp case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac case "$row_key" in ''|*[!0-9-]*) return 1 ;; esac - receipt="$STATE/.secondmate-wake-stall-receipt-$task-$row_key" + root="$STATE/.secondmate-wake-stall-receipts" + task_dir="$root/$task" + if [ -e "$root" ] || [ -L "$root" ]; then + [ -d "$root" ] && [ ! -L "$root" ] || return 1 + else + mkdir "$root" || return 1 + chmod 0700 "$root" || return 1 + fi + if [ -e "$task_dir" ] || [ -L "$task_dir" ]; then + [ -d "$task_dir" ] && [ ! -L "$task_dir" ] || return 1 + else + mkdir "$task_dir" || return 1 + chmod 0700 "$task_dir" || return 1 + fi + receipt="$task_dir/$row_key" [ "$(cat "$receipt" 2>/dev/null || true)" != "$row_key" ] || return 0 - tmp=$(mktemp "$STATE/.secondmate-wake-stall-receipt.XXXXXX") || return 1 + tmp=$(mktemp "$task_dir/.receipt.XXXXXX") || return 1 if ! printf '%s\n' "$row_key" > "$tmp" || ! chmod 0600 "$tmp" \ || ! _fm_atomic_replace "$tmp" "$receipt"; then rm -f -- "$tmp" diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 27f2507acdb..cb2641c68be 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -321,7 +321,7 @@ secondmate_oldest_queue_row() { # # only this home's marker so a later row can be observed. secondmate_wake_stall_tick() { local now=$(( $(date +%s) )) threshold=$SECONDMATE_WAKE_STALL_SECS - local meta task kind remote_host home queue row epoch seq row_key marker receipt notify_key queued age reason + local meta task kind remote_host home queue row epoch seq row_key marker receipt receipt_dir notify_key queued age reason case "$threshold" in ''|*[!0-9]*|0) threshold=60 ;; esac for meta in "$STATE"/*.meta; do [ -e "$meta" ] || continue @@ -339,8 +339,13 @@ secondmate_wake_stall_tick() { queue="$home/state/.wake-queue" row=$(secondmate_oldest_queue_row "$queue") marker="$STATE/.secondmate-wake-stall-$task" + receipt_dir="$STATE/.secondmate-wake-stall-receipts/$task" if [ -z "$row" ]; then - rm -f "$marker" "$STATE"/.secondmate-wake-stall-receipt-"$task"-* + rm -f "$marker" + if [ -e "$receipt_dir" ] || [ -L "$receipt_dir" ]; then + [ -d "$receipt_dir" ] && [ ! -L "$receipt_dir" ] || return 1 + rm -rf -- "$receipt_dir" || return 1 + fi continue fi IFS=$(printf '\t') read -r epoch seq _row_kind _row_key _row_payload </dev/null || true)" = "$row_key" ] && continue [ "$(cat "$receipt" 2>/dev/null || true)" = "$row_key" ] && continue notify_key="secondmate-wake-loop-$task-$row_key" diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 9822a070d39..d98d9b64f9f 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -90,7 +90,7 @@ EOF } test_failed_wake_retries_when_the_item_is_already_present() { - local home="$TMP_ROOT/retry-wake-main" sub="$TMP_ROOT/retry-wake-sub" out rc=0 + local home="$TMP_ROOT/retry-wake-main" sub="$TMP_ROOT/retry-wake-sub" out corr rc=0 setup_homes "$home" "$sub" rm -f "$home/state/design.meta" mkdir -p "$sub/data" @@ -106,6 +106,9 @@ EOF [ "$rc" -ne 0 ] || fail "handoff without a receiver endpoint reported success" assert_contains "$out" "receiver was not woken" "missing receiver failure was not observable" assert_grep 'retry-item' "$sub/data/backlog.md" "failed wake lost the durably handed-off item" + corr=$(cut -d: -f2- "$home/state/.backlog-handoff-design.wake-pending") + assert_absent "$home/state/pending-replies/.delivery-confirmed-$corr" \ + "missing endpoint was recorded as an attempted delivery" cat > "$home/state/design.meta" < "$empty/.fm-secondmate-home" + printf 'ios-ui\n' > "$stalled/.fm-secondmate-home" + printf 'window=firstmate:fm-ios\nkind=secondmate\nhome=%s\n' "$empty" > "$state/ios.meta" + printf 'window=firstmate:fm-ios-ui\nkind=secondmate\nhome=%s\n' "$stalled" > "$state/ios-ui.meta" + : > "$empty/state/.wake-queue" + epoch=$(( $(date +%s) - 10 )) + printf '%s\t9\tcheck\trouted\tcheck: routed row\n' "$epoch" > "$stalled/state/.wake-queue" + row_before="$dir/foreign-before" + cp "$stalled/state/.wake-queue" "$row_before" + append_wake "$state" check "secondmate-wake-loop-ios-ui-$epoch-9" \ + "check: secondmate wake-loop stalled: mate=ios-ui row=9 age=10s" \ + || fail "could not seed the ios-ui stall publication" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2> "$dir/drain.err" \ + || fail "ios-ui stall publication could not be drained" + ack_drain_err "$state" "$dir/drain.err" \ + || fail "ios-ui stall publication could not be acknowledged" + + fakebin="$dir/fakebin" + round=1 + while [ "$round" -le 2 ]; do + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 \ + > "$dir/watch-$round.out" 2> "$dir/watch-$round.err" || true + ! grep -F 'secondmate wake-loop stalled' "$dir/watch-$round.out" >/dev/null \ + || fail "empty ios queue erased ios-ui idempotency on checkpoint $round" + round=$((round + 1)) + done + [ ! -s "$state/.wake-queue" ] \ + || fail "overlapping mate ids re-published the acknowledged ios-ui stall" + cmp -s "$row_before" "$stalled/state/.wake-queue" \ + || fail "overlapping mate receipt checks changed the foreign row" + pass "empty prefix mate cleanup preserves another mate's stall receipt" +} + test_drain_asserts_watcher_liveness() { local dir state err identity dir=$(make_case drain-liveness) @@ -915,6 +960,7 @@ test_historical_annotation_skips_announced_status() { test_self_held_lock_reclaims_instead_of_deadlocking test_secondmate_foreign_queue_stall_is_one_shot_and_read_only test_acknowledged_stall_publication_survives_pre_marker_crash +test_empty_prefix_mate_preserves_other_mate_receipt test_self_announced_append_guards test_historical_annotation_skips_announced_status test_concurrent_append_and_drain From 4779409f4ded06c9cf26c395b7a01047952ec7c7 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:36:38 -0700 Subject: [PATCH 08/24] no-mistakes(review): Reset known-undelivered wake attempts for durable retries --- bin/fm-backlog-handoff.sh | 4 --- bin/fm-pending-reply-lib.sh | 15 ++++++++++ bin/fm-send.sh | 24 ++++++++++------ bin/fm-watch.sh | 1 + tests/fm-backlog-handoff.test.sh | 48 ++++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 13 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index bc4fd9fc2ce..5f0720b72b8 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -390,10 +390,6 @@ wake_secondmate_receiver() { # printf 'error: secondmate %s has non-secondmate endpoint metadata; backlog is durable but the receiver was not woken\n' "$id" >&2 return 1 } - fm_pending_reply_prepare_delivery "$STATE" "$corr" || { - printf 'error: receiver wake delivery for secondmate %s could not be prepared\n' "$id" >&2 - return 1 - } out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_ROOT_OVERRIDE="$FM_ROOT" \ FM_PENDING_REPLY_EXISTING_CORR="$corr" \ "$SCRIPT_DIR/fm-send.sh" "$id" "$RECEIVER_WAKE_MESSAGE" 2>&1) || rc=$? diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 5453585d0e2..091722fc500 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -412,6 +412,21 @@ fm_pending_reply_reconcile_delivery() { # return 1 } +fm_pending_reply_reset_known_undelivered() { # + local state=$1 corr=$2 rec delivered phase marker entry + rec=$(fm_pending_reply_path "$state" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -z "$delivered" ] || return 1 + phase=$(fm_pending_reply_get "$rec" phase) + [ "$phase" = awaiting_report ] || return 1 + marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + entry=$(cat "$marker" 2>/dev/null || true) + case "$entry" in attempted=*) rm -f -- "$marker" ;; *) return 1 ;; esac +} + # Drop an undelivered expectation after a failed send so transport failure does # not masquerade as a missed report later. fm_pending_reply_discard_undelivered() { # diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 512df6245c7..1e92fbea7ef 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -376,6 +376,14 @@ MARK_FROM_FIRSTMATE=0 PENDING_REPLY_CORR= PENDING_REPLY_CREATED=0 TARGET_TASK_ID= +fm_send_known_undelivered_cleanup() { + [ -n "$PENDING_REPLY_CORR" ] || return 0 + if [ "$PENDING_REPLY_CREATED" = 1 ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" + else + fm_pending_reply_reset_known_undelivered "$STATE" "$PENDING_REPLY_CORR" + fi +} if [ -n "$TARGET_SELECTOR" ] && [ -n "$TARGET_META" ] && [ "$(fm_meta_get "$TARGET_META" kind)" = secondmate ]; then MARK_FROM_FIRSTMATE=1 TARGET_TASK_ID=$(fm_send_id_from_meta "$TARGET_META") @@ -548,9 +556,9 @@ else PENDING_REPLY_CREATED=1 fi fm_pending_reply_embed_corr "$MESSAGE" "$PENDING_REPLY_CORR" MESSAGE - if [ "$PENDING_REPLY_CREATED" = 1 ] \ - && ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + if ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then + [ "$PENDING_REPLY_CREATED" != 1 ] \ + || fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true echo "error: failed to durably prepare pending-reply delivery for $TARGET_TASK_ID" >&2 exit 1 fi @@ -608,9 +616,8 @@ else echo "error: text delivery to remote secondmate $TARGET_REMOTE_ID is unknown; do not resend - same-host reconciliation is required" >&2 exit 1 fi - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + fm_send_known_undelivered_cleanup || \ + echo "error: known-undelivered pending-reply state could not be reset for $TARGET_TASK_ID" >&2 echo "error: text not sent to $T ($TARGET_BACKEND send failed; tried $RESOLUTION_TRIED)" >&2 exit 1 fi @@ -618,9 +625,8 @@ else empty) ;; send-failed) - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + fm_send_known_undelivered_cleanup || \ + echo "error: known-undelivered pending-reply state could not be reset for $TARGET_TASK_ID" >&2 echo "error: text not sent to $T ($TARGET_BACKEND send failed; tried $RESOLUTION_TRIED)" >&2 exit 1 ;; diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index cb2641c68be..b1ddb1f3e70 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -323,6 +323,7 @@ secondmate_wake_stall_tick() { local now=$(( $(date +%s) )) threshold=$SECONDMATE_WAKE_STALL_SECS local meta task kind remote_host home queue row epoch seq row_key marker receipt receipt_dir notify_key queued age reason case "$threshold" in ''|*[!0-9]*|0) threshold=60 ;; esac + # Endpoint metadata admits this queue-loop check; secondmate-liveness owns registered mates whose endpoint is missing or dead. for meta in "$STATE"/*.meta; do [ -e "$meta" ] || continue kind=$(fm_meta_get "$meta" kind) diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index d98d9b64f9f..ae10864a043 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -126,6 +126,53 @@ EOF pass "a failed receiver wake is loud and retries from an already-present handoff" } +test_known_receiver_failure_remains_retryable_after_grace() { + local home="$TMP_ROOT/known-fail-main" sub="$TMP_ROOT/known-fail-sub" + local basebin rejectbin="$TMP_ROOT/known-fail-reject" out corr phase rc=0 + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$rejectbin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] known-fail - retry after known receiver rejection (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + basebin=$(make_fake_tmux "$TMP_ROOT/known-fail-fake") + cat > "$rejectbin/tmux" <<'SH' +#!/usr/bin/env bash +[ "${1:-}" != send-keys ] || exit 1 +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$rejectbin/tmux" + + out=$(PATH="$rejectbin:$basebin:$PATH" FM_BASE_TMUX="$basebin/tmux" \ + FM_HOME="$home" FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/known-fail-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/known-fail-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design known-fail 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "known receiver rejection reported handoff success" + assert_grep 'known-fail' "$sub/data/backlog.md" "known receiver rejection lost the durable item" + corr=$(cut -d: -f2- "$home/state/.backlog-handoff-design.wake-pending") + assert_absent "$home/state/pending-replies/.delivery-confirmed-$corr" \ + "known receiver rejection retained an attempted-delivery marker" + FM_PENDING_REPLY_NOW=9999999999 bash -c ' + . "$1" + fm_pending_reply_reconcile_delivery "$2" "$3" >/dev/null 2>&1 || true + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$home/state" "$corr" + phase=$(sed -n 's/^phase=//p' "$home/state/pending-replies/$corr") + [ "$phase" = awaiting_report ] \ + || fail "known receiver rejection aged into unretryable phase $phase" + + : > "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design known-fail \ + > "$TMP_ROOT/known-fail-retry.out" 2>&1 \ + || fail "known receiver rejection did not retry: $(cat "$TMP_ROOT/known-fail-retry.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "known receiver rejection retry did not wake the receiver" + pass "a known receiver failure stays retryable after reconciliation grace" +} + test_move_crash_keeps_wake_pending_for_recovery() { local home="$TMP_ROOT/move-crash-main" sub="$TMP_ROOT/move-crash-sub" local fakebin="$TMP_ROOT/move-crash-fakebin" real_tasks rc=0 @@ -965,6 +1012,7 @@ EOF test_handoff_wakes_live_local_receiver test_failed_wake_retries_when_the_item_is_already_present +test_known_receiver_failure_remains_retryable_after_grace test_move_crash_keeps_wake_pending_for_recovery test_delivery_confirmation_crash_does_not_resend test_concurrent_local_handoffs_serialize_move_and_wake From 698e6b97e25b4b0fe35136766f2fd806cd810fa5 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:40:04 -0700 Subject: [PATCH 09/24] no-mistakes(review): Refuse duplicate sends for unresolved delivery attempts --- bin/fm-pending-reply-lib.sh | 13 +++++++++ bin/fm-send.sh | 5 ++++ tests/fm-backlog-handoff.test.sh | 47 ++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 091722fc500..4339a7db58f 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -412,6 +412,19 @@ fm_pending_reply_reconcile_delivery() { # return 1 } +fm_pending_reply_delivery_attempt_unresolved() { # + local state=$1 corr=$2 rec delivered marker entry + rec=$(fm_pending_reply_path "$state" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -z "$delivered" ] || return 1 + marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + entry=$(cat "$marker" 2>/dev/null || true) + case "$entry" in attempted=*) return 0 ;; esac + return 1 +} + fm_pending_reply_reset_known_undelivered() { # local state=$1 corr=$2 rec delivered phase marker entry rec=$(fm_pending_reply_path "$state" "$corr") diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 1e92fbea7ef..99594b03506 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -556,6 +556,11 @@ else PENDING_REPLY_CREATED=1 fi fm_pending_reply_embed_corr "$MESSAGE" "$PENDING_REPLY_CORR" MESSAGE + if [ "$PENDING_REPLY_CREATED" != 1 ] \ + && fm_pending_reply_delivery_attempt_unresolved "$STATE" "$PENDING_REPLY_CORR"; then + echo "error: pending-reply delivery for $TARGET_TASK_ID is unresolved; refusing to resend correlation $PENDING_REPLY_CORR" >&2 + exit 1 + fi if ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then [ "$PENDING_REPLY_CREATED" != 1 ] \ || fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index ae10864a043..e02c5c4af86 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -274,6 +274,52 @@ SH pass "a post-confirmation crash reconciles delivery without resending" } +test_unresolved_delivery_attempt_refuses_immediate_resend() { + local home="$TMP_ROOT/attempt-crash-main" sub="$TMP_ROOT/attempt-crash-sub" + local fakebin="$TMP_ROOT/attempt-crash-fakebin" real_mv rc=0 wake_count out + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] attempt-crash - do not resend an unresolved delivery (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_mv=$(command -v mv) + cat > "$fakebin/mv" <<'SH' +#!/usr/bin/env bash +for arg in "$@"; do + if [ -f "$arg" ] && grep -q '^confirmed=' "$arg" 2>/dev/null; then + kill -KILL "$PPID" + exit 1 + fi +done +exec "$FM_REAL_MV" "$@" +SH + chmod +x "$fakebin/mv" + : > "$TMP_ROOT/default-tmux.log" + + set +e + PATH="$fakebin:$PATH" FM_REAL_MV="$real_mv" FM_HOME="$home" \ + "$ROOT/bin/fm-backlog-handoff.sh" design attempt-crash \ + > "$TMP_ROOT/attempt-crash.out" 2>&1 + rc=$? + set +e + [ "$rc" -ne 0 ] || fail "unresolved-attempt crash fixture unexpectedly reported success" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log") + [ "$wake_count" -eq 1 ] || fail "unresolved-attempt crash did not deliver exactly one receiver wake" + + rc=0 + out=$(FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design attempt-crash 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "immediate retry resent or accepted an unresolved delivery attempt" + assert_contains "$out" 'delivery for design is unresolved; refusing to resend correlation' \ + "immediate retry did not report the unresolved delivery boundary" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$wake_count" ] \ + || fail "immediate retry duplicated the unresolved receiver wake" + pass "an unresolved delivery attempt refuses an immediate duplicate wake" +} + test_concurrent_local_handoffs_serialize_move_and_wake() { local home="$TMP_ROOT/concurrent-main" sub="$TMP_ROOT/concurrent-sub" local basebin blockbin="$TMP_ROOT/concurrent-blockbin" first second i wake_count @@ -1015,6 +1061,7 @@ test_failed_wake_retries_when_the_item_is_already_present test_known_receiver_failure_remains_retryable_after_grace test_move_crash_keeps_wake_pending_for_recovery test_delivery_confirmation_crash_does_not_resend +test_unresolved_delivery_attempt_refuses_immediate_resend test_concurrent_local_handoffs_serialize_move_and_wake test_local_teardown_waits_for_handoff_wake test_body_moves_when_followed_by_another_item From 1ec2ea286e0b13f8c1bed7c249701085edccb313 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:46:33 -0700 Subject: [PATCH 10/24] no-mistakes(review): Atomically restore retryability after reconciled send failures --- bin/fm-pending-reply-lib.sh | 40 ++++++++++++++++--- tests/fm-backlog-handoff.test.sh | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 4339a7db58f..c8ca5d430ae 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -372,7 +372,7 @@ fm_pending_reply_mark_delivery_unknown() { # fm_pending_reply_set "$rec" phase delivery_unknown } -fm_pending_reply_reconcile_delivery() { # +_fm_pending_reply_reconcile_delivery_locked() { # local state=$1 corr=$2 rec delivered marker entry delivery_state value epoch local grace now age phase rec=$(fm_pending_reply_path "$state" "$corr") @@ -412,6 +412,18 @@ fm_pending_reply_reconcile_delivery() { # return 1 } +fm_pending_reply_reconcile_delivery() { # + local state=$1 corr=$2 lock rc=0 + local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK + STATE=$state + lock="$state/.pending-reply-$corr.lock" + . "$_FM_PENDING_REPLY_LIB_DIR/fm-wake-lib.sh" + fm_lock_acquire_wait "$lock" || return 1 + _fm_pending_reply_reconcile_delivery_locked "$@" || rc=$? + fm_lock_release "$lock" + return "$rc" +} + fm_pending_reply_delivery_attempt_unresolved() { # local state=$1 corr=$2 rec delivered marker entry rec=$(fm_pending_reply_path "$state" "$corr") @@ -426,18 +438,36 @@ fm_pending_reply_delivery_attempt_unresolved() { # } fm_pending_reply_reset_known_undelivered() { # + local state=$1 corr=$2 lock rc=0 + local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK + STATE=$state + lock="$state/.pending-reply-$corr.lock" + . "$_FM_PENDING_REPLY_LIB_DIR/fm-wake-lib.sh" + fm_lock_acquire_wait "$lock" || return 1 + _fm_pending_reply_reset_known_undelivered_locked "$@" || rc=$? + fm_lock_release "$lock" + return "$rc" +} + +_fm_pending_reply_reset_known_undelivered_locked() { # local state=$1 corr=$2 rec delivered phase marker entry rec=$(fm_pending_reply_path "$state" "$corr") [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 delivered=$(fm_pending_reply_get "$rec" delivered_epoch) [ -z "$delivered" ] || return 1 phase=$(fm_pending_reply_get "$rec" phase) - [ "$phase" = awaiting_report ] || return 1 + case "$phase" in awaiting_report|delivery_unknown) ;; *) return 1 ;; esac marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") - [ -e "$marker" ] || [ -L "$marker" ] || return 0 + [ -e "$marker" ] || [ -L "$marker" ] || { + [ "$phase" = awaiting_report ] + return $? + } [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 entry=$(cat "$marker" 2>/dev/null || true) - case "$entry" in attempted=*) rm -f -- "$marker" ;; *) return 1 ;; esac + case "$entry" in attempted=*) ;; *) return 1 ;; esac + [ "$phase" = awaiting_report ] \ + || fm_pending_reply_set "$rec" phase awaiting_report || return 1 + rm -f -- "$marker" } # Drop an undelivered expectation after a failed send so transport failure does @@ -1077,7 +1107,7 @@ _fm_pending_reply_maybe_escalate_locked() { # [ -f "$rec" ] || return 1 phase=$(fm_pending_reply_get "$rec" phase) if [ "$phase" = delivery_unknown ]; then - fm_pending_reply_reconcile_delivery "$state" "$corr" || true + _fm_pending_reply_reconcile_delivery_locked "$state" "$corr" || true phase=$(fm_pending_reply_get "$rec" phase) [ "$phase" = delivery_unknown ] || return 0 fi diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index e02c5c4af86..9f649a49889 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -173,6 +173,73 @@ SH pass "a known receiver failure stays retryable after reconciliation grace" } +test_known_failure_restores_retry_after_reconciliation_race() { + local home="$TMP_ROOT/reconcile-race-main" sub="$TMP_ROOT/reconcile-race-sub" + local basebin blockbin="$TMP_ROOT/reconcile-race-block" handoff i corr phase + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$blockbin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] reconcile-race - retry after concurrent reconciliation (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + basebin=$(make_fake_tmux "$TMP_ROOT/reconcile-race-fake") + cat > "$blockbin/tmux" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = send-keys ]; then + touch "$FM_RECONCILE_RACE_ENTERED" + while [ ! -f "$FM_RECONCILE_RACE_RELEASE" ]; do sleep 0.02; done + exit 1 +fi +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$blockbin/tmux" + + PATH="$blockbin:$basebin:$PATH" FM_BASE_TMUX="$basebin/tmux" FM_HOME="$home" \ + FM_RECONCILE_RACE_ENTERED="$TMP_ROOT/reconcile-race.entered" \ + FM_RECONCILE_RACE_RELEASE="$TMP_ROOT/reconcile-race.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/reconcile-race-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/reconcile-race-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design reconcile-race \ + > "$TMP_ROOT/reconcile-race.out" 2>&1 & + handoff=$! + i=0 + while [ ! -f "$TMP_ROOT/reconcile-race.entered" ]; do + kill -0 "$handoff" 2>/dev/null || fail "reconciliation-race handoff exited before backend delivery" + i=$((i + 1)) + [ "$i" -le 250 ] || fail "reconciliation-race handoff never reached backend delivery" + sleep 0.02 + done + corr=$(cut -d: -f2- "$home/state/.backlog-handoff-design.wake-pending") + FM_PENDING_REPLY_NOW=9999999999 bash -c ' + . "$1" + fm_pending_reply_reconcile_delivery "$2" "$3" + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$home/state" "$corr" \ + || fail "concurrent watcher fixture did not reconcile the aged attempt" + phase=$(sed -n 's/^phase=//p' "$home/state/pending-replies/$corr") + [ "$phase" = delivery_unknown ] || fail "aged in-flight attempt did not become delivery_unknown" + touch "$TMP_ROOT/reconcile-race.release" + if wait "$handoff"; then + fail "known backend failure after reconciliation reported success" + fi + phase=$(sed -n 's/^phase=//p' "$home/state/pending-replies/$corr") + [ "$phase" = awaiting_report ] \ + || fail "known backend failure did not restore retryable phase after reconciliation" + assert_absent "$home/state/pending-replies/.delivery-confirmed-$corr" \ + "known backend failure retained its aged attempted marker" + + : > "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design reconcile-race \ + > "$TMP_ROOT/reconcile-race-retry.out" 2>&1 \ + || fail "reconciliation-race handoff did not retry: $(cat "$TMP_ROOT/reconcile-race-retry.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "reconciliation-race retry did not wake the receiver" + pass "known failure restores retryability after concurrent reconciliation" +} + test_move_crash_keeps_wake_pending_for_recovery() { local home="$TMP_ROOT/move-crash-main" sub="$TMP_ROOT/move-crash-sub" local fakebin="$TMP_ROOT/move-crash-fakebin" real_tasks rc=0 @@ -1059,6 +1126,7 @@ EOF test_handoff_wakes_live_local_receiver test_failed_wake_retries_when_the_item_is_already_present test_known_receiver_failure_remains_retryable_after_grace +test_known_failure_restores_retry_after_reconciliation_race test_move_crash_keeps_wake_pending_for_recovery test_delivery_confirmation_crash_does_not_resend test_unresolved_delivery_attempt_refuses_immediate_resend From e1196e146265b84d200a8bec00669460e55d2548 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 00:50:14 -0700 Subject: [PATCH 11/24] no-mistakes(review): Serialize delivery confirmation with reconciliation --- bin/fm-pending-reply-lib.sh | 12 +++++++++ tests/fm-pending-reply.test.sh | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index c8ca5d430ae..36f1a774042 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -344,6 +344,18 @@ fm_pending_reply_prepare_delivery() { # } fm_pending_reply_confirm_delivery() { # + local state=$1 corr=$2 lock rc=0 + local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK + STATE=$state + lock="$state/.pending-reply-$corr.lock" + . "$_FM_PENDING_REPLY_LIB_DIR/fm-wake-lib.sh" + fm_lock_acquire_wait "$lock" || return 1 + _fm_pending_reply_confirm_delivery_locked "$@" || rc=$? + fm_lock_release "$lock" + return "$rc" +} + +_fm_pending_reply_confirm_delivery_locked() { # local state=$1 corr=$2 now marker marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") if ! fm_pending_reply_prepare_delivery "$state" "$corr"; then diff --git a/tests/fm-pending-reply.test.sh b/tests/fm-pending-reply.test.sh index 4457ae6bb76..64526a3eea3 100755 --- a/tests/fm-pending-reply.test.sh +++ b/tests/fm-pending-reply.test.sh @@ -664,6 +664,54 @@ test_delivery_confirmation_fallback_reconciles() { pass "delivery confirmation fallback reconciles durably" } +test_delivery_confirmation_serializes_with_reconciliation() { + ( + local home state corr rec calls entered release confirm_pid reconcile_pid count i + home=$(setup_parent delivery-confirm-reconcile-race) + state="$home/state" + export FM_PENDING_REPLY_NOW=5900 + corr=$(fm_pending_reply_create "$home" "$state" hibit "serialized delivery") + rec=$(fm_pending_reply_path "$state" "$corr") + calls="$home/mark-delivered.calls" + entered="$home/mark-delivered.entered" + release="$home/mark-delivered.release" + fm_pending_reply_mark_delivered() { + local pending_state=$1 pending_corr=$2 epoch=$3 pending_rec phase + printf '%s\n' "$BASHPID" >> "$calls" + : > "$entered" + while [ ! -e "$release" ]; do /bin/sleep 0.01; done + pending_rec=$(fm_pending_reply_path "$pending_state" "$pending_corr") + fm_pending_reply_set "$pending_rec" delivered_epoch "$epoch" || return 1 + phase=$(fm_pending_reply_get "$pending_rec" phase) + [ "$phase" != delivery_unknown ] \ + || fm_pending_reply_set "$pending_rec" phase awaiting_report + } + fm_pending_reply_confirm_delivery "$state" "$corr" & + confirm_pid=$! + for i in $(seq 1 100); do + [ -e "$entered" ] && break + /bin/sleep 0.01 + done + [ -e "$entered" ] || fail "delivery confirmation did not reach its commit boundary" + fm_pending_reply_reconcile_delivery "$state" "$corr" & + reconcile_pid=$! + /bin/sleep 0.1 + : > "$release" + wait "$confirm_pid" || fail "delivery confirmation should commit" + wait "$reconcile_pid" || fail "reconciliation should observe committed delivery" + count=$(wc -l < "$calls" | tr -d ' ') + [ "$count" = 1 ] \ + || fail "confirmation and reconciliation raced through $count delivery commits" + [ "$(fm_pending_reply_get "$rec" delivered_epoch)" = 5900 ] \ + || fail "serialized confirmation should retain delivered_epoch" + [ "$(phase_of "$state" "$corr")" = awaiting_report ] \ + || fail "serialized confirmation should retain awaiting_report phase" + [ ! -e "$(fm_pending_reply_delivery_confirmation_path "$state" "$corr")" ] \ + || fail "serialized delivery marker should be removed" + ) || fail "delivery confirmation serialization regression failed" + pass "delivery confirmation serializes with reconciliation" +} + test_unrelated_and_stale_corr_cannot_resolve() { local home state corr other home=$(setup_parent stale-corr) @@ -1197,6 +1245,7 @@ test_concurrent_escalation_yields_to_late_reply test_transport_success_is_not_reply_success test_undelivered_records_are_scan_immutable test_delivery_confirmation_fallback_reconciles +test_delivery_confirmation_serializes_with_reconciliation test_unrelated_and_stale_corr_cannot_resolve test_restart_preserves_expectation_and_parent_destination test_wrong_home_detected_not_acknowledged From 32dcfdb79cdfcc3be6d850f9cf39c7b33c15f7f8 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 01:02:57 -0700 Subject: [PATCH 12/24] no-mistakes(document): Document routed wake and stall supervision --- .agents/skills/bootstrap-diagnostics/SKILL.md | 4 ++-- .agents/skills/secondmate-provisioning/SKILL.md | 4 +++- bin/fm-backlog-handoff.sh | 11 +++++++---- bin/fm-watch.sh | 5 +++++ docs/architecture.md | 13 +++++++++---- docs/configuration.md | 6 ++++-- docs/herdr-backend.md | 4 ++-- docs/remote-secondmates.md | 4 ++-- docs/scripts.md | 4 ++-- 9 files changed, 36 insertions(+), 19 deletions(-) diff --git a/.agents/skills/bootstrap-diagnostics/SKILL.md b/.agents/skills/bootstrap-diagnostics/SKILL.md index 95932444f83..0aad8846387 100644 --- a/.agents/skills/bootstrap-diagnostics/SKILL.md +++ b/.agents/skills/bootstrap-diagnostics/SKILL.md @@ -53,8 +53,8 @@ When any diagnostic needs captain attention, report the plain consequence and re - `SECONDMATE_SYNC: secondmate : skipped: ` - secondmate convergence left a live home on its existing checkout because the home was dirty, diverged, unsafe, on the wrong branch, missing its placement-specific target commit, unreachable, or otherwise not fast-forwardable, or because inherited local-material propagation failed; bootstrap continued, but inspect the reason because the secondmate's tracked instructions, inherited settings, or shared captain preferences may be stale after a primary update. - `SECONDMATE_LIVENESS: secondmate : skipped: |respawn failed after : ` - the session-start liveness sweep could not guarantee that the registered secondmate is running a real agent process. Investigate the reason because that secondmate is not guaranteed live. -- `SECONDMATE_HANDOFF: secondmate : pending delivery: item(s)` - queued work has already left the main dispatchable backlog and remains safe in the named remote route's backlog-format outbox. - Preserve that outbox and rerun `bin/fm-backlog-handoff.sh --resume-pending` after same-host connectivity returns; never re-add or dispatch the items from the main backlog. +- `SECONDMATE_HANDOFF: secondmate : pending delivery: item(s)` - queued work has already left the main dispatchable backlog and remains safe in the named remote route's backlog-format outbox, pending backlog receipt or receiver-wake confirmation. + Preserve that outbox and rerun `bin/fm-backlog-handoff.sh --resume-pending` after the route or endpoint problem is resolved; never re-add or dispatch the items from the main backlog. An unsafe-outbox variant requires path and file-type inspection before any retry. - `NUDGE_SECONDMATES: secondmate : send failed: ` - secondmate convergence changed a running home's loaded instructions or inherited config, but the deterministic `fm-send.sh fm-` re-read nudge failed. Inspect the reason, keep the pending marker under `state/.secondmate-nudge-pending/` intact, and rerun session start after the endpoint or metadata issue is fixed so bootstrap can retry the exact same marked send on the same local or remote route. diff --git a/.agents/skills/secondmate-provisioning/SKILL.md b/.agents/skills/secondmate-provisioning/SKILL.md index b878c6f7658..07428f7b8fd 100644 --- a/.agents/skills/secondmate-provisioning/SKILL.md +++ b/.agents/skills/secondmate-provisioning/SKILL.md @@ -189,7 +189,9 @@ After seeding, run this handoff for the new secondmate's in-scope queued items. For an existing or inherited domain, complete record intake first so no already-shipped plan row is handed off as open work. For a local route, the helper resolves and validates the secondmate home from `data/secondmates.md`, then delegates the item move to `tasks-axi mv` (the single owner of the backlog format), which moves each named item - and a whole connected set, blocker plus dependents, atomically - from the main `data/backlog.md` into the secondmate home's `data/backlog.md`. For a remote route, the same helper first moves the dependency-closed set atomically from the main backlog into `data/handoff/.outbox.md`, then transfers that backlog-format outbox through `fm-on.sh` and lets the remote home's `fm-backlog-receive.sh` move every not-already-present key under the destination lock. -The outbox is the whole recovery record: its presence means delivery is unfinished, `--resume-pending` safely re-delivers it, and confirmed receipt removes it. +After a new local placement or a remote outbox receipt becomes durable, the helper sends one marked routed-work instruction through the receiving secondmate's recorded endpoint; missing or failed delivery makes the command fail loudly with the moved work intact, and the same handoff command retries known-undelivered wake intent without moving an already-present item again. +An unresolved delivery attempt is never blindly resent. +For a remote route, the outbox remains until both backlog receipt and receiver wake are confirmed; `--resume-pending` retries unfinished outboxes, while the script header owns its stable wake-correlation recovery state. There is no two-phase handoff journal and no tasks-axi release beyond the already-required atomic `mv` capability. Bootstrap retries pending outboxes when mutation is authorized and emits `SECONDMATE_HANDOFF:` for any that remain. This delegated route remains required when `config/backlog-backend=manual`, which controls only routine firstmate backlog edits. diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 5f0720b72b8..0770e866474 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -50,10 +50,13 @@ # Remote routes use an outbox handoff: one atomic local tasks-axi mv removes the # selected set from the dispatchable backlog into data/handoff/.outbox.md, # then an idempotent confined transfer and fm-backlog-receive.sh deliver it. -# A present outbox is the whole recovery record. No two-phase journal exists. -# Every successful delivery also sends one marked wake to the receiving endpoint. -# A missing endpoint or a live endpoint that rejects the wake makes the handoff -# fail with the delivered backlog intact. +# A present outbox remains the remote retry trigger until backlog receipt and +# receiver wake are both confirmed; a companion pending-reply correlation makes +# crash recovery reconcile an attempted or confirmed wake instead of blindly +# resending it. No two-phase journal exists. +# Every successful backlog delivery also sends one marked wake to the receiving +# endpoint. A missing endpoint or a live endpoint that rejects the wake makes the +# handoff fail with the delivered backlog intact. # Usage: fm-backlog-handoff.sh ... # fm-backlog-handoff.sh --resume-pending set -eu diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index b1ddb1f3e70..d417713d2e4 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -68,6 +68,11 @@ # check: inactive-outcome bounded poll-loop reconciliation found a suspicious # inactive terminal outcome that still lacks its durable # upstream receipt +# check: secondmate wake-loop stalled: mate= row= age=s +# the oldest valid row in an endpoint-recorded local +# secondmate home's durable wake queue exceeded +# FM_SECONDMATE_WAKE_STALL_SECS; observation is read-only +# and one parent receipt suppresses repeats for that row # For normal supervision, resume the session-start primary-harness protocol # after each printed reason. Direct duplicate invocations of this script still # no-op through the watcher singleton lock. diff --git a/docs/architecture.md b/docs/architecture.md index 0f1cf8dd9ac..e0b5affa08e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,11 +14,15 @@ Repeated provably-working stale escalations on the same unchanged pane add an es A pane holding a file newer than the start of its own quiet window, anywhere in the worktree recorded for that task, is deferred instead of escalated, because a crew writing source, then tests, then documentation behind a static pane is liveness that neither pane quietness nor the run step can show. That deferral re-surfaces on the same `FM_PAUSE_RESURFACE_SECS` cadence as a declared wait, with a reason naming the write evidence rather than a wedge, and it is bounded to one pruned, depth-bounded, wall-clock-bounded walk (`FM_WORKTREE_WRITE_PRUNE`, `FM_WORKTREE_WRITE_MAXDEPTH`, `FM_WORKTREE_WRITE_TIMEOUT`) taken only in the branch that was about to escalate, never on every poll. Every absence of write evidence, including a missing worktree record, a torn-down worktree, a walk that outlives its wall-clock bound on a hung mount, and a failed walk, leaves the existing escalation schedule untouched, so a crew that writes nothing still escalates exactly as before. -A secondmate is never probed at all, because the worktree recorded for it is a provisioned firstmate home whose own supervision keeps writing inside it whether or not the mate produces anything, so its panes keep escalating on the unchanged schedule. +A secondmate's recorded worktree is never probed for write activity, because it is a provisioned firstmate home whose own supervision keeps writing inside it whether or not the mate produces anything, so its panes keep escalating on the unchanged schedule. A busy pane is otherwise exempt from staleness, but only until its latest `state/.turn-ended` marker reaches `FM_BUSY_TURN_MAX_SECS`, or its `state/.meta` spawn record reaches that age before any turn completes; past that bound it is routed through the same wedge escalation, with the identical reason, escalation count, worktree-write deferral, and `demand-deep-inspection` marker, for inspection only - never an automatic interrupt, signal, or restart. A crew that declared an external wait (`paused:`) or a verified captain-held transfer is the one exception to that bound: its busy verdict supplies liveness while identifying the long-running foreground call as the declared wait, so it takes the bounded `FM_PAUSE_RESURFACE_SECS` recheck instead of a wedge escalation. Lifting the declaration restores the unchanged busy-pane wedge path, while a pane that is no longer busy returns to the existing idle declared-wait classification. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) only after generation-bound recovery evidence is published, so an interrupted watcher or handling turn can be recovered without losing the queue record. +Agent endpoint liveness and queue-consumption liveness are separate: on each poll, the primary watcher reads the oldest valid row from every endpoint-recorded local secondmate home's durable wake queue without locking, consuming, or rewriting that foreign queue. +Once that row reaches `FM_SECONDMATE_WAKE_STALL_SECS`, the primary appends one keyed `check` wake naming the mate, row sequence, and observed age; parent receipts and queued-key deduplication suppress repeats for the same row across watcher and handling crashes, while empty and younger queues remain silent. +Endpointless registered mates remain outside this scan because startup secondmate-liveness owns dead or missing endpoint recovery, and remote homes retain their host-local supervision boundary. +`tests/fm-wake-queue.test.sh` pins the notification, idempotence, quiet-queue, and byte-for-byte foreign-row preservation guarantees. When a canonical validated PR poll returns exactly `merged`, the watcher appends that durable notification before publishing a private receipt bound to the poll's registration, bytes, file identities, metadata, provider, URL, and task ID. The receipt makes retirement safely retryable across restarts: fixed-path recovery revalidates the same evidence, removes the runnable check first, removes its registration and data sidecars, removes the receipt last, and preserves task metadata including `pr=` and `pr_head=`. A concurrent replacement remains armed, every non-merged or invalid observation remains unchanged, and retirement never performs task or persistent-secondmate cleanup. @@ -220,9 +224,10 @@ Secondmates are idle by default: after startup recovery reconciles only work alr When called with `FM_HOME=` or when `FM_HOME` is already set to the active firstmate home, metadata-routed `fm-send.sh` requests to a live `kind=secondmate` use the live-charter-compatible `from-firstmate` carrier owned by `bin/fm-operational-input.sh`, so the secondmate returns terse answers through status lines and detailed answers through docs plus status pointers instead of replying only in its own chat. The parent guards every marked request against a missing correlated report without reading the secondmate conversation; `bin/fm-pending-reply-lib.sh` owns the correlation, recovery, escalation, and retention contract. Explicit backend-target sends and direct human typing stay unmarked, so captain intervention in a secondmate pane remains conversational. -After seeding a secondmate, `fm-backlog-handoff.sh` validates the fleet-specific handoff, then atomically delegates already-judged in-scope queued item moves to `tasks-axi mv` so the domain queue starts in the right place. -Remote routes move that dependency-closed set into a non-dispatchable backlog-format outbox before transfer, then use an idempotent remote receive under the destination backlog's own lock. -The outbox is the complete retry record, so no two-phase journal or transport-level retry is needed. +After seeding a secondmate, `fm-backlog-handoff.sh` validates the fleet-specific handoff, atomically delegates already-judged in-scope queued item moves to `tasks-axi mv`, and then sends a marked routed-work wake through the receiver's recorded endpoint. +A durable move with a missing, failed, or unresolved wake is reported as failure rather than success; rerunning the same handoff recovers known-undelivered wake intent without moving the item again, while an unresolved delivery is never blindly resent. +Remote routes move that dependency-closed set into a non-dispatchable backlog-format outbox before transfer, then use an idempotent remote receive under the destination backlog's own lock and retain the outbox until the receiver wake is confirmed. +The script header owns the wake correlation and recovery mechanics; `tests/fm-backlog-handoff.test.sh` and `tests/fm-remote-backlog-handoff.test.sh` pin the local and remote delivery boundaries. An unreachable remote host is unknown rather than dead, preserves its route and durable work, and is never failed over or relaunched locally. Idle secondmate panes are healthy; teardown is explicit and refuses while the secondmate home has in-flight work unless the captain has approved discard with `--force`. diff --git a/docs/configuration.md b/docs/configuration.md index c9d0d293a52..dce6f73e6a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -37,7 +37,7 @@ This preference is local to each Firstmate home and is not part of secondmate in The tracked `.tasks.toml` pins the default `tasks-axi` markdown backend to `data/backlog.md`, with `done_keep = 10` and an archive at `data/done-archive.md`. When the default backend is selected and compatible `tasks-axi` is on `PATH`, firstmate uses its verbs for routine backlog mutations. -Secondmate handoffs are separate and unconditional: `fm-backlog-handoff.sh` keeps only its own fleet-level validation and always delegates the item move to `tasks-axi mv`, the single owner of the backlog format. +Secondmate handoffs bypass that routine-backend choice: `fm-backlog-handoff.sh` keeps only its own fleet-level validation, delegates the item move to `tasks-axi mv`, and requires a verified receiver wake after a new move becomes durable. It moves in-scope `## Queued` items only and refuses `## In flight` and historical `## Done` records, which stay with their home for pruning or archiving. Handoff item bodies must use at least two leading spaces, and the helper refuses a selected item with a single-space or tab-indented continuation rather than risk orphaning it. Because bootstrap requires `tasks-axi` on `PATH` on every profile, that delegation works fleet-wide, and the `config/backlog-backend=manual` knob governs firstmate's own hand-editing of its backlog, not this validated helper. @@ -181,7 +181,8 @@ The lease is held under the secondmate id until explicit retirement or seed roll Teardown of a leased home fails closed if `treehouse return` cannot release the lease; plain-clone homes with no treehouse pool slot are removed directly. Secondmate routes cover `no-mistakes` and `direct-PR` projects; `local-only` projects remain main-firstmate work. For `no-mistakes` projects, seeding initializes only projects newly cloned into a secondmate home and refuses to mutate a preexisting clone that is not already initialized. -After creating a secondmate, move existing main-backlog queued items that you have judged in-scope with `fm-backlog-handoff.sh ...`; it is idempotent and refuses In flight, Done, or non-secondmate homes. +After creating a secondmate, move existing main-backlog queued items that you have judged in-scope with `fm-backlog-handoff.sh ...`; it refuses In flight, Done, or non-secondmate homes, and a new move succeeds only after waking the recorded receiver. +If the wake fails, the moved item remains durable and rerunning the same handoff retries it idempotently. Set `FM_SECONDMATE_CHARTER` to seed from inline charter text when no filled charter brief exists; set `FM_SECONDMATE_SCOPE` when the routing scope should differ from the charter text. The seeded home's `data/charter.md` owns the standard secondmate lifecycle and escalation contract; the route file points to it through the existing `home:` field instead of adding another pointer. Each seed writes an `.fm-secondmate-home` identity marker at the home root, alongside a durable `.fm-secondmate-parent` record of the home's route to its parent (see "Provision a route" in [`docs/remote-secondmates.md`](remote-secondmates.md)). @@ -679,6 +680,7 @@ FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a declared external wait or verified captain-held transfer takes the FM_PAUSE_RESURFACE_SECS recheck below instead FM_PAUSE_RESURFACE_SECS=3600 # seconds before the watcher re-surfaces a declared external wait or verified captain-held transfer for a recheck, including a live busy pane past FM_BUSY_TURN_MAX_SECS; the away-mode daemon uses the same setting for a declared external wait or verified captain-held transfer +FM_SECONDMATE_WAKE_STALL_SECS=60 # minimum age of the oldest valid foreign wake-queue row before an endpoint-recorded local secondmate produces one durable parent wake-loop-stall notification; zero or invalid values use 60 FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WORKTREE_WRITE_PRUNE='.git node_modules .venv venv __pycache__ .mypy_cache .pytest_cache .ruff_cache .tox target dist build .next .cache vendor' # directory names the wedge detector's task-worktree write probe skips; the default keeps .git out so a supervisor's own read-only git command can never look like crew progress; set it to the empty string to prune nothing, which widens the probe to the whole depth-bounded tree rather than disabling it FM_WORKTREE_WRITE_MAXDEPTH=6 # depth that same probe walks below the recorded worktree; it runs only at the moment a wedge escalation would otherwise fire, never on every poll; no probe knob applies to a secondmate, whose recorded worktree is a provisioned home the probe skips entirely diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index a8ee9556774..f8a9bbd701d 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -269,7 +269,7 @@ A structurally gone pane becomes `missing`, a restored agent-less shell becomes Unlike tmux process-name inspection, native registration can classify Pi without guessing from a generic interpreter name. The session-start sweep uses this probe. -Mid-session secondmate liveness is not implemented because idle secondmates are deliberately exempt from stale-pane escalation and need a separate periodic identity signal. +Mid-session secondmate agent-process liveness is not implemented because idle secondmates are deliberately exempt from stale-pane escalation and need a separate periodic identity signal. ## Push events and polling fallback @@ -320,7 +320,7 @@ Tests use thin compatibility wrappers in `tests/herdr-test-safety.sh` and never - Mutable labels can collide; they are never placement or destructive authority. - A Firstmate outside Herdr cannot resolve a launcher workspace, so a colliding home label refuses new spawns until the collision is cleared. - Ghost and placeholder recognition uses ANSI de-emphasis when available; an unstyled glyph row carrying trailing non-idle text fails safely to `unknown`. -- Mid-session secondmate liveness is not implemented. +- Mid-session secondmate agent-process liveness is not implemented. - Only tmux and Herdr can host the away-mode supervisor terminal. ## Regression entry points diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index c5f471875d5..b5ac0f9db3b 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -204,8 +204,8 @@ bin/fm-backlog-handoff.sh ... For a remote route, `tasks-axi mv` first moves the dependency-closed set atomically from the primary backlog into `data/handoff/.outbox.md`. The outbox is then copied to the remote handoff scratch directory and `fm-backlog-receive.sh` atomically ingests every destination-absent key under the remote backlog's own lock. -Confirmed receipt removes the outbox. -An existing outbox is the complete retry record, and `--resume-pending` safely re-delivers it. +After receipt, the helper sends a marked routed-work instruction through the recorded remote endpoint and removes the outbox only after that wake is confirmed. +A failed wake leaves the remote backlog intact and the outbox available for `--resume-pending`; an unresolved send is reported without a blind resend. Bootstrap retries pending outboxes and emits `SECONDMATE_HANDOFF:` only when one remains. There is no two-phase journal and no additional tasks-axi release requirement. diff --git a/docs/scripts.md b/docs/scripts.md index 3359c32e6c8..c724eabb9c0 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -24,7 +24,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-remote-job-worker.sh` | Long-lived remote queue worker for tracked `fm-*.sh` commands in the account runtime | | `fm-remote-job-reap-orphans.sh` | Stop remote job workers left running by a pruned code root, never one whose checkout still exists | | `fm-remote-doctor.sh` | Check, and with `--fix` repair, one remote account's second-mate readiness (remote job worker, Herdr, Aqua launch agents, PATH, and required tools) | -| `fm-backlog-handoff.sh` | Validate and delegate queued backlog-item moves into a secondmate home | +| `fm-backlog-handoff.sh` | Move queued backlog items into a secondmate home and durably wake its recorded receiver | | `fm-backlog-receive.sh` | Idempotently ingest one confined remote handoff outbox through tasks-axi | | `fm-captain-hold.sh` | Hold tasks for the captain, record the captain's answers, gate investigation completion, and report record divergence between the status log and the backlog | | `fm-decision-hold.sh` | One-release compatibility shim mapping the retired decision commands onto fm-captain-hold.sh | @@ -72,7 +72,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-gate-refuse-lib.sh` | Shared no-mistakes gate-context refusal for fleet lifecycle entrypoints | | `fm-watch-arm.sh` | Verified home-scoped watcher arm wrapper with loud cycle endings and bounded lifecycle ledger | | `fm-watch-checkpoint.sh` | Run one bounded foreground watcher checkpoint for Codex-style supervision | -| `fm-watch.sh` | Singleton-safe always-on watcher: absorb benign wakes, queue and exit on actionable ones | +| `fm-watch.sh` | Singleton-safe watcher: absorb benign wakes, detect stalled local-secondmate wake queues, and exit on actionable ones | | `fm-inactive-reconcile.sh` | Reconcile long-inactive direct crewmate terminal outcomes without forge access | | `fm-afk-start.sh` | Run the common sourceable away-mode daemon entry in the foreground | | `fm-afk-launch.sh` | Own away-mode entry, exit, rollback, and any backend terminal lifecycle | From 1c875a9d80932fb0e38c39ea240836347d66dc86 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 01:05:31 -0700 Subject: [PATCH 13/24] no-mistakes(lint): Fix ShellCheck expansion and subshell warnings --- bin/fm-wake-lib.sh | 2 +- tests/fm-pending-reply.test.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 9035108b7d1..5fd51fe9d59 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -1211,7 +1211,7 @@ fm_wake_commit_secondmate_stall_receipts_through() { # rest=${key%-*} epoch=${rest##*-} task=${rest#secondmate-wake-loop-} - task=${task%-$epoch} + task=${task%-"$epoch"} case "$seq" in ''|*[!0-9]*) return 1 ;; esac case "$epoch" in ''|*[!0-9]*) return 1 ;; esac case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac diff --git a/tests/fm-pending-reply.test.sh b/tests/fm-pending-reply.test.sh index 64526a3eea3..5a14bafcad9 100755 --- a/tests/fm-pending-reply.test.sh +++ b/tests/fm-pending-reply.test.sh @@ -669,6 +669,8 @@ test_delivery_confirmation_serializes_with_reconciliation() { local home state corr rec calls entered release confirm_pid reconcile_pid count i home=$(setup_parent delivery-confirm-reconcile-race) state="$home/state" + # This fixture clock is intentionally scoped to the isolated subshell. + # shellcheck disable=SC2030,SC2031 export FM_PENDING_REPLY_NOW=5900 corr=$(fm_pending_reply_create "$home" "$state" hibit "serialized delivery") rec=$(fm_pending_reply_path "$state" "$corr") @@ -687,6 +689,8 @@ test_delivery_confirmation_serializes_with_reconciliation() { || fm_pending_reply_set "$pending_rec" phase awaiting_report } fm_pending_reply_confirm_delivery "$state" "$corr" & + # The background PID is consumed within this isolated test subshell. + # shellcheck disable=SC2031 confirm_pid=$! for i in $(seq 1 100); do [ -e "$entered" ] && break @@ -694,6 +698,8 @@ test_delivery_confirmation_serializes_with_reconciliation() { done [ -e "$entered" ] || fail "delivery confirmation did not reach its commit boundary" fm_pending_reply_reconcile_delivery "$state" "$corr" & + # The background PID is consumed within this isolated test subshell. + # shellcheck disable=SC2031 reconcile_pid=$! /bin/sleep 0.1 : > "$release" From 039392f810f61d21cdf3593493a1a80d66ae71b2 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 01:29:58 -0700 Subject: [PATCH 14/24] no-mistakes: apply CI fixes --- bin/fm-backlog-handoff.sh | 8 +++++++- tests/fm-backlog-handoff.test.sh | 21 +++++++++++++++++++-- tests/fm-secondmate-lifecycle-e2e.test.sh | 8 ++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 0770e866474..65bcbeae684 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -756,8 +756,14 @@ fi WAKE_PENDING_BEFORE=0 WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" +# A surviving marker belongs to an earlier invocation. Resolve its delivery +# before creating the correlation for this new move; otherwise a confirmed old +# correlation could make the new routed work appear woken without any submit. if [ -e "$WAKE_PENDING_MARKER" ] || [ -L "$WAKE_PENDING_MARKER" ]; then - WAKE_PENDING_BEFORE=1 + wake_pending_secondmate_receiver "$ID" || { + echo "error: previous receiver wake for secondmate $ID is unresolved; nothing new was moved" >&2 + exit 1 + } fi receiver_wake_mark_pending "$ID" || { echo "error: receiver wake state for secondmate $ID could not be recorded; nothing was moved" >&2 diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 9f649a49889..376945dbdbc 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -331,14 +331,31 @@ SH *) fail "post-confirmation crash lost its stable delivery correlation" ;; esac + # Route different work before explicitly retrying the crashed invocation. The + # completed old correlation must be reconciled, but must not stand in as the + # delivery proof for this new durable move. + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] after-crash - requires its own receiver wake (repo: alpha) + +## Done +EOF + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design after-crash \ + > "$TMP_ROOT/after-confirm-crash.out" 2>&1 \ + || fail "new handoff after a confirmation crash failed: $(cat "$TMP_ROOT/after-confirm-crash.out")" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$((wake_count + 1))" ] \ + || fail "completed stale correlation suppressed or duplicated the new handoff wake" + assert_grep 'after-crash' "$sub/data/backlog.md" \ + "new item after a confirmation crash was not durably handed off" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design confirm-crash \ > "$TMP_ROOT/confirm-crash-retry.out" 2>&1 \ || fail "post-confirmation crash recovery failed: $(cat "$TMP_ROOT/confirm-crash-retry.out")" - [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$wake_count" ] \ + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$((wake_count + 1))" ] \ || fail "post-confirmation crash recovery duplicated the receiver wake" assert_absent "$home/state/.backlog-handoff-design.wake-pending" \ "post-confirmation crash recovery left wake state pending" - pass "a post-confirmation crash reconciles delivery without resending" + pass "a post-confirmation crash reconciles once without suppressing a later handoff wake" } test_unresolved_delivery_attempt_refuses_immediate_resend() { diff --git a/tests/fm-secondmate-lifecycle-e2e.test.sh b/tests/fm-secondmate-lifecycle-e2e.test.sh index 9c9555f1cf8..41c0ce3d623 100755 --- a/tests/fm-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-secondmate-lifecycle-e2e.test.sh @@ -171,7 +171,9 @@ phase_handoff() { - [x] old-task - shipped thing - local main (merged 2026-06-19) EOF local out before - out=$(FM_HOME="$HOME_DIR" "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y) \ + out=$(PATH="$FAKEBIN:$PATH" FM_HOME="$HOME_DIR" FM_FAKE_TMUX_LOG="$LOG" \ + FM_FAKE_TMUX_CAPTURE="$PANE" \ + "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y) \ || fail "handoff failed for in-scope items" assert_contains "$out" "handed off 2 item(s) to design" "handoff did not report the moved items" @@ -187,7 +189,9 @@ EOF # Idempotent: a second handoff neither errors nor duplicates, and leaves main alone. before=$(cat "$HOME_DIR/data/backlog.md") - FM_HOME="$HOME_DIR" "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y >/dev/null 2>&1 \ + PATH="$FAKEBIN:$PATH" FM_HOME="$HOME_DIR" FM_FAKE_TMUX_LOG="$LOG" \ + FM_FAKE_TMUX_CAPTURE="$PANE" \ + "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y >/dev/null 2>&1 \ || fail "idempotent re-run failed" [ "$(grep -cF -- '- [ ] feat-x - add feature x (repo: alpha)' "$SUB/data/backlog.md")" -eq 1 ] \ || fail "idempotent re-run duplicated feat-x in the subhome backlog" From 2204c97495649defcae12c79ac4675d31a8e5be0 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 01:51:51 -0700 Subject: [PATCH 15/24] no-mistakes: apply CI fixes --- bin/fm-backlog-handoff.sh | 12 ++++++++++++ tests/fm-remote-backlog-handoff.test.sh | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 65bcbeae684..c841be083a6 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -600,6 +600,18 @@ remote_handoff() { # return 1 done < <(backlog_key_noncanonical_body_lines "$MAIN_BACKLOG" "$key") done + # Do not append a fresh handoff to an older recovery batch. In particular, a + # confirmed wake can survive when outbox cleanup fails; if new work were + # staged into that outbox, the old confirmation would suppress the wake for + # the new work. Finish receipt, wake reconciliation, and cleanup for the old + # batch first. A failure leaves the fresh items dispatchable in main. + if [ "${#to_move[@]}" -gt 0 ] && [ -f "$outbox" ] \ + && [ "$(outbox_item_count "$outbox")" -gt 0 ]; then + remote_deliver_outbox "$id" "$outbox" || { + echo "error: previous remote handoff for secondmate $id could not be completed; nothing new was staged" >&2 + return 1 + } + fi seed_backlog_scaffold "$outbox" if [ "${#to_move[@]}" -gt 0 ]; then if ! mv_out=$(tasks-axi mv "${to_move[@]}" --file "$MAIN_BACKLOG" --to "$outbox" 2>&1); then diff --git a/tests/fm-remote-backlog-handoff.test.sh b/tests/fm-remote-backlog-handoff.test.sh index 203e5a303c7..1b95e5d425c 100755 --- a/tests/fm-remote-backlog-handoff.test.sh +++ b/tests/fm-remote-backlog-handoff.test.sh @@ -386,13 +386,20 @@ esac wakes_after=$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG") [ "$wakes_after" -eq $((wakes_before + 1)) ] \ || fail "remote cleanup failure did not perform exactly one receiver wake" -handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending >/dev/null \ - || fail "remote cleanup retry did not converge" -[ "$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG")" -eq "$wakes_after" ] \ - || fail "remote cleanup retry duplicated a confirmed receiver wake" +write_backlog '- [ ] after-cleanup - fresh work after confirmed cleanup failure (repo: alpha)' +handoff_env "$ROOT/bin/fm-backlog-handoff.sh" ios after-cleanup >/dev/null \ + || fail "fresh handoff did not converge an older confirmed cleanup failure" +[ "$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG")" -eq $((wakes_after + 1)) ] \ + || fail "fresh handoff reused the older confirmed wake instead of waking its receiver" +[ "$(grep -cF cleanup-retry "$REMOTE/data/backlog.md")" -eq 1 ] \ + || fail "cleanup recovery lost or duplicated the older delivered item" +[ "$(grep -cF after-cleanup "$REMOTE/data/backlog.md")" -eq 1 ] \ + || fail "fresh handoff after cleanup recovery was lost or duplicated" +assert_absent "$PARENT/data/handoff/ios.outbox.md" \ + "fresh handoff left the recovered outbox pending" assert_absent "$PARENT/state/.backlog-handoff-ios.wake-pending" \ - "remote cleanup retry left confirmed wake state behind" -pass "remote cleanup recovery does not duplicate a confirmed receiver wake" + "fresh handoff left confirmed wake state behind" +pass "fresh remote work gets a new wake after confirmed cleanup recovery" write_backlog '- [ ] route-race - remains dispatchable through retirement (repo: alpha)' registry_lock="$PARENT/state/.secondmate-registry.lock" From dfb66603d6032ab52c423dee1afd1b3579c25503 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 02:12:58 -0700 Subject: [PATCH 16/24] no-mistakes(review): Retire stale wake state and defer pre-move wakes --- bin/fm-backlog-handoff.sh | 85 ++++++++++++++----- bin/fm-teardown.sh | 78 +++++++++++++++++ tests/fm-backlog-handoff.test.sh | 50 +++++++++++ ...fm-remote-secondmate-lifecycle-e2e.test.sh | 16 ++++ tests/fm-secondmate-lifecycle-e2e.test.sh | 18 +++- 5 files changed, 226 insertions(+), 21 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index c841be083a6..deedda13521 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -321,6 +321,7 @@ receiver_wake_state_write() { # case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac case "$value" in pending|confirmed) ;; + prepared:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; pending:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; confirmed:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; *) return 1 ;; @@ -332,14 +333,15 @@ receiver_wake_state_write() { # fi } -receiver_wake_mark_pending() { # - local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec +receiver_wake_mark() { # + local id=$1 wake_phase=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec + case "$wake_phase" in prepared|pending) ;; *) return 1 ;; esac if [ -e "$marker" ] || [ -L "$marker" ]; then [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 value=$(cat "$marker" 2>/dev/null || true) case "$value" in - pending:*) - corr=${value#pending:} + prepared:*|pending:*) + corr=${value#*:} rec=$(fm_pending_reply_path "$STATE" "$corr") [ -f "$rec" ] && [ ! -L "$rec" ] \ && [ "$(fm_pending_reply_get "$rec" task_id)" = "$id" ] @@ -350,12 +352,44 @@ receiver_wake_mark_pending() { # esac fi corr=$(fm_pending_reply_create "$FM_HOME" "$STATE" "$id" "$RECEIVER_WAKE_MESSAGE") || return 1 - if ! receiver_wake_state_write "$id" "pending:$corr"; then + if ! receiver_wake_state_write "$id" "$wake_phase:$corr"; then fm_pending_reply_discard_undelivered "$STATE" "$corr" || true return 1 fi } +receiver_wake_mark_pending() { # + receiver_wake_mark "$1" pending +} + +receiver_wake_mark_prepared() { # + receiver_wake_mark "$1" prepared +} + +receiver_wake_discard_prepared() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + prepared:*) corr=${value#prepared:} ;; + *) return 1 ;; + esac + fm_pending_reply_discard_undelivered "$STATE" "$corr" || return 1 + rm -f -- "$marker" +} + +receiver_wake_promote_prepared() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + prepared:*) corr=${value#prepared:} ;; + pending:*) return 0 ;; + *) return 1 ;; + esac + receiver_wake_state_write "$id" "pending:$corr" +} + receiver_wake_discard_pending() { # local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 @@ -414,6 +448,10 @@ wake_pending_secondmate_receiver() { # [retain-confirmed] value=$(cat "$marker" 2>/dev/null || true) case "$value" in confirmed|confirmed:*) return 0 ;; + prepared|prepared:*) + printf 'error: receiver wake for secondmate %s was prepared before its backlog became durable\n' "$id" >&2 + return 1 + ;; pending) receiver_wake_mark_pending "$id" || return 1 value=$(cat "$marker" 2>/dev/null || true) @@ -744,6 +782,10 @@ fi if [ "${#TO_MOVE[@]}" -eq 0 ]; then echo "nothing to move: ${ALREADY[*]:-no keys} already present in $SUB_BACKLOG" + WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" + case "$(cat "$WAKE_PENDING_MARKER" 2>/dev/null || true)" in + prepared:*) receiver_wake_promote_prepared "$ID" || exit 1 ;; + esac wake_pending_secondmate_receiver "$ID" || exit 1 exit 0 fi @@ -766,18 +808,19 @@ if ! fm_tasks_axi_compatible; then exit 1 fi -WAKE_PENDING_BEFORE=0 WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" -# A surviving marker belongs to an earlier invocation. Resolve its delivery -# before creating the correlation for this new move; otherwise a confirmed old -# correlation could make the new routed work appear woken without any submit. if [ -e "$WAKE_PENDING_MARKER" ] || [ -L "$WAKE_PENDING_MARKER" ]; then - wake_pending_secondmate_receiver "$ID" || { - echo "error: previous receiver wake for secondmate $ID is unresolved; nothing new was moved" >&2 - exit 1 - } + case "$(cat "$WAKE_PENDING_MARKER" 2>/dev/null || true)" in + prepared:*) receiver_wake_discard_prepared "$ID" || exit 1 ;; + *) + wake_pending_secondmate_receiver "$ID" || { + echo "error: previous receiver wake for secondmate $ID is unresolved; nothing new was moved" >&2 + exit 1 + } + ;; + esac fi -receiver_wake_mark_pending "$ID" || { +receiver_wake_mark_prepared "$ID" || { echo "error: receiver wake state for secondmate $ID could not be recorded; nothing was moved" >&2 exit 1 } @@ -802,12 +845,10 @@ if ! MV_OUT=$(tasks-axi mv "${TO_MOVE[@]}" --file "$MAIN_BACKLOG" --to "$SUB_BAC if [ "$SUB_CREATED" -eq 1 ]; then rm -f "$SUB_BACKLOG" fi - if [ "$WAKE_PENDING_BEFORE" -eq 0 ]; then - receiver_wake_discard_pending "$ID" || { - echo "error: tasks-axi mv failed and receiver wake state could not be cleared" >&2 - exit 1 - } - fi + receiver_wake_discard_prepared "$ID" || { + echo "error: tasks-axi mv failed and receiver wake state could not be cleared" >&2 + exit 1 + } if [ -n "$MV_OUT" ]; then printf '%s\n' "$MV_OUT" >&2 fi @@ -817,6 +858,10 @@ fi echo "handed off ${#TO_MOVE[@]} item(s) to $ID: ${TO_MOVE[*]}" echo " into $SUB_BACKLOG" +receiver_wake_promote_prepared "$ID" || { + echo "error: handed off work to secondmate $ID, but durable receiver wake state could not be recorded" >&2 + exit 1 +} wake_pending_secondmate_receiver "$ID" || exit 1 if [ "${#ALREADY[@]}" -gt 0 ]; then echo " already present (skipped): ${ALREADY[*]}" diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 54a29c7b3c3..62a593f6ffd 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -166,6 +166,8 @@ SUB_HOME_PARENT_MARKER=".fm-secondmate-parent" . "$SCRIPT_DIR/fm-secondmate-parent-lib.sh" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-pending-reply-lib.sh +. "$SCRIPT_DIR/fm-pending-reply-lib.sh" # shellcheck source=bin/fm-nm-run-lib.sh . "$SCRIPT_DIR/fm-nm-run-lib.sh" if [ "$#" -lt 1 ] || ! fm_task_id_path_safe "$1"; then @@ -240,6 +242,77 @@ REMOTE_REGISTRY_LOCK= REMOTE_REPLY_LIFECYCLE_LOCK= LOCAL_HANDOFF_LOCK= LOCAL_REGISTRY_LOCK= +HANDOFF_WAKE_RETIRE_MARKER= +HANDOFF_WAKE_RETIRE_VALUE= +HANDOFF_WAKE_RETIRE_CORR= + +handoff_wake_retire_validate() { + local marker="$STATE/.backlog-handoff-$ID.wake-pending" value corr rec confirmation + HANDOFF_WAKE_RETIRE_MARKER= + HANDOFF_WAKE_RETIRE_VALUE= + HANDOFF_WAKE_RETIRE_CORR= + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] || { + echo "REFUSED: receiver wake state for secondmate $ID is unsafe" >&2 + return 1 + } + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + pending|confirmed) ;; + prepared:*|pending:*|confirmed:*) + corr=${value#*:} + printf '%s' "$corr" | grep -Eq '^[a-f0-9]{16}$' || { + echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 + return 1 + } + rec=$(fm_pending_reply_path "$STATE" "$corr") + if [ -e "$rec" ] || [ -L "$rec" ]; then + [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$ID" ] || { + echo "REFUSED: receiver wake correlation for secondmate $ID is unsafe or belongs to another task" >&2 + return 1 + } + fi + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") + if [ -e "$confirmation" ] || [ -L "$confirmation" ]; then + [ -f "$confirmation" ] && [ ! -L "$confirmation" ] || { + echo "REFUSED: receiver wake delivery state for secondmate $ID is unsafe" >&2 + return 1 + } + fi + HANDOFF_WAKE_RETIRE_CORR=$corr + ;; + *) + echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 + return 1 + ;; + esac + HANDOFF_WAKE_RETIRE_MARKER=$marker + HANDOFF_WAKE_RETIRE_VALUE=$value +} + +handoff_wake_retire() { + local marker=$HANDOFF_WAKE_RETIRE_MARKER corr=$HANDOFF_WAKE_RETIRE_CORR lock rec confirmation rc=0 + [ -n "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] \ + && [ "$(cat "$marker" 2>/dev/null || true)" = "$HANDOFF_WAKE_RETIRE_VALUE" ] || return 1 + if [ -n "$corr" ]; then + lock="$STATE/.pending-reply-$corr.lock" + fm_lock_acquire_wait "$lock" || return 1 + rec=$(fm_pending_reply_path "$STATE" "$corr") + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") + if { [ ! -e "$rec" ] && [ ! -L "$rec" ]; } \ + || { [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$ID" ]; }; then + rm -f -- "$confirmation" "$rec" "$marker" || rc=$? + else + rc=1 + fi + fm_lock_release "$lock" + return "$rc" + fi + rm -f -- "$marker" +} remote_teardown_locks_release() { if [ -n "$REMOTE_REPLY_LIFECYCLE_LOCK" ]; then @@ -352,6 +425,7 @@ remote_secondmate_teardown() { [ "$route_host" = "$remote_host" ] && [ "$route_root" = "$remote_root" ] && [ "$route_home" = "$remote_home" ] \ || { echo "REFUSED: remote secondmate metadata does not match its registry route" >&2; return 1; } [ -z "$FORCE" ] || [ "$FORCE" = --force ] || { echo "error: invalid teardown option: $FORCE" >&2; return 2; } + handoff_wake_retire_validate || return 1 remote_recovery_paths_validate initial || return 1 if [ "$FORCE" != --force ] && [ "$REMOTE_OUTBOX_PRESENT" -eq 1 ]; then echo "REFUSED: remote secondmate $ID still has a pending backlog outbox; deliver it or explicitly discard with --force" >&2 @@ -401,6 +475,8 @@ remote_secondmate_teardown() { fi remote_pending_replies_cleanup \ || { echo "error: remote pending-reply cleanup failed; preserving the local route for retry" >&2; return 1; } + handoff_wake_retire \ + || { echo "error: remote receiver wake cleanup failed; preserving the local route for retry" >&2; return 1; } tmp="$SECONDMATE_REG.tmp.$$" grep -vE "^- $ID( |$)" "$SECONDMATE_REG" > "$tmp" || true mv -f -- "$tmp" "$SECONDMATE_REG" @@ -2298,6 +2374,7 @@ if [ "$KIND" = secondmate ]; then fm_lock_acquire_wait "$LOCAL_REGISTRY_LOCK" || exit 1 LOCAL_HANDOFF_LOCK="$STATE/.backlog-handoff-$ID.lock" fm_lock_acquire_wait "$LOCAL_HANDOFF_LOCK" || exit 1 + handoff_wake_retire_validate || exit 1 [ -n "$HOME_PATH" ] || HOME_PATH=$WT validate_firstmate_home_for_removal "$HOME_PATH" "secondmate home" "$ID" >/dev/null || exit 1 if [ "$FORCE" = "--force" ]; then @@ -2560,6 +2637,7 @@ fi if [ "$KIND" = secondmate ]; then [ -n "$HOME_PATH" ] || HOME_PATH=$WT remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID" || exit $? + handoff_wake_retire || { echo "error: receiver wake cleanup failed; preserving the secondmate route for retry" >&2; exit 1; } remove_secondmate_registry_entry "$ID" fi remove_grok_turnend_auth "$STATE" "$ID" || exit 1 diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 376945dbdbc..dc7c465c18a 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -291,6 +291,55 @@ SH pass "a post-move crash preserves wake intent for an idempotent retry" } +test_pre_move_crash_does_not_wake_until_move_lands() { + local home="$TMP_ROOT/pre-move-crash-main" sub="$TMP_ROOT/pre-move-crash-sub" + local fakebin="$TMP_ROOT/pre-move-crash-fakebin" real_tasks rc=0 wake_count + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] pre-move-crash - wake only after durable move (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_tasks=$(command -v tasks-axi) + cat > "$fakebin/tasks-axi" <<'SH' +#!/usr/bin/env bash +case " $* " in + *" --file "*" --to "*) + if [ "${1:-}" = mv ]; then + handoff_pid=$(ps -o ppid= -p "$PPID" | tr -d '[:space:]') + kill -KILL "$handoff_pid" + sleep 1 + fi + ;; +esac +exec "$FM_REAL_TASKS_AXI" "$@" +SH + chmod +x "$fakebin/tasks-axi" + : > "$TMP_ROOT/default-tmux.log" + + set +e + FM_REAL_TASKS_AXI="$real_tasks" PATH="$fakebin:$PATH" FM_HOME="$home" \ + "$ROOT/bin/fm-backlog-handoff.sh" design pre-move-crash > "$TMP_ROOT/pre-move-crash.out" 2>&1 + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "pre-move crash fixture unexpectedly reported success" + assert_grep 'pre-move-crash' "$home/data/backlog.md" "pre-move crash changed the source backlog" + assert_no_grep 'pre-move-crash' "$sub/data/backlog.md" "pre-move crash changed the destination backlog" + assert_present "$home/state/.backlog-handoff-design.wake-pending" \ + "pre-move crash lost its prepared wake intent" + + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design pre-move-crash \ + > "$TMP_ROOT/pre-move-crash-retry.out" 2>&1 \ + || fail "pre-move crash recovery failed: $(cat "$TMP_ROOT/pre-move-crash-retry.out")" + assert_grep 'pre-move-crash' "$sub/data/backlog.md" "pre-move crash recovery did not move the item" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log") + [ "$wake_count" -eq 1 ] || fail "pre-move crash recovery emitted $wake_count receiver wakes" + pass "a pre-move crash wakes only after retry makes the item durable" +} + test_delivery_confirmation_crash_does_not_resend() { local home="$TMP_ROOT/confirm-crash-main" sub="$TMP_ROOT/confirm-crash-sub" local fakebin="$TMP_ROOT/confirm-crash-fakebin" real_sleep rc=0 wake_count @@ -1145,6 +1194,7 @@ test_failed_wake_retries_when_the_item_is_already_present test_known_receiver_failure_remains_retryable_after_grace test_known_failure_restores_retry_after_reconciliation_race test_move_crash_keeps_wake_pending_for_recovery +test_pre_move_crash_does_not_wake_until_move_lands test_delivery_confirmation_crash_does_not_resend test_unresolved_delivery_attempt_refuses_immediate_resend test_concurrent_local_handoffs_serialize_move_and_wake diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index 9e6bfbba4d4..e7e84a95407 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -1149,6 +1149,19 @@ assert_present "$REMOTE_HOME" "unsafe pending-replies retirement removed the rem assert_present "$TMP_ROOT/external-pending/escape" "unsafe retirement removed an external pending reply" rm -f "$PARENT/state/pending-replies" mv "$PARENT/state/pending-replies.safe" "$PARENT/state/pending-replies" +retired_wake_corr=$(FM_HOME="$PARENT" bash -c ' + . "$1" + fm_pending_reply_create "$2" "$2/state" ios "New routed work is in your backlog." +' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$PARENT") \ + || fail "could not seed remote receiver wake retirement state" +retired_wake_rec="$PARENT/state/pending-replies/$retired_wake_corr" +FM_HOME="$PARENT" bash -c ' + . "$1" + fm_pending_reply_set "$2" phase resolved + fm_pending_reply_set "$2" delivered_epoch 1 +' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$retired_wake_rec" \ + || fail "could not settle remote receiver wake retirement state" +printf 'confirmed:%s\n' "$retired_wake_corr" > "$PARENT/state/.backlog-handoff-ios.wake-pending" handoff_lock="$PARENT/state/.backlog-handoff-ios.lock" FM_HOME="$PARENT" /bin/bash -c ' . "$1" @@ -1199,6 +1212,9 @@ if ! wait "$teardown_pid"; then fi assert_absent "$REMOTE_HOME" "remote retirement did not remove the remote home" assert_absent "$PARENT/state/ios.meta" "remote retirement did not remove parent metadata" +assert_absent "$PARENT/state/.backlog-handoff-ios.wake-pending" \ + "remote retirement left receiver wake state that could poison a replacement route" +assert_absent "$retired_wake_rec" "remote retirement left the retired receiver wake correlation" assert_no_grep '- ios ' "$PARENT/data/secondmates.md" "remote retirement did not remove the registry route" jq -e --arg workspace "$SIBLING_WORKSPACE" --arg pane "$SIBLING_PANE" ' any(.workspaces[]; .workspace_id == $workspace and .label == "2ndmate-macos") diff --git a/tests/fm-secondmate-lifecycle-e2e.test.sh b/tests/fm-secondmate-lifecycle-e2e.test.sh index 41c0ce3d623..c4cb31d06f1 100755 --- a/tests/fm-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-secondmate-lifecycle-e2e.test.sh @@ -214,7 +214,20 @@ phase_recovery() { } phase_teardown() { - local teardown_out + local teardown_out corr rec + corr=$(FM_HOME="$HOME_DIR" bash -c ' + . "$1" + fm_pending_reply_create "$2" "$2/state" design "New routed work is in your backlog." + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$HOME_DIR") \ + || fail "could not seed receiver wake retirement state" + rec="$HOME_DIR/state/pending-replies/$corr" + FM_HOME="$HOME_DIR" bash -c ' + . "$1" + fm_pending_reply_set "$2" phase resolved + fm_pending_reply_set "$2" delivered_epoch 1 + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$rec" \ + || fail "could not settle receiver wake retirement state" + printf 'confirmed:%s\n' "$corr" > "$HOME_DIR/state/.backlog-handoff-design.wake-pending" : > "$LOG" teardown_out=$(PATH="$FAKEBIN:$PATH" FM_HOME="$HOME_DIR" FM_FAKE_TMUX_LOG="$LOG" FM_FAKE_TMUX_CAPTURE="$PANE" \ "$ROOT/bin/fm-teardown.sh" design 2>&1) \ @@ -223,6 +236,9 @@ phase_teardown() { && fail "secondmate teardown emitted a main-backlog completion reminder" assert_absent "$SUB" "teardown did not remove the retired secondmate home" assert_absent "$HOME_DIR/state/design.meta" "teardown did not clear the parent meta" + assert_absent "$HOME_DIR/state/.backlog-handoff-design.wake-pending" \ + "teardown left receiver wake state that could poison a replacement route" + assert_absent "$rec" "teardown left the retired receiver wake correlation" assert_no_grep '- design ' "$HOME_DIR/data/secondmates.md" "teardown did not remove the registry route" # The parent's source projects are untouched (no write through a parent home). assert_present "$HOME_DIR/projects/alpha" "teardown disturbed a parent project" From 7217c316edd6b34d9f4f28284d13f6140b83a0fa Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 02:20:15 -0700 Subject: [PATCH 17/24] no-mistakes(review): Secure markers, bind batches, and preserve teardown routes --- bin/fm-backlog-handoff.sh | 60 ++++++++++++++++++++++++-------- bin/fm-teardown.sh | 46 ++++++++++++++---------- bin/fm-wake-lib.sh | 16 +++++++++ bin/fm-watch.sh | 5 ++- tests/fm-backlog-handoff.test.sh | 59 +++++++++++++++++++++++++++++++ tests/fm-wake-queue.test.sh | 38 ++++++++++++++++++++ 6 files changed, 190 insertions(+), 34 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index deedda13521..050b61b76b2 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -316,14 +316,25 @@ warn_stale_public_commitments() { # ... # verified submit and failure semantics. A seeded but not-yet-spawned home is a # valid handoff destination, but its missing endpoint is reported rather than # pretending the task was started. +receiver_wake_batch_id() { # ... + local digest + if command -v shasum >/dev/null 2>&1; then + digest=$(printf '%s\n' "$@" | LC_ALL=C sort | shasum -a 256 2>/dev/null | awk '{print $1}') + else + digest=$(printf '%s\n' "$@" | LC_ALL=C sort | sha256sum 2>/dev/null | awk '{print $1}') + fi + printf '%s' "$digest" | grep -Eq '^[a-f0-9]{64}$' || return 1 + printf '%s' "${digest:0:16}" +} + receiver_wake_state_write() { # local id=$1 value=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" tmp case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac case "$value" in pending|confirmed) ;; - prepared:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; - pending:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; - confirmed:[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]) ;; + prepared:*) printf '%s' "$value" | grep -Eq '^prepared:[a-f0-9]{16}:[a-f0-9]{16}$' || return 1 ;; + pending:*) printf '%s' "$value" | grep -Eq '^pending:[a-f0-9]{16}$' || return 1 ;; + confirmed:*) printf '%s' "$value" | grep -Eq '^confirmed:[a-f0-9]{16}$' || return 1 ;; *) return 1 ;; esac tmp=$(umask 077; mktemp "$STATE/.backlog-handoff-wake.XXXXXX") || return 1 @@ -333,8 +344,9 @@ receiver_wake_state_write() { # fi } -receiver_wake_mark() { # - local id=$1 wake_phase=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec +receiver_wake_mark() { # [batch-id] + local id=$1 wake_phase=$2 batch=${3:-} marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec + local wake_state case "$wake_phase" in prepared|pending) ;; *) return 1 ;; esac if [ -e "$marker" ] || [ -L "$marker" ]; then [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 @@ -342,6 +354,7 @@ receiver_wake_mark() { # case "$value" in prepared:*|pending:*) corr=${value#*:} + corr=${corr%%:*} rec=$(fm_pending_reply_path "$STATE" "$corr") [ -f "$rec" ] && [ ! -L "$rec" ] \ && [ "$(fm_pending_reply_get "$rec" task_id)" = "$id" ] @@ -352,7 +365,12 @@ receiver_wake_mark() { # esac fi corr=$(fm_pending_reply_create "$FM_HOME" "$STATE" "$id" "$RECEIVER_WAKE_MESSAGE") || return 1 - if ! receiver_wake_state_write "$id" "$wake_phase:$corr"; then + wake_state="$wake_phase:$corr" + if [ "$wake_phase" = prepared ]; then + printf '%s' "$batch" | grep -Eq '^[a-f0-9]{16}$' || return 1 + wake_state="$wake_state:$batch" + fi + if ! receiver_wake_state_write "$id" "$wake_state"; then fm_pending_reply_discard_undelivered "$STATE" "$corr" || true return 1 fi @@ -362,8 +380,8 @@ receiver_wake_mark_pending() { # receiver_wake_mark "$1" pending } -receiver_wake_mark_prepared() { # - receiver_wake_mark "$1" prepared +receiver_wake_mark_prepared() { # + receiver_wake_mark "$1" prepared "$2" } receiver_wake_discard_prepared() { # @@ -371,19 +389,25 @@ receiver_wake_discard_prepared() { # [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 value=$(cat "$marker" 2>/dev/null || true) case "$value" in - prepared:*) corr=${value#prepared:} ;; + prepared:*) + corr=${value#prepared:} + corr=${corr%%:*} + ;; *) return 1 ;; esac fm_pending_reply_discard_undelivered "$STATE" "$corr" || return 1 rm -f -- "$marker" } -receiver_wake_promote_prepared() { # - local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr +receiver_wake_promote_prepared() { # + local id=$1 batch=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 value=$(cat "$marker" 2>/dev/null || true) case "$value" in - prepared:*) corr=${value#prepared:} ;; + prepared:*:"$batch") + corr=${value#prepared:} + corr=${corr%%:*} + ;; pending:*) return 0 ;; *) return 1 ;; esac @@ -780,11 +804,17 @@ if [ "$FAILED" -ne 0 ]; then exit 1 fi +REQUESTED_BATCH=$(receiver_wake_batch_id "$@") || { + echo "error: receiver wake batch identity could not be recorded; nothing was moved" >&2 + exit 1 +} + if [ "${#TO_MOVE[@]}" -eq 0 ]; then echo "nothing to move: ${ALREADY[*]:-no keys} already present in $SUB_BACKLOG" WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" case "$(cat "$WAKE_PENDING_MARKER" 2>/dev/null || true)" in - prepared:*) receiver_wake_promote_prepared "$ID" || exit 1 ;; + prepared:*:"$REQUESTED_BATCH") receiver_wake_promote_prepared "$ID" "$REQUESTED_BATCH" || exit 1 ;; + prepared:*) receiver_wake_discard_prepared "$ID" || exit 1 ;; esac wake_pending_secondmate_receiver "$ID" || exit 1 exit 0 @@ -820,7 +850,7 @@ if [ -e "$WAKE_PENDING_MARKER" ] || [ -L "$WAKE_PENDING_MARKER" ]; then ;; esac fi -receiver_wake_mark_prepared "$ID" || { +receiver_wake_mark_prepared "$ID" "$REQUESTED_BATCH" || { echo "error: receiver wake state for secondmate $ID could not be recorded; nothing was moved" >&2 exit 1 } @@ -858,7 +888,7 @@ fi echo "handed off ${#TO_MOVE[@]} item(s) to $ID: ${TO_MOVE[*]}" echo " into $SUB_BACKLOG" -receiver_wake_promote_prepared "$ID" || { +receiver_wake_promote_prepared "$ID" "$REQUESTED_BATCH" || { echo "error: handed off work to secondmate $ID, but durable receiver wake state could not be recorded" >&2 exit 1 } diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 62a593f6ffd..48ded155996 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -259,34 +259,44 @@ handoff_wake_retire_validate() { value=$(cat "$marker" 2>/dev/null || true) case "$value" in pending|confirmed) ;; - prepared:*|pending:*|confirmed:*) + prepared:*) + corr=${value#prepared:} + corr=${corr%%:*} + printf '%s' "$value" | grep -Eq '^prepared:[a-f0-9]{16}:[a-f0-9]{16}$' || { + echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 + return 1 + } + ;; + pending:*|confirmed:*) corr=${value#*:} printf '%s' "$corr" | grep -Eq '^[a-f0-9]{16}$' || { echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 return 1 } - rec=$(fm_pending_reply_path "$STATE" "$corr") - if [ -e "$rec" ] || [ -L "$rec" ]; then - [ -f "$rec" ] && [ ! -L "$rec" ] \ - && [ "$(fm_pending_reply_get "$rec" task_id)" = "$ID" ] || { - echo "REFUSED: receiver wake correlation for secondmate $ID is unsafe or belongs to another task" >&2 - return 1 - } - fi - confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") - if [ -e "$confirmation" ] || [ -L "$confirmation" ]; then - [ -f "$confirmation" ] && [ ! -L "$confirmation" ] || { - echo "REFUSED: receiver wake delivery state for secondmate $ID is unsafe" >&2 - return 1 - } - fi - HANDOFF_WAKE_RETIRE_CORR=$corr ;; *) echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 return 1 ;; esac + if [ -n "$corr" ]; then + rec=$(fm_pending_reply_path "$STATE" "$corr") + if [ -e "$rec" ] || [ -L "$rec" ]; then + [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$ID" ] || { + echo "REFUSED: receiver wake correlation for secondmate $ID is unsafe or belongs to another task" >&2 + return 1 + } + fi + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") + if [ -e "$confirmation" ] || [ -L "$confirmation" ]; then + [ -f "$confirmation" ] && [ ! -L "$confirmation" ] || { + echo "REFUSED: receiver wake delivery state for secondmate $ID is unsafe" >&2 + return 1 + } + fi + HANDOFF_WAKE_RETIRE_CORR=$corr + fi HANDOFF_WAKE_RETIRE_MARKER=$marker HANDOFF_WAKE_RETIRE_VALUE=$value } @@ -2636,8 +2646,8 @@ if [ "$BACKEND" = herdr ]; then fi if [ "$KIND" = secondmate ]; then [ -n "$HOME_PATH" ] || HOME_PATH=$WT - remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID" || exit $? handoff_wake_retire || { echo "error: receiver wake cleanup failed; preserving the secondmate route for retry" >&2; exit 1; } + remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID" || exit $? remove_secondmate_registry_entry "$ID" fi remove_grok_turnend_auth "$STATE" "$ID" || exit 1 diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 5fd51fe9d59..8ce2195ac8d 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -1176,6 +1176,22 @@ fm_wake_queued_keys_locked() { "$FM_WAKE_QUEUE" 2>/dev/null || true } +fm_wake_secondmate_stall_marker_write() { # + local task=$1 row_key=$2 marker tmp + case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$row_key" in ''|*[!0-9-]*) return 1 ;; esac + marker="$STATE/.secondmate-wake-stall-$task" + if [ -e "$marker" ] || [ -L "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + fi + tmp=$(mktemp "$STATE/.secondmate-wake-stall.XXXXXX") || return 1 + if ! printf '%s\n' "$row_key" > "$tmp" || ! chmod 0600 "$tmp" \ + || ! _fm_atomic_replace "$tmp" "$marker"; then + rm -f -- "$tmp" + return 1 + fi +} + fm_wake_secondmate_stall_receipt_write() { # local task=$1 row_key=$2 root task_dir receipt tmp case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index d417713d2e4..1d883caf8bb 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -363,6 +363,9 @@ EOF [ "$age" -ge "$threshold" ] || continue row_key="$epoch-$seq" receipt="$receipt_dir/$row_key" + if [ -e "$marker" ] || [ -L "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + fi [ "$(cat "$marker" 2>/dev/null || true)" = "$row_key" ] && continue [ "$(cat "$receipt" 2>/dev/null || true)" = "$row_key" ] && continue notify_key="secondmate-wake-loop-$task-$row_key" @@ -372,7 +375,7 @@ EOF fm_wake_append check "$notify_key" "$reason" || return 1 fi fm_wake_secondmate_stall_receipt_write "$task" "$row_key" || return 1 - printf '%s\n' "$row_key" > "$marker" + fm_wake_secondmate_stall_marker_write "$task" "$row_key" || return 1 wake "$reason" done return 0 diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index dc7c465c18a..80cdc2e2d1d 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -331,6 +331,22 @@ SH assert_present "$home/state/.backlog-handoff-design.wake-pending" \ "pre-move crash lost its prepared wake intent" + cat > "$sub/data/backlog.md" <<'EOF' +## Queued +- [ ] unrelated-ready - already durable from another handoff (repo: alpha) + +## Done +EOF + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design unrelated-ready \ + > "$TMP_ROOT/pre-move-unrelated.out" 2>&1 \ + || fail "unrelated already-present handoff failed: $(cat "$TMP_ROOT/pre-move-unrelated.out")" + [ ! -s "$TMP_ROOT/default-tmux.log" ] \ + || fail "unrelated already-present work promoted another batch's prepared wake" + assert_grep 'pre-move-crash' "$home/data/backlog.md" \ + "unrelated handoff changed the prepared batch's source item" + assert_absent "$home/state/.backlog-handoff-design.wake-pending" \ + "unrelated handoff retained another batch's prepared wake" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design pre-move-crash \ > "$TMP_ROOT/pre-move-crash-retry.out" 2>&1 \ || fail "pre-move crash recovery failed: $(cat "$TMP_ROOT/pre-move-crash-retry.out")" @@ -582,6 +598,48 @@ SH pass "local teardown waits for the routed move and receiver wake" } +test_local_teardown_preserves_home_when_wake_retirement_fails() { + local home="$TMP_ROOT/teardown-wake-fail-main" sub="$TMP_ROOT/teardown-wake-fail-sub" + local fakebin rm_bin="$TMP_ROOT/teardown-wake-fail-rm" real_rm corr rc=0 + setup_homes "$home" "$sub" + printf 'project=%s\n' "$ROOT" >> "$home/state/design.meta" + mkdir -p "$sub/data" "$rm_bin" + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + corr=$(FM_HOME="$home" bash -c ' + . "$1" + fm_pending_reply_create "$2" "$2/state" design "New routed work is in your backlog." + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$home") \ + || fail "could not seed teardown wake retirement failure" + printf 'pending:%s\n' "$corr" > "$home/state/.backlog-handoff-design.wake-pending" + real_rm=$(command -v rm) + cat > "$rm_bin/rm" <<'SH' +#!/usr/bin/env bash +for arg in "$@"; do + [ "$arg" != "$FM_FAIL_WAKE_MARKER" ] || exit 1 +done +exec "$FM_REAL_RM" "$@" +SH + chmod +x "$rm_bin/rm" + fakebin=$(make_fake_tmux "$TMP_ROOT/teardown-wake-fail-fake") + + set +e + PATH="$rm_bin:$fakebin:$PATH" FM_REAL_RM="$real_rm" \ + FM_FAIL_WAKE_MARKER="$home/state/.backlog-handoff-design.wake-pending" \ + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-wake-fail-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-wake-fail-fake/pane.txt" \ + "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-wake-fail.out" 2>&1 + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "teardown ignored receiver wake retirement failure" + assert_present "$sub" "teardown removed the home before receiver wake retirement succeeded" + assert_present "$home/state/design.meta" "teardown removed route metadata after wake retirement failure" + assert_grep '- design ' "$home/data/secondmates.md" \ + "teardown removed the registry route after wake retirement failure" + pass "local teardown preserves its route when receiver wake retirement fails" +} + # Exact multi-line block extract: header matching key plus following body lines # (indented lines and blank separators between paragraphs), stopping at the next # item header or unindented section heading (column-0 ##). @@ -1199,6 +1257,7 @@ test_delivery_confirmation_crash_does_not_resend test_unresolved_delivery_attempt_refuses_immediate_resend test_concurrent_local_handoffs_serialize_move_and_wake test_local_teardown_waits_for_handoff_wake +test_local_teardown_preserves_home_when_wake_retirement_fails test_body_moves_when_followed_by_another_item test_body_moves_when_followed_by_section_heading test_multi_paragraph_body_with_internal_blanks_moves_whole diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 1d3b7e34468..2b994fcbe05 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -317,6 +317,43 @@ SH pass "foreign secondmate queue stalls notify once, remain byte-stable, and stay quiet when empty or healthy" } +test_secondmate_stall_marker_rejects_symlink() { + local dir state sub fakebin marker outside expected + dir=$(make_case secondmate-stall-marker-symlink) + state="$dir/state" + sub="$dir/secondmate" + mkdir -p "$sub/state" + printf 'mate\n' > "$sub/.fm-secondmate-home" + printf 'window=firstmate:fm-mate\nkind=secondmate\nhome=%s\n' "$sub" > "$state/mate.meta" + printf '%s\t7\tcheck\trouted\tcheck: routed row\n' "$(( $(date +%s) - 10 ))" > "$sub/state/.wake-queue" + outside="$dir/outside" + expected='must remain unchanged' + printf '%s\n' "$expected" > "$outside" + marker="$state/.secondmate-wake-stall-mate" + ln -s "$outside" "$marker" + fakebin="$dir/fakebin" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +case "${1:-}" in + list-windows) printf '%s\n' 'firstmate:fm-mate' ;; + capture-pane) : ;; + display-message) printf '0\n' ;; + *) exit 0 ;; +esac +SH + chmod +x "$fakebin/tmux" + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 \ + FM_SIGNAL_GRACE=0 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 \ + > "$dir/watch.out" 2> "$dir/watch.err" || true + [ "$(cat "$outside")" = "$expected" ] || fail "stall marker write followed an unsafe symlink" + [ -L "$marker" ] || fail "stall marker write replaced rather than rejected an unsafe path" + [ ! -s "$state/.wake-queue" ] || fail "unsafe stall marker path still published a parent notification" + pass "secondmate stall markers reject symlinks without touching their targets" +} + test_acknowledged_stall_publication_survives_pre_marker_crash() { local dir state sub fakebin out epoch row_before dir=$(make_case secondmate-stall-crash) @@ -959,6 +996,7 @@ test_historical_annotation_skips_announced_status() { test_self_held_lock_reclaims_instead_of_deadlocking test_secondmate_foreign_queue_stall_is_one_shot_and_read_only +test_secondmate_stall_marker_rejects_symlink test_acknowledged_stall_publication_survives_pre_marker_crash test_empty_prefix_mate_preserves_other_mate_receipt test_self_announced_append_guards From ab9a115d17649739ed406a3e34bb6c11835412ed Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 02:23:29 -0700 Subject: [PATCH 18/24] no-mistakes(review): Preserve unresolved prepared wakes across unrelated handoffs --- bin/fm-backlog-handoff.sh | 7 +++++-- tests/fm-backlog-handoff.test.sh | 31 ++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 050b61b76b2..be3acd7409c 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -810,12 +810,15 @@ REQUESTED_BATCH=$(receiver_wake_batch_id "$@") || { } if [ "${#TO_MOVE[@]}" -eq 0 ]; then - echo "nothing to move: ${ALREADY[*]:-no keys} already present in $SUB_BACKLOG" WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" case "$(cat "$WAKE_PENDING_MARKER" 2>/dev/null || true)" in prepared:*:"$REQUESTED_BATCH") receiver_wake_promote_prepared "$ID" "$REQUESTED_BATCH" || exit 1 ;; - prepared:*) receiver_wake_discard_prepared "$ID" || exit 1 ;; + prepared:*) + echo "error: a prepared receiver wake for secondmate $ID belongs to a different routed batch; retry that original handoff before handling ${ALREADY[*]}" >&2 + exit 1 + ;; esac + echo "nothing to move: ${ALREADY[*]:-no keys} already present in $SUB_BACKLOG" wake_pending_secondmate_receiver "$ID" || exit 1 exit 0 fi diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 80cdc2e2d1d..abec09d0c42 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -242,7 +242,7 @@ SH test_move_crash_keeps_wake_pending_for_recovery() { local home="$TMP_ROOT/move-crash-main" sub="$TMP_ROOT/move-crash-sub" - local fakebin="$TMP_ROOT/move-crash-fakebin" real_tasks rc=0 + local fakebin="$TMP_ROOT/move-crash-fakebin" real_tasks rc=0 prepared_state setup_homes "$home" "$sub" mkdir -p "$sub/data" "$fakebin" cat > "$home/data/backlog.md" <<'EOF' @@ -279,6 +279,23 @@ SH assert_grep 'crash-item' "$sub/data/backlog.md" "post-move crash did not leave the item durable" assert_present "$home/state/.backlog-handoff-design.wake-pending" \ "post-move crash lost receiver wake intent" + prepared_state=$(cat "$home/state/.backlog-handoff-design.wake-pending") + cat > "$sub/data/backlog.md" <<'EOF' +## Queued +- [ ] crash-item - survive the post-move crash (repo: alpha) +- [ ] unrelated-ready - already durable from another handoff (repo: alpha) + +## Done +EOF + rc=0 + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design unrelated-ready \ + > "$TMP_ROOT/move-crash-unrelated.out" 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "unrelated handoff discarded a post-move prepared wake" + assert_contains "$(cat "$TMP_ROOT/move-crash-unrelated.out")" \ + 'belongs to a different routed batch' \ + "unrelated handoff did not surface the unresolved prepared batch" + [ "$(cat "$home/state/.backlog-handoff-design.wake-pending")" = "$prepared_state" ] \ + || fail "unrelated handoff changed the post-move prepared wake" : > "$TMP_ROOT/default-tmux.log" FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design crash-item \ @@ -337,15 +354,19 @@ SH ## Done EOF + rc=0 FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design unrelated-ready \ - > "$TMP_ROOT/pre-move-unrelated.out" 2>&1 \ - || fail "unrelated already-present handoff failed: $(cat "$TMP_ROOT/pre-move-unrelated.out")" + > "$TMP_ROOT/pre-move-unrelated.out" 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "unrelated handoff accepted another batch's prepared wake" + assert_contains "$(cat "$TMP_ROOT/pre-move-unrelated.out")" \ + 'belongs to a different routed batch' \ + "unrelated handoff did not report the prepared batch conflict" [ ! -s "$TMP_ROOT/default-tmux.log" ] \ || fail "unrelated already-present work promoted another batch's prepared wake" assert_grep 'pre-move-crash' "$home/data/backlog.md" \ "unrelated handoff changed the prepared batch's source item" - assert_absent "$home/state/.backlog-handoff-design.wake-pending" \ - "unrelated handoff retained another batch's prepared wake" + assert_present "$home/state/.backlog-handoff-design.wake-pending" \ + "unrelated handoff discarded another batch's prepared wake" FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design pre-move-crash \ > "$TMP_ROOT/pre-move-crash-retry.out" 2>&1 \ From f155ae8e51bf1bb6af60de419d1e6bb6fcdafa29 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 02:26:52 -0700 Subject: [PATCH 19/24] no-mistakes(review): Preserve prepared wakes before unrelated moving handoffs --- bin/fm-backlog-handoff.sh | 6 +++++- tests/fm-backlog-handoff.test.sh | 15 +++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index be3acd7409c..7cb6974ccef 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -844,7 +844,11 @@ fi WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" if [ -e "$WAKE_PENDING_MARKER" ] || [ -L "$WAKE_PENDING_MARKER" ]; then case "$(cat "$WAKE_PENDING_MARKER" 2>/dev/null || true)" in - prepared:*) receiver_wake_discard_prepared "$ID" || exit 1 ;; + prepared:*:"$REQUESTED_BATCH") receiver_wake_discard_prepared "$ID" || exit 1 ;; + prepared:*) + echo "error: a prepared receiver wake for secondmate $ID belongs to a different routed batch; retry that original handoff before moving ${TO_MOVE[*]}" >&2 + exit 1 + ;; *) wake_pending_secondmate_receiver "$ID" || { echo "error: previous receiver wake for secondmate $ID is unresolved; nothing new was moved" >&2 diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index abec09d0c42..ced92143cab 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -280,22 +280,25 @@ SH assert_present "$home/state/.backlog-handoff-design.wake-pending" \ "post-move crash lost receiver wake intent" prepared_state=$(cat "$home/state/.backlog-handoff-design.wake-pending") - cat > "$sub/data/backlog.md" <<'EOF' + cat > "$home/data/backlog.md" <<'EOF' ## Queued -- [ ] crash-item - survive the post-move crash (repo: alpha) -- [ ] unrelated-ready - already durable from another handoff (repo: alpha) +- [ ] unrelated-move - still waiting in the main backlog (repo: alpha) ## Done EOF rc=0 - FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design unrelated-ready \ + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design unrelated-move \ > "$TMP_ROOT/move-crash-unrelated.out" 2>&1 || rc=$? - [ "$rc" -ne 0 ] || fail "unrelated handoff discarded a post-move prepared wake" + [ "$rc" -ne 0 ] || fail "unrelated moving handoff discarded a post-move prepared wake" assert_contains "$(cat "$TMP_ROOT/move-crash-unrelated.out")" \ 'belongs to a different routed batch' \ "unrelated handoff did not surface the unresolved prepared batch" [ "$(cat "$home/state/.backlog-handoff-design.wake-pending")" = "$prepared_state" ] \ - || fail "unrelated handoff changed the post-move prepared wake" + || fail "unrelated moving handoff changed the post-move prepared wake" + assert_grep 'unrelated-move' "$home/data/backlog.md" \ + "unrelated moving handoff changed its source item before resolving the older wake" + assert_no_grep 'unrelated-move' "$sub/data/backlog.md" \ + "unrelated moving handoff moved work despite the unresolved older wake" : > "$TMP_ROOT/default-tmux.log" FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design crash-item \ From 4ae17f04dd83859dd7f4204fccc772559f8e6589 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 02:33:48 -0700 Subject: [PATCH 20/24] no-mistakes(document): Document prepared wake batch ownership --- bin/fm-backlog-handoff.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 7cb6974ccef..110e277944f 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -53,7 +53,11 @@ # A present outbox remains the remote retry trigger until backlog receipt and # receiver wake are both confirmed; a companion pending-reply correlation makes # crash recovery reconcile an attempted or confirmed wake instead of blindly -# resending it. No two-phase journal exists. +# resending it. A prepared local wake is bound to the exact sorted +# requested-key batch; an unrelated handoff to that mate refuses until the +# original batch is retried, +# so it cannot discard wake intent for work that already moved. No two-phase +# journal exists. # Every successful backlog delivery also sends one marked wake to the receiving # endpoint. A missing endpoint or a live endpoint that rejects the wake makes the # handoff fail with the delivered backlog intact. From e53661cabe460d2f6eb4e15d803684bde0e08dd6 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 02:53:13 -0700 Subject: [PATCH 21/24] no-mistakes: apply CI fixes --- tests/fm-gotmp.test.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/fm-gotmp.test.sh b/tests/fm-gotmp.test.sh index f247b7ff3dc..248d3f28c43 100755 --- a/tests/fm-gotmp.test.sh +++ b/tests/fm-gotmp.test.sh @@ -57,6 +57,7 @@ make_fake_root() { ln -s "$ROOT/bin/fm-backend.sh" "$fake/bin/fm-backend.sh" ln -s "$ROOT/bin/backends/tmux.sh" "$fake/bin/backends/tmux.sh" ln -s "$ROOT/bin/fm-tmux-lib.sh" "$fake/bin/fm-tmux-lib.sh" + ln -s "$ROOT/bin/fm-cursor-lib.sh" "$fake/bin/fm-cursor-lib.sh" ln -s "$ROOT/bin/fm-composer-lib.sh" "$fake/bin/fm-composer-lib.sh" ln -s "$ROOT/bin/fm-nm-run-lib.sh" "$fake/bin/fm-nm-run-lib.sh" # fm-lock-lib.sh: teardown sources it for the shared lock-staleness proof. @@ -81,6 +82,11 @@ make_fake_root() { ln -s "$ROOT/bin/fm-x-lib.sh" "$fake/bin/fm-x-lib.sh" ln -s "$ROOT/bin/fm-secondmate-registry-lib.sh" "$fake/bin/fm-secondmate-registry-lib.sh" ln -s "$ROOT/bin/fm-secondmate-parent-lib.sh" "$fake/bin/fm-secondmate-parent-lib.sh" + # Receiver-wake retirement sources the pending-reply library, which in turn + # requires the marker helper even for this ordinary-task teardown fixture. + ln -s "$ROOT/bin/fm-pending-reply-lib.sh" "$fake/bin/fm-pending-reply-lib.sh" + ln -s "$ROOT/bin/fm-marker-lib.sh" "$fake/bin/fm-marker-lib.sh" + ln -s "$ROOT/bin/fm-operational-input.sh" "$fake/bin/fm-operational-input.sh" # fm-guard.sh: stub (teardown calls it with `|| true`). cat > "$fake/bin/fm-guard.sh" <<'SH' #!/usr/bin/env bash @@ -141,6 +147,7 @@ test_teardown_skips_gracefully_without_tasktmp() { ln -s "$ROOT/bin/fm-backend.sh" "$fake/bin/fm-backend.sh" ln -s "$ROOT/bin/backends/tmux.sh" "$fake/bin/backends/tmux.sh" ln -s "$ROOT/bin/fm-tmux-lib.sh" "$fake/bin/fm-tmux-lib.sh" + ln -s "$ROOT/bin/fm-cursor-lib.sh" "$fake/bin/fm-cursor-lib.sh" ln -s "$ROOT/bin/fm-composer-lib.sh" "$fake/bin/fm-composer-lib.sh" ln -s "$ROOT/bin/fm-nm-run-lib.sh" "$fake/bin/fm-nm-run-lib.sh" ln -s "$ROOT/bin/fm-lock-lib.sh" "$fake/bin/fm-lock-lib.sh" @@ -162,6 +169,9 @@ test_teardown_skips_gracefully_without_tasktmp() { ln -s "$ROOT/bin/fm-x-lib.sh" "$fake/bin/fm-x-lib.sh" ln -s "$ROOT/bin/fm-secondmate-registry-lib.sh" "$fake/bin/fm-secondmate-registry-lib.sh" ln -s "$ROOT/bin/fm-secondmate-parent-lib.sh" "$fake/bin/fm-secondmate-parent-lib.sh" + ln -s "$ROOT/bin/fm-pending-reply-lib.sh" "$fake/bin/fm-pending-reply-lib.sh" + ln -s "$ROOT/bin/fm-marker-lib.sh" "$fake/bin/fm-marker-lib.sh" + ln -s "$ROOT/bin/fm-operational-input.sh" "$fake/bin/fm-operational-input.sh" cat > "$fake/bin/fm-guard.sh" <<'SH' #!/usr/bin/env bash exit 0 From 92a8dc045d4fd01bbbe6099b9960e3589ea9e211 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 03:12:55 -0700 Subject: [PATCH 22/24] no-mistakes: apply CI fixes --- bin/fm-teardown.sh | 5 ++- tests/fm-backlog-handoff.test.sh | 61 +++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 48ded155996..39b68f101d2 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -2646,8 +2646,11 @@ if [ "$BACKEND" = herdr ]; then fi if [ "$KIND" = secondmate ]; then [ -n "$HOME_PATH" ] || HOME_PATH=$WT - handoff_wake_retire || { echo "error: receiver wake cleanup failed; preserving the secondmate route for retry" >&2; exit 1; } + # Keep the durable wake intact until home retirement succeeds. If home + # removal fails, the registered receiver and its routed backlog still have + # their recovery signal; a teardown retry can remove both in order. remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID" || exit $? + handoff_wake_retire || { echo "error: receiver wake cleanup failed; preserving the secondmate route for retry" >&2; exit 1; } remove_secondmate_registry_entry "$ID" fi remove_grok_turnend_auth "$STATE" "$ID" || exit 1 diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index ced92143cab..e0194495c18 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -622,46 +622,67 @@ SH pass "local teardown waits for the routed move and receiver wake" } -test_local_teardown_preserves_home_when_wake_retirement_fails() { - local home="$TMP_ROOT/teardown-wake-fail-main" sub="$TMP_ROOT/teardown-wake-fail-sub" - local fakebin rm_bin="$TMP_ROOT/teardown-wake-fail-rm" real_rm corr rc=0 +test_local_teardown_preserves_wake_when_home_removal_fails() { + local home="$TMP_ROOT/teardown-home-fail-main" sub="$TMP_ROOT/teardown-home-fail-sub" + local fakebin rm_bin="$TMP_ROOT/teardown-home-fail-rm" real_rm corr rc=0 marker rec fail_home + local marker_before="$TMP_ROOT/teardown-home-fail-marker.before" + local rec_before="$TMP_ROOT/teardown-home-fail-record.before" setup_homes "$home" "$sub" printf 'project=%s\n' "$ROOT" >> "$home/state/design.meta" mkdir -p "$sub/data" "$rm_bin" - printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + printf '## Queued\n- [ ] still-routed - preserve its wake (repo: alpha)\n\n## Done\n' > "$sub/data/backlog.md" corr=$(FM_HOME="$home" bash -c ' . "$1" fm_pending_reply_create "$2" "$2/state" design "New routed work is in your backlog." ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$home") \ - || fail "could not seed teardown wake retirement failure" - printf 'pending:%s\n' "$corr" > "$home/state/.backlog-handoff-design.wake-pending" + || fail "could not seed teardown wake state" + marker="$home/state/.backlog-handoff-design.wake-pending" + rec="$home/state/pending-replies/$corr" + printf 'pending:%s\n' "$corr" > "$marker" + cp -p -- "$marker" "$marker_before" + cp -p -- "$rec" "$rec_before" real_rm=$(command -v rm) + fail_home=$(cd "$sub" && pwd -P) cat > "$rm_bin/rm" <<'SH' #!/usr/bin/env bash for arg in "$@"; do - [ "$arg" != "$FM_FAIL_WAKE_MARKER" ] || exit 1 + [ "$arg" != "$FM_FAIL_HOME" ] || exit 1 done exec "$FM_REAL_RM" "$@" SH chmod +x "$rm_bin/rm" - fakebin=$(make_fake_tmux "$TMP_ROOT/teardown-wake-fail-fake") + fakebin=$(make_fake_tmux "$TMP_ROOT/teardown-home-fail-fake") set +e - PATH="$rm_bin:$fakebin:$PATH" FM_REAL_RM="$real_rm" \ - FM_FAIL_WAKE_MARKER="$home/state/.backlog-handoff-design.wake-pending" \ + PATH="$rm_bin:$fakebin:$PATH" FM_REAL_RM="$real_rm" FM_FAIL_HOME="$fail_home" \ FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ - FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-wake-fail-tmux.log" \ - FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-wake-fail-fake/pane.txt" \ - "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-wake-fail.out" 2>&1 + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-home-fail-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-home-fail-fake/pane.txt" \ + "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-home-fail.out" 2>&1 rc=$? set -e - [ "$rc" -ne 0 ] || fail "teardown ignored receiver wake retirement failure" - assert_present "$sub" "teardown removed the home before receiver wake retirement succeeded" - assert_present "$home/state/design.meta" "teardown removed route metadata after wake retirement failure" - assert_grep '- design ' "$home/data/secondmates.md" \ - "teardown removed the registry route after wake retirement failure" - pass "local teardown preserves its route when receiver wake retirement fails" + [ "$rc" -ne 0 ] || fail "teardown ignored the receiver-home removal failure" + assert_present "$sub" "failed teardown did not preserve the receiver home" + assert_grep 'still-routed' "$sub/data/backlog.md" "failed teardown lost routed backlog work" + cmp -s "$marker_before" "$marker" \ + || fail "failed home removal changed the pending wake marker" + cmp -s "$rec_before" "$rec" \ + || fail "failed home removal changed the pending wake correlation" + assert_present "$home/state/design.meta" "failed teardown removed route metadata" + assert_grep '- design ' "$home/data/secondmates.md" "failed teardown removed the registry route" + + PATH="$fakebin:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-home-fail-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-home-fail-fake/pane.txt" \ + "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-home-retry.out" 2>&1 \ + || fail "teardown retry did not retire the preserved wake: $(cat "$TMP_ROOT/teardown-home-retry.out")" + assert_absent "$sub" "teardown retry left the receiver home" + assert_absent "$marker" "teardown retry left the pending wake marker" + assert_absent "$rec" "teardown retry left the pending wake correlation" + assert_no_grep '- design ' "$home/data/secondmates.md" "teardown retry left the registry route" + pass "failed local home removal preserves its wake and a retry retires both" } # Exact multi-line block extract: header matching key plus following body lines @@ -1281,7 +1302,7 @@ test_delivery_confirmation_crash_does_not_resend test_unresolved_delivery_attempt_refuses_immediate_resend test_concurrent_local_handoffs_serialize_move_and_wake test_local_teardown_waits_for_handoff_wake -test_local_teardown_preserves_home_when_wake_retirement_fails +test_local_teardown_preserves_wake_when_home_removal_fails test_body_moves_when_followed_by_another_item test_body_moves_when_followed_by_section_heading test_multi_paragraph_body_with_internal_blanks_moves_whole From a0428b35b9c3b49a1065a72822b6872eeacfac59 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 03:31:27 -0700 Subject: [PATCH 23/24] no-mistakes(review): Make local wake retirement recoverable --- bin/fm-teardown.sh | 143 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 137 insertions(+), 6 deletions(-) diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 39b68f101d2..d50539d2b7a 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -196,6 +196,10 @@ teardown_release_locks() { fm_lock_release "${DESCENDANT_LOCK_PATHS[$i]}" || true done DESCENDANT_LOCK_PATHS=() + if [ -n "${HANDOFF_WAKE_RETIRE_LOCK:-}" ]; then + fm_lock_release "$HANDOFF_WAKE_RETIRE_LOCK" || true + HANDOFF_WAKE_RETIRE_LOCK= + fi if [ -n "${LOCAL_HANDOFF_LOCK:-}" ]; then fm_lock_release "$LOCAL_HANDOFF_LOCK" || true LOCAL_HANDOFF_LOCK= @@ -245,6 +249,8 @@ LOCAL_REGISTRY_LOCK= HANDOFF_WAKE_RETIRE_MARKER= HANDOFF_WAKE_RETIRE_VALUE= HANDOFF_WAKE_RETIRE_CORR= +HANDOFF_WAKE_RETIRE_LOCK= +HANDOFF_WAKE_RETIRE_STAGE= handoff_wake_retire_validate() { local marker="$STATE/.backlog-handoff-$ID.wake-pending" value corr rec confirmation @@ -324,6 +330,123 @@ handoff_wake_retire() { rm -f -- "$marker" } +handoff_wake_retire_stage_restore() { + local stage=$HANDOFF_WAKE_RETIRE_STAGE marker rec confirmation name destination + [ -n "$stage" ] || return 0 + marker="$STATE/.backlog-handoff-$ID.wake-pending" + rec= + confirmation= + if [ -n "$HANDOFF_WAKE_RETIRE_CORR" ]; then + rec=$(fm_pending_reply_path "$STATE" "$HANDOFF_WAKE_RETIRE_CORR") + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$HANDOFF_WAKE_RETIRE_CORR") + fi + for name in record confirmation marker; do + [ -e "$stage/$name" ] || continue + case "$name" in + record) destination=$rec ;; + confirmation) destination=$confirmation ;; + marker) destination=$marker ;; + esac + [ -n "$destination" ] && [ ! -e "$destination" ] && [ ! -L "$destination" ] \ + && mv -- "$stage/$name" "$destination" || return 1 + done + rm -f -- "$stage/corr" || return 1 + rmdir -- "$stage" || return 1 + if [ -n "$HANDOFF_WAKE_RETIRE_LOCK" ]; then + fm_lock_release "$HANDOFF_WAKE_RETIRE_LOCK" || return 1 + HANDOFF_WAKE_RETIRE_LOCK= + fi + HANDOFF_WAKE_RETIRE_STAGE= +} + +handoff_wake_retire_stage_commit() { + local stage=$HANDOFF_WAKE_RETIRE_STAGE retired + [ -n "$stage" ] || return 0 + retired="$stage.retired.$$" + [ ! -e "$retired" ] && [ ! -L "$retired" ] || return 1 + mv -- "$stage" "$retired" || return 1 + HANDOFF_WAKE_RETIRE_STAGE= + if [ -n "$HANDOFF_WAKE_RETIRE_LOCK" ]; then + fm_lock_release "$HANDOFF_WAKE_RETIRE_LOCK" || return 1 + HANDOFF_WAKE_RETIRE_LOCK= + fi + rm -rf -- "$retired" || echo "warning: retired receiver wake state remains at $retired" >&2 +} + +handoff_wake_retire_stage_recover() { + local home=$1 stage="$STATE/.backlog-handoff-$ID.wake-retiring" corr + [ -e "$stage" ] || [ -L "$stage" ] || return 0 + [ -d "$stage" ] && [ ! -L "$stage" ] || { + echo "REFUSED: receiver wake retirement state for secondmate $ID is unsafe" >&2 + return 1 + } + if [ ! -e "$stage/corr" ] && [ ! -L "$stage/corr" ]; then + rmdir -- "$stage" 2>/dev/null && return 0 + echo "REFUSED: receiver wake retirement state for secondmate $ID is incomplete" >&2 + return 1 + fi + [ -f "$stage/corr" ] && [ ! -L "$stage/corr" ] || { + echo "REFUSED: receiver wake retirement state for secondmate $ID is unsafe" >&2 + return 1 + } + corr=$(cat "$stage/corr" 2>/dev/null || true) + [ -z "$corr" ] || printf '%s' "$corr" | grep -Eq '^[a-f0-9]{16}$' || { + echo "REFUSED: receiver wake retirement correlation for secondmate $ID is invalid" >&2 + return 1 + } + local staged + for staged in "$stage/marker" "$stage/record" "$stage/confirmation"; do + [ ! -e "$staged" ] && [ ! -L "$staged" ] && continue + [ -f "$staged" ] && [ ! -L "$staged" ] || { + echo "REFUSED: receiver wake retirement state for secondmate $ID is unsafe" >&2 + return 1 + } + done + HANDOFF_WAKE_RETIRE_CORR=$corr + HANDOFF_WAKE_RETIRE_STAGE=$stage + if [ -n "$corr" ]; then + HANDOFF_WAKE_RETIRE_LOCK="$STATE/.pending-reply-$corr.lock" + fm_lock_acquire_wait "$HANDOFF_WAKE_RETIRE_LOCK" || return 1 + fi + if [ -e "$home" ] || [ -L "$home" ]; then + handoff_wake_retire_stage_restore + else + handoff_wake_retire_stage_commit + fi +} + +handoff_wake_retire_stage() { + local stage="$STATE/.backlog-handoff-$ID.wake-retiring" marker=$HANDOFF_WAKE_RETIRE_MARKER + local corr=$HANDOFF_WAKE_RETIRE_CORR rec confirmation + [ -n "$marker" ] || return 0 + [ ! -e "$stage" ] && [ ! -L "$stage" ] || return 1 + (umask 077; mkdir -- "$stage") || return 1 + HANDOFF_WAKE_RETIRE_STAGE=$stage + printf '%s\n' "$corr" > "$stage/corr" || { handoff_wake_retire_stage_restore || true; return 1; } + if [ -n "$corr" ]; then + HANDOFF_WAKE_RETIRE_LOCK="$STATE/.pending-reply-$corr.lock" + fm_lock_acquire_wait "$HANDOFF_WAKE_RETIRE_LOCK" || { + HANDOFF_WAKE_RETIRE_LOCK= + handoff_wake_retire_stage_restore || true + return 1 + } + rec=$(fm_pending_reply_path "$STATE" "$corr") + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") + if [ -e "$rec" ] && ! mv -- "$rec" "$stage/record"; then + handoff_wake_retire_stage_restore || true + return 1 + fi + if [ -e "$confirmation" ] && ! mv -- "$confirmation" "$stage/confirmation"; then + handoff_wake_retire_stage_restore || true + return 1 + fi + fi + if ! mv -- "$marker" "$stage/marker"; then + handoff_wake_retire_stage_restore || true + return 1 + fi +} + remote_teardown_locks_release() { if [ -n "$REMOTE_REPLY_LIFECYCLE_LOCK" ]; then fm_lock_release "$REMOTE_REPLY_LIFECYCLE_LOCK" @@ -2384,8 +2507,9 @@ if [ "$KIND" = secondmate ]; then fm_lock_acquire_wait "$LOCAL_REGISTRY_LOCK" || exit 1 LOCAL_HANDOFF_LOCK="$STATE/.backlog-handoff-$ID.lock" fm_lock_acquire_wait "$LOCAL_HANDOFF_LOCK" || exit 1 - handoff_wake_retire_validate || exit 1 [ -n "$HOME_PATH" ] || HOME_PATH=$WT + handoff_wake_retire_stage_recover "$HOME_PATH" || exit 1 + handoff_wake_retire_validate || exit 1 validate_firstmate_home_for_removal "$HOME_PATH" "secondmate home" "$ID" >/dev/null || exit 1 if [ "$FORCE" = "--force" ]; then validate_firstmate_home_children_removal "$HOME_PATH" || exit 1 @@ -2646,11 +2770,18 @@ if [ "$BACKEND" = herdr ]; then fi if [ "$KIND" = secondmate ]; then [ -n "$HOME_PATH" ] || HOME_PATH=$WT - # Keep the durable wake intact until home retirement succeeds. If home - # removal fails, the registered receiver and its routed backlog still have - # their recovery signal; a teardown retry can remove both in order. - remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID" || exit $? - handoff_wake_retire || { echo "error: receiver wake cleanup failed; preserving the secondmate route for retry" >&2; exit 1; } + handoff_wake_retire_stage \ + || { echo "error: receiver wake cleanup could not be staged; preserving the secondmate home and route" >&2; exit 1; } + if remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID"; then + : + else + rc=$? + handoff_wake_retire_stage_restore \ + || echo "error: receiver wake restoration failed; recovery state remains at $HANDOFF_WAKE_RETIRE_STAGE" >&2 + exit "$rc" + fi + handoff_wake_retire_stage_commit \ + || { echo "error: receiver wake cleanup failed; preserving the secondmate route for retry" >&2; exit 1; } remove_secondmate_registry_entry "$ID" fi remove_grok_turnend_auth "$STATE" "$ID" || exit 1 From 8727e97cc4d2031c55adaf578437e8f58919fd21 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Sun, 23 Aug 2026 03:37:18 -0700 Subject: [PATCH 24/24] no-mistakes(document): Clarify handoff recovery and teardown documentation --- bin/fm-backlog-handoff.sh | 11 +++++------ bin/fm-pending-reply-lib.sh | 4 ++++ bin/fm-teardown.sh | 8 ++++++-- docs/configuration.md | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 110e277944f..fa729c9d1b6 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -55,12 +55,11 @@ # crash recovery reconcile an attempted or confirmed wake instead of blindly # resending it. A prepared local wake is bound to the exact sorted # requested-key batch; an unrelated handoff to that mate refuses until the -# original batch is retried, -# so it cannot discard wake intent for work that already moved. No two-phase -# journal exists. -# Every successful backlog delivery also sends one marked wake to the receiving -# endpoint. A missing endpoint or a live endpoint that rejects the wake makes the -# handoff fail with the delivered backlog intact. +# original batch is retried, so it cannot discard wake intent for work that +# already moved. No two-phase journal exists. +# Every newly durable backlog delivery also sends one marked wake to the +# receiving endpoint. A missing endpoint or a live endpoint that rejects the +# wake makes the handoff fail with the delivered backlog intact. # Usage: fm-backlog-handoff.sh ... # fm-backlog-handoff.sh --resume-pending set -eu diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 36f1a774042..b32fff8672c 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -449,6 +449,10 @@ fm_pending_reply_delivery_attempt_unresolved() { # return 1 } +# A definitive backend rejection makes the existing correlation retryable again. +# Reconciliation may have aged the same attempted sidecar to delivery_unknown +# while the backend call was in flight, so both undelivered phases converge here +# under the per-correlation lock; a confirmed delivery can never be reset. fm_pending_reply_reset_known_undelivered() { # local state=$1 corr=$2 lock rc=0 local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index d50539d2b7a..d83ce4d0567 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -50,8 +50,12 @@ # is the approved discard path that prevalidates child removal targets, locks each # descendant home's task set before enumeration, and holds those locks through # child cleanup. Contention refuses the complete forced teardown before child -# mutation. It then discards child work, kills child runtime endpoints, and removes -# the retired home. Removing a leased home releases its durable treehouse lease so the pool slot is freed, +# mutation. Local and remote retirement serialize their destructive phase with +# that mate's backlog-handoff lock under the registry lock. Pending handoff wake +# state is retired with the home, and local removal failure restores that state +# before preserving the route for retry. Teardown then discards child work, kills +# child runtime endpoints, and removes the retired home. Removing a leased home +# releases its durable treehouse lease so the pool slot is freed, # never left leased forever. If the treehouse return fails, teardown leaves the # leased home and state in place instead of hiding a still-held lease. # Usage: fm-teardown.sh [--force] diff --git a/docs/configuration.md b/docs/configuration.md index dce6f73e6a6..0406ce83d02 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -182,7 +182,7 @@ Teardown of a leased home fails closed if `treehouse return` cannot release the Secondmate routes cover `no-mistakes` and `direct-PR` projects; `local-only` projects remain main-firstmate work. For `no-mistakes` projects, seeding initializes only projects newly cloned into a secondmate home and refuses to mutate a preexisting clone that is not already initialized. After creating a secondmate, move existing main-backlog queued items that you have judged in-scope with `fm-backlog-handoff.sh ...`; it refuses In flight, Done, or non-secondmate homes, and a new move succeeds only after waking the recorded receiver. -If the wake fails, the moved item remains durable and rerunning the same handoff retries it idempotently. +If the wake is known to have failed, the moved item remains durable and rerunning the same handoff retries it idempotently; an unresolved delivery is reported and never blindly resent. Set `FM_SECONDMATE_CHARTER` to seed from inline charter text when no filled charter brief exists; set `FM_SECONDMATE_SCOPE` when the routing scope should differ from the charter text. The seeded home's `data/charter.md` owns the standard secondmate lifecycle and escalation contract; the route file points to it through the existing `home:` field instead of adding another pointer. Each seed writes an `.fm-secondmate-home` identity marker at the home root, alongside a durable `.fm-secondmate-parent` record of the home's route to its parent (see "Provision a route" in [`docs/remote-secondmates.md`](remote-secondmates.md)).