From 4e4e4f3867e095aeeccb46b5cbe7c48c8e9c2547 Mon Sep 17 00:00:00 2001 From: JTInventory Date: Thu, 13 Aug 2026 21:21:32 +0000 Subject: [PATCH 001/163] feat: replay inactive terminal crew outcomes --- bin/fm-inactive-reconcile.sh | 374 ++++++++++++++++++++++++++++++ bin/fm-pending-reply-lib.sh | 92 ++++++++ bin/fm-send.sh | 10 + bin/fm-session-start.sh | 9 +- bin/fm-spawn.sh | 7 + bin/fm-wake-drain.sh | 28 ++- bin/fm-watch.sh | 8 + docs/architecture.md | 2 + docs/scripts.md | 1 + tests/fm-inactive-outcome.test.sh | 303 ++++++++++++++++++++++++ 10 files changed, 831 insertions(+), 3 deletions(-) create mode 100755 bin/fm-inactive-reconcile.sh create mode 100755 tests/fm-inactive-outcome.test.sh diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh new file mode 100755 index 00000000000..a2693253dce --- /dev/null +++ b/bin/fm-inactive-reconcile.sh @@ -0,0 +1,374 @@ +#!/usr/bin/env bash +# Replay an authoritative terminal crew state that stayed quiet long enough to +# be missed by the normal watcher. This is a reporting layer only: it never +# closes an endpoint, returns a slot, removes a worktree, or changes a PR. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-worker-isolation-lib.sh +. "$SCRIPT_DIR/fm-worker-isolation-lib.sh" +if [ "${FM_SESSION_LOCK_BOOTSTRAP:-0}" != 1 ]; then + fm_worker_refuse_primary_operation "inactive outcome reconciliation" || exit 1 +fi +# shellcheck source=bin/fm-wake-lib.sh +. "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-pending-reply-lib.sh +. "$SCRIPT_DIR/fm-pending-reply-lib.sh" + +FM_HOME="${FM_HOME:-$FM_ROOT}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +FM_CREW_STATE_BIN="${FM_CREW_STATE_BIN:-$SCRIPT_DIR/fm-crew-state.sh}" +OUTCOME_DIR="$STATE/terminal-outcomes" +SCAN_LOCK="$STATE/.inactive-outcome-reconcile.lock" +SCAN_MARKER="$STATE/.inactive-outcome-reconcile" +SCAN_CURSOR="$STATE/.inactive-outcome-reconcile.cursor" +ROUTE_MARKER="$FM_HOME/.fm-secondmate-home" +CHILD_LOCK_HELD=0 +CHILD_LOCK= + +bounded_secs() { + local value=$1 fallback=$2 minimum=$3 maximum=$4 + case "$value" in ''|*[!0-9]*) value=$fallback ;; esac + [ "$value" -lt "$minimum" ] && value=$minimum + [ "$value" -gt "$maximum" ] && value=$maximum + printf '%s' "$value" +} + +RECONCILE_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_SECS:-900}" 900 60 1800) +SCAN_BUDGET_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_BUDGET_SECS:-10}" 10 1 300) + +meta_value() { # + awk -F= -v wanted="$2" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$1" 2>/dev/null +} + +file_mtime() { + if [ "$(uname -s 2>/dev/null)" = Darwin ]; then + stat -f '%m' "$1" 2>/dev/null + else + stat -c '%Y' "$1" 2>/dev/null + fi +} + +latest_activity() { + local id=$1 meta="$STATE/$1.meta" path m latest=0 + for path in "$meta" "$STATE/$id.status" "$STATE/$id.turn-ended"; do + if [ ! -e "$path" ] || [ -L "$path" ]; then + continue + fi + m=$(file_mtime "$path") || continue + [ "$m" -gt "$latest" ] && latest=$m + done + printf '%s' "$latest" +} + +hash_text() { + if command -v shasum >/dev/null 2>&1; then + printf '%s' "$1" | shasum -a 256 | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$1" | sha256sum | awk '{print $1}' + else + printf '%s' "$1" | cksum | awk '{print $1}' + fi +} + +single_line() { + printf '%s' "$1" | tr '\t\r\n' ' ' +} + +valid_task_id() { + printf '%s' "$1" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$' +} + +is_secondmate_home() { + local marker=$ROUTE_MARKER value + if [ ! -e "$marker" ]; then + return 1 + fi + [ -f "$marker" ] && [ ! -L "$marker" ] || return 2 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + ''|*[!A-Za-z0-9._-]*) return 2 ;; + esac + return 0 +} + +herdr_identity_allowed() { # + local meta=$1 backend session window + backend=$(meta_value "$meta" backend) + [ "$backend" = herdr ] || return 0 + session=$(meta_value "$meta" herdr_session) + window=$(meta_value "$meta" window) + case "$session" in + firstmate) : ;; + default|DEFAULT|CAPTAIN|captain) return 1 ;; + *) return 1 ;; + esac + case "$window" in firstmate:*) return 0 ;; esac + return 1 +} + +queue_contains() { # + local key=$1 + [ -f "$FM_WAKE_QUEUE" ] || return 1 + awk -F '\t' -v wanted="$key" '$4 == wanted { found=1 } END { exit(found ? 0 : 1) }' "$FM_WAKE_QUEUE" 2>/dev/null +} + +receipt_path() { # + printf '%s/%s.%s' "$OUTCOME_DIR" "$1" "$2" +} + +receipt_field() { # + meta_value "$1" "$2" +} + +receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE + local pending tmp + pending=$(receipt_path "$FP" pending) + RECEIPT_CREATED=0 + for suffix in pending presented reported; do + local existing + existing=$(receipt_path "$FP" "$suffix") + [ ! -L "$existing" ] || return 1 + [ -e "$existing" ] || continue + return 0 + done + mkdir -p "$OUTCOME_DIR" || return 1 + tmp=$(mktemp "$OUTCOME_DIR/.receipt.XXXXXX") || return 1 + chmod 600 "$tmp" 2>/dev/null || true + { + printf 'schema=fm-jt-terminal-outcome.v1\n' + printf 'fingerprint=%s\n' "$FP" + printf 'task_id=%s\n' "$ID" + printf 'incarnation=%s\n' "$INC" + printf 'outcome=%s\n' "$OUTCOME" + printf 'terminal_source=%s\n' "$SOURCE" + printf 'terminal_snapshot=%s\n' "$SNAPSHOT" + printf 'kind=%s\n' "$KIND" + printf 'created_epoch=%s\n' "$(date +%s)" + } > "$tmp" || { rm -f "$tmp"; return 1; } + # ln is an exclusive, same-filesystem publication. A concurrent scanner can + # therefore never replace a receipt for another incarnation. + if ln "$tmp" "$pending" 2>/dev/null; then + rm -f "$tmp" + RECEIPT_CREATED=1 + else + rm -f "$tmp" + [ -e "$pending" ] || return 1 + fi + return 0 +} + +read_incarnation() { # + local meta=$1 id=$2 token tasktmp window worktree seed + token=$(meta_value "$meta" spawn_incarnation) + case "$token" in + ''|legacy-unknown|*[!A-Za-z0-9._:-]*) token= ;; + esac + if [ -n "$token" ]; then + printf '%s' "$token" + return 0 + fi + # Legacy metadata has no incarnation token. Prefer the per-task temp root, + # then bind the fallback to the old endpoint/worktree identity. This is only + # a compatibility boundary; a new spawn always writes spawn_incarnation. + tasktmp=$(meta_value "$meta" tasktmp) + window=$(meta_value "$meta" window) + worktree=$(meta_value "$meta" worktree) + if [ -n "$tasktmp" ]; then + seed="legacy|tasktmp=$tasktmp" + else + seed="legacy|window=$window|worktree=$worktree" + fi + printf 'legacy-%s' "$(hash_text "$seed" | cut -c1-32)" +} + +child_cleanup() { + local status=$? + if [ "$CHILD_LOCK_HELD" = 1 ]; then + fm_lock_release "$CHILD_LOCK" || true + fi + exit "$status" +} + +reconcile_child() { + local id=$1 meta="$STATE/$1.meta" kind backend now activity age line outcome source + local snapshot token key route_rc + valid_task_id "$id" || return 0 + [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 + kind=$(meta_value "$meta" kind) + [ "$kind" = secondmate ] && return 0 + case "$kind" in ''|ship|scout) ;; *) return 0 ;; esac + herdr_identity_allowed "$meta" || return 0 + now=$(date +%s) + activity=$(latest_activity "$id") + [ "$activity" -gt 0 ] || return 0 + age=$((now - activity)) + [ "$age" -ge "$RECONCILE_SECS" ] || return 0 + + CHILD_LOCK="$STATE/.spawn-$id.lock" + FM_LOCK_WAIT_SECS=${FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS:-30} + fm_lock_acquire_wait "$CHILD_LOCK" || return 0 + CHILD_LOCK_HELD=1 + trap child_cleanup EXIT INT TERM + # Teardown/relaunch can replace or remove metadata only after the same lock is + # released. Re-read it after acquiring the lock so the snapshot belongs to the + # current incarnation. + [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 + kind=$(meta_value "$meta" kind) + [ "$kind" = secondmate ] && return 0 + herdr_identity_allowed "$meta" || return 0 + FM_CREW_STATE_NM_TIMEOUT=${FM_INACTIVE_OUTCOME_STATE_TIMEOUT_SECS:-10} \ + "$FM_CREW_STATE_BIN" "$id" > "$STATE/.$id.inactive-state.$$" 2>/dev/null || return 0 + line=$(cat "$STATE/.$id.inactive-state.$$" 2>/dev/null || true) + rm -f "$STATE/.$id.inactive-state.$$" + case "$line" in *$'\n'*) return 0 ;; esac + case "$line" in + state:\ done\ *|state:\ failed\ *) ;; + *) return 0 ;; + esac + case "$line" in + *'source: none'*|*'source: status-log'*|*'state: unknown'*|*'occupancy unknown'*) return 0 ;; + esac + outcome=${line#state: } + outcome=${outcome%% *} + case "$outcome" in done|failed) ;; *) return 0 ;; esac + source=$(printf '%s\n' "$line" | sed -n 's/.*source: \([^ ·]*\).*/\1/p') + [ -n "$source" ] && [ "$source" != none ] || return 0 + snapshot=$(single_line "$line") + INC=$(read_incarnation "$meta" "$id") + FP=$(hash_text "$id|$INC|$outcome|$snapshot") + ID=$id + OUTCOME=$outcome + SNAPSHOT=$snapshot + SOURCE=$source + KIND=${kind:-ship} + if is_secondmate_home; then + route_rc=$? + [ "$route_rc" = 0 ] || return 0 + fm_pending_reply_secondmate_route_validate "$FM_HOME" || return 0 + KIND=secondmate + else + [ "$?" = 1 ] || return 0 + KIND=${kind:-ship} + fi + receipt_write || return 1 + key="inactive-outcome:$FP" + if [ "$RECEIPT_CREATED" = 1 ] || [ -f "$(receipt_path "$FP" pending)" ]; then + if ! queue_contains "$key"; then + fm_wake_append check "$key" "inactive terminal outcome: task=$id state=$outcome fingerprint=$FP" || return 1 + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" + fi + fi + return 0 +} + +ack_receipt() { # + local key=$1 fp rec id kind outcome parent_status corr line target + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 0 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + rec=$(receipt_path "$fp" pending) + [ -f "$rec" ] && [ ! -L "$rec" ] || return 0 + [ "$(receipt_field "$rec" fingerprint)" = "$fp" ] || return 2 + id=$(receipt_field "$rec" task_id) + kind=$(receipt_field "$rec" kind) + outcome=$(receipt_field "$rec" outcome) + if [ "$kind" = secondmate ]; then + fm_pending_reply_secondmate_route_validate "$FM_HOME" || return 2 + parent_status=$FM_PENDING_ROUTE_PARENT_STATUS + corr=$FM_PENDING_ROUTE_CORR + line="$outcome [corr=$corr]: inactive terminal outcome replayed: task=$id fingerprint=$fp" + mkdir -p "$(dirname "$parent_status")" || return 2 + if ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then + printf '%s\n' "$line" >> "$parent_status" || return 2 + fi + target=$(receipt_path "$fp" reported) + else + target=$(receipt_path "$fp" presented) + fi + [ ! -L "$target" ] || return 2 + [ ! -e "$target" ] || { rm -f "$rec"; return 0; } + mv "$rec" "$target" || return 2 + return 0 +} + +scan_locked() { + local startup=${1:-0} marker_mtime now age cursor meta id found=0 started=1 + local scan_started remaining rc complete=1 + marker_mtime=$(file_mtime "$SCAN_MARKER" 2>/dev/null || true) + now=$(date +%s) + if [ "$startup" != 1 ] && [ -n "$marker_mtime" ]; then + age=$((now - marker_mtime)) + [ "$age" -ge "$RECONCILE_SECS" ] || return 0 + fi + scan_started=$now + cursor=$(cat "$SCAN_CURSOR" 2>/dev/null || true) + if [ -n "$cursor" ] && { [ ! -f "$STATE/$cursor.meta" ] || [ -L "$STATE/$cursor.meta" ]; }; then + cursor= + fi + if [ -n "$cursor" ]; then started=0; fi + for meta in "$STATE"/*.meta; do + if [ ! -f "$meta" ] || [ -L "$meta" ]; then + continue + fi + id=$(basename "$meta" .meta) + valid_task_id "$id" || continue + if [ "$started" = 0 ]; then + [ "$id" = "$cursor" ] || continue + started=1 + continue + fi + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + if [ "$remaining" -le 0 ]; then + complete=0 + break + fi + printf '%s\n' "$id" > "$SCAN_CURSOR" + rc=0 + FM_LOCK_WAIT_SECS="$remaining" timeout --foreground "$remaining" \ + "$SCRIPT_DIR/fm-inactive-reconcile.sh" _child "$id" || rc=$? + if [ "$rc" -ne 0 ]; then + complete=0 + break + fi + found=1 + done + [ "$complete" = 1 ] && rm -f "$SCAN_CURSOR" + date +%s > "$SCAN_MARKER" + [ "$found" = 1 ] || true + return 0 +} + +scan() { + local startup=${1:-0} + mkdir -p "$STATE" || return 1 + fm_lock_acquire_wait "$SCAN_LOCK" || return 1 + trap 'fm_lock_release "$SCAN_LOCK" || true' EXIT INT TERM + scan_locked "$startup" + rc=$? + fm_lock_release "$SCAN_LOCK" || true + trap - EXIT INT TERM + return "$rc" +} + +case "${1:-}" in + scan) + if [ "${2:-}" = --startup ]; then + scan 1 + else + scan 0 + fi + ;; + ack) + [ -n "${2:-}" ] || exit 2 + ack_receipt "$2" + ;; + _child) + [ -n "${2:-}" ] || exit 2 + reconcile_child "$2" + ;; + *) + echo "usage: fm-inactive-reconcile.sh scan [--startup] | ack " >&2 + exit 2 + ;; +esac diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 230cbd279a6..32b7dcef6df 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -481,6 +481,98 @@ fm_pending_reply_get() { # grep "^${key}=" "$rec" 2>/dev/null | tail -1 | cut -d= -f2- || true } +# JT secondmate outcome routing uses the existing parent-owned pending-reply +# record and its corr= acknowledgement grammar. This small marker only binds a +# secondmate home to that already-owned record; it is not a second parent +# protocol. The marker is intentionally fail-closed when any field is missing, +# malformed, or no longer matches the live parent record. +fm_pending_reply_secondmate_route_path() { # + printf '%s/state/.fm-jt-parent-route' "$1" +} + +fm_pending_reply_secondmate_route_write() { # + local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 + local marker tmp parent_abs state_abs status_path + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + case "$secondmate_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + state_abs=$(cd "$parent_state" 2>/dev/null && pwd -P) || return 1 + status_path="$state_abs/$secondmate_id.status" + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + tmp="$marker.tmp.${BASHPID:-$$}" + { + printf 'schema=fm-jt-parent-route.v1\n' + printf 'secondmate_id=%s\n' "$secondmate_id" + printf 'parent_home=%s\n' "$parent_abs" + printf 'parent_status=%s\n' "$status_path" + printf 'corr_id=%s\n' "$corr" + } > "$tmp" || { rm -f "$tmp"; return 1; } + chmod 600 "$tmp" 2>/dev/null || true + mv -f "$tmp" "$marker" +} + +fm_pending_reply_secondmate_route_validate() { # + local secondmate_home=$1 marker line key value schema marker_id secondmate_id + local parent_home parent_status corr parent_abs state_abs expected_status rec + local phase delivered record_home record_status record_task record_corr home_marker + FM_PENDING_ROUTE_PARENT_STATUS= + FM_PENDING_ROUTE_CORR= + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + schema='' secondmate_id='' parent_home='' parent_status='' corr='' + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + *=*) key=${line%%=*}; value=${line#*=} ;; + *) return 1 ;; + esac + case "$key" in + schema) [ -z "$schema" ] || return 1; schema=$value ;; + secondmate_id) [ -z "$secondmate_id" ] || return 1; secondmate_id=$value ;; + parent_home) [ -z "$parent_home" ] || return 1; parent_home=$value ;; + parent_status) [ -z "$parent_status" ] || return 1; parent_status=$value ;; + corr_id) [ -z "$corr" ] || return 1; corr=$value ;; + *) return 1 ;; + esac + done < "$marker" + [ "$schema" = fm-jt-parent-route.v1 ] || return 1 + home_marker="$secondmate_home/.fm-secondmate-home" + [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 + marker_id=$(cat "$home_marker" 2>/dev/null || true) + case "$marker_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + [ "$secondmate_id" = "$marker_id" ] || return 1 + case "$parent_home" in /*) ;; *) return 1 ;; esac + case "$parent_status" in /*) ;; *) return 1 ;; esac + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + state_abs=$(cd "$parent_abs/state" 2>/dev/null && pwd -P) || return 1 + expected_status="$state_abs/$secondmate_id.status" + [ "$parent_status" = "$expected_status" ] || return 1 + rec=$(fm_pending_reply_active_path "$state_abs" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + record_task=$(fm_pending_reply_get "$rec" task_id) + record_home=$(fm_pending_reply_get "$rec" parent_home) + record_status=$(fm_pending_reply_get "$rec" parent_status) + record_corr=$(fm_pending_reply_get "$rec" corr_id) + [ "$record_task" = "$secondmate_id" ] || return 1 + [ "$record_home" = "$parent_abs" ] || return 1 + [ "$record_status" = "$expected_status" ] || return 1 + [ "$record_corr" = "$corr" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -n "$delivered" ] || return 1 + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in + awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated) ;; + *) return 1 ;; + esac + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_PARENT_STATUS=$expected_status + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_CORR=$corr + return 0 +} + fm_pending_reply_corr_reusable() { # local state=$1 corr=$2 task_id=$3 rec phase printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 5e885f9937e..6d6ae66875d 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -88,6 +88,7 @@ MARK_FROM_FIRSTMATE=0 PENDING_REPLY_CORR= PENDING_REPLY_CREATED=0 TARGET_TASK_ID= +TARGET_HOME= case "$RAW_TARGET" in fm-*) meta="$STATE/${RAW_TARGET#fm-}.meta" @@ -140,6 +141,15 @@ else echo "error: failed to durably prepare pending-reply delivery for $TARGET_TASK_ID" >&2 exit 1 fi + TARGET_HOME=$(fm_meta_get "$meta" home) + if ! fm_pending_reply_secondmate_route_write \ + "$TARGET_HOME" "$FM_HOME" "$STATE" "$TARGET_TASK_ID" "$PENDING_REPLY_CORR"; then + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + fi + echo "error: failed to bind the secondmate pending-reply route for $TARGET_TASK_ID" >&2 + exit 1 + fi fi # Slash commands open a completion popup in some TUIs (verified on codex); # submitting too fast selects nothing, so give the popup time to settle before diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index b381c5c1c8d..a0ba17fdee2 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -362,7 +362,14 @@ if [ "$READ_ONLY" -eq 1 ]; then GUARD_OUT=$(FM_GUARD_READ_ONLY=1 "$SCRIPT_DIR/fm-guard.sh" 2>&1) [ -n "$GUARD_OUT" ] && printf '%s\n' "$GUARD_OUT" else - DRAIN_OUT=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) + # Reconcile quiet terminal children while this session owns the fleet lock; + # the following drain then presents and acknowledges the resulting durable + # receipts in the same turn. + INACTIVE_OUT=$( + FM_SESSION_LOCK_BOOTSTRAP=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" scan --startup 2>&1 || true + ) + [ -z "$INACTIVE_OUT" ] || printf '%s\n' "$INACTIVE_OUT" + DRAIN_OUT=$(FM_SESSION_LOCK_BOOTSTRAP=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) if [ -n "$DRAIN_OUT" ]; then printf '%s\n' "$DRAIN_OUT" else diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index a7eab45715c..b072d0c9738 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -197,6 +197,7 @@ HERDR_FLAT_ABORT_LABEL= HERDR_FLAT_ABORT_UNCERTAINTY_FILE= SPAWN_TASK_LOCK= SPAWN_TASK_LOCK_HELD=0 +SPAWN_INCARNATION= SPAWN_ENDPOINT_CREATED=0 SPAWN_WORKTREE_LEASED=0 SPAWN_WORKTREE_PROVEN=0 @@ -316,6 +317,7 @@ spawn_abort_recovery_meta() { echo "kind=${KIND:-ship}" echo "mode=${MODE:-no-mistakes}" echo "yolo=${YOLO:-off}" + echo "spawn_incarnation=${SPAWN_INCARNATION:-legacy-unknown}" echo "tasktmp=${TASK_TMP:-}" echo "model=${MODEL:-default}" echo "effort=${EFFORT:-default}" @@ -376,6 +378,7 @@ spawn_endpoint_recovery_meta() { printf 'kind=%s\n' "${KIND:-ship}" printf 'mode=%s\n' "${MODE:-no-mistakes}" printf 'yolo=%s\n' "${YOLO:-off}" + printf 'spawn_incarnation=%s\n' "${SPAWN_INCARNATION:-legacy-unknown}" printf 'backend=tmux\n' printf 'endpoint_recovery=1\n' printf 'spawn_state=aborted\n' @@ -403,6 +406,7 @@ spawn_endpoint_recovery_reservation() { printf 'kind=%s\n' "${KIND:-ship}" printf 'mode=%s\n' "${MODE:-no-mistakes}" printf 'yolo=%s\n' "${YOLO:-off}" + printf 'spawn_incarnation=%s\n' "${SPAWN_INCARNATION:-legacy-unknown}" printf 'backend=tmux\n' printf 'endpoint_recovery=1\n' printf 'endpoint_recovery_pending=1\n' @@ -877,6 +881,7 @@ spawn_abort_cleanup() { echo "kind=$KIND" echo "mode=${MODE:-no-mistakes}" echo "yolo=${YOLO:-off}" + echo "spawn_incarnation=${SPAWN_INCARNATION:-legacy-unknown}" echo "tasktmp=${TASK_TMP:-}" echo "model=${MODEL:-default}" echo "effort=${EFFORT:-default}" @@ -1044,6 +1049,7 @@ if ! fm_lock_try_acquire "$SPAWN_TASK_LOCK"; then exit 1 fi SPAWN_TASK_LOCK_HELD=1 +SPAWN_INCARNATION="s$(date +%s)-${BASHPID:-$$}-$RANDOM" HERDR_FLAT_ABORT_UNCERTAINTY_FILE="$STATE/$ID.herdr-cleanup-uncertain" if [ -e "$HERDR_FLAT_ABORT_UNCERTAINTY_FILE" ] || [ -L "$HERDR_FLAT_ABORT_UNCERTAINTY_FILE" ]; then echo "error: unresolved Herdr cleanup uncertainty for $ID at $HERDR_FLAT_ABORT_UNCERTAINTY_FILE; refusing another spawn" >&2 @@ -2120,6 +2126,7 @@ chmod 600 "$META_TMP" || { rm -f "$META_TMP"; exit 1; } echo "kind=$KIND" echo "mode=$MODE" echo "yolo=$YOLO" + echo "spawn_incarnation=$SPAWN_INCARNATION" echo "tasktmp=$TASK_TMP" echo "model=${MODEL:-default}" echo "effort=${EFFORT:-default}" diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index c5a51cfa8d2..8605af26f05 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -5,11 +5,14 @@ set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=bin/fm-worker-isolation-lib.sh . "$SCRIPT_DIR/fm-worker-isolation-lib.sh" -fm_worker_refuse_primary_operation "wake drain" || exit 1 +if [ "${FM_SESSION_LOCK_BOOTSTRAP:-0}" != 1 ]; then + fm_worker_refuse_primary_operation "wake drain" || exit 1 +fi # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" DRAIN_TMP= +DRAIN_DEDUPED= DRAIN_LOCK_HELD=false # Defense in depth for the watcher re-arm chain: this script runs at the top of @@ -33,6 +36,7 @@ cleanup() { if [ "$status" -ne 0 ] && [ "$DRAIN_LOCK_HELD" = true ] && [ -n "$DRAIN_TMP" ] && [ -e "$DRAIN_TMP" ]; then fm_wake_restore_queue "$DRAIN_TMP" || true fi + [ -z "$DRAIN_DEDUPED" ] || rm -f "$DRAIN_DEDUPED" || true if [ "$DRAIN_LOCK_HELD" = true ]; then fm_lock_release "$FM_WAKE_QUEUE_LOCK" fi @@ -56,12 +60,32 @@ if [ ! -s "$FM_WAKE_QUEUE" ]; then fi DRAIN_TMP="$STATE/.wake-queue.drain.$(fm_current_pid)" +DRAIN_DEDUPED="$STATE/.wake-queue.deduped.$(fm_current_pid)" rm -f "$DRAIN_TMP" +rm -f "$DRAIN_DEDUPED" mv "$FM_WAKE_QUEUE" "$DRAIN_TMP" || exit 1 : > "$FM_WAKE_QUEUE" || exit 1 -fm_wake_print_deduped "$DRAIN_TMP" || exit "$?" +fm_wake_print_deduped "$DRAIN_TMP" > "$DRAIN_DEDUPED" || exit "$?" +cat "$DRAIN_DEDUPED" +# Inactive-outcome rows are acknowledged only after their matching durable +# receipt is presented. The locked session-start/watcher context authorizes the +# helper; a failed correlation leaves the original drained rows restorable. +while IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload; do + case "$_key" in + inactive-outcome:*) + "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" || { + ack_status=$? + # 1 means the receipt was already acknowledged or is not ours. Any + # other failure keeps the drained row durable for a later turn. + [ "$ack_status" = 1 ] || exit "$ack_status" + } + ;; + esac +done < "$DRAIN_DEDUPED" rm -f "$DRAIN_TMP" DRAIN_TMP= +rm -f "$DRAIN_DEDUPED" +DRAIN_DEDUPED= assert_watcher_liveness exit 0 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 6251a5a2724..fc648680d71 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -589,6 +589,14 @@ while :; do # No conversation scraping; unresolved records are never silently expired. fm_pending_reply_tick "$STATE" || true + # The helper owns its bounded cadence and receipt idempotence. A non-empty + # result means it appended an inactive-outcome wake, so surface that wake in + # this watcher turn without probing panes or scraping secondmate chat here. + inactive_out=$(FM_SESSION_LOCK_BOOTSTRAP=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" scan 2>/dev/null || true) + if [ -n "$inactive_out" ]; then + wake "check: inactive terminal outcome replay queued" + fi + # Slow per-task checks (firstmate writes these, e.g. a merged-PR poll). # Time-based via .last-check mtime so the cadence survives watcher restarts. # Evaluated BEFORE the signal scan: wake() exits the cycle, so a check placed diff --git a/docs/architecture.md b/docs/architecture.md index b978e5815d7..41330e9c135 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,6 +24,7 @@ Its `--restart` mode signals only the watcher recorded in the current home's `st A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, or if tasks are in flight and that watcher stops running or queued wakes are waiting to be drained. The drain script calls that guard after emptying the queue, which avoids repeating the queued-wakes warning for records it just consumed while still warning on stale watcher liveness. It leads with prominent bordered banners for the tangle and no-watcher cases so they cannot be skimmed past. +The locked session-start path and watcher cadence also run `fm-inactive-reconcile.sh`. It asks `fm-crew-state.sh` for the canonical current state of quiet direct children and publishes one atomic receipt plus one `check` wake for an authoritative `done` or `failed` result. Draining the matching `inactive-outcome:` wake acknowledges the receipt; this layer never closes panes, returns slots, removes worktrees, or changes PRs. New metadata carries a `spawn_incarnation` created under `.spawn-.lock`; legacy metadata without it falls back to a hash of `tasktmp`, or the recorded window and worktree, so relaunches remain distinguishable when possible. A presence-gated sub-supervisor (`bin/fm-supervise-daemon.sh`) extends this for walk-away supervision: the `/afk` skill activates it, after which the watcher reverts to daemon-managed one-shot mode and the daemon self-handles routine wakes in bash. The watcher and daemon share `bin/fm-classify-lib.sh`, so captain-relevant status verbs and signal, stale, and heartbeat-scan classification stay consistent in both modes. @@ -61,6 +62,7 @@ Seeding is transactional: if validation, cloning, initialization, or registry up The same project may appear in multiple secondmate homes when their scopes differ, such as issue triage versus feature development. Secondmates are idle by default: after startup recovery reconciles only work already in their own home, an empty queue waits silently for routed tasks, and they never self-initiate surveys or audits. Bare `fm-send.sh fm-` requests to a live `kind=secondmate` are prefixed with the from-firstmate marker from `bin/fm-marker-lib.sh`, so the secondmate returns terse answers through status lines and detailed answers through docs plus status pointers instead of replying only in its own chat. +Secondmate inactive-outcome receipts use the existing parent-owned pending-reply record and `corr=` status correlation through a validated `state/.fm-jt-parent-route` marker. Missing or malformed routes fail closed; the parent watcher remains the only status owner, and no secondmate conversation is scraped. Explicit `session:window` sends and direct human typing stay unmarked, so captain intervention in a secondmate pane remains conversational. After seeding a secondmate, `fm-backlog-handoff.sh` moves already-judged in-scope queued items from the main backlog into that secondmate home so the domain queue starts in the right place. Idle secondmate panes are healthy; teardown is explicit and refuses while the secondmate home has in-flight work unless the captain has approved discard with `--force`. diff --git a/docs/scripts.md b/docs/scripts.md index 30f78c80aed..6eb46816b8e 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -26,6 +26,7 @@ Each file also starts with a short header comment. | `fm-supervise-daemon.sh` | Presence-gated sub-supervisor for walk-away (`/afk`) supervision: wraps `fm-watch.sh`, uses the shared wake classifier, self-handles routine wakes in bash, and escalates only captain-relevant events as one verified, batched, single-line digest prefixed with a sentinel marker | | `fm-crew-state.sh` | Print one stable current-state line for a crew by reconciling its matching no-mistakes run-step, even when the pane has closed, with pane and status-log fallback | | `fm-numeric-lib.sh` | Shared bounded nonnegative-integer parser with decimal leading-zero handling, sourced by `fm-crew-state.sh` | +| `fm-inactive-reconcile.sh` | Scan this home's direct ordinary children on a bounded cadence, publish fingerprinted inactive terminal receipts, queue one replay wake, and acknowledge it during wake drain | | `fm-tangle-lib.sh` | Shared default-branch resolution and primary-checkout tangle classification sourced by bootstrap and guard | | `fm-ff-lib.sh` | Shared guarded fast-forward helper for `/updatefirstmate` origin pulls and no-fetch local secondmate syncs | | `fm-tasks-axi-lib.sh` | Shared `tasks-axi` compatibility probe sourced by bootstrap and teardown | diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh new file mode 100755 index 00000000000..888c917109c --- /dev/null +++ b/tests/fm-inactive-outcome.test.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Focused behavior tests for inactive terminal-outcome replay. +set -u + +# A crewmate's inherited environment is deliberately refused by the production +# helper. Re-exec the behavior suite once with a clean parent ancestry so its +# throwaway firstmate homes exercise the primary-only path. +if [ "${FM_INACTIVE_TEST_CLEAN:-0}" != 1 ]; then + exec env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_PRIMARY_ATTESTATION FM_INACTIVE_TEST_CLEAN=1 bash "$0" "$@" +fi + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RECON="$ROOT/bin/fm-inactive-reconcile.sh" +DRAIN="$ROOT/bin/fm-wake-drain.sh" +TMP_ROOT=$(fm_test_tmproot fm-inactive-outcome) +trap fm_test_cleanup EXIT +CASE_DIR= +CASE_ROOT= +CASE_HOME= +CASE_FAKEBIN= + +new_case() { + local name=$1 dir root home state fakebin + dir="$TMP_ROOT/$name" + root="$dir/root" + home="$dir/home" + state="$home/state" + fakebin="$dir/fakebin" + mkdir -p "$state" "$home/data" "$home/config" "$fakebin" + git init -q -b main "$root" + git -C "$root" commit -q --allow-empty -m init + cat > "$fakebin/fm-crew-state.sh" <<'SH' +#!/usr/bin/env bash +set -u +id=${1:-} +key=$(printf '%s' "$id" | tr -c 'A-Za-z0-9' '_' | tr '[:lower:]' '[:upper:]') +var="FM_FAKE_CREW_STATE_$key" +printf '%s\n' "${!var:-${FM_FAKE_CREW_STATE:-state: unknown · source: none · fake default}}" +SH + chmod +x "$fakebin/fm-crew-state.sh" + for tool in gh gh-axi curl; do + cat > "$fakebin/$tool" <<'SH' +#!/usr/bin/env bash +set -u +[ -z "${FM_NETWORK_LOG:-}" ] || printf '%s %s\n' "$(basename "$0")" "$*" >> "$FM_NETWORK_LOG" +exit 97 +SH + chmod +x "$fakebin/$tool" + done + CASE_DIR=$dir + CASE_ROOT=$root + CASE_HOME=$home + CASE_FAKEBIN=$fakebin +} + +scan() { + local root=$1 home=$2 fakebin=$3 startup=${4:-} + env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME -u FM_PRIMARY_ATTESTATION \ + PATH="$fakebin:$PATH" \ + FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ + FM_SESSION_LOCK_BOOTSTRAP=1 \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + "$RECON" scan "$startup" +} + +drain() { + local root=$1 home=$2 fakebin=$3 + env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME -u FM_PRIMARY_ATTESTATION \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_SESSION_LOCK_BOOTSTRAP=1 \ + FM_STATE_OVERRIDE="$home/state" "$DRAIN" +} + +write_meta() { + local state=$1 id=$2 token=$3 kind=${4:-ship} backend=${5:-tmux} window + window=${6:-tmux:fm-$id} + local file="$state/$id.meta" + fm_write_meta "$file" \ + "window=$window" "worktree=$state/work-$id" "project=$state/work-$id" \ + "harness=echo" "kind=$kind" "mode=$kind" "yolo=off" \ + "backend=$backend" "spawn_incarnation=$token" + mkdir -p "$state/work-$id" + printf 'working: fixture\n' > "$state/$id.status" + : > "$state/$id.turn-ended" + touch -d '2 minutes ago' "$file" "$state/$id.status" "$state/$id.turn-ended" +} + +receipt_count() { + local state=$1 suffix=$2 + find "$state/terminal-outcomes" -maxdepth 1 -type f -name "*.$suffix" 2>/dev/null | wc -l | tr -d ' ' +} + +queue_count() { + local state=$1 + [ -f "$state/.wake-queue" ] || { printf '0'; return 0; } + awk -F '\t' '$3 == "check" && $4 ~ /^inactive-outcome:/ { n++ } END { print n + 0 }' \ + "$state/.wake-queue" 2>/dev/null +} + +test_done_and_failed_are_replayed_once() { + local dir root home fakebin state + new_case done-failed + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" done-x1 inc-done + write_meta "$state" failed-x1 inc-failed + export FM_FAKE_CREW_STATE_DONE_X1='state: done · source: pane · pane is quiet' + export FM_FAKE_CREW_STATE_FAILED_X1='state: failed · source: run-step · checks failed' + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 2 ] || fail "done and failed outcomes did not create two pending receipts" + [ "$(queue_count "$state")" = 2 ] || fail "done and failed outcomes did not create two wakes" + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 2 ] || fail "rescan duplicated inactive outcome receipts" + [ "$(queue_count "$state")" = 2 ] || fail "rescan duplicated inactive outcome wakes" + drain "$root" "$home" "$fakebin" >/dev/null + [ "$(receipt_count "$state" pending)" = 0 ] || fail "drain did not acknowledge pending receipts" + [ "$(receipt_count "$state" presented)" = 2 ] || fail "drain did not preserve two presented receipts" + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" presented)" = 2 ] || fail "presented receipts were replayed" + unset FM_FAKE_CREW_STATE_DONE_X1 FM_FAKE_CREW_STATE_FAILED_X1 + pass "done and failed inactive outcomes are replayed once and acknowledged on drain" +} + +test_reused_task_id_gets_new_fingerprint() { + local dir root home fakebin state + new_case reused-id + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" reused-x1 incarnation-old + export FM_FAKE_CREW_STATE_REUSED_X1='state: done · source: pane · first run quiet' + scan "$root" "$home" "$fakebin" --startup >/dev/null + sed -i 's/^spawn_incarnation=.*/spawn_incarnation=incarnation-new/' "$state/reused-x1.meta" + touch -d '2 minutes ago' "$state/reused-x1.meta" + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 2 ] || fail "reused task id did not create a new incarnation receipt" + [ "$(queue_count "$state")" = 2 ] || fail "reused task id did not create a new fingerprinted wake" + unset FM_FAKE_CREW_STATE_REUSED_X1 + pass "reused task ids are separated by the spawn incarnation" +} + +test_relaunch_and_teardown_races_recheck_under_spawn_lock() { + local dir root home fakebin state holder scanner ready release + new_case races + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" relaunch-x1 old-inc + export FM_FAKE_CREW_STATE_RELAUNCH_X1='state: done · source: pane · relaunch race' + ready="$dir/ready" + release="$dir/release" + ( + FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ + bash -c '. "$1/bin/fm-wake-lib.sh"; fm_lock_acquire_wait "$2/.spawn-relaunch-x1.lock"; : > "$3"; while [ ! -e "$4" ]; do sleep 0.01; done; fm_lock_release "$2/.spawn-relaunch-x1.lock"' _ "$ROOT" "$state" "$ready" "$release" + ) & + holder=$! + for _ in $(seq 1 100); do [ -e "$ready" ] && break; sleep 0.01; done + scan "$root" "$home" "$fakebin" --startup >"$dir/relaunch.scan.out" 2>&1 & + scanner=$! + sleep 1 + sed -i 's/^spawn_incarnation=.*/spawn_incarnation=new-inc/' "$state/relaunch-x1.meta" + : > "$release" + wait "$holder" || fail "spawn-lock relaunch fixture failed" + wait "$scanner" || fail "relaunch reconciliation fixture failed" + grep -F 'incarnation=new-inc' "$state"/terminal-outcomes/*.pending >/dev/null || fail "relaunch race used stale incarnation" + + write_meta "$state" teardown-x1 teardown-inc + export FM_FAKE_CREW_STATE_TEARDOWN_X1='state: done · source: pane · teardown race' + ready="$dir/ready-teardown" + release="$dir/release-teardown" + ( + FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ + bash -c '. "$1/bin/fm-wake-lib.sh"; fm_lock_acquire_wait "$2/.spawn-teardown-x1.lock"; : > "$3"; while [ ! -e "$4" ]; do sleep 0.01; done; fm_lock_release "$2/.spawn-teardown-x1.lock"' _ "$ROOT" "$state" "$ready" "$release" + ) & + holder=$! + for _ in $(seq 1 100); do [ -e "$ready" ] && break; sleep 0.01; done + scan "$root" "$home" "$fakebin" --startup >"$dir/teardown.scan.out" 2>&1 & + scanner=$! + sleep 1 + rm -f "$state/teardown-x1.meta" "$state/teardown-x1.status" "$state/teardown-x1.turn-ended" + : > "$release" + wait "$holder" || fail "spawn-lock teardown fixture failed" + wait "$scanner" || fail "teardown reconciliation fixture failed" + [ "$(find "$state/terminal-outcomes" -type f -name '*teardown-x1*.pending' 2>/dev/null | wc -l | tr -d ' ')" = 0 ] || fail "teardown race created a receipt after meta removal" + unset FM_FAKE_CREW_STATE_RELAUNCH_X1 FM_FAKE_CREW_STATE_TEARDOWN_X1 + pass "relaunch and teardown races recheck metadata under the spawn lock" +} + +test_herdr_identity_and_default_captain_refusal() { + local dir root home fakebin state + new_case herdr-identity + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" herdr-good good-inc ship herdr firstmate:pane + printf 'herdr_session=firstmate\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-good.meta" + write_meta "$state" herdr-default default-inc ship herdr default:pane + printf 'herdr_session=default\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-default.meta" + write_meta "$state" herdr-captain captain-inc ship herdr CAPTAIN:pane + printf 'herdr_session=CAPTAIN\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-captain.meta" + touch -d '2 minutes ago' "$state/herdr-good.meta" "$state/herdr-default.meta" "$state/herdr-captain.meta" + export FM_FAKE_CREW_STATE_HERDR_GOOD='state: done · source: pane · dedicated session quiet' + export FM_FAKE_CREW_STATE_HERDR_DEFAULT='state: done · source: pane · must refuse' + export FM_FAKE_CREW_STATE_HERDR_CAPTAIN='state: done · source: pane · must refuse' + export FM_NETWORK_LOG="$dir/network.log" + PATH="$fakebin:$PATH" scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 1 ] || fail "Herdr dedicated session was not accepted while default/CAPTAIN were refused" + [ ! -s "$dir/network.log" ] || fail "inactive reconciliation made a forge/network call" + unset FM_FAKE_CREW_STATE_HERDR_GOOD FM_FAKE_CREW_STATE_HERDR_DEFAULT FM_FAKE_CREW_STATE_HERDR_CAPTAIN FM_NETWORK_LOG + pass "Herdr uses the dedicated firstmate identity and refuses default/CAPTAIN" +} + +test_occupancy_unknown_is_not_terminal() { + local dir root home fakebin state + new_case occupancy-unknown + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" unknown-x1 unknown-inc + export FM_FAKE_CREW_STATE_UNKNOWN_X1='state: unknown · source: none · occupancy unknown' + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 0 ] || fail "occupancy unknown was treated as terminal" + [ "$(queue_count "$state")" = 0 ] || fail "occupancy unknown created an actionable wake" + unset FM_FAKE_CREW_STATE_UNKNOWN_X1 + pass "occupancy unknown remains non-terminal" +} + +test_status_log_terminal_is_not_replayed() { + local dir root home fakebin state + new_case stale-status-log + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" stale-x1 stale-inc + export FM_FAKE_CREW_STATE_STALE_X1='state: done · source: status-log · stale done event' + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 0 ] || fail "stale status-log done was treated as terminal" + [ "$(queue_count "$state")" = 0 ] || fail "stale status-log done created an actionable wake" + unset FM_FAKE_CREW_STATE_STALE_X1 + pass "status-log terminal output remains fail-closed" +} + +test_valid_secondmate_route_reports_parent_once() { + local dir root home fakebin state child_home child_state parent_status corr rec + new_case secondmate-route-valid + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$state/pending-replies" + printf 'sm-valid\n' > "$child_home/.fm-secondmate-home" + write_meta "$child_state" child-x1 child-inc + corr=0123456789abcdef + parent_status="$state/sm-valid.status" + rec="$state/pending-replies/$corr" + fm_write_meta "$rec" \ + schema=fm-pending-reply.v1 corr_id="$corr" task_id=sm-valid \ + parent_home="$home" parent_status="$parent_status" delivered_epoch=1 phase=awaiting_report + export FM_FAKE_CREW_STATE_CHILD_X1='state: failed · source: pane · child quiet' + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-valid\nparent_home=%s\nparent_status=%s\ncorr_id=%s\n' \ + "$home" "$parent_status" "$corr" > "$child_state/.fm-jt-parent-route" + scan "$root" "$child_home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "valid secondmate route did not create a pending receipt" + drain "$root" "$child_home" "$fakebin" >/dev/null + [ "$(receipt_count "$child_state" reported)" = 1 ] || fail "valid secondmate route was not reported" + grep -F "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status" >/dev/null \ + || fail "valid secondmate route did not append the correlated parent status" + drain "$root" "$child_home" "$fakebin" >/dev/null + [ "$(grep -Fc "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status")" = 1 ] \ + || fail "secondmate parent report was duplicated" + unset FM_FAKE_CREW_STATE_CHILD_X1 + pass "valid secondmate outcomes use the parent status correlation exactly once" +} + +test_malformed_or_missing_secondmate_route_fails_closed() { + local dir root home fakebin state child_home child_state parent_status + new_case secondmate-route + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + child_home="$dir/secondmate-home" + child_state="$child_home/state" + mkdir -p "$child_state" "$child_home/data" "$child_home/config" + printf 'sm-x1\n' > "$child_home/.fm-secondmate-home" + write_meta "$child_state" child-x1 child-inc + export FM_FAKE_CREW_STATE_CHILD_X1='state: failed · source: pane · child quiet' + scan "$root" "$child_home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "missing secondmate parent route was not fail-closed" + parent_status="$home/state/parent-x1.status" + printf 'schema=fm-jt-parent-route.v1\nparent_home=%s\nparent_status=%s\ninvalid=\n' "$home" "$parent_status" > "$child_state/.fm-jt-parent-route" + scan "$root" "$child_home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "malformed secondmate parent route was not fail-closed" + [ ! -e "$parent_status" ] || fail "malformed secondmate route wrote parent status" + unset FM_FAKE_CREW_STATE_CHILD_X1 + pass "malformed and missing secondmate parent routes fail closed without chat scraping" +} + +test_done_and_failed_are_replayed_once +test_reused_task_id_gets_new_fingerprint +test_relaunch_and_teardown_races_recheck_under_spawn_lock +test_herdr_identity_and_default_captain_refusal +test_occupancy_unknown_is_not_terminal +test_status_log_terminal_is_not_replayed +test_valid_secondmate_route_reports_parent_once +test_malformed_or_missing_secondmate_route_fails_closed From c7ce8e1cbc97d25cbc7d29f1491450dbfad93f6f Mon Sep 17 00:00:00 2001 From: tests Date: Thu, 13 Aug 2026 21:47:41 +0000 Subject: [PATCH 002/163] no-mistakes(review): Fixed inactive routing, rollback, and timeout portability --- bin/fm-inactive-reconcile.sh | 18 +++++- bin/fm-pending-reply-lib.sh | 94 ++++++++++++++++++++++++++++++- bin/fm-send.sh | 15 +++++ bin/fm-wake-drain.sh | 22 ++++++-- tests/fm-inactive-outcome.test.sh | 88 +++++++++++++++++++++++++++++ 5 files changed, 230 insertions(+), 7 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index a2693253dce..c7df7ec7ae0 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -113,6 +113,20 @@ queue_contains() { # awk -F '\t' -v wanted="$key" '$4 == wanted { found=1 } END { exit(found ? 0 : 1) }' "$FM_WAKE_QUEUE" 2>/dev/null } +run_bounded_child() { # [args...] + local seconds=$1 + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + elif command -v perl >/dev/null 2>&1; then + perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$seconds" "$@" + else + return 125 + fi +} + receipt_path() { # printf '%s/%s.%s' "$OUTCOME_DIR" "$1" "$2" } @@ -288,6 +302,8 @@ ack_receipt() { # [ ! -L "$target" ] || return 2 [ ! -e "$target" ] || { rm -f "$rec"; return 0; } mv "$rec" "$target" || return 2 + [ "$kind" = secondmate ] || return 0 + fm_pending_reply_secondmate_route_clear "$FM_HOME" "$corr" || true return 0 } @@ -325,7 +341,7 @@ scan_locked() { fi printf '%s\n' "$id" > "$SCAN_CURSOR" rc=0 - FM_LOCK_WAIT_SECS="$remaining" timeout --foreground "$remaining" \ + FM_LOCK_WAIT_SECS="$remaining" run_bounded_child "$remaining" \ "$SCRIPT_DIR/fm-inactive-reconcile.sh" _child "$id" || rc=$? if [ "$rc" -ne 0 ]; then complete=0 diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 32b7dcef6df..8611f65b7f9 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -490,13 +490,21 @@ fm_pending_reply_secondmate_route_path() { # printf '%s/state/.fm-jt-parent-route' "$1" } +fm_pending_reply_secondmate_route_lock_path() { # + printf '%s/state/.fm-jt-parent-route.lock' "$1" +} + fm_pending_reply_secondmate_route_write() { # local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 - local marker tmp parent_abs state_abs status_path + local marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 + local existing_schema existing_id existing_home existing_status existing_corr + local existing_state existing_record existing_phase marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 case "$secondmate_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + marker_id=$(cat "$secondmate_home/.fm-secondmate-home" 2>/dev/null || true) + [ "$marker_id" = "$secondmate_id" ] || return 1 parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 state_abs=$(cd "$parent_state" 2>/dev/null && pwd -P) || return 1 status_path="$state_abs/$secondmate_id.status" @@ -510,7 +518,89 @@ fm_pending_reply_secondmate_route_write() { # < printf 'corr_id=%s\n' "$corr" } > "$tmp" || { rm -f "$tmp"; return 1; } chmod 600 "$tmp" 2>/dev/null || true - mv -f "$tmp" "$marker" + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + if ! fm_lock_acquire_wait "$route_lock"; then + rm -f "$tmp" + return 1 + fi + if [ -e "$marker" ] || [ -L "$marker" ]; then + if [ -f "$marker" ] && [ ! -L "$marker" ] \ + && [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ]; then + existing_schema=$(fm_pending_reply_get "$marker" schema) + existing_id=$(fm_pending_reply_get "$marker" secondmate_id) + existing_home=$(fm_pending_reply_get "$marker" parent_home) + existing_status=$(fm_pending_reply_get "$marker" parent_status) + existing_corr=$(fm_pending_reply_get "$marker" corr_id) + if [ "$existing_schema" = fm-jt-parent-route.v1 ] \ + && [ "$existing_id" = "$secondmate_id" ] \ + && [ "$existing_home" = "$parent_abs" ] \ + && [ "$existing_status" = "$status_path" ] \ + && [ "$existing_corr" = "$corr" ]; then + rm -f "$tmp" + fm_lock_release "$route_lock" || return 1 + return 0 + fi + if [ "$existing_schema" != fm-jt-parent-route.v1 ] \ + || [ "$existing_id" != "$secondmate_id" ]; then + route_status=1 + fi + case "$existing_home" in /*) ;; *) route_status=1 ;; esac + case "$existing_status" in /*) ;; *) route_status=1 ;; esac + printf '%s' "$existing_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || route_status=1 + if [ "$route_status" = 0 ]; then + existing_state=$(cd "$existing_home/state" 2>/dev/null && pwd -P) || route_status=1 + fi + if [ "$route_status" = 0 ] \ + && [ "$existing_status" != "$existing_state/$existing_id.status" ]; then + route_status=1 + fi + if [ "$route_status" = 0 ]; then + if [ -f "$(fm_pending_reply_active_path "$existing_state" "$existing_corr")" ]; then + existing_record=$(fm_pending_reply_active_path "$existing_state" "$existing_corr") + elif [ -f "$(fm_pending_reply_history_dir "$existing_state")/$existing_corr" ]; then + existing_record="$(fm_pending_reply_history_dir "$existing_state")/$existing_corr" + else + route_status=1 + fi + fi + if [ "$route_status" = 0 ]; then + existing_phase=$(fm_pending_reply_get "$existing_record" phase) + case "$existing_phase" in + resolved|retired) ;; + *) route_status=1 ;; + esac + fi + else + route_status=1 + fi + fi + if [ "$route_status" = 0 ]; then + mv -f "$tmp" "$marker" || route_status=1 + fi + [ "$route_status" = 0 ] || rm -f "$tmp" + fm_lock_release "$route_lock" || route_status=1 + return "$route_status" +} + +fm_pending_reply_secondmate_route_clear() { # + local secondmate_home=$1 corr=$2 marker route_lock existing_corr status=0 + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + fm_lock_acquire_wait "$route_lock" || return 1 + if [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" != 5 ]; then + status=1 + else + existing_corr=$(fm_pending_reply_get "$marker" corr_id) + [ "$existing_corr" = "$corr" ] || status=1 + fi + if [ "$status" = 0 ]; then + rm -f "$marker" || status=1 + fi + fm_lock_release "$route_lock" || status=1 + return "$status" } fm_pending_reply_secondmate_route_validate() { # diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 6d6ae66875d..bcc37475fa3 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -89,6 +89,14 @@ PENDING_REPLY_CORR= PENDING_REPLY_CREATED=0 TARGET_TASK_ID= TARGET_HOME= + +clear_new_pending_route() { + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ + && [ -n "$TARGET_HOME" ]; then + fm_pending_reply_secondmate_route_clear "$TARGET_HOME" "$PENDING_REPLY_CORR" || true + fi +} + case "$RAW_TARGET" in fm-*) meta="$STATE/${RAW_TARGET#fm-}.meta" @@ -137,6 +145,7 @@ else fm_pending_reply_embed_corr "$MESSAGE" "$PENDING_REPLY_CORR" MESSAGE if [ "$PENDING_REPLY_CREATED" = 1 ] \ && ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then + clear_new_pending_route fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true echo "error: failed to durably prepare pending-reply delivery for $TARGET_TASK_ID" >&2 exit 1 @@ -144,6 +153,7 @@ else TARGET_HOME=$(fm_meta_get "$meta" home) if ! fm_pending_reply_secondmate_route_write \ "$TARGET_HOME" "$FM_HOME" "$STATE" "$TARGET_TASK_ID" "$PENDING_REPLY_CORR"; then + clear_new_pending_route if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true fi @@ -181,6 +191,7 @@ else # Type once, submit, verify. Lenient: only a positively-confirmed swallow # (text still in the composer) is an error; an unreadable pane is assumed sent. if ! verdict=$(fm_backend_send_text_submit "$TARGET_BACKEND" "$T" "$MESSAGE" "$retries" "$sleep_s" "$settle"); then + clear_new_pending_route if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true fi @@ -194,6 +205,7 @@ else sleep "$settle" final_after_pending=1 if ! verdict=$(fm_backend_submit_enter "$TARGET_BACKEND" "$T" 1 "$sleep_s" "$MESSAGE"); then + clear_new_pending_route if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true fi @@ -203,6 +215,7 @@ else fi case "$verdict" in pending) + clear_new_pending_route if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true fi @@ -210,6 +223,7 @@ else exit 1 ;; send-failed) + clear_new_pending_route if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true fi @@ -218,6 +232,7 @@ else ;; unknown) if [ "$final_after_pending" = 1 ]; then + clear_new_pending_route if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true fi diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 8605af26f05..d20d357ff6e 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -13,6 +13,7 @@ fi DRAIN_TMP= DRAIN_DEDUPED= +DRAIN_RESTORE= DRAIN_LOCK_HELD=false # Defense in depth for the watcher re-arm chain: this script runs at the top of @@ -33,9 +34,15 @@ assert_watcher_liveness() { # shellcheck disable=SC2317,SC2329 # Invoked by trap handlers below. cleanup() { local status=$? - if [ "$status" -ne 0 ] && [ "$DRAIN_LOCK_HELD" = true ] && [ -n "$DRAIN_TMP" ] && [ -e "$DRAIN_TMP" ]; then - fm_wake_restore_queue "$DRAIN_TMP" || true + if [ "$status" -ne 0 ] && [ "$DRAIN_LOCK_HELD" = true ]; then + if [ -n "$DRAIN_RESTORE" ] && [ -e "$DRAIN_RESTORE" ]; then + fm_wake_restore_queue "$DRAIN_RESTORE" || true + elif [ -n "$DRAIN_TMP" ] && [ -e "$DRAIN_TMP" ]; then + fm_wake_restore_queue "$DRAIN_TMP" || true + fi fi + [ -z "$DRAIN_TMP" ] || rm -f "$DRAIN_TMP" || true + [ -z "$DRAIN_RESTORE" ] || rm -f "$DRAIN_RESTORE" || true [ -z "$DRAIN_DEDUPED" ] || rm -f "$DRAIN_DEDUPED" || true if [ "$DRAIN_LOCK_HELD" = true ]; then fm_lock_release "$FM_WAKE_QUEUE_LOCK" @@ -71,14 +78,21 @@ cat "$DRAIN_DEDUPED" # Inactive-outcome rows are acknowledged only after their matching durable # receipt is presented. The locked session-start/watcher context authorizes the # helper; a failed correlation leaves the original drained rows restorable. -while IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload; do +drain_line=0 +while IFS= read -r drain_row || [ -n "$drain_row" ]; do + drain_line=$((drain_line + 1)) + IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" case "$_key" in inactive-outcome:*) "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" || { ack_status=$? # 1 means the receipt was already acknowledged or is not ours. Any # other failure keeps the drained row durable for a later turn. - [ "$ack_status" = 1 ] || exit "$ack_status" + if [ "$ack_status" != 1 ]; then + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + exit "$ack_status" + fi } ;; esac diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 888c917109c..f885b048223 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -126,6 +126,27 @@ test_done_and_failed_are_replayed_once() { pass "done and failed inactive outcomes are replayed once and acknowledged on drain" } +test_portable_timeout_runner_is_used() { + local dir root home fakebin state + new_case portable-timeout + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" portable-x1 portable-inc + cat > "$fakebin/timeout" <<'SH' +#!/usr/bin/env bash +set -u +[ "${1:-}" != --foreground ] || exit 91 +shift +exec "$@" +SH + chmod +x "$fakebin/timeout" + export FM_FAKE_CREW_STATE_PORTABLE_X1='state: done · source: pane · portable timeout' + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 1 ] || fail "portable timeout runner did not reconcile the child" + unset FM_FAKE_CREW_STATE_PORTABLE_X1 + pass "inactive scan uses the portable timeout invocation" +} + test_reused_task_id_gets_new_fingerprint() { local dir root home fakebin state new_case reused-id @@ -263,6 +284,7 @@ test_valid_secondmate_route_reports_parent_once() { [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "valid secondmate route did not create a pending receipt" drain "$root" "$child_home" "$fakebin" >/dev/null [ "$(receipt_count "$child_state" reported)" = 1 ] || fail "valid secondmate route was not reported" + [ ! -e "$child_state/.fm-jt-parent-route" ] || fail "reported secondmate route remained installed" grep -F "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status" >/dev/null \ || fail "valid secondmate route did not append the correlated parent status" drain "$root" "$child_home" "$fakebin" >/dev/null @@ -272,6 +294,69 @@ test_valid_secondmate_route_reports_parent_once() { pass "valid secondmate outcomes use the parent status correlation exactly once" } +test_concurrent_secondmate_routes_are_rejected() { + local dir root home fakebin state child_home child_state marker corr_one corr_two + new_case secondmate-route-concurrent + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + marker="$child_state/.fm-jt-parent-route" + corr_one=0123456789abcdef + corr_two=1123456789abcdef + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$state/pending-replies" + printf 'sm-concurrent\n' > "$child_home/.fm-secondmate-home" + fm_write_meta "$state/pending-replies/$corr_one" \ + schema=fm-pending-reply.v1 corr_id="$corr_one" task_id=sm-concurrent \ + parent_home="$home" parent_status="$state/sm-concurrent.status" delivered_epoch=1 phase=awaiting_report + fm_write_meta "$state/pending-replies/$corr_two" \ + schema=fm-pending-reply.v1 corr_id="$corr_two" task_id=sm-concurrent \ + parent_home="$home" parent_status="$state/sm-concurrent.status" delivered_epoch=1 phase=awaiting_report + route_write() { + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-concurrent "$1" + } + route_write "$corr_one" || fail "initial secondmate route was not written" + marker_before=$(cat "$marker") + if route_write "$corr_two"; then + fail "concurrent secondmate route was silently replaced" + fi + [ "$(cat "$marker")" = "$marker_before" ] || fail "concurrent route rejection changed the active marker" + pass "concurrent secondmate routes fail closed without overwriting" +} + +test_drain_restores_only_unprocessed_rows() { + local dir root home fakebin state first second + new_case drain-rollback + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + first=1111111111111111 + second=2222222222222222 + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$first.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$first" task_id=first-x1 \ + incarnation=first-inc outcome=done terminal_source=pane terminal_snapshot=done kind=ship + fm_write_meta "$state/terminal-outcomes/$second.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$second" task_id=second-x1 \ + incarnation=second-inc outcome=failed terminal_source=pane terminal_snapshot=failed kind=secondmate + printf 'sm-rollback\n' > "$home/.fm-secondmate-home" + printf '1\t1\tcheck\tinactive-outcome:%s\tfirst\n2\t2\tcheck\tinactive-outcome:%s\tsecond\n' \ + "$first" "$second" > "$state/.wake-queue" + if drain "$root" "$home" "$fakebin" >"$dir/drain.out" 2>&1; then + fail "drain accepted a malformed secondmate route" + fi + [ -e "$state/terminal-outcomes/$first.presented" ] || fail "presented receipt was not acknowledged" + [ ! -e "$state/terminal-outcomes/$first.pending" ] || fail "presented receipt remained pending" + [ -e "$state/terminal-outcomes/$second.pending" ] || fail "failed receipt was lost" + [ "$(awk -F '\t' '$4 == "inactive-outcome:'"$first"'" { n++ } END { print n + 0 }' "$state/.wake-queue")" = 0 ] \ + || fail "already-presented receipt was requeued" + [ "$(awk -F '\t' '$4 == "inactive-outcome:'"$second"'" { n++ } END { print n + 0 }' "$state/.wake-queue")" = 1 ] \ + || fail "unprocessed receipt was not requeued" + pass "drain rollback preserves only unprocessed inactive outcomes" +} + test_malformed_or_missing_secondmate_route_fails_closed() { local dir root home fakebin state child_home child_state parent_status new_case secondmate-route @@ -294,10 +379,13 @@ test_malformed_or_missing_secondmate_route_fails_closed() { } test_done_and_failed_are_replayed_once +test_portable_timeout_runner_is_used test_reused_task_id_gets_new_fingerprint test_relaunch_and_teardown_races_recheck_under_spawn_lock test_herdr_identity_and_default_captain_refusal test_occupancy_unknown_is_not_terminal test_status_log_terminal_is_not_replayed test_valid_secondmate_route_reports_parent_once +test_concurrent_secondmate_routes_are_rejected +test_drain_restores_only_unprocessed_rows test_malformed_or_missing_secondmate_route_fails_closed From b9bf10a8663e6ddd8f4f85e79918a5978e567495 Mon Sep 17 00:00:00 2001 From: tests Date: Thu, 13 Aug 2026 22:12:05 +0000 Subject: [PATCH 003/163] no-mistakes(review): Propagated scan failures and rejected unsafe state paths --- bin/fm-inactive-reconcile.sh | 68 ++++++++++++++++++++++++++----- tests/fm-inactive-outcome.test.sh | 64 +++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 10 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index c7df7ec7ae0..65d3c91cca1 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -10,18 +10,53 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ "${FM_SESSION_LOCK_BOOTSTRAP:-0}" != 1 ]; then fm_worker_refuse_primary_operation "inactive outcome reconciliation" || exit 1 fi + +FM_ROOT="${FM_ROOT_OVERRIDE:-${FM_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-${STATE:-$FM_HOME/state}}" +OUTCOME_DIR="$STATE/terminal-outcomes" +SCAN_MARKER="$STATE/.inactive-outcome-reconcile" +SCAN_CURSOR="$STATE/.inactive-outcome-reconcile.cursor" +FM_WAKE_QUEUE="${FM_WAKE_QUEUE:-$STATE/.wake-queue}" + +inactive_state_path_is_safe() { + local path=$1 kind=$2 + [ ! -L "$path" ] || return 1 + if [ -e "$path" ]; then + case "$kind" in + dir) [ -d "$path" ] || return 1 ;; + file) [ -f "$path" ] || return 1 ;; + *) return 1 ;; + esac + fi +} + +inactive_state_preflight() { + if [ -L "$STATE" ] || { [ -e "$STATE" ] && [ ! -d "$STATE" ]; }; then + return 1 + fi + if [ ! -e "$STATE" ] && ! mkdir -p "$STATE"; then + return 1 + fi + inactive_state_path_is_safe "$STATE" dir || return 1 + inactive_state_path_is_safe "$OUTCOME_DIR" dir || return 1 + inactive_state_path_is_safe "$SCAN_MARKER" file || return 1 + inactive_state_path_is_safe "$SCAN_CURSOR" file || return 1 + inactive_state_path_is_safe "$FM_WAKE_QUEUE" file || return 1 +} + +inactive_state_preflight || { + echo "error: inactive reconciliation state must be local regular state under $FM_HOME" >&2 + exit 1 +} + # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" # shellcheck source=bin/fm-pending-reply-lib.sh . "$SCRIPT_DIR/fm-pending-reply-lib.sh" -FM_HOME="${FM_HOME:-$FM_ROOT}" -STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" FM_CREW_STATE_BIN="${FM_CREW_STATE_BIN:-$SCRIPT_DIR/fm-crew-state.sh}" -OUTCOME_DIR="$STATE/terminal-outcomes" SCAN_LOCK="$STATE/.inactive-outcome-reconcile.lock" -SCAN_MARKER="$STATE/.inactive-outcome-reconcile" -SCAN_CURSOR="$STATE/.inactive-outcome-reconcile.cursor" ROUTE_MARKER="$FM_HOME/.fm-secondmate-home" CHILD_LOCK_HELD=0 CHILD_LOCK= @@ -137,6 +172,7 @@ receipt_field() { # receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE local pending tmp + inactive_state_preflight || return 1 pending=$(receipt_path "$FP" pending) RECEIPT_CREATED=0 for suffix in pending presented reported; do @@ -309,7 +345,8 @@ ack_receipt() { # scan_locked() { local startup=${1:-0} marker_mtime now age cursor meta id found=0 started=1 - local scan_started remaining rc complete=1 + local scan_started remaining rc complete=1 scan_failed=0 + inactive_state_preflight || return 1 marker_mtime=$(file_mtime "$SCAN_MARKER" 2>/dev/null || true) now=$(date +%s) if [ "$startup" != 1 ] && [ -n "$marker_mtime" ]; then @@ -339,25 +376,36 @@ scan_locked() { complete=0 break fi - printf '%s\n' "$id" > "$SCAN_CURSOR" rc=0 FM_LOCK_WAIT_SECS="$remaining" run_bounded_child "$remaining" \ "$SCRIPT_DIR/fm-inactive-reconcile.sh" _child "$id" || rc=$? if [ "$rc" -ne 0 ]; then complete=0 + scan_failed=1 + break + fi + if printf '%s\n' "$id" > "$SCAN_CURSOR"; then + : + else + rc=$? + complete=0 + scan_failed=1 break fi found=1 done - [ "$complete" = 1 ] && rm -f "$SCAN_CURSOR" - date +%s > "$SCAN_MARKER" + [ "$scan_failed" = 0 ] || return "$rc" + if [ "$complete" = 1 ]; then + rm -f "$SCAN_CURSOR" || return 1 + fi + date +%s > "$SCAN_MARKER" || return 1 [ "$found" = 1 ] || true return 0 } scan() { local startup=${1:-0} - mkdir -p "$STATE" || return 1 + inactive_state_preflight || return 1 fm_lock_acquire_wait "$SCAN_LOCK" || return 1 trap 'fm_lock_release "$SCAN_LOCK" || true' EXIT INT TERM scan_locked "$startup" diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index f885b048223..28fc3d610b3 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -147,6 +147,68 @@ SH pass "inactive scan uses the portable timeout invocation" } +test_scan_failure_retries_without_advancing_cadence() { + local dir root home fakebin state wake_dir wake_removed + new_case scan-failure + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + wake_dir="$dir/wake" + wake_removed="$dir/wake-removed" + mkdir -p "$wake_dir" + : > "$wake_dir/queue" + write_meta "$state" first-x1 first-inc + write_meta "$state" second-x1 second-inc + cat > "$fakebin/fm-crew-state.sh" <<'SH' +#!/usr/bin/env bash +set -u +if [ "${1:-}" = second-x1 ] && [ "${FM_BREAK_QUEUE:-0}" = 1 ]; then + mv "${FM_WAKE_QUEUE_DIR}" "${FM_WAKE_QUEUE_REMOVED}" +fi +printf 'state: done · source: pane · scan retry\n' +SH + chmod +x "$fakebin/fm-crew-state.sh" + export FM_WAKE_QUEUE="$wake_dir/queue" FM_WAKE_QUEUE_LOCK="$wake_dir/lock" + export FM_WAKE_QUEUE_DIR="$wake_dir" FM_WAKE_QUEUE_REMOVED="$wake_removed" FM_BREAK_QUEUE=1 + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "child scan failure was reported as success" + fi + [ ! -e "$state/.inactive-outcome-reconcile" ] || fail "failed scan advanced the cadence marker" + [ "$(receipt_count "$state" pending)" = 2 ] || fail "durable receipts were not retained across wake publication failure" + grep -l '^task_id=first-x1$' "$state"/terminal-outcomes/*.pending >/dev/null \ + || fail "successful child receipt was not retained" + [ "$(cat "$state/.inactive-outcome-reconcile.cursor")" = first-x1 ] || fail "cursor did not preserve the last successful child" + mv "$wake_removed" "$wake_dir" + export FM_BREAK_QUEUE=0 + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "retry after child failure did not complete" + [ "$(receipt_count "$state" pending)" = 2 ] || fail "failed child was skipped on retry" + unset FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK FM_WAKE_QUEUE_DIR FM_WAKE_QUEUE_REMOVED FM_BREAK_QUEUE + pass "scan failures preserve retry state and cadence" +} + +test_state_paths_reject_symlinks_and_non_directories() { + local dir root home fakebin state real_state + new_case symlink-state + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + real_state="$dir/real-state" + mv "$state" "$real_state" + ln -s "$real_state" "$state" + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "symlinked state path was accepted" + fi + [ ! -e "$real_state/.inactive-outcome-reconcile" ] || fail "symlinked state received a cadence marker" + + new_case file-state + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + mv "$state" "$dir/state-directory" + printf 'not a directory\n' > "$state" + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "non-directory state path was accepted" + fi + pass "inactive reconciliation rejects unsafe state paths" +} + test_reused_task_id_gets_new_fingerprint() { local dir root home fakebin state new_case reused-id @@ -380,6 +442,8 @@ test_malformed_or_missing_secondmate_route_fails_closed() { test_done_and_failed_are_replayed_once test_portable_timeout_runner_is_used +test_scan_failure_retries_without_advancing_cadence +test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint test_relaunch_and_teardown_races_recheck_under_spawn_lock test_herdr_identity_and_default_captain_refusal From 65ae712e239536072f5c22ea1cb4111fe4d3fe3b Mon Sep 17 00:00:00 2001 From: tests Date: Thu, 13 Aug 2026 23:12:34 +0000 Subject: [PATCH 004/163] no-mistakes(review): Fixed inactive replay isolation, routing, receipts, and bounded scans --- .github/workflows/ci.yml | 1 + bin/fm-inactive-reconcile.sh | 207 +++++++++++++++++++----- bin/fm-pending-reply-lib.sh | 97 +++++++++++- bin/fm-session-start.sh | 14 +- bin/fm-wake-drain.sh | 32 +++- bin/fm-wake-lib.sh | 38 ++++- bin/fm-watch.sh | 5 +- tests/fm-inactive-outcome.test.sh | 251 ++++++++++++++++++++++++++---- 8 files changed, 547 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9fff23c2b4..39d32d0859a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: run: | set -eu bash tests/fm-worker-isolation.test.sh + bash tests/fm-inactive-outcome.test.sh bash tests/fm-watch-session.test.sh bash tests/fm-slot-occupant-proof.test.sh FM_TEARDOWN_TEST_FOCUS=s1 bash tests/fm-teardown.test.sh diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 65d3c91cca1..d561892f6dc 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -7,9 +7,7 @@ set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=bin/fm-worker-isolation-lib.sh . "$SCRIPT_DIR/fm-worker-isolation-lib.sh" -if [ "${FM_SESSION_LOCK_BOOTSTRAP:-0}" != 1 ]; then - fm_worker_refuse_primary_operation "inactive outcome reconciliation" || exit 1 -fi +fm_worker_refuse_primary_operation "inactive outcome reconciliation" || exit 1 FM_ROOT="${FM_ROOT_OVERRIDE:-${FM_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}}" FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" @@ -142,12 +140,6 @@ herdr_identity_allowed() { # return 1 } -queue_contains() { # - local key=$1 - [ -f "$FM_WAKE_QUEUE" ] || return 1 - awk -F '\t' -v wanted="$key" '$4 == wanted { found=1 } END { exit(found ? 0 : 1) }' "$FM_WAKE_QUEUE" 2>/dev/null -} - run_bounded_child() { # [args...] local seconds=$1 shift @@ -171,15 +163,34 @@ receipt_field() { # } receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE - local pending tmp + local pending tmp existing inactive_state_preflight || return 1 pending=$(receipt_path "$FP" pending) RECEIPT_CREATED=0 for suffix in pending presented reported; do - local existing existing=$(receipt_path "$FP" "$suffix") [ ! -L "$existing" ] || return 1 [ -e "$existing" ] || continue + [ -f "$existing" ] || return 1 + [ "$(receipt_field "$existing" schema)" = fm-jt-terminal-outcome.v1 ] || return 1 + [ "$(receipt_field "$existing" fingerprint)" = "$FP" ] || return 1 + [ "$(receipt_field "$existing" task_id)" = "$ID" ] || return 1 + [ "$(receipt_field "$existing" incarnation)" = "$INC" ] || return 1 + [ "$(receipt_field "$existing" outcome)" = "$OUTCOME" ] || return 1 + [ "$(receipt_field "$existing" terminal_source)" = "$SOURCE" ] || return 1 + [ "$(receipt_field "$existing" terminal_snapshot)" = "$SNAPSHOT" ] || return 1 + [ "$(receipt_field "$existing" kind)" = "$KIND" ] || return 1 + if [ "$KIND" = secondmate ]; then + [ "$(receipt_field "$existing" parent_task_id)" = "${FM_PENDING_ROUTE_SECOND_MATE_ID:-}" ] || return 1 + [ "$(receipt_field "$existing" parent_home)" = "${FM_PENDING_ROUTE_PARENT_HOME:-}" ] || return 1 + [ "$(receipt_field "$existing" parent_status)" = "${FM_PENDING_ROUTE_PARENT_STATUS:-}" ] || return 1 + [ "$(receipt_field "$existing" parent_corr)" = "${FM_PENDING_ROUTE_CORR:-}" ] || return 1 + else + [ -z "$(receipt_field "$existing" parent_task_id)" ] || return 1 + [ -z "$(receipt_field "$existing" parent_home)" ] || return 1 + [ -z "$(receipt_field "$existing" parent_status)" ] || return 1 + [ -z "$(receipt_field "$existing" parent_corr)" ] || return 1 + fi return 0 done mkdir -p "$OUTCOME_DIR" || return 1 @@ -194,6 +205,10 @@ receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE printf 'terminal_source=%s\n' "$SOURCE" printf 'terminal_snapshot=%s\n' "$SNAPSHOT" printf 'kind=%s\n' "$KIND" + printf 'parent_task_id=%s\n' "${FM_PENDING_ROUTE_SECOND_MATE_ID:-}" + printf 'parent_home=%s\n' "${FM_PENDING_ROUTE_PARENT_HOME:-}" + printf 'parent_status=%s\n' "${FM_PENDING_ROUTE_PARENT_STATUS:-}" + printf 'parent_corr=%s\n' "${FM_PENDING_ROUTE_CORR:-}" printf 'created_epoch=%s\n' "$(date +%s)" } > "$tmp" || { rm -f "$tmp"; return 1; } # ln is an exclusive, same-filesystem publication. A concurrent scanner can @@ -208,6 +223,60 @@ receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE return 0 } +presentation_marker_path() { # + printf '%s/%s.presented-marker' "$OUTCOME_DIR" "$1" +} + +presentation_mark() { # + local key=$1 fp marker pending presented reported tmp existing + [ "${FM_INACTIVE_ACK_FROM_DRAIN:-0}" = 1 ] || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + inactive_state_preflight || return 2 + marker=$(presentation_marker_path "$fp") + [ ! -L "$marker" ] || return 2 + if [ -e "$marker" ]; then + [ -f "$marker" ] || return 2 + return 1 + fi + pending=$(receipt_path "$fp" pending) + presented=$(receipt_path "$fp" presented) + reported=$(receipt_path "$fp" reported) + for existing in "$pending" "$presented" "$reported"; do + [ ! -L "$existing" ] || return 2 + if [ -e "$existing" ]; then + [ -f "$existing" ] || return 2 + [ "$(receipt_field "$existing" fingerprint)" = "$fp" ] || return 2 + fi + done + if [ ! -f "$pending" ]; then + [ -f "$presented" ] || [ -f "$reported" ] || return 2 + return 1 + fi + tmp=$(mktemp "$OUTCOME_DIR/.presentation.XXXXXX") || return 2 + printf '%s\n' "$fp" > "$tmp" || { rm -f "$tmp"; return 2; } + if ln "$tmp" "$marker" 2>/dev/null; then + rm -f "$tmp" + return 0 + fi + rm -f "$tmp" + [ -f "$marker" ] && [ ! -L "$marker" ] && return 1 + return 2 +} + +presentation_clear() { # + local key=$1 fp marker + [ "${FM_INACTIVE_ACK_FROM_DRAIN:-0}" = 1 ] || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + inactive_state_preflight || return 2 + marker=$(presentation_marker_path "$fp") + [ ! -L "$marker" ] || return 2 + [ -e "$marker" ] || return 0 + [ -f "$marker" ] || return 2 + rm -f "$marker" +} + read_incarnation() { # local meta=$1 id=$2 token tasktmp window worktree seed token=$(meta_value "$meta" spawn_incarnation) @@ -242,19 +311,13 @@ child_cleanup() { reconcile_child() { local id=$1 meta="$STATE/$1.meta" kind backend now activity age line outcome source - local snapshot token key route_rc + local snapshot token key route_rc state_tmp state_rc valid_task_id "$id" || return 0 [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 kind=$(meta_value "$meta" kind) [ "$kind" = secondmate ] && return 0 case "$kind" in ''|ship|scout) ;; *) return 0 ;; esac herdr_identity_allowed "$meta" || return 0 - now=$(date +%s) - activity=$(latest_activity "$id") - [ "$activity" -gt 0 ] || return 0 - age=$((now - activity)) - [ "$age" -ge "$RECONCILE_SECS" ] || return 0 - CHILD_LOCK="$STATE/.spawn-$id.lock" FM_LOCK_WAIT_SECS=${FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS:-30} fm_lock_acquire_wait "$CHILD_LOCK" || return 0 @@ -267,10 +330,22 @@ reconcile_child() { kind=$(meta_value "$meta" kind) [ "$kind" = secondmate ] && return 0 herdr_identity_allowed "$meta" || return 0 + now=$(date +%s) + activity=$(latest_activity "$id") + [ "$activity" -gt 0 ] || return 0 + age=$((now - activity)) + [ "$age" -ge "$RECONCILE_SECS" ] || return 0 + state_tmp=$(mktemp "$STATE/.$id.inactive-state.XXXXXX") || return 1 + [ -f "$state_tmp" ] && [ ! -L "$state_tmp" ] || { rm -f "$state_tmp"; return 1; } + state_rc=0 FM_CREW_STATE_NM_TIMEOUT=${FM_INACTIVE_OUTCOME_STATE_TIMEOUT_SECS:-10} \ - "$FM_CREW_STATE_BIN" "$id" > "$STATE/.$id.inactive-state.$$" 2>/dev/null || return 0 - line=$(cat "$STATE/.$id.inactive-state.$$" 2>/dev/null || true) - rm -f "$STATE/.$id.inactive-state.$$" + "$FM_CREW_STATE_BIN" "$id" > "$state_tmp" 2>/dev/null || state_rc=$? + line= + if [ "$state_rc" -eq 0 ]; then + line=$(cat "$state_tmp" 2>/dev/null) || state_rc=$? + fi + rm -f "$state_tmp" || [ "$state_rc" -ne 0 ] || state_rc=1 + [ "$state_rc" -eq 0 ] || return "$state_rc" case "$line" in *$'\n'*) return 0 ;; esac case "$line" in state:\ done\ *|state:\ failed\ *) ;; @@ -304,8 +379,9 @@ reconcile_child() { receipt_write || return 1 key="inactive-outcome:$FP" if [ "$RECEIPT_CREATED" = 1 ] || [ -f "$(receipt_path "$FP" pending)" ]; then - if ! queue_contains "$key"; then - fm_wake_append check "$key" "inactive terminal outcome: task=$id state=$outcome fingerprint=$FP" || return 1 + fm_wake_append_if_absent FM_WAKE_APPEND_CREATED check "$key" \ + "inactive terminal outcome: task=$id state=$outcome fingerprint=$FP" || return 1 + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" fi fi @@ -313,21 +389,48 @@ reconcile_child() { } ack_receipt() { # - local key=$1 fp rec id kind outcome parent_status corr line target + local key=$1 fp rec id kind outcome parent_task_id parent_home parent_status corr line target existing marker + [ "${FM_INACTIVE_ACK_FROM_DRAIN:-0}" = 1 ] || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 0 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac rec=$(receipt_path "$fp" pending) - [ -f "$rec" ] && [ ! -L "$rec" ] || return 0 + [ ! -L "$rec" ] || return 2 + if [ ! -e "$rec" ]; then + for existing in "$(receipt_path "$fp" presented)" "$(receipt_path "$fp" reported)"; do + [ ! -L "$existing" ] || return 2 + if [ -e "$existing" ]; then + [ -f "$existing" ] || return 2 + [ "$(receipt_field "$existing" fingerprint)" = "$fp" ] || return 2 + return 1 + fi + done + return 2 + fi + [ -f "$rec" ] || return 2 + marker=$(presentation_marker_path "$fp") + [ -f "$marker" ] && [ ! -L "$marker" ] || return 2 + [ "$(cat "$marker" 2>/dev/null || true)" = "$fp" ] || return 2 [ "$(receipt_field "$rec" fingerprint)" = "$fp" ] || return 2 + [ "$(receipt_field "$rec" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 id=$(receipt_field "$rec" task_id) kind=$(receipt_field "$rec" kind) + parent_task_id=$(receipt_field "$rec" parent_task_id) outcome=$(receipt_field "$rec" outcome) + case "$kind" in ship|scout|secondmate) ;; *) return 2 ;; esac if [ "$kind" = secondmate ]; then - fm_pending_reply_secondmate_route_validate "$FM_HOME" || return 2 - parent_status=$FM_PENDING_ROUTE_PARENT_STATUS - corr=$FM_PENDING_ROUTE_CORR + parent_home=$(receipt_field "$rec" parent_home) + parent_status=$(receipt_field "$rec" parent_status) + corr=$(receipt_field "$rec" parent_corr) + [ -n "$parent_task_id" ] || return 2 + fm_pending_reply_secondmate_receipt_validate \ + "$FM_HOME" "$parent_task_id" "$parent_home" "$parent_status" "$corr" || return 2 line="$outcome [corr=$corr]: inactive terminal outcome replayed: task=$id fingerprint=$fp" - mkdir -p "$(dirname "$parent_status")" || return 2 + [ ! -L "$parent_status" ] || return 2 + if [ -e "$parent_status" ]; then + [ -f "$parent_status" ] || return 2 + else + : > "$parent_status" || return 2 + fi if ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then printf '%s\n' "$line" >> "$parent_status" || return 2 fi @@ -336,15 +439,19 @@ ack_receipt() { # target=$(receipt_path "$fp" presented) fi [ ! -L "$target" ] || return 2 - [ ! -e "$target" ] || { rm -f "$rec"; return 0; } + if [ -e "$target" ]; then + [ -f "$target" ] || return 2 + [ "$(receipt_field "$target" fingerprint)" = "$fp" ] || return 2 + rm -f "$rec" || return 2 + return 1 + fi mv "$rec" "$target" || return 2 - [ "$kind" = secondmate ] || return 0 - fm_pending_reply_secondmate_route_clear "$FM_HOME" "$corr" || true + presentation_clear "$key" || return 2 return 0 } scan_locked() { - local startup=${1:-0} marker_mtime now age cursor meta id found=0 started=1 + local startup=${1:-0} marker_mtime now age cursor meta id started=1 cursor_seen=1 local scan_started remaining rc complete=1 scan_failed=0 inactive_state_preflight || return 1 marker_mtime=$(file_mtime "$SCAN_MARKER" 2>/dev/null || true) @@ -359,7 +466,14 @@ scan_locked() { cursor= fi if [ -n "$cursor" ]; then started=0; fi - for meta in "$STATE"/*.meta; do + [ -n "$cursor" ] && cursor_seen=0 + while IFS= read -r -d '' meta; do + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + if [ "$remaining" -le 0 ]; then + complete=0 + break + fi if [ ! -f "$meta" ] || [ -L "$meta" ]; then continue fi @@ -368,14 +482,9 @@ scan_locked() { if [ "$started" = 0 ]; then [ "$id" = "$cursor" ] || continue started=1 + cursor_seen=1 continue fi - now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) - if [ "$remaining" -le 0 ]; then - complete=0 - break - fi rc=0 FM_LOCK_WAIT_SECS="$remaining" run_bounded_child "$remaining" \ "$SCRIPT_DIR/fm-inactive-reconcile.sh" _child "$id" || rc=$? @@ -392,14 +501,18 @@ scan_locked() { scan_failed=1 break fi - found=1 - done + done < <( + find "$STATE" \( -type d ! -path "$STATE" -prune \) -o \ + \( -type f -name '*.meta' -print0 \) + ) [ "$scan_failed" = 0 ] || return "$rc" + if [ "$complete" = 1 ] && [ "$cursor_seen" = 0 ]; then + return 1 + fi if [ "$complete" = 1 ]; then rm -f "$SCAN_CURSOR" || return 1 fi date +%s > "$SCAN_MARKER" || return 1 - [ "$found" = 1 ] || true return 0 } @@ -427,6 +540,14 @@ case "${1:-}" in [ -n "${2:-}" ] || exit 2 ack_receipt "$2" ;; + presentation) + [ -n "${2:-}" ] || exit 2 + presentation_mark "$2" + ;; + presentation-clear) + [ -n "${2:-}" ] || exit 2 + presentation_clear "$2" + ;; _child) [ -n "${2:-}" ] || exit 2 reconcile_child "$2" diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 8611f65b7f9..89099d3f7da 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -505,9 +505,17 @@ fm_pending_reply_secondmate_route_write() { # < case "$secondmate_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac marker_id=$(cat "$secondmate_home/.fm-secondmate-home" 2>/dev/null || true) [ "$marker_id" = "$secondmate_id" ] || return 1 + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 + [ -d "$parent_home/state" ] && [ ! -L "$parent_home/state" ] || return 1 parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + [ -d "$parent_state" ] && [ ! -L "$parent_state" ] || return 1 state_abs=$(cd "$parent_state" 2>/dev/null && pwd -P) || return 1 + [ "$state_abs" = "$parent_abs/state" ] || return 1 status_path="$state_abs/$secondmate_id.status" + [ ! -L "$status_path" ] || return 1 + if [ -e "$status_path" ]; then + [ -f "$status_path" ] || return 1 + fi printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 tmp="$marker.tmp.${BASHPID:-$$}" { @@ -607,8 +615,11 @@ fm_pending_reply_secondmate_route_validate() { # local secondmate_home=$1 marker line key value schema marker_id secondmate_id local parent_home parent_status corr parent_abs state_abs expected_status rec local phase delivered record_home record_status record_task record_corr home_marker + local seen_schema=0 seen_secondmate_id=0 seen_parent_home=0 seen_parent_status=0 seen_corr=0 FM_PENDING_ROUTE_PARENT_STATUS= + FM_PENDING_ROUTE_PARENT_HOME= FM_PENDING_ROUTE_CORR= + FM_PENDING_ROUTE_SECOND_MATE_ID= marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 schema='' secondmate_id='' parent_home='' parent_status='' corr='' @@ -618,14 +629,17 @@ fm_pending_reply_secondmate_route_validate() { # *) return 1 ;; esac case "$key" in - schema) [ -z "$schema" ] || return 1; schema=$value ;; - secondmate_id) [ -z "$secondmate_id" ] || return 1; secondmate_id=$value ;; - parent_home) [ -z "$parent_home" ] || return 1; parent_home=$value ;; - parent_status) [ -z "$parent_status" ] || return 1; parent_status=$value ;; - corr_id) [ -z "$corr" ] || return 1; corr=$value ;; + schema) [ "$seen_schema" = 0 ] || return 1; seen_schema=1; schema=$value ;; + secondmate_id) [ "$seen_secondmate_id" = 0 ] || return 1; seen_secondmate_id=1; secondmate_id=$value ;; + parent_home) [ "$seen_parent_home" = 0 ] || return 1; seen_parent_home=1; parent_home=$value ;; + parent_status) [ "$seen_parent_status" = 0 ] || return 1; seen_parent_status=1; parent_status=$value ;; + corr_id) [ "$seen_corr" = 0 ] || return 1; seen_corr=1; corr=$value ;; *) return 1 ;; esac done < "$marker" + [ "$seen_schema" = 1 ] && [ "$seen_secondmate_id" = 1 ] \ + && [ "$seen_parent_home" = 1 ] && [ "$seen_parent_status" = 1 ] \ + && [ "$seen_corr" = 1 ] || return 1 [ "$schema" = fm-jt-parent-route.v1 ] || return 1 home_marker="$secondmate_home/.fm-secondmate-home" [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 @@ -635,11 +649,21 @@ fm_pending_reply_secondmate_route_validate() { # case "$parent_home" in /*) ;; *) return 1 ;; esac case "$parent_status" in /*) ;; *) return 1 ;; esac printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + [ -d "$parent_abs/state" ] && [ ! -L "$parent_abs/state" ] || return 1 state_abs=$(cd "$parent_abs/state" 2>/dev/null && pwd -P) || return 1 + [ -d "$state_abs/pending-replies" ] && [ ! -L "$state_abs/pending-replies" ] || return 1 + if [ -e "$state_abs/pending-replies/history" ]; then + [ -d "$state_abs/pending-replies/history" ] && [ ! -L "$state_abs/pending-replies/history" ] || return 1 + fi expected_status="$state_abs/$secondmate_id.status" [ "$parent_status" = "$expected_status" ] || return 1 - rec=$(fm_pending_reply_active_path "$state_abs" "$corr") + [ ! -L "$expected_status" ] || return 1 + if [ -e "$expected_status" ]; then + [ -f "$expected_status" ] || return 1 + fi + rec="$state_abs/pending-replies/$corr" [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 record_task=$(fm_pending_reply_get "$rec" task_id) record_home=$(fm_pending_reply_get "$rec" parent_home) @@ -653,13 +677,72 @@ fm_pending_reply_secondmate_route_validate() { # [ -n "$delivered" ] || return 1 phase=$(fm_pending_reply_get "$rec" phase) case "$phase" in - awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated) ;; + awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; *) return 1 ;; esac # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_PARENT_HOME=$parent_abs + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_PARENT_STATUS=$expected_status # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_CORR=$corr + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_SECOND_MATE_ID=$secondmate_id + return 0 +} + +fm_pending_reply_secondmate_receipt_validate() { # + local secondmate_home=$1 secondmate_id=$2 parent_home=$3 parent_status=$4 corr=$5 + local home_marker marker_id parent_abs state_abs expected_status rec active_rec history_rec + local record_task record_home record_status record_corr delivered phase + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + home_marker="$secondmate_home/.fm-secondmate-home" + [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 + marker_id=$(cat "$home_marker" 2>/dev/null || true) + [ "$marker_id" = "$secondmate_id" ] || return 1 + case "$parent_home" in /*) ;; *) return 1 ;; esac + case "$parent_status" in /*) ;; *) return 1 ;; esac + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + [ -d "$parent_abs/state" ] && [ ! -L "$parent_abs/state" ] || return 1 + state_abs=$(cd "$parent_abs/state" 2>/dev/null && pwd -P) || return 1 + [ -d "$state_abs/pending-replies" ] && [ ! -L "$state_abs/pending-replies" ] || return 1 + if [ -e "$state_abs/pending-replies/history" ]; then + [ -d "$state_abs/pending-replies/history" ] && [ ! -L "$state_abs/pending-replies/history" ] || return 1 + fi + expected_status="$state_abs/$secondmate_id.status" + [ "$parent_status" = "$expected_status" ] || return 1 + [ ! -L "$parent_status" ] || return 1 + if [ -e "$parent_status" ]; then + [ -f "$parent_status" ] || return 1 + fi + active_rec="$state_abs/pending-replies/$corr" + history_rec="$state_abs/pending-replies/history/$corr" + if [ -f "$active_rec" ] && [ ! -L "$active_rec" ]; then + rec=$active_rec + elif [ -f "$history_rec" ] && [ ! -L "$history_rec" ]; then + rec=$history_rec + else + return 1 + fi + [ "$(fm_pending_reply_get "$rec" schema)" = fm-pending-reply.v1 ] || return 1 + record_task=$(fm_pending_reply_get "$rec" task_id) + record_home=$(fm_pending_reply_get "$rec" parent_home) + record_status=$(fm_pending_reply_get "$rec" parent_status) + record_corr=$(fm_pending_reply_get "$rec" corr_id) + [ "$record_task" = "$secondmate_id" ] || return 1 + [ "$record_home" = "$parent_abs" ] || return 1 + [ "$record_status" = "$expected_status" ] || return 1 + [ "$record_corr" = "$corr" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -n "$delivered" ] || return 1 + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in + awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; + *) return 1 ;; + esac return 0 } diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index a0ba17fdee2..7b65fba9ef1 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -362,19 +362,17 @@ if [ "$READ_ONLY" -eq 1 ]; then GUARD_OUT=$(FM_GUARD_READ_ONLY=1 "$SCRIPT_DIR/fm-guard.sh" 2>&1) [ -n "$GUARD_OUT" ] && printf '%s\n' "$GUARD_OUT" else - # Reconcile quiet terminal children while this session owns the fleet lock; - # the following drain then presents and acknowledges the resulting durable - # receipts in the same turn. - INACTIVE_OUT=$( - FM_SESSION_LOCK_BOOTSTRAP=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" scan --startup 2>&1 || true - ) - [ -z "$INACTIVE_OUT" ] || printf '%s\n' "$INACTIVE_OUT" - DRAIN_OUT=$(FM_SESSION_LOCK_BOOTSTRAP=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) + DRAIN_OUT=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) if [ -n "$DRAIN_OUT" ]; then printf '%s\n' "$DRAIN_OUT" else printf '(no queued wakes)\n' fi + if ! INACTIVE_OUT=$("$SCRIPT_DIR/fm-inactive-reconcile.sh" scan --startup 2>&1); then + printf '%s\n' "$INACTIVE_OUT" >&2 + exit 1 + fi + [ -z "$INACTIVE_OUT" ] || printf '%s\n' "$INACTIVE_OUT" fi # --- 4. supervision operating instructions ---------------------------------- diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index d20d357ff6e..f6a96b142c7 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -5,9 +5,7 @@ set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=bin/fm-worker-isolation-lib.sh . "$SCRIPT_DIR/fm-worker-isolation-lib.sh" -if [ "${FM_SESSION_LOCK_BOOTSTRAP:-0}" != 1 ]; then - fm_worker_refuse_primary_operation "wake drain" || exit 1 -fi +fm_worker_refuse_primary_operation "wake drain" || exit 1 # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" @@ -74,7 +72,6 @@ mv "$FM_WAKE_QUEUE" "$DRAIN_TMP" || exit 1 : > "$FM_WAKE_QUEUE" || exit 1 fm_wake_print_deduped "$DRAIN_TMP" > "$DRAIN_DEDUPED" || exit "$?" -cat "$DRAIN_DEDUPED" # Inactive-outcome rows are acknowledged only after their matching durable # receipt is presented. The locked session-start/watcher context authorizes the # helper; a failed correlation leaves the original drained rows restorable. @@ -84,7 +81,25 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" case "$_key" in inactive-outcome:*) - "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" || { + presentation_status=0 + FM_INACTIVE_ACK_FROM_DRAIN=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" presentation "$_key" || presentation_status=$? + case "$presentation_status" in + 0) + if ! printf '%s\n' "$drain_row"; then + FM_INACTIVE_ACK_FROM_DRAIN=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" presentation-clear "$_key" || true + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + exit 1 + fi + ;; + 1) ;; + *) + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + exit "$presentation_status" + ;; + esac + FM_INACTIVE_ACK_FROM_DRAIN=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" || { ack_status=$? # 1 means the receipt was already acknowledged or is not ours. Any # other failure keeps the drained row durable for a later turn. @@ -95,6 +110,13 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do fi } ;; + *) + if ! printf '%s\n' "$drain_row"; then + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + exit 1 + fi + ;; esac done < "$DRAIN_DEDUPED" rm -f "$DRAIN_TMP" diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index f97d430c100..0836ef568e4 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -632,7 +632,7 @@ fm_wake_clean_field() { LC_ALL=C tr '\t\r\n' ' ' } -fm_wake_append() { +fm_wake_append_locked() { local kind=$1 key=$2 payload=$3 clean_key clean_payload epoch seq seq_file status case "$kind" in signal|stale|check|heartbeat) ;; @@ -645,10 +645,6 @@ fm_wake_append() { seq_file="$STATE/.wake-queue.seq" status=0 - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || { - printf 'fm_wake_append: could not serialize the wake queue; refusing to append unlocked\n' >&2 - return 1 - } seq=$(cat "$seq_file" 2>/dev/null || echo 0) case "$seq" in ''|*[!0-9]*) seq=0 ;; @@ -658,7 +654,37 @@ fm_wake_append() { if [ "$status" -eq 0 ]; then printf '%s\t%s\t%s\t%s\t%s\n' "$epoch" "$seq" "$kind" "$clean_key" "$clean_payload" >> "$FM_WAKE_QUEUE" || status=$? fi - fm_lock_release "$FM_WAKE_QUEUE_LOCK" + return "$status" +} + +fm_wake_append() { + local status + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || { + printf 'fm_wake_append: could not serialize the wake queue; refusing to append unlocked\n' >&2 + return 1 + } + fm_wake_append_locked "$@" + status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 + return "$status" +} + +fm_wake_append_if_absent() { # + local result_var=$1 kind=$2 key=$3 payload=$4 status=0 + FM_WAKE_APPEND_CREATED=0 + case "$result_var" in ''|*[!A-Za-z0-9_]*) return 2 ;; esac + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || { + printf 'fm_wake_append_if_absent: could not serialize the wake queue; refusing to append unlocked\n' >&2 + return 1 + } + if [ -f "$FM_WAKE_QUEUE" ] && awk -F '\t' -v wanted="$key" '$4 == wanted { found=1 } END { exit(found ? 0 : 1) }' "$FM_WAKE_QUEUE" 2>/dev/null; then + printf -v "$result_var" '%s' 0 + else + fm_wake_append_locked "$kind" "$key" "$payload" + status=$? + [ "$status" -eq 0 ] && { FM_WAKE_APPEND_CREATED=1; printf -v "$result_var" '%s' 1; } + fi + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 return "$status" } diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index fc648680d71..5a2990efa22 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -592,7 +592,10 @@ while :; do # The helper owns its bounded cadence and receipt idempotence. A non-empty # result means it appended an inactive-outcome wake, so surface that wake in # this watcher turn without probing panes or scraping secondmate chat here. - inactive_out=$(FM_SESSION_LOCK_BOOTSTRAP=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" scan 2>/dev/null || true) + if ! inactive_out=$("$SCRIPT_DIR/fm-inactive-reconcile.sh" scan 2>&1); then + printf '%s\n' "$inactive_out" >&2 + exit 1 + fi if [ -n "$inactive_out" ]; then wake "check: inactive terminal outcome replay queued" fi diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 28fc3d610b3..ffc4aaec95c 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -22,6 +22,8 @@ CASE_DIR= CASE_ROOT= CASE_HOME= CASE_FAKEBIN= +CASE_THREAD= +CASE_TOKEN= new_case() { local name=$1 dir root home state fakebin @@ -33,6 +35,8 @@ new_case() { mkdir -p "$state" "$home/data" "$home/config" "$fakebin" git init -q -b main "$root" git -C "$root" commit -q --allow-empty -m init + cp "$ROOT/AGENTS.md" "$root/AGENTS.md" + mkdir -p "$root/bin" "$home/projects" cat > "$fakebin/fm-crew-state.sh" <<'SH' #!/usr/bin/env bash set -u @@ -42,6 +46,28 @@ var="FM_FAKE_CREW_STATE_$key" printf '%s\n' "${!var:-${FM_FAKE_CREW_STATE:-state: unknown · source: none · fake default}}" SH chmod +x "$fakebin/fm-crew-state.sh" + cat > "$fakebin/ps" <<'SH' +#!/usr/bin/env bash +set -u +pid= +previous= +for argument in "$@"; do + [ "$previous" = -p ] && pid=$argument + previous=$argument +done +case "$*" in + *"comm="*|*"args="*) + if [ "$pid" = "${FM_FAKE_HARNESS_PID:-}" ]; then + printf '%s\n' claude + else + printf '%s\n' bash + fi + ;; + *"ppid="*) printf '%s\n' "${FM_FAKE_HARNESS_PID:-1}" ;; + *) exit 1 ;; +esac +SH + chmod +x "$fakebin/ps" for tool in gh gh-axi curl; do cat > "$fakebin/$tool" <<'SH' #!/usr/bin/env bash @@ -55,25 +81,50 @@ SH CASE_ROOT=$root CASE_HOME=$home CASE_FAKEBIN=$fakebin + CASE_THREAD="inactive-${name//[^A-Za-z0-9]/-}" +} + +prepare_primary_proof() { + local root=$1 home=$2 fakebin=$3 state="$home/state" token + mkdir -p "$state" "$home/projects" + if [ ! -f "$state/.primary-attestation" ]; then + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_PRIMARY_ATTESTATION -u FM_ROOT -u STATE \ + FM_HOME="$home" FM_ROOT_OVERRIDE="$root" FM_STATE_OVERRIDE="$state" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" PATH="$fakebin:$PATH" \ + bash -c '. "$1"; fm_worker_primary_attestation_prepare' _ \ + "$ROOT/bin/fm-worker-isolation-lib.sh" ) || fail "primary proof setup failed" + fi + token=$(awk -F= '$1 == "token" {print substr($0, index($0, "=") + 1); exit}' \ + "$state/.primary-attestation") + [ -n "$token" ] || fail "primary proof token was not persisted" + printf '%s|codex:%s|fallback\n' "$$" "$CASE_THREAD" > "$state/.lock" + CASE_TOKEN=$token } scan() { local root=$1 home=$2 fakebin=$3 startup=${4:-} - env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME -u FM_PRIMARY_ATTESTATION \ - PATH="$fakebin:$PATH" \ - FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ - FM_SESSION_LOCK_BOOTSTRAP=1 \ - FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ - FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ - "$RECON" scan "$startup" + if [ -d "$home/state" ] && [ ! -L "$home/state" ]; then + prepare_primary_proof "$root" "$home" "$fakebin" + fi + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" \ + FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ + FM_FAKE_HARNESS_PID="$$" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + "$RECON" scan "$startup" ) } drain() { local root=$1 home=$2 fakebin=$3 - env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME -u FM_PRIMARY_ATTESTATION \ - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_SESSION_LOCK_BOOTSTRAP=1 \ - FM_STATE_OVERRIDE="$home/state" "$DRAIN" + if [ -d "$home/state" ] && [ ! -L "$home/state" ]; then + prepare_primary_proof "$root" "$home" "$fakebin" + fi + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$home/state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" "$DRAIN" ) } write_meta() { @@ -90,6 +141,23 @@ write_meta() { touch -d '2 minutes ago' "$file" "$state/$id.status" "$state/$id.turn-ended" } +write_legacy_meta() { + local state=$1 id=$2 kind=${3:-ship} backend=${4:-tmux} window + window="tmux:fm-$id" + fm_write_meta "$state/$id.meta" \ + "window=$window" "worktree=$state/work-$id" "project=$state/work-$id" \ + "harness=echo" "kind=$kind" "mode=$kind" "yolo=off" "backend=$backend" + mkdir -p "$state/work-$id" + printf 'working: fixture\n' > "$state/$id.status" + : > "$state/$id.turn-ended" + touch -d '2 minutes ago' "$state/$id.meta" "$state/$id.status" "$state/$id.turn-ended" +} + +receipt_value() { + local file=$1 key=$2 + awk -F= -v wanted="$key" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$file" +} + receipt_count() { local state=$1 suffix=$2 find "$state/terminal-outcomes" -maxdepth 1 -type f -name "*.$suffix" 2>/dev/null | wc -l | tr -d ' ' @@ -103,7 +171,7 @@ queue_count() { } test_done_and_failed_are_replayed_once() { - local dir root home fakebin state + local dir root home fakebin state rec task fingerprint new_case done-failed dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -114,6 +182,34 @@ test_done_and_failed_are_replayed_once() { scan "$root" "$home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$state" pending)" = 2 ] || fail "done and failed outcomes did not create two pending receipts" [ "$(queue_count "$state")" = 2 ] || fail "done and failed outcomes did not create two wakes" + for rec in "$state"/terminal-outcomes/*.pending; do + task=$(receipt_value "$rec" task_id) + fingerprint=$(basename "$rec" .pending) + [ "$(receipt_value "$rec" schema)" = fm-jt-terminal-outcome.v1 ] || fail "receipt schema was not durable" + [ "$(receipt_value "$rec" fingerprint)" = "$fingerprint" ] || fail "receipt fingerprint did not bind its filename" + [ "$(receipt_value "$rec" incarnation)" = inc-done ] || [ "$(receipt_value "$rec" incarnation)" = inc-failed ] \ + || fail "receipt lost its spawn incarnation" + case "$task" in + done-x1) [ "$(receipt_value "$rec" outcome)" = done ] || fail "done receipt outcome was incorrect" ;; + failed-x1) [ "$(receipt_value "$rec" outcome)" = failed ] || fail "failed receipt outcome was incorrect" ;; + *) fail "receipt persisted an unexpected task id: $task" ;; + esac + [ "$(receipt_value "$rec" terminal_source)" != "" ] || fail "receipt lost terminal source" + [ "$(receipt_value "$rec" terminal_snapshot)" != "" ] || fail "receipt lost terminal snapshot" + [ "$(receipt_value "$rec" parent_home)" = "" ] || fail "firstmate receipt invented a parent route" + done + rec=$(find "$state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + fingerprint=$(basename "$rec" .pending) + if ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + "$RECON" ack "inactive-outcome:$fingerprint" >/dev/null 2>&1 ); then + fail "direct inactive acknowledgement bypassed the wake drain" + fi + [ -f "$rec" ] || fail "direct inactive acknowledgement removed its pending receipt" + touch -d '2 minutes ago' "$state/.inactive-outcome-reconcile" + scan "$root" "$home" "$fakebin" >/dev/null || fail "normal watcher cadence scan failed" scan "$root" "$home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$state" pending)" = 2 ] || fail "rescan duplicated inactive outcome receipts" [ "$(queue_count "$state")" = 2 ] || fail "rescan duplicated inactive outcome wakes" @@ -161,22 +257,34 @@ test_scan_failure_retries_without_advancing_cadence() { cat > "$fakebin/fm-crew-state.sh" <<'SH' #!/usr/bin/env bash set -u -if [ "${1:-}" = second-x1 ] && [ "${FM_BREAK_QUEUE:-0}" = 1 ]; then - mv "${FM_WAKE_QUEUE_DIR}" "${FM_WAKE_QUEUE_REMOVED}" +if [ "${FM_BREAK_QUEUE:-0}" = 1 ]; then + if [ -e "${FM_BREAK_QUEUE_MARKER:-}" ]; then + mv "${FM_WAKE_QUEUE_DIR}" "${FM_WAKE_QUEUE_REMOVED}" + else + : > "${FM_BREAK_QUEUE_MARKER}" + fi fi printf 'state: done · source: pane · scan retry\n' SH chmod +x "$fakebin/fm-crew-state.sh" export FM_WAKE_QUEUE="$wake_dir/queue" FM_WAKE_QUEUE_LOCK="$wake_dir/lock" - export FM_WAKE_QUEUE_DIR="$wake_dir" FM_WAKE_QUEUE_REMOVED="$wake_removed" FM_BREAK_QUEUE=1 + export FM_WAKE_QUEUE_DIR="$wake_dir" FM_WAKE_QUEUE_REMOVED="$wake_removed" \ + FM_BREAK_QUEUE_MARKER="$dir/scan-first" FM_BREAK_QUEUE=1 if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then fail "child scan failure was reported as success" fi [ ! -e "$state/.inactive-outcome-reconcile" ] || fail "failed scan advanced the cadence marker" [ "$(receipt_count "$state" pending)" = 2 ] || fail "durable receipts were not retained across wake publication failure" + [ ! -e "$state"/.first-x1.inactive-state.* ] || fail "failed crew-state scan leaked its temporary output" + [ ! -e "$state"/.second-x1.inactive-state.* ] || fail "failed crew-state scan leaked its temporary output" grep -l '^task_id=first-x1$' "$state"/terminal-outcomes/*.pending >/dev/null \ - || fail "successful child receipt was not retained" - [ "$(cat "$state/.inactive-outcome-reconcile.cursor")" = first-x1 ] || fail "cursor did not preserve the last successful child" + || fail "first child receipt was not retained" + grep -l '^task_id=second-x1$' "$state"/terminal-outcomes/*.pending >/dev/null \ + || fail "second child receipt was not retained" + case "$(cat "$state/.inactive-outcome-reconcile.cursor")" in + first-x1|second-x1) ;; + *) fail "cursor did not preserve the last successful child" ;; + esac mv "$wake_removed" "$wake_dir" export FM_BREAK_QUEUE=0 scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "retry after child failure did not complete" @@ -210,7 +318,7 @@ test_state_paths_reject_symlinks_and_non_directories() { } test_reused_task_id_gets_new_fingerprint() { - local dir root home fakebin state + local dir root home fakebin state rec first_fp second_fp new_case reused-id dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -222,10 +330,46 @@ test_reused_task_id_gets_new_fingerprint() { scan "$root" "$home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$state" pending)" = 2 ] || fail "reused task id did not create a new incarnation receipt" [ "$(queue_count "$state")" = 2 ] || fail "reused task id did not create a new fingerprinted wake" + for rec in "$state"/terminal-outcomes/*.pending; do + [ "$(receipt_value "$rec" task_id)" = reused-x1 ] || fail "reused task receipt lost its task id" + case "$(receipt_value "$rec" incarnation)" in + incarnation-old|incarnation-new) ;; + *) fail "reused task receipt lost its incarnation" ;; + esac + done + first_fp= + second_fp= + for rec in "$state"/terminal-outcomes/*.pending; do + if [ -z "$first_fp" ]; then + first_fp=$(basename "$rec" .pending) + else + second_fp=$(basename "$rec" .pending) + fi + done + [ -n "$first_fp" ] && [ -n "$second_fp" ] && [ "$first_fp" != "$second_fp" ] \ + || fail "reused task receipts did not retain distinct fingerprints" unset FM_FAKE_CREW_STATE_REUSED_X1 pass "reused task ids are separated by the spawn incarnation" } +test_legacy_metadata_uses_stable_fallback() { + local dir root home fakebin state rec incarnation + new_case legacy-fallback + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_legacy_meta "$state" legacy-x1 + export FM_FAKE_CREW_STATE_LEGACY_X1='state: done · source: pane · legacy quiet' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "legacy metadata scan failed" + rec=$(find "$state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + [ -n "$rec" ] || fail "legacy metadata did not create a receipt" + incarnation=$(receipt_value "$rec" incarnation) + case "$incarnation" in legacy-*) ;; *) fail "legacy metadata lacked a documented fallback incarnation" ;; esac + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "legacy metadata rescan failed" + [ "$(receipt_count "$state" pending)" = 1 ] || fail "legacy fallback was not stable across rescans" + unset FM_FAKE_CREW_STATE_LEGACY_X1 + pass "legacy metadata receives a stable fallback incarnation" +} + test_relaunch_and_teardown_races_recheck_under_spawn_lock() { local dir root home fakebin state holder scanner ready release new_case races @@ -235,9 +379,12 @@ test_relaunch_and_teardown_races_recheck_under_spawn_lock() { export FM_FAKE_CREW_STATE_RELAUNCH_X1='state: done · source: pane · relaunch race' ready="$dir/ready" release="$dir/release" + prepare_primary_proof "$root" "$home" "$fakebin" ( - FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ - bash -c '. "$1/bin/fm-wake-lib.sh"; fm_lock_acquire_wait "$2/.spawn-relaunch-x1.lock"; : > "$3"; while [ ! -e "$4" ]; do sleep 0.01; done; fm_lock_release "$2/.spawn-relaunch-x1.lock"' _ "$ROOT" "$state" "$ready" "$release" + cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + PATH="$fakebin:$PATH" bash -c '. "$1/bin/fm-wake-lib.sh"; fm_lock_acquire_wait "$2/.spawn-relaunch-x1.lock"; : > "$3"; while [ ! -e "$4" ]; do sleep 0.01; done; fm_lock_release "$2/.spawn-relaunch-x1.lock"' _ "$ROOT" "$state" "$ready" "$release" ) & holder=$! for _ in $(seq 1 100); do [ -e "$ready" ] && break; sleep 0.01; done @@ -248,6 +395,10 @@ test_relaunch_and_teardown_races_recheck_under_spawn_lock() { : > "$release" wait "$holder" || fail "spawn-lock relaunch fixture failed" wait "$scanner" || fail "relaunch reconciliation fixture failed" + [ "$(find "$state/terminal-outcomes" -type f -name '*.pending' 2>/dev/null | wc -l | tr -d ' ')" = 0 ] \ + || fail "fresh relaunch was replayed before its quiet period" + touch -d '2 minutes ago' "$state/relaunch-x1.meta" + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "aged relaunch reconciliation failed" grep -F 'incarnation=new-inc' "$state"/terminal-outcomes/*.pending >/dev/null || fail "relaunch race used stale incarnation" write_meta "$state" teardown-x1 teardown-inc @@ -255,8 +406,10 @@ test_relaunch_and_teardown_races_recheck_under_spawn_lock() { ready="$dir/ready-teardown" release="$dir/release-teardown" ( - FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ - bash -c '. "$1/bin/fm-wake-lib.sh"; fm_lock_acquire_wait "$2/.spawn-teardown-x1.lock"; : > "$3"; while [ ! -e "$4" ]; do sleep 0.01; done; fm_lock_release "$2/.spawn-teardown-x1.lock"' _ "$ROOT" "$state" "$ready" "$release" + cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + PATH="$fakebin:$PATH" bash -c '. "$1/bin/fm-wake-lib.sh"; fm_lock_acquire_wait "$2/.spawn-teardown-x1.lock"; : > "$3"; while [ ! -e "$4" ]; do sleep 0.01; done; fm_lock_release "$2/.spawn-teardown-x1.lock"' _ "$ROOT" "$state" "$ready" "$release" ) & holder=$! for _ in $(seq 1 100); do [ -e "$ready" ] && break; sleep 0.01; done @@ -272,6 +425,20 @@ test_relaunch_and_teardown_races_recheck_under_spawn_lock() { pass "relaunch and teardown races recheck metadata under the spawn lock" } +test_parent_home_secondmate_records_are_skipped() { + local dir root home fakebin state + new_case parent-home-skip + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" parent-sm-x1 parent-inc secondmate + export FM_FAKE_CREW_STATE_PARENT_SM_X1='state: done · source: pane · parent record' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "parent-home scan failed" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "parent-home secondmate record was replayed" + [ "$(queue_count "$state")" = 0 ] || fail "parent-home secondmate record queued a wake" + unset FM_FAKE_CREW_STATE_PARENT_SM_X1 + pass "parent-home secondmate records stay outside inactive replay" +} + test_herdr_identity_and_default_captain_refusal() { local dir root home fakebin state new_case herdr-identity @@ -288,7 +455,8 @@ test_herdr_identity_and_default_captain_refusal() { export FM_FAKE_CREW_STATE_HERDR_DEFAULT='state: done · source: pane · must refuse' export FM_FAKE_CREW_STATE_HERDR_CAPTAIN='state: done · source: pane · must refuse' export FM_NETWORK_LOG="$dir/network.log" - PATH="$fakebin:$PATH" scan "$root" "$home" "$fakebin" --startup >/dev/null + PATH="$fakebin:$PATH" scan "$root" "$home" "$fakebin" --startup >/dev/null \ + || fail "Herdr identity scan failed" [ "$(receipt_count "$state" pending)" = 1 ] || fail "Herdr dedicated session was not accepted while default/CAPTAIN were refused" [ ! -s "$dir/network.log" ] || fail "inactive reconciliation made a forge/network call" unset FM_FAKE_CREW_STATE_HERDR_GOOD FM_FAKE_CREW_STATE_HERDR_DEFAULT FM_FAKE_CREW_STATE_HERDR_CAPTAIN FM_NETWORK_LOG @@ -302,7 +470,8 @@ test_occupancy_unknown_is_not_terminal() { state="$home/state" write_meta "$state" unknown-x1 unknown-inc export FM_FAKE_CREW_STATE_UNKNOWN_X1='state: unknown · source: none · occupancy unknown' - scan "$root" "$home" "$fakebin" --startup >/dev/null + scan "$root" "$home" "$fakebin" --startup >/dev/null \ + || fail "occupancy scan failed" [ "$(receipt_count "$state" pending)" = 0 ] || fail "occupancy unknown was treated as terminal" [ "$(queue_count "$state")" = 0 ] || fail "occupancy unknown created an actionable wake" unset FM_FAKE_CREW_STATE_UNKNOWN_X1 @@ -316,7 +485,8 @@ test_status_log_terminal_is_not_replayed() { state="$home/state" write_meta "$state" stale-x1 stale-inc export FM_FAKE_CREW_STATE_STALE_X1='state: done · source: status-log · stale done event' - scan "$root" "$home" "$fakebin" --startup >/dev/null + scan "$root" "$home" "$fakebin" --startup >/dev/null \ + || fail "status-log scan failed" [ "$(receipt_count "$state" pending)" = 0 ] || fail "stale status-log done was treated as terminal" [ "$(queue_count "$state")" = 0 ] || fail "stale status-log done created an actionable wake" unset FM_FAKE_CREW_STATE_STALE_X1 @@ -324,7 +494,7 @@ test_status_log_terminal_is_not_replayed() { } test_valid_secondmate_route_reports_parent_once() { - local dir root home fakebin state child_home child_state parent_status corr rec + local dir root home fakebin state child_home child_state parent_status corr rec outside new_case secondmate-route-valid dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -344,9 +514,25 @@ test_valid_secondmate_route_reports_parent_once() { "$home" "$parent_status" "$corr" > "$child_state/.fm-jt-parent-route" scan "$root" "$child_home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "valid secondmate route did not create a pending receipt" - drain "$root" "$child_home" "$fakebin" >/dev/null + rec=$(find "$child_state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + [ "$(receipt_value "$rec" parent_task_id)" = sm-valid ] || fail "secondmate receipt did not persist its parent task identity" + [ "$(receipt_value "$rec" parent_corr)" = "$corr" ] || fail "secondmate receipt did not persist its parent correlation" + [ "$(receipt_value "$rec" parent_home)" = "$home" ] || fail "secondmate receipt did not persist its parent home" + [ "$(receipt_value "$rec" parent_status)" = "$parent_status" ] || fail "secondmate receipt did not persist its parent status path" + outside="$dir/outside-status" + printf 'outside\n' > "$outside" + ln -s "$outside" "$parent_status" + if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then + fail "secondmate acknowledgement followed a parent status symlink" + fi + [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "symlinked parent status lost the pending receipt" + rm -f "$parent_status" + if ! drain "$root" "$child_home" "$fakebin" >"$dir/second-drain.out" 2>&1; then + cat "$dir/second-drain.out" >&2 + fail "valid secondmate route drain failed after symlink removal" + fi [ "$(receipt_count "$child_state" reported)" = 1 ] || fail "valid secondmate route was not reported" - [ ! -e "$child_state/.fm-jt-parent-route" ] || fail "reported secondmate route remained installed" + [ -f "$child_state/.fm-jt-parent-route" ] || fail "reported secondmate route was cleared before its parent lifecycle resolved" grep -F "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status" >/dev/null \ || fail "valid secondmate route did not append the correlated parent status" drain "$root" "$child_home" "$fakebin" >/dev/null @@ -436,6 +622,13 @@ test_malformed_or_missing_secondmate_route_fails_closed() { scan "$root" "$child_home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "malformed secondmate parent route was not fail-closed" [ ! -e "$parent_status" ] || fail "malformed secondmate route wrote parent status" + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=\nsecondmate_id=sm-x1\nparent_home=%s\nparent_status=%s\ncorr_id=0123456789abcdef\n' \ + "$home" "$parent_status" > "$child_state/.fm-jt-parent-route" + fm_write_meta "$home/state/pending-replies/0123456789abcdef" \ + schema=fm-pending-reply.v1 corr_id=0123456789abcdef task_id=sm-x1 \ + parent_home="$home" parent_status="$parent_status" delivered_epoch=1 phase=awaiting_report + scan "$root" "$child_home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "duplicate route fields were accepted" unset FM_FAKE_CREW_STATE_CHILD_X1 pass "malformed and missing secondmate parent routes fail closed without chat scraping" } @@ -445,7 +638,9 @@ test_portable_timeout_runner_is_used test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint +test_legacy_metadata_uses_stable_fallback test_relaunch_and_teardown_races_recheck_under_spawn_lock +test_parent_home_secondmate_records_are_skipped test_herdr_identity_and_default_captain_refusal test_occupancy_unknown_is_not_terminal test_status_log_terminal_is_not_replayed From 8794471c3327dc0318ac2e5f400395eaa734793a Mon Sep 17 00:00:00 2001 From: tests Date: Thu, 13 Aug 2026 23:52:30 +0000 Subject: [PATCH 005/163] no-mistakes(review): Harden inactive replay claims and secondmate history routing --- bin/fm-inactive-reconcile.sh | 186 +++++++++++++++++++----------- bin/fm-pending-reply-lib.sh | 28 +++-- bin/fm-wake-drain.sh | 32 ++--- tests/fm-inactive-outcome.test.sh | 184 +++++++++++++++++++++++++++-- 4 files changed, 325 insertions(+), 105 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index d561892f6dc..7fd8b9cbc78 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -114,10 +114,11 @@ valid_task_id() { is_secondmate_home() { local marker=$ROUTE_MARKER value + [ ! -L "$marker" ] || return 2 if [ ! -e "$marker" ]; then return 1 fi - [ -f "$marker" ] && [ ! -L "$marker" ] || return 2 + [ -f "$marker" ] || return 2 value=$(cat "$marker" 2>/dev/null || true) case "$value" in ''|*[!A-Za-z0-9._-]*) return 2 ;; @@ -162,6 +163,103 @@ receipt_field() { # meta_value "$1" "$2" } +claim_path() { # + printf '%s/.%s.claim' "$OUTCOME_DIR" "$1" +} + +claim_field() { # + meta_value "$1" "$2" +} + +drain_claim_owner() { + local row=$1 owner parent_pid drain_file drain_dir state_dir + owner=$(fm_lock_link_owner "$FM_WAKE_QUEUE_LOCK" 2>/dev/null || true) + parent_pid=${PPID:-} + drain_file=${FM_WAKE_DRAIN_FILE:-} + [ -n "$owner" ] && [ -n "$parent_pid" ] && [ -n "$row" ] && [ -n "$drain_file" ] || return 1 + [ "$(cat "$owner/pid" 2>/dev/null || true)" = "$parent_pid" ] || return 1 + fm_lock_points_to_owner "$FM_WAKE_QUEUE_LOCK" "$owner" || return 1 + [ -f "$drain_file" ] && [ ! -L "$drain_file" ] || return 1 + drain_dir=$(cd "$(dirname "$drain_file")" 2>/dev/null && pwd -P) || return 1 + state_dir=$(cd "$STATE" 2>/dev/null && pwd -P) || return 1 + [ "$drain_dir" = "$state_dir" ] || return 1 + [ "$(basename "$drain_file")" = ".wake-queue.deduped.$parent_pid" ] || return 1 + awk -v wanted="$row" '$0 == wanted { found=1; exit } END { exit !found }' "$drain_file" +} + +claim_validate() { # + local claim=$1 fp=$2 row=$3 state + [ -f "$claim" ] && [ ! -L "$claim" ] || return 1 + [ "$(claim_field "$claim" schema)" = fm-inactive-outcome-claim.v1 ] || return 1 + [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 1 + [ "$(claim_field "$claim" row)" = "$row" ] || return 1 + state=$(claim_field "$claim" state) + case "$state" in reserved|presented) printf '%s' "$state" ;; *) return 1 ;; esac +} + +claim_reserve() { # + local key=$1 row=$2 fp claim tmp state existing + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + inactive_state_preflight || return 2 + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + if [ -e "$claim" ]; then + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presented ] && return 1 + return 0 + fi + mkdir -p "$OUTCOME_DIR" || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + printf 'schema=fm-inactive-outcome-claim.v1\n' + printf 'fingerprint=%s\n' "$fp" + printf 'row=%s\n' "$row" + printf 'state=reserved\n' + printf 'created_epoch=%s\n' "$(date +%s)" + } > "$tmp" || { rm -f "$tmp"; return 2; } + if ln "$tmp" "$claim" 2>/dev/null; then + rm -f "$tmp" + return 0 + fi + rm -f "$tmp" + [ -e "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presented ] && return 1 + return 0 +} + +claim_mark_presented() { # + local key=$1 row=$2 fp claim tmp line + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + [ "$(claim_validate "$claim" "$fp" "$row")" = reserved ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in state=*) printf 'state=presented\n' ;; *) printf '%s\n' "$line" ;; esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_remove() { # + local key=$1 row=$2 fp claim + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + [ -e "$claim" ] || return 0 + claim_validate "$claim" "$fp" "$row" >/dev/null || return 2 + rm -f "$claim" +} + receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE local pending tmp existing inactive_state_preflight || return 1 @@ -223,60 +321,6 @@ receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE return 0 } -presentation_marker_path() { # - printf '%s/%s.presented-marker' "$OUTCOME_DIR" "$1" -} - -presentation_mark() { # - local key=$1 fp marker pending presented reported tmp existing - [ "${FM_INACTIVE_ACK_FROM_DRAIN:-0}" = 1 ] || return 2 - case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac - case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac - inactive_state_preflight || return 2 - marker=$(presentation_marker_path "$fp") - [ ! -L "$marker" ] || return 2 - if [ -e "$marker" ]; then - [ -f "$marker" ] || return 2 - return 1 - fi - pending=$(receipt_path "$fp" pending) - presented=$(receipt_path "$fp" presented) - reported=$(receipt_path "$fp" reported) - for existing in "$pending" "$presented" "$reported"; do - [ ! -L "$existing" ] || return 2 - if [ -e "$existing" ]; then - [ -f "$existing" ] || return 2 - [ "$(receipt_field "$existing" fingerprint)" = "$fp" ] || return 2 - fi - done - if [ ! -f "$pending" ]; then - [ -f "$presented" ] || [ -f "$reported" ] || return 2 - return 1 - fi - tmp=$(mktemp "$OUTCOME_DIR/.presentation.XXXXXX") || return 2 - printf '%s\n' "$fp" > "$tmp" || { rm -f "$tmp"; return 2; } - if ln "$tmp" "$marker" 2>/dev/null; then - rm -f "$tmp" - return 0 - fi - rm -f "$tmp" - [ -f "$marker" ] && [ ! -L "$marker" ] && return 1 - return 2 -} - -presentation_clear() { # - local key=$1 fp marker - [ "${FM_INACTIVE_ACK_FROM_DRAIN:-0}" = 1 ] || return 2 - case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac - case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac - inactive_state_preflight || return 2 - marker=$(presentation_marker_path "$fp") - [ ! -L "$marker" ] || return 2 - [ -e "$marker" ] || return 0 - [ -f "$marker" ] || return 2 - rm -f "$marker" -} - read_incarnation() { # local meta=$1 id=$2 token tasktmp window worktree seed token=$(meta_value "$meta" spawn_incarnation) @@ -389,10 +433,13 @@ reconcile_child() { } ack_receipt() { # - local key=$1 fp rec id kind outcome parent_task_id parent_home parent_status corr line target existing marker - [ "${FM_INACTIVE_ACK_FROM_DRAIN:-0}" = 1 ] || return 2 + local key=$1 row=${2:-} fp rec id kind outcome parent_task_id parent_home parent_status corr line target existing claim_state + [ -n "$row" ] || return 2 + drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 0 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + claim_state=$(claim_validate "$(claim_path "$fp")" "$fp" "$row") || return 2 + [ "$claim_state" = presented ] || return 2 rec=$(receipt_path "$fp" pending) [ ! -L "$rec" ] || return 2 if [ ! -e "$rec" ]; then @@ -401,15 +448,13 @@ ack_receipt() { # if [ -e "$existing" ]; then [ -f "$existing" ] || return 2 [ "$(receipt_field "$existing" fingerprint)" = "$fp" ] || return 2 + claim_remove "$key" "$row" || return 2 return 1 fi done return 2 fi [ -f "$rec" ] || return 2 - marker=$(presentation_marker_path "$fp") - [ -f "$marker" ] && [ ! -L "$marker" ] || return 2 - [ "$(cat "$marker" 2>/dev/null || true)" = "$fp" ] || return 2 [ "$(receipt_field "$rec" fingerprint)" = "$fp" ] || return 2 [ "$(receipt_field "$rec" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 id=$(receipt_field "$rec" task_id) @@ -443,10 +488,11 @@ ack_receipt() { # [ -f "$target" ] || return 2 [ "$(receipt_field "$target" fingerprint)" = "$fp" ] || return 2 rm -f "$rec" || return 2 + claim_remove "$key" "$row" || return 2 return 1 fi mv "$rec" "$target" || return 2 - presentation_clear "$key" || return 2 + claim_remove "$key" "$row" || return 2 return 0 } @@ -537,23 +583,23 @@ case "${1:-}" in fi ;; ack) - [ -n "${2:-}" ] || exit 2 - ack_receipt "$2" + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + ack_receipt "$2" "$3" ;; - presentation) - [ -n "${2:-}" ] || exit 2 - presentation_mark "$2" + claim) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_reserve "$2" "$3" ;; - presentation-clear) - [ -n "${2:-}" ] || exit 2 - presentation_clear "$2" + presented) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_presented "$2" "$3" ;; _child) [ -n "${2:-}" ] || exit 2 reconcile_child "$2" ;; *) - echo "usage: fm-inactive-reconcile.sh scan [--startup] | ack " >&2 + echo "usage: fm-inactive-reconcile.sh scan [--startup] | ack " >&2 exit 2 ;; esac diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 89099d3f7da..ae4710ae9be 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -613,7 +613,7 @@ fm_pending_reply_secondmate_route_clear() { # fm_pending_reply_secondmate_route_validate() { # local secondmate_home=$1 marker line key value schema marker_id secondmate_id - local parent_home parent_status corr parent_abs state_abs expected_status rec + local parent_home parent_status corr parent_abs state_abs expected_status rec active_rec history_rec history_dir local phase delivered record_home record_status record_task record_corr home_marker local seen_schema=0 seen_secondmate_id=0 seen_parent_home=0 seen_parent_status=0 seen_corr=0 FM_PENDING_ROUTE_PARENT_STATUS= @@ -654,8 +654,9 @@ fm_pending_reply_secondmate_route_validate() { # [ -d "$parent_abs/state" ] && [ ! -L "$parent_abs/state" ] || return 1 state_abs=$(cd "$parent_abs/state" 2>/dev/null && pwd -P) || return 1 [ -d "$state_abs/pending-replies" ] && [ ! -L "$state_abs/pending-replies" ] || return 1 - if [ -e "$state_abs/pending-replies/history" ]; then - [ -d "$state_abs/pending-replies/history" ] && [ ! -L "$state_abs/pending-replies/history" ] || return 1 + history_dir=$(fm_pending_reply_history_dir "$state_abs") + if [ -e "$history_dir" ] || [ -L "$history_dir" ]; then + [ -d "$history_dir" ] && [ ! -L "$history_dir" ] || return 1 fi expected_status="$state_abs/$secondmate_id.status" [ "$parent_status" = "$expected_status" ] || return 1 @@ -663,8 +664,16 @@ fm_pending_reply_secondmate_route_validate() { # if [ -e "$expected_status" ]; then [ -f "$expected_status" ] || return 1 fi - rec="$state_abs/pending-replies/$corr" - [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + active_rec="$state_abs/pending-replies/$corr" + history_rec="$history_dir/$corr" + if [ -e "$active_rec" ] || [ -L "$active_rec" ]; then + [ -f "$active_rec" ] && [ ! -L "$active_rec" ] || return 1 + rec=$active_rec + elif [ -f "$history_rec" ] && [ ! -L "$history_rec" ]; then + rec=$history_rec + else + return 1 + fi record_task=$(fm_pending_reply_get "$rec" task_id) record_home=$(fm_pending_reply_get "$rec" parent_home) record_status=$(fm_pending_reply_get "$rec" parent_status) @@ -693,7 +702,7 @@ fm_pending_reply_secondmate_route_validate() { # fm_pending_reply_secondmate_receipt_validate() { # local secondmate_home=$1 secondmate_id=$2 parent_home=$3 parent_status=$4 corr=$5 - local home_marker marker_id parent_abs state_abs expected_status rec active_rec history_rec + local home_marker marker_id parent_abs state_abs expected_status rec active_rec history_rec history_dir local record_task record_home record_status record_corr delivered phase [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 @@ -709,8 +718,9 @@ fm_pending_reply_secondmate_receipt_validate() { # /dev/null && pwd -P) || return 1 [ -d "$state_abs/pending-replies" ] && [ ! -L "$state_abs/pending-replies" ] || return 1 - if [ -e "$state_abs/pending-replies/history" ]; then - [ -d "$state_abs/pending-replies/history" ] && [ ! -L "$state_abs/pending-replies/history" ] || return 1 + history_dir=$(fm_pending_reply_history_dir "$state_abs") + if [ -e "$history_dir" ] || [ -L "$history_dir" ]; then + [ -d "$history_dir" ] && [ ! -L "$history_dir" ] || return 1 fi expected_status="$state_abs/$secondmate_id.status" [ "$parent_status" = "$expected_status" ] || return 1 @@ -719,7 +729,7 @@ fm_pending_reply_secondmate_receipt_validate() { # "$DRAIN_DEDUPED" || exit "$?" # Inactive-outcome rows are acknowledged only after their matching durable -# receipt is presented. The locked session-start/watcher context authorizes the -# helper; a failed correlation leaves the original drained rows restorable. +# receipt is presented. A one-time claim binds the receipt to this locked drain. drain_line=0 while IFS= read -r drain_row || [ -n "$drain_row" ]; do drain_line=$((drain_line + 1)) IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" case "$_key" in inactive-outcome:*) - presentation_status=0 - FM_INACTIVE_ACK_FROM_DRAIN=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" presentation "$_key" || presentation_status=$? - case "$presentation_status" in + claim_status=0 + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" claim "$_key" "$drain_row" || claim_status=$? + case "$claim_status" in 0) if ! printf '%s\n' "$drain_row"; then - FM_INACTIVE_ACK_FROM_DRAIN=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" presentation-clear "$_key" || true - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" + awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + exit 1 + fi + if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$_key" "$drain_row"; then + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 fi ;; 1) ;; *) - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 - exit "$presentation_status" + exit "$claim_status" ;; esac - FM_INACTIVE_ACK_FROM_DRAIN=1 "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" || { + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" "$drain_row" || { ack_status=$? # 1 means the receipt was already acknowledged or is not ours. Any # other failure keeps the drained row durable for a later turn. if [ "$ack_status" != 1 ]; then - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit "$ack_status" fi @@ -112,7 +116,7 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do ;; *) if ! printf '%s\n' "$drain_row"; then - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$(fm_current_pid)" + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 fi diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index ffc4aaec95c..4d5bc457196 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -158,6 +158,17 @@ receipt_value() { awk -F= -v wanted="$key" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$file" } +receipt_fingerprint() { + local value=$1 + if command -v shasum >/dev/null 2>&1; then + printf '%s' "$value" | shasum -a 256 | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$value" | sha256sum | awk '{print $1}' + else + printf '%s' "$value" | cksum | awk '{print $1}' + fi +} + receipt_count() { local state=$1 suffix=$2 find "$state/terminal-outcomes" -maxdepth 1 -type f -name "*.$suffix" 2>/dev/null | wc -l | tr -d ' ' @@ -187,11 +198,19 @@ test_done_and_failed_are_replayed_once() { fingerprint=$(basename "$rec" .pending) [ "$(receipt_value "$rec" schema)" = fm-jt-terminal-outcome.v1 ] || fail "receipt schema was not durable" [ "$(receipt_value "$rec" fingerprint)" = "$fingerprint" ] || fail "receipt fingerprint did not bind its filename" - [ "$(receipt_value "$rec" incarnation)" = inc-done ] || [ "$(receipt_value "$rec" incarnation)" = inc-failed ] \ - || fail "receipt lost its spawn incarnation" case "$task" in - done-x1) [ "$(receipt_value "$rec" outcome)" = done ] || fail "done receipt outcome was incorrect" ;; - failed-x1) [ "$(receipt_value "$rec" outcome)" = failed ] || fail "failed receipt outcome was incorrect" ;; + done-x1) + [ "$(receipt_value "$rec" incarnation)" = inc-done ] || fail "done receipt used the wrong incarnation" + [ "$(receipt_value "$rec" outcome)" = done ] || fail "done receipt outcome was incorrect" + [ "$(receipt_value "$rec" terminal_snapshot)" = 'state: done · source: pane · pane is quiet' ] || fail "done receipt snapshot was not exact" + [ "$fingerprint" = "$(receipt_fingerprint 'done-x1|inc-done|done|state: done · source: pane · pane is quiet')" ] || fail "done receipt fingerprint was not bound to its fields" + ;; + failed-x1) + [ "$(receipt_value "$rec" incarnation)" = inc-failed ] || fail "failed receipt used the wrong incarnation" + [ "$(receipt_value "$rec" outcome)" = failed ] || fail "failed receipt outcome was incorrect" + [ "$(receipt_value "$rec" terminal_snapshot)" = 'state: failed · source: run-step · checks failed' ] || fail "failed receipt snapshot was not exact" + [ "$fingerprint" = "$(receipt_fingerprint 'failed-x1|inc-failed|failed|state: failed · source: run-step · checks failed')" ] || fail "failed receipt fingerprint was not bound to its fields" + ;; *) fail "receipt persisted an unexpected task id: $task" ;; esac [ "$(receipt_value "$rec" terminal_source)" != "" ] || fail "receipt lost terminal source" @@ -318,7 +337,7 @@ test_state_paths_reject_symlinks_and_non_directories() { } test_reused_task_id_gets_new_fingerprint() { - local dir root home fakebin state rec first_fp second_fp + local dir root home fakebin state rec first_fp second_fp incarnation fingerprint new_case reused-id dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -332,10 +351,12 @@ test_reused_task_id_gets_new_fingerprint() { [ "$(queue_count "$state")" = 2 ] || fail "reused task id did not create a new fingerprinted wake" for rec in "$state"/terminal-outcomes/*.pending; do [ "$(receipt_value "$rec" task_id)" = reused-x1 ] || fail "reused task receipt lost its task id" - case "$(receipt_value "$rec" incarnation)" in - incarnation-old|incarnation-new) ;; - *) fail "reused task receipt lost its incarnation" ;; - esac + incarnation=$(receipt_value "$rec" incarnation) + case "$incarnation" in incarnation-old|incarnation-new) ;; *) fail "reused task receipt lost its incarnation" ;; esac + [ "$(receipt_value "$rec" outcome)" = done ] || fail "reused task receipt lost its outcome" + [ "$(receipt_value "$rec" terminal_snapshot)" = 'state: done · source: pane · first run quiet' ] || fail "reused task receipt snapshot was not exact" + fingerprint=$(basename "$rec" .pending) + [ "$fingerprint" = "$(receipt_fingerprint "reused-x1|$incarnation|done|state: done · source: pane · first run quiet")" ] || fail "reused task fingerprint was not bound to its fields" done first_fp= second_fp= @@ -352,6 +373,132 @@ test_reused_task_id_gets_new_fingerprint() { pass "reused task ids are separated by the spawn incarnation" } +test_spawn_publishes_incarnation_token() { + local dir root home fakebin state project worktree tmux_state pane_pid out status meta token + new_case spawn-contract + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + cp -a "$ROOT/bin/." "$root/bin/" + project="$dir/project" + worktree="$dir/worktree" + tmux_state="$dir/tmux-window-name" + git init -q -b main "$project" + git -C "$project" commit -q --allow-empty -m init + git -C "$project" worktree add -q --detach "$worktree" + mkdir -p "$home/data/spawn-contract" "$home/projects" "$home/config" + printf 'spawn contract brief\n' > "$home/data/spawn-contract/brief.md" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u + case "$*" in + *"#{pane_current_path}"*) printf '%s\n' "${FM_FAKE_PANE_PATH:-}"; exit 0 ;; + *"#{pane_pid}"*) printf '%s\n' "${FM_FAKE_PANE_PID:-}"; exit 0 ;; + *"#{window_name}"*) cat "$FM_FAKE_TMUX_STATE"; exit 0 ;; +esac +case "${1:-}" in + display-message|list-windows|has-session|new-session|send-keys|kill-window|set-window-option) exit 0 ;; + new-window) printf '%s\n' '@42'; exit 0 ;; + rename-window) printf '%s\n' "${@: -1}" > "$FM_FAKE_TMUX_STATE"; exit 0 ;; +esac +exit 0 +SH + chmod +x "$fakebin/tmux" + cat > "$fakebin/treehouse" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$fakebin/treehouse" + : > "$tmux_state" + ( cd "$worktree" && exec sleep 30 ) >/dev/null 2>&1 & + pane_pid=$! + prepare_primary_proof "$root" "$home" "$fakebin" + out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_PROJECTS_OVERRIDE="$home/projects" \ + FM_CONFIG_OVERRIDE="$home/config" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_SPAWN_NO_GUARD=1 \ + FM_FAKE_PANE_PATH="$worktree" FM_FAKE_PANE_PID="$pane_pid" FM_FAKE_TMUX_STATE="$tmux_state" TMUX=fake,1,0 \ + FM_SPAWN_WT_WAIT_SECS=3 "$root/bin/fm-spawn.sh" spawn-contract "$project" \ + --harness codex 2>&1) + status=$? + kill "$pane_pid" 2>/dev/null || true + [ "$status" = 0 ] || fail "public fm-spawn path failed: $out" + meta="$state/spawn-contract.meta" + [ -f "$meta" ] || fail "public fm-spawn path did not publish metadata" + token=$(receipt_value "$meta" spawn_incarnation) + case "$token" in ''|legacy-unknown) fail "public fm-spawn path published no incarnation token" ;; esac + grep -F 'spawn_incarnation=' "$meta" >/dev/null || fail "spawn metadata omitted its incarnation field" + pass "public fm-spawn publishes the incarnation token in task metadata" +} + +test_session_start_drains_before_inactive_scan() { + local dir root home fakebin state out status wake_line inactive_line + new_case session-start-wiring + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + cp -a "$ROOT/bin/." "$root/bin/" + write_meta "$state" session-x1 session-inc + export FM_FAKE_CREW_STATE_SESSION_X1='state: done · source: pane · session wiring' + printf '1\t1\tsignal\ttask-before\tqueued before inactive scan\n' > "$state/.wake-queue" + prepare_primary_proof "$root" "$home" "$fakebin" + out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ + FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux "$root/bin/fm-session-start.sh" 2>&1) + status=$? + [ "$status" = 0 ] || fail "session-start integration path failed: $out" + wake_line=$(printf '%s\n' "$out" | grep -n '^1[[:space:]]\+1[[:space:]]\+signal[[:space:]]\+task-before' | head -1 | cut -d: -f1) + inactive_line=$(printf '%s\n' "$out" | grep -n 'queued inactive outcome: task=session-x1' | head -1 | cut -d: -f1) + [ -n "$wake_line" ] && [ -n "$inactive_line" ] && [ "$wake_line" -lt "$inactive_line" ] \ + || fail "session-start did not drain the existing wake before inactive reconciliation" + [ "$(receipt_count "$state" pending)" = 1 ] || fail "session-start did not run inactive reconciliation" + unset FM_FAKE_CREW_STATE_SESSION_X1 + pass "session-start drains existing wakes before inactive reconciliation" +} + +test_watcher_runs_inactive_cadence() { + local dir root home fakebin state out status + new_case watcher-wiring + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + cp -a "$ROOT/bin/." "$root/bin/" + write_meta "$state" watcher-x1 watcher-inc + export FM_FAKE_CREW_STATE_WATCHER_X1='state: failed · source: pane · watcher wiring' + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u +case "$*" in + *"#{pane_current_path}"*) printf '%s\n' "${FM_FAKE_PANE_PATH:-}" ;; + *"#{pane_pid}"*) printf '%s\n' "${FM_FAKE_HARNESS_PID:-$$}" ;; + *"#{window_name}"*) printf '%s\n' firstmate ;; + capture-pane) : ;; +esac +exit 0 +SH + chmod +x "$fakebin/tmux" + prepare_primary_proof "$root" "$home" "$fakebin" + out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ + FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 FM_FAKE_PANE_PATH="$home" \ + FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ + "$root/bin/fm-watch.sh" 2>&1) + status=$? + [ "$status" = 0 ] || fail "watcher cadence failed while surfacing the inactive outcome wake" + printf '%s\n' "$out" | grep -F 'check: inactive terminal outcome replay queued' >/dev/null \ + || fail "watcher did not surface the inactive reconciliation result" + [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher cadence did not create the inactive receipt" + [ "$(queue_count "$state")" = 1 ] || fail "watcher cadence did not retain the inactive outcome wake" + unset FM_FAKE_CREW_STATE_WATCHER_X1 + pass "watcher cadence runs inactive reconciliation and surfaces its wake" +} + test_legacy_metadata_uses_stable_fallback() { local dir root home fakebin state rec incarnation new_case legacy-fallback @@ -494,18 +641,20 @@ test_status_log_terminal_is_not_replayed() { } test_valid_secondmate_route_reports_parent_once() { - local dir root home fakebin state child_home child_state parent_status corr rec outside + local dir root home fakebin state child_home child_state parent_status corr rec outside parent_history new_case secondmate-route-valid dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" child_home="$dir/secondmate-home" child_state="$child_home/state" - mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$state/pending-replies" + mkdir -p "$child_state" "$child_home/data" "$child_home/config" \ + "$state/pending-replies" "$state/pending-reply-history" printf 'sm-valid\n' > "$child_home/.fm-secondmate-home" write_meta "$child_state" child-x1 child-inc corr=0123456789abcdef parent_status="$state/sm-valid.status" - rec="$state/pending-replies/$corr" + parent_history="$state/pending-reply-history" + rec="$parent_history/$corr" fm_write_meta "$rec" \ schema=fm-pending-reply.v1 corr_id="$corr" task_id=sm-valid \ parent_home="$home" parent_status="$parent_status" delivered_epoch=1 phase=awaiting_report @@ -629,6 +778,14 @@ test_malformed_or_missing_secondmate_route_fails_closed() { parent_home="$home" parent_status="$parent_status" delivered_epoch=1 phase=awaiting_report scan "$root" "$child_home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "duplicate route fields were accepted" + rm -f "$child_home/.fm-jt-parent-route" + rm -f "$child_home/.fm-secondmate-home" + ln -s "$dir/missing-secondmate-marker" "$child_home/.fm-secondmate-home" + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-x1\nparent_home=%s\nparent_status=%s\ncorr_id=0123456789abcdef\n' \ + "$home" "$parent_status" > "$child_state/.fm-jt-parent-route" + scan "$root" "$child_home" "$fakebin" --startup >/dev/null \ + || fail "dangling secondmate marker scan failed" + [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "dangling secondmate marker was treated as an ordinary home" unset FM_FAKE_CREW_STATE_CHILD_X1 pass "malformed and missing secondmate parent routes fail closed without chat scraping" } @@ -638,6 +795,9 @@ test_portable_timeout_runner_is_used test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint +test_spawn_publishes_incarnation_token +test_session_start_drains_before_inactive_scan +test_watcher_runs_inactive_cadence test_legacy_metadata_uses_stable_fallback test_relaunch_and_teardown_races_recheck_under_spawn_lock test_parent_home_secondmate_records_are_skipped From b65deda293690064417e3996621f6d2b91c77ad6 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 02:05:45 +0000 Subject: [PATCH 006/163] no-mistakes(review): Hardened inactive replay and secondmate route recovery --- bin/fm-inactive-reconcile.sh | 141 ++++++++++++++----- bin/fm-pending-reply-lib.sh | 29 ++-- bin/fm-wake-drain.sh | 4 +- bin/fm-wake-lib.sh | 16 ++- tests/fm-inactive-outcome.test.sh | 223 +++++++++++++++++++++++++++--- 5 files changed, 344 insertions(+), 69 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 7fd8b9cbc78..e37c07416f1 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -62,6 +62,8 @@ CHILD_LOCK= bounded_secs() { local value=$1 fallback=$2 minimum=$3 maximum=$4 case "$value" in ''|*[!0-9]*) value=$fallback ;; esac + while [ "${value#0}" != "$value" ]; do value=${value#0}; done + [ -n "$value" ] || value=0 [ "$value" -lt "$minimum" ] && value=$minimum [ "$value" -gt "$maximum" ] && value=$maximum printf '%s' "$value" @@ -74,6 +76,17 @@ meta_value() { # awk -F= -v wanted="$2" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$1" 2>/dev/null } +meta_value_unique() { # + awk -F= -v wanted="$2" ' + $1 == wanted { count++; value=substr($0, index($0, "=") + 1) } + END { + if (count == 1) { print value; exit 0 } + if (count == 0) exit 1 + exit 2 + } + ' "$1" 2>/dev/null +} + file_mtime() { if [ "$(uname -s 2>/dev/null)" = Darwin ]; then stat -f '%m' "$1" 2>/dev/null @@ -127,11 +140,17 @@ is_secondmate_home() { } herdr_identity_allowed() { # - local meta=$1 backend session window - backend=$(meta_value "$meta" backend) + local meta=$1 backend session window rc + if backend=$(meta_value_unique "$meta" backend); then + : + else + rc=$? + [ "$rc" = 1 ] && return 0 + return 1 + fi [ "$backend" = herdr ] || return 0 - session=$(meta_value "$meta" herdr_session) - window=$(meta_value "$meta" window) + session=$(meta_value_unique "$meta" herdr_session) || return 1 + window=$(meta_value_unique "$meta" window) || return 1 case "$session" in firstmate) : ;; default|DEFAULT|CAPTAIN|captain) return 1 ;; @@ -144,12 +163,14 @@ herdr_identity_allowed() { # run_bounded_child() { # [args...] local seconds=$1 shift - if command -v timeout >/dev/null 2>&1; then + if [ "${FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT:-0}" != 1 ] \ + && command -v timeout >/dev/null 2>&1; then timeout "$seconds" "$@" - elif command -v gtimeout >/dev/null 2>&1; then + elif [ "${FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT:-0}" != 1 ] \ + && command -v gtimeout >/dev/null 2>&1; then gtimeout "$seconds" "$@" elif command -v perl >/dev/null 2>&1; then - perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$seconds" "$@" + perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV or exit 127 } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; my $status = $?; exit(($status & 127) ? 128 + ($status & 127) : ($status >> 8))' "$seconds" "$@" else return 125 fi @@ -160,7 +181,7 @@ receipt_path() { # } receipt_field() { # - meta_value "$1" "$2" + meta_value_unique "$1" "$2" } claim_path() { # @@ -168,7 +189,7 @@ claim_path() { # } claim_field() { # - meta_value "$1" "$2" + meta_value_unique "$1" "$2" } drain_claim_owner() { @@ -198,7 +219,7 @@ claim_validate() { # } claim_reserve() { # - local key=$1 row=$2 fp claim tmp state existing + local key=$1 row=$2 fp claim tmp state existing old_row line drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -206,8 +227,22 @@ claim_reserve() { # claim=$(claim_path "$fp") [ ! -L "$claim" ] || return 2 if [ -e "$claim" ]; then - state=$(claim_validate "$claim" "$fp" "$row") || return 2 - [ "$state" = presented ] && return 1 + [ -f "$claim" ] || return 2 + [ "$(claim_field "$claim" schema)" = fm-inactive-outcome-claim.v1 ] || return 2 + [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 2 + state=$(claim_field "$claim" state) + case "$state" in presented) return 1 ;; reserved) ;; *) return 2 ;; esac + old_row=$(claim_field "$claim" row) + [ -n "$old_row" ] || return 2 + if [ "$old_row" != "$row" ]; then + tmp=$(mktemp "$OUTCOME_DIR/.claim-row.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in row=*) printf 'row=%s\n' "$row" ;; *) printf '%s\n' "$line" ;; esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } + fi return 0 fi mkdir -p "$OUTCOME_DIR" || return 2 @@ -321,9 +356,34 @@ receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE return 0 } +publish_receipt_and_wake() { + local status=0 + FM_WAKE_APPEND_CREATED=0 + if ! fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"; then + receipt_write || return 1 + return 1 + fi + if receipt_write; then + if [ -f "$(receipt_path "$FP" pending)" ]; then + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$ID state=$OUTCOME fingerprint=$FP" || status=$? + fi + else + status=$? + fi + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 + return "$status" +} + read_incarnation() { # - local meta=$1 id=$2 token tasktmp window worktree seed - token=$(meta_value "$meta" spawn_incarnation) + local meta=$1 id=$2 token tasktmp window worktree seed rc + if token=$(meta_value_unique "$meta" spawn_incarnation); then + : + else + rc=$? + [ "$rc" = 1 ] || return 1 + token= + fi case "$token" in ''|legacy-unknown|*[!A-Za-z0-9._:-]*) token= ;; esac @@ -404,7 +464,7 @@ reconcile_child() { source=$(printf '%s\n' "$line" | sed -n 's/.*source: \([^ ·]*\).*/\1/p') [ -n "$source" ] && [ "$source" != none ] || return 0 snapshot=$(single_line "$line") - INC=$(read_incarnation "$meta" "$id") + INC=$(read_incarnation "$meta" "$id") || return 0 FP=$(hash_text "$id|$INC|$outcome|$snapshot") ID=$id OUTCOME=$outcome @@ -420,20 +480,16 @@ reconcile_child() { [ "$?" = 1 ] || return 0 KIND=${kind:-ship} fi - receipt_write || return 1 key="inactive-outcome:$FP" - if [ "$RECEIPT_CREATED" = 1 ] || [ -f "$(receipt_path "$FP" pending)" ]; then - fm_wake_append_if_absent FM_WAKE_APPEND_CREATED check "$key" \ - "inactive terminal outcome: task=$id state=$outcome fingerprint=$FP" || return 1 - if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then - printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" - fi + publish_receipt_and_wake || return 1 + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" fi return 0 } ack_receipt() { # - local key=$1 row=${2:-} fp rec id kind outcome parent_task_id parent_home parent_status corr line target existing claim_state + local key=$1 row=${2:-} fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing claim_state [ -n "$row" ] || return 2 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 0 ;; esac @@ -459,8 +515,12 @@ ack_receipt() { # [ "$(receipt_field "$rec" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 id=$(receipt_field "$rec" task_id) kind=$(receipt_field "$rec" kind) + incarnation=$(receipt_field "$rec" incarnation) parent_task_id=$(receipt_field "$rec" parent_task_id) outcome=$(receipt_field "$rec" outcome) + snapshot=$(receipt_field "$rec" terminal_snapshot) + expected_fp=$(hash_text "$id|$incarnation|$outcome|$snapshot") + [ "$expected_fp" = "$fp" ] || return 2 case "$kind" in ship|scout|secondmate) ;; *) return 2 ;; esac if [ "$kind" = secondmate ]; then parent_home=$(receipt_field "$rec" parent_home) @@ -469,15 +529,19 @@ ack_receipt() { # [ -n "$parent_task_id" ] || return 2 fm_pending_reply_secondmate_receipt_validate \ "$FM_HOME" "$parent_task_id" "$parent_home" "$parent_status" "$corr" || return 2 - line="$outcome [corr=$corr]: inactive terminal outcome replayed: task=$id fingerprint=$fp" - [ ! -L "$parent_status" ] || return 2 - if [ -e "$parent_status" ]; then - [ -f "$parent_status" ] || return 2 + if [ "$FM_PENDING_ROUTE_PHASE" = resolved ] || [ "$FM_PENDING_ROUTE_PHASE" = retired ]; then + fm_pending_reply_secondmate_route_clear "$FM_HOME" "$corr" || return 2 else - : > "$parent_status" || return 2 - fi - if ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then - printf '%s\n' "$line" >> "$parent_status" || return 2 + line="$outcome [corr=$corr]: inactive terminal outcome replayed: task=$id fingerprint=$fp" + [ ! -L "$parent_status" ] || return 2 + if [ -e "$parent_status" ]; then + [ -f "$parent_status" ] || return 2 + else + : > "$parent_status" || return 2 + fi + if ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then + printf '%s\n' "$line" >> "$parent_status" || return 2 + fi fi target=$(receipt_path "$fp" reported) else @@ -498,7 +562,7 @@ ack_receipt() { # scan_locked() { local startup=${1:-0} marker_mtime now age cursor meta id started=1 cursor_seen=1 - local scan_started remaining rc complete=1 scan_failed=0 + local scan_started remaining rc complete=1 scan_failed=0 find_tmp inactive_state_preflight || return 1 marker_mtime=$(file_mtime "$SCAN_MARKER" 2>/dev/null || true) now=$(date +%s) @@ -513,6 +577,13 @@ scan_locked() { fi if [ -n "$cursor" ]; then started=0; fi [ -n "$cursor" ] && cursor_seen=0 + find_tmp=$(mktemp "$STATE/.inactive-outcome-find.XXXXXX") || return 1 + [ -f "$find_tmp" ] && [ ! -L "$find_tmp" ] || { rm -f "$find_tmp"; return 1; } + if ! find "$STATE" \( -type d ! -path "$STATE" -prune \) -o \ + \( -type f -name '*.meta' -print0 \) > "$find_tmp"; then + rm -f "$find_tmp" + return 1 + fi while IFS= read -r -d '' meta; do now=$(date +%s) remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) @@ -547,10 +618,8 @@ scan_locked() { scan_failed=1 break fi - done < <( - find "$STATE" \( -type d ! -path "$STATE" -prune \) -o \ - \( -type f -name '*.meta' -print0 \) - ) + done < "$find_tmp" + rm -f "$find_tmp" || return 1 [ "$scan_failed" = 0 ] || return "$rc" if [ "$complete" = 1 ] && [ "$cursor_seen" = 0 ]; then return 1 diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index ae4710ae9be..a71c6402f65 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -496,14 +496,16 @@ fm_pending_reply_secondmate_route_lock_path() { # fm_pending_reply_secondmate_route_write() { # local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 - local marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 + local marker home_marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 local existing_schema existing_id existing_home existing_status existing_corr local existing_state existing_record existing_phase marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + home_marker="$secondmate_home/.fm-secondmate-home" + [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 case "$secondmate_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac - marker_id=$(cat "$secondmate_home/.fm-secondmate-home" 2>/dev/null || true) + marker_id=$(cat "$home_marker" 2>/dev/null || true) [ "$marker_id" = "$secondmate_id" ] || return 1 [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 [ -d "$parent_home/state" ] && [ ! -L "$parent_home/state" ] || return 1 @@ -517,20 +519,22 @@ fm_pending_reply_secondmate_route_write() { # < [ -f "$status_path" ] || return 1 fi printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 - tmp="$marker.tmp.${BASHPID:-$$}" + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + if ! fm_lock_acquire_wait "$route_lock"; then + return 1 + fi + tmp=$(mktemp "$secondmate_home/state/.fm-jt-parent-route.XXXXXX") || { + fm_lock_release "$route_lock" || true + return 1 + } { printf 'schema=fm-jt-parent-route.v1\n' printf 'secondmate_id=%s\n' "$secondmate_id" printf 'parent_home=%s\n' "$parent_abs" printf 'parent_status=%s\n' "$status_path" printf 'corr_id=%s\n' "$corr" - } > "$tmp" || { rm -f "$tmp"; return 1; } + } > "$tmp" || { rm -f "$tmp"; fm_lock_release "$route_lock" || true; return 1; } chmod 600 "$tmp" 2>/dev/null || true - route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") - if ! fm_lock_acquire_wait "$route_lock"; then - rm -f "$tmp" - return 1 - fi if [ -e "$marker" ] || [ -L "$marker" ]; then if [ -f "$marker" ] && [ ! -L "$marker" ] \ && [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ]; then @@ -620,6 +624,7 @@ fm_pending_reply_secondmate_route_validate() { # FM_PENDING_ROUTE_PARENT_HOME= FM_PENDING_ROUTE_CORR= FM_PENDING_ROUTE_SECOND_MATE_ID= + FM_PENDING_ROUTE_PHASE= marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 schema='' secondmate_id='' parent_home='' parent_status='' corr='' @@ -674,6 +679,7 @@ fm_pending_reply_secondmate_route_validate() { # else return 1 fi + [ "$(fm_pending_reply_get "$rec" schema)" = fm-pending-reply.v1 ] || return 1 record_task=$(fm_pending_reply_get "$rec" task_id) record_home=$(fm_pending_reply_get "$rec" parent_home) record_status=$(fm_pending_reply_get "$rec" parent_status) @@ -689,6 +695,10 @@ fm_pending_reply_secondmate_route_validate() { # awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; *) return 1 ;; esac + if [ "$phase" = resolved ] || [ "$phase" = retired ]; then + fm_pending_reply_secondmate_route_clear "$secondmate_home" "$corr" || return 1 + return 1 + fi # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_PARENT_HOME=$parent_abs # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh @@ -753,6 +763,7 @@ fm_pending_reply_secondmate_receipt_validate() { # = start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 fi - if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$_key" "$drain_row"; then + if ! printf '%s\n' "$drain_row"; then DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 0836ef568e4..e4d7a903867 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -669,14 +669,10 @@ fm_wake_append() { return "$status" } -fm_wake_append_if_absent() { # +fm_wake_append_if_absent_locked() { # local result_var=$1 kind=$2 key=$3 payload=$4 status=0 FM_WAKE_APPEND_CREATED=0 case "$result_var" in ''|*[!A-Za-z0-9_]*) return 2 ;; esac - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || { - printf 'fm_wake_append_if_absent: could not serialize the wake queue; refusing to append unlocked\n' >&2 - return 1 - } if [ -f "$FM_WAKE_QUEUE" ] && awk -F '\t' -v wanted="$key" '$4 == wanted { found=1 } END { exit(found ? 0 : 1) }' "$FM_WAKE_QUEUE" 2>/dev/null; then printf -v "$result_var" '%s' 0 else @@ -684,6 +680,16 @@ fm_wake_append_if_absent() { # status=$? [ "$status" -eq 0 ] && { FM_WAKE_APPEND_CREATED=1; printf -v "$result_var" '%s' 1; } fi + return "$status" +} + +fm_wake_append_if_absent() { # + local status=0 + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || { + printf 'fm_wake_append_if_absent: could not serialize the wake queue; refusing to append unlocked\n' >&2 + return 1 + } + fm_wake_append_if_absent_locked "$@" || status=$? fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 return "$status" } diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 4d5bc457196..8d67a135753 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -41,6 +41,12 @@ new_case() { #!/usr/bin/env bash set -u id=${1:-} +if [ "${FM_FAKE_CREW_STATE_SIGNAL:-0}" = 1 ]; then + kill -TERM $$ +fi +if [ "${FM_FAKE_CREW_STATE_EXIT:-0}" != 0 ]; then + exit "$FM_FAKE_CREW_STATE_EXIT" +fi key=$(printf '%s' "$id" | tr -c 'A-Za-z0-9' '_' | tr '[:lower:]' '[:upper:]') var="FM_FAKE_CREW_STATE_$key" printf '%s\n' "${!var:-${FM_FAKE_CREW_STATE:-state: unknown · source: none · fake default}}" @@ -56,6 +62,7 @@ for argument in "$@"; do previous=$argument done case "$*" in + *"lstart="*|*"command="*) exec /usr/bin/ps "$@" ;; *"comm="*|*"args="*) if [ "$pid" = "${FM_FAKE_HARNESS_PID:-}" ]; then printf '%s\n' claude @@ -102,6 +109,28 @@ prepare_primary_proof() { CASE_TOKEN=$token } +prepare_watcher_protocol() { + local root=$1 home=$2 state=$3 pid_start pid_identity watch arm + watch="$root/bin/fm-watch.sh" + arm="$root/bin/fm-watch-arm.sh" + pid_start=$(FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c '. "$1/bin/fm-wake-lib.sh"; fm_pid_start "$2"' _ "$ROOT" "$$") + pid_identity=$(FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c '. "$1/bin/fm-wake-lib.sh"; fm_pid_identity "$2"' _ "$ROOT" "$$") + mkdir -p "$state/.watch.lock" "$state/.watch-arm.lock" + printf '%s\n' "$$" > "$state/.watch.lock/pid" + printf '%s\n' "$home" > "$state/.watch.lock/fm-home" + printf '%s\n' "$watch" > "$state/.watch.lock/watcher-path" + printf '%s\n' "$pid_start" > "$state/.watch.lock/pid-start" + printf '%s\n' "$pid_identity" > "$state/.watch.lock/pid-identity" + printf '%s\n' pending-reply-ticket-v3 > "$state/.watch.lock/pending-reply-protocol" + printf '%s\n' "$$" > "$state/.watch-arm.lock/pid" + printf '%s\n' "$home" > "$state/.watch-arm.lock/fm-home" + printf '%s\n' "$arm" > "$state/.watch-arm.lock/owner-path" + printf '%s\n' "$pid_start" > "$state/.watch-arm.lock/pid-start" + printf '%s\n' "$pid_identity" > "$state/.watch-arm.lock/pid-identity" +} + scan() { local root=$1 home=$2 fakebin=$3 startup=${4:-} if [ -d "$home/state" ] && [ ! -L "$home/state" ]; then @@ -112,7 +141,8 @@ scan() { FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ FM_FAKE_HARNESS_PID="$$" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ - FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + FM_INACTIVE_OUTCOME_SECS="${FM_INACTIVE_OUTCOME_SECS:-60}" \ + FM_INACTIVE_OUTCOME_BUDGET_SECS="${FM_INACTIVE_OUTCOME_BUDGET_SECS:-10}" \ "$RECON" scan "$startup" ) } @@ -237,6 +267,7 @@ test_done_and_failed_are_replayed_once() { [ "$(receipt_count "$state" presented)" = 2 ] || fail "drain did not preserve two presented receipts" scan "$root" "$home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$state" presented)" = 2 ] || fail "presented receipts were replayed" + [ "$(queue_count "$state")" = 0 ] || fail "presented receipts caused a duplicate wake on rescan" unset FM_FAKE_CREW_STATE_DONE_X1 FM_FAKE_CREW_STATE_FAILED_X1 pass "done and failed inactive outcomes are replayed once and acknowledged on drain" } @@ -258,10 +289,106 @@ SH export FM_FAKE_CREW_STATE_PORTABLE_X1='state: done · source: pane · portable timeout' scan "$root" "$home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$state" pending)" = 1 ] || fail "portable timeout runner did not reconcile the child" - unset FM_FAKE_CREW_STATE_PORTABLE_X1 + export FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT=1 + export FM_FAKE_CREW_STATE_EXIT=7 + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "portable timeout runner hid a non-zero child status" + fi + unset FM_FAKE_CREW_STATE_PORTABLE_X1 FM_FAKE_CREW_STATE_EXIT FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT pass "inactive scan uses the portable timeout invocation" } +test_portable_timeout_preserves_signal_failure() { + local dir root home fakebin state + new_case portable-timeout-signal + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" signal-x1 signal-inc + export FM_FAKE_CREW_STATE_SIGNAL=1 FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT=1 + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "portable timeout runner converted a signal failure into success" + fi + [ ! -e "$state/.inactive-outcome-reconcile" ] || fail "signal failure advanced the cadence marker" + unset FM_FAKE_CREW_STATE_SIGNAL FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT + pass "portable timeout preserves signal failures" +} + +test_leading_zero_cadence_is_normalized() { + local dir root home fakebin state + new_case leading-zero-cadence + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" zero-x1 zero-inc + export FM_FAKE_CREW_STATE_ZERO_X1='state: done · source: pane · leading zero' + export FM_INACTIVE_OUTCOME_SECS=0080 FM_INACTIVE_OUTCOME_BUDGET_SECS=0010 + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "leading-zero cadence aborted the scan" + [ "$(receipt_count "$state" pending)" = 1 ] || fail "leading-zero cadence did not reconcile the child" + unset FM_FAKE_CREW_STATE_ZERO_X1 FM_INACTIVE_OUTCOME_SECS FM_INACTIVE_OUTCOME_BUDGET_SECS + pass "leading-zero cadence values are normalized before arithmetic" +} + +test_find_failure_propagates_without_advancing_scan() { + local dir root home fakebin state + new_case find-failure + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" find-x1 find-inc + cat > "$fakebin/find" <<'SH' +#!/usr/bin/env bash +exit 42 +SH + chmod +x "$fakebin/find" + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "find enumeration failure was reported as success" + fi + [ ! -e "$state/.inactive-outcome-reconcile" ] || fail "find failure advanced the cadence marker" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "find failure created a receipt" + pass "find enumeration failures propagate and preserve retry state" +} + +test_ack_recomputes_fingerprint_from_receipt_fields() { + local dir root home fakebin state rec fingerprint + new_case fingerprint-binding + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" fingerprint-x1 fingerprint-inc + export FM_FAKE_CREW_STATE_FINGERPRINT_X1='state: done · source: pane · original snapshot' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "fingerprint fixture scan failed" + rec=$(find "$state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + fingerprint=$(basename "$rec" .pending) + sed -i 's/^terminal_snapshot=.*/terminal_snapshot=tampered snapshot/' "$rec" + if drain "$root" "$home" "$fakebin" >/dev/null 2>&1; then + fail "drain accepted a receipt whose snapshot no longer matched its fingerprint" + fi + [ -f "$rec" ] || fail "fingerprint mismatch removed the pending receipt" + [ "$(queue_count "$state")" = 1 ] || fail "fingerprint mismatch did not preserve the wake for retry" + [ "$(receipt_value "$rec" fingerprint)" = "$fingerprint" ] || fail "fingerprint fixture changed its filename binding" + unset FM_FAKE_CREW_STATE_FINGERPRINT_X1 + pass "drain recomputes the receipt fingerprint from bound fields" +} + +test_reserved_claim_recovers_to_a_new_wake_row() { + local dir root home fakebin state fingerprint old_row new_row + new_case claim-recovery + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'claim-x1|claim-inc|done|state: done · source: pane · claim recovery') + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=claim-x1 \ + incarnation=claim-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · claim recovery' kind=ship + old_row='1 1 check inactive-outcome:'"$fingerprint"$'\told row' + new_row='2 2 check inactive-outcome:'"$fingerprint"$'\tnew row' + printf 'schema=fm-inactive-outcome-claim.v1\nfingerprint=%s\nrow=%s\nstate=reserved\ncreated_epoch=1\n' \ + "$fingerprint" "$old_row" > "$state/terminal-outcomes/.$fingerprint.claim" + printf '%s\n' "$new_row" > "$state/.wake-queue" + drain "$root" "$home" "$fakebin" >/dev/null || fail "drain did not recover a reserved claim" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] || fail "recovered claim did not present its receipt" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "recovered claim was not retired" + pass "reserved inactive claims recover when the wake row is recreated" +} + test_scan_failure_retries_without_advancing_cadence() { local dir root home fakebin state wake_dir wake_removed new_case scan-failure @@ -454,6 +581,9 @@ test_session_start_drains_before_inactive_scan() { inactive_line=$(printf '%s\n' "$out" | grep -n 'queued inactive outcome: task=session-x1' | head -1 | cut -d: -f1) [ -n "$wake_line" ] && [ -n "$inactive_line" ] && [ "$wake_line" -lt "$inactive_line" ] \ || fail "session-start did not drain the existing wake before inactive reconciliation" + [ "$(awk -F '\t' '$4 == "task-before" { n++ } END { print n + 0 }' "$state/.wake-queue")" = 0 ] \ + || fail "session-start left the pre-existing wake queued" + [ "$(queue_count "$state")" = 1 ] || fail "session-start did not queue exactly one inactive wake after draining" [ "$(receipt_count "$state" pending)" = 1 ] || fail "session-start did not run inactive reconciliation" unset FM_FAKE_CREW_STATE_SESSION_X1 pass "session-start drains existing wakes before inactive reconciliation" @@ -593,20 +723,28 @@ test_herdr_identity_and_default_captain_refusal() { state="$home/state" write_meta "$state" herdr-good good-inc ship herdr firstmate:pane printf 'herdr_session=firstmate\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-good.meta" + printf 'herdr_session=default\n' >> "$state/herdr-good.meta" + write_meta "$state" herdr-unique unique-inc ship herdr firstmate:pane + printf 'herdr_session=firstmate\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-unique.meta" write_meta "$state" herdr-default default-inc ship herdr default:pane printf 'herdr_session=default\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-default.meta" write_meta "$state" herdr-captain captain-inc ship herdr CAPTAIN:pane printf 'herdr_session=CAPTAIN\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-captain.meta" - touch -d '2 minutes ago' "$state/herdr-good.meta" "$state/herdr-default.meta" "$state/herdr-captain.meta" + touch -d '2 minutes ago' "$state/herdr-good.meta" "$state/herdr-unique.meta" \ + "$state/herdr-default.meta" "$state/herdr-captain.meta" export FM_FAKE_CREW_STATE_HERDR_GOOD='state: done · source: pane · dedicated session quiet' + export FM_FAKE_CREW_STATE_HERDR_UNIQUE='state: done · source: pane · unique session quiet' export FM_FAKE_CREW_STATE_HERDR_DEFAULT='state: done · source: pane · must refuse' export FM_FAKE_CREW_STATE_HERDR_CAPTAIN='state: done · source: pane · must refuse' export FM_NETWORK_LOG="$dir/network.log" PATH="$fakebin:$PATH" scan "$root" "$home" "$fakebin" --startup >/dev/null \ || fail "Herdr identity scan failed" - [ "$(receipt_count "$state" pending)" = 1 ] || fail "Herdr dedicated session was not accepted while default/CAPTAIN were refused" + [ "$(receipt_count "$state" pending)" = 1 ] || fail "Herdr duplicate identity was accepted or default/CAPTAIN were not refused" + grep -l '^task_id=herdr-unique$' "$state"/terminal-outcomes/*.pending >/dev/null \ + || fail "unique Herdr identity was not accepted" [ ! -s "$dir/network.log" ] || fail "inactive reconciliation made a forge/network call" - unset FM_FAKE_CREW_STATE_HERDR_GOOD FM_FAKE_CREW_STATE_HERDR_DEFAULT FM_FAKE_CREW_STATE_HERDR_CAPTAIN FM_NETWORK_LOG + unset FM_FAKE_CREW_STATE_HERDR_GOOD FM_FAKE_CREW_STATE_HERDR_UNIQUE \ + FM_FAKE_CREW_STATE_HERDR_DEFAULT FM_FAKE_CREW_STATE_HERDR_CAPTAIN FM_NETWORK_LOG pass "Herdr uses the dedicated firstmate identity and refuses default/CAPTAIN" } @@ -641,26 +779,42 @@ test_status_log_terminal_is_not_replayed() { } test_valid_secondmate_route_reports_parent_once() { - local dir root home fakebin state child_home child_state parent_status corr rec outside parent_history + local dir root home fakebin state child_home child_state parent_status corr rec outside parent_record send_out new_case secondmate-route-valid dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" + cp -a "$ROOT/bin/." "$root/bin/" child_home="$dir/secondmate-home" child_state="$child_home/state" mkdir -p "$child_state" "$child_home/data" "$child_home/config" \ "$state/pending-replies" "$state/pending-reply-history" printf 'sm-valid\n' > "$child_home/.fm-secondmate-home" + write_meta "$state" sm-valid parent-inc secondmate tmux firstmate:fm-sm-valid + printf 'home=%s\n' "$child_home" >> "$state/sm-valid.meta" write_meta "$child_state" child-x1 child-inc - corr=0123456789abcdef parent_status="$state/sm-valid.status" - parent_history="$state/pending-reply-history" - rec="$parent_history/$corr" - fm_write_meta "$rec" \ - schema=fm-pending-reply.v1 corr_id="$corr" task_id=sm-valid \ - parent_home="$home" parent_status="$parent_status" delivered_epoch=1 phase=awaiting_report + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u +case "$*" in + *"#{cursor_y}"*) printf '1\n' ;; + *capture-pane*) : ;; + *) : ;; +esac +exit 0 +SH + chmod +x "$fakebin/tmux" + prepare_primary_proof "$root" "$home" "$fakebin" + prepare_watcher_protocol "$root" "$home" "$state" + send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ + fm-sm-valid "parent request" 2>&1) || fail "public fm-send route setup failed: $send_out" + corr=$(basename "$(find "$state/pending-replies" -maxdepth 1 -type f | head -1)") + [ -n "$corr" ] || fail "public fm-send did not create a correlation record" export FM_FAKE_CREW_STATE_CHILD_X1='state: failed · source: pane · child quiet' - printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-valid\nparent_home=%s\nparent_status=%s\ncorr_id=%s\n' \ - "$home" "$parent_status" "$corr" > "$child_state/.fm-jt-parent-route" scan "$root" "$child_home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "valid secondmate route did not create a pending receipt" rec=$(find "$child_state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) @@ -670,6 +824,7 @@ test_valid_secondmate_route_reports_parent_once() { [ "$(receipt_value "$rec" parent_status)" = "$parent_status" ] || fail "secondmate receipt did not persist its parent status path" outside="$dir/outside-status" printf 'outside\n' > "$outside" + rm -f "$parent_status" ln -s "$outside" "$parent_status" if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then fail "secondmate acknowledgement followed a parent status symlink" @@ -687,12 +842,16 @@ test_valid_secondmate_route_reports_parent_once() { drain "$root" "$child_home" "$fakebin" >/dev/null [ "$(grep -Fc "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status")" = 1 ] \ || fail "secondmate parent report was duplicated" + parent_record="$state/pending-replies/$corr" + sed -i 's/^phase=.*/phase=resolved/' "$parent_record" + scan "$root" "$child_home" "$fakebin" --startup >/dev/null || fail "resolved history lifecycle scan failed" + [ ! -e "$child_state/.fm-jt-parent-route" ] || fail "resolved parent history left a stale route marker" unset FM_FAKE_CREW_STATE_CHILD_X1 pass "valid secondmate outcomes use the parent status correlation exactly once" } test_concurrent_secondmate_routes_are_rejected() { - local dir root home fakebin state child_home child_state marker corr_one corr_two + local dir root home fakebin state child_home child_state marker corr_one corr_two outside new_case secondmate-route-concurrent dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -715,8 +874,24 @@ test_concurrent_secondmate_routes_are_rejected() { '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ _ "$ROOT" "$child_home" "$home" "$state" sm-concurrent "$1" } + mv "$child_home/.fm-secondmate-home" "$dir/secondmate-marker" + ln -s "$dir/secondmate-marker" "$child_home/.fm-secondmate-home" + if route_write "$corr_one"; then + fail "symlinked secondmate home marker was accepted" + fi + rm -f "$child_home/.fm-secondmate-home" + mv "$dir/secondmate-marker" "$child_home/.fm-secondmate-home" route_write "$corr_one" || fail "initial secondmate route was not written" marker_before=$(cat "$marker") + outside="$dir/temp-target" + printf 'protected\n' > "$outside" + if env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; ln -s "$7" "$2/state/.fm-jt-parent-route.tmp.${BASHPID}"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-concurrent "$corr_two" "$outside"; then + fail "second route replaced an active route unexpectedly" + fi + [ "$(cat "$outside")" = protected ] || fail "route publication followed a pre-created temporary symlink" if route_write "$corr_two"; then fail "concurrent secondmate route was silently replaced" fi @@ -729,8 +904,8 @@ test_drain_restores_only_unprocessed_rows() { new_case drain-rollback dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" - first=1111111111111111 - second=2222222222222222 + first=$(receipt_fingerprint 'first-x1|first-inc|done|done') + second=$(receipt_fingerprint 'second-x1|second-inc|failed|failed') mkdir -p "$state/terminal-outcomes" fm_write_meta "$state/terminal-outcomes/$first.pending" \ schema=fm-jt-terminal-outcome.v1 fingerprint="$first" task_id=first-x1 \ @@ -761,6 +936,7 @@ test_malformed_or_missing_secondmate_route_fails_closed() { child_home="$dir/secondmate-home" child_state="$child_home/state" mkdir -p "$child_state" "$child_home/data" "$child_home/config" + mkdir -p "$home/state/pending-replies" printf 'sm-x1\n' > "$child_home/.fm-secondmate-home" write_meta "$child_state" child-x1 child-inc export FM_FAKE_CREW_STATE_CHILD_X1='state: failed · source: pane · child quiet' @@ -771,6 +947,14 @@ test_malformed_or_missing_secondmate_route_fails_closed() { scan "$root" "$child_home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "malformed secondmate parent route was not fail-closed" [ ! -e "$parent_status" ] || fail "malformed secondmate route wrote parent status" + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-x1\nparent_home=%s\nparent_status=%s\ncorr_id=0123456789abcdef\n' \ + "$home" "$parent_status" > "$child_state/.fm-jt-parent-route" + fm_write_meta "$home/state/pending-replies/0123456789abcdef" \ + schema=wrong-schema corr_id=0123456789abcdef task_id=sm-x1 \ + parent_home="$home" parent_status="$parent_status" delivered_epoch=1 phase=awaiting_report + scan "$root" "$child_home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "malformed parent schema was accepted" + rm -f "$child_state/.fm-jt-parent-route" printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=\nsecondmate_id=sm-x1\nparent_home=%s\nparent_status=%s\ncorr_id=0123456789abcdef\n' \ "$home" "$parent_status" > "$child_state/.fm-jt-parent-route" fm_write_meta "$home/state/pending-replies/0123456789abcdef" \ @@ -792,6 +976,11 @@ test_malformed_or_missing_secondmate_route_fails_closed() { test_done_and_failed_are_replayed_once test_portable_timeout_runner_is_used +test_portable_timeout_preserves_signal_failure +test_leading_zero_cadence_is_normalized +test_find_failure_propagates_without_advancing_scan +test_ack_recomputes_fingerprint_from_receipt_fields +test_reserved_claim_recovers_to_a_new_wake_row test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint From 10b8a8f5d03296adc86bffe5caebc0b186f38d92 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 02:47:27 +0000 Subject: [PATCH 007/163] no-mistakes(review): Hardened inactive replay and history route recovery --- bin/fm-inactive-reconcile.sh | 153 ++++++++++++++++++++++++------ bin/fm-pending-reply-lib.sh | 14 ++- bin/fm-wake-drain.sh | 7 +- bin/fm-watch.sh | 8 ++ tests/fm-inactive-outcome.test.sh | 125 ++++++++++++++++++++++-- 5 files changed, 265 insertions(+), 42 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index e37c07416f1..b2e12a24b93 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -215,7 +215,7 @@ claim_validate() { # [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 1 [ "$(claim_field "$claim" row)" = "$row" ] || return 1 state=$(claim_field "$claim" state) - case "$state" in reserved|presented) printf '%s' "$state" ;; *) return 1 ;; esac + case "$state" in reserved|presenting|presented) printf '%s' "$state" ;; *) return 1 ;; esac } claim_reserve() { # @@ -231,7 +231,7 @@ claim_reserve() { # [ "$(claim_field "$claim" schema)" = fm-inactive-outcome-claim.v1 ] || return 2 [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 2 state=$(claim_field "$claim" state) - case "$state" in presented) return 1 ;; reserved) ;; *) return 2 ;; esac + case "$state" in presented|presenting|reserved) ;; *) return 2 ;; esac old_row=$(claim_field "$claim" row) [ -n "$old_row" ] || return 2 if [ "$old_row" != "$row" ]; then @@ -243,6 +243,7 @@ claim_reserve() { # [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } fi + [ "$state" = presented ] && return 1 return 0 fi mkdir -p "$OUTCOME_DIR" || return 2 @@ -266,6 +267,28 @@ claim_reserve() { # return 0 } +claim_mark_presenting() { # + local key=$1 row=$2 fp claim state tmp line + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + case "$state" in + presenting) return 0 ;; + reserved) ;; + *) return 2 ;; + esac + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in state=*) printf 'state=presenting\n' ;; *) printf '%s\n' "$line" ;; esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + claim_mark_presented() { # local key=$1 row=$2 fp claim tmp line drain_claim_owner "$row" || return 2 @@ -273,7 +296,7 @@ claim_mark_presented() { # case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac claim=$(claim_path "$fp") [ ! -L "$claim" ] || return 2 - [ "$(claim_validate "$claim" "$fp" "$row")" = reserved ] || return 2 + [ "$(claim_validate "$claim" "$fp" "$row")" = presenting ] || return 2 tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 chmod 600 "$tmp" 2>/dev/null || true while IFS= read -r line || [ -n "$line" ]; do @@ -375,6 +398,44 @@ publish_receipt_and_wake() { return "$status" } +receipt_existing_core() { + local suffix existing expected_fp existing_kind parent_id parent_home parent_status parent_corr + RECEIPT_EXISTING_SUFFIX= + expected_fp=$(hash_text "$ID|$INC|$OUTCOME|$SNAPSHOT") + [ "$expected_fp" = "$FP" ] || return 1 + for suffix in pending presented reported; do + existing=$(receipt_path "$FP" "$suffix") + [ ! -L "$existing" ] || return 2 + [ -e "$existing" ] || continue + [ -f "$existing" ] || return 2 + [ "$(receipt_field "$existing" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 + [ "$(receipt_field "$existing" fingerprint)" = "$FP" ] || return 2 + [ "$(receipt_field "$existing" task_id)" = "$ID" ] || return 2 + [ "$(receipt_field "$existing" incarnation)" = "$INC" ] || return 2 + [ "$(receipt_field "$existing" outcome)" = "$OUTCOME" ] || return 2 + [ "$(receipt_field "$existing" terminal_source)" = "$SOURCE" ] || return 2 + [ "$(receipt_field "$existing" terminal_snapshot)" = "$SNAPSHOT" ] || return 2 + existing_kind=$(receipt_field "$existing" kind) + case "$existing_kind" in ship|scout) parent_id=$(receipt_field "$existing" parent_task_id); parent_home=$(receipt_field "$existing" parent_home); parent_status=$(receipt_field "$existing" parent_status); parent_corr=$(receipt_field "$existing" parent_corr); [ -z "$parent_id" ] && [ -z "$parent_home" ] && [ -z "$parent_status" ] && [ -z "$parent_corr" ] || return 2 ;; secondmate) parent_id=$(receipt_field "$existing" parent_task_id); parent_home=$(receipt_field "$existing" parent_home); parent_status=$(receipt_field "$existing" parent_status); parent_corr=$(receipt_field "$existing" parent_corr); [ -n "$parent_id" ] && case "$parent_home" in /*) ;; *) return 2 ;; esac && case "$parent_status" in /*) ;; *) return 2 ;; esac && printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 2 ;; *) return 2 ;; esac + RECEIPT_EXISTING_SUFFIX=$suffix + return 0 + done + return 1 +} + +republish_existing_receipt_wake() { + local existing task outcome status=0 + existing=$(receipt_path "$FP" pending) + task=$(receipt_field "$existing" task_id) + outcome=$(receipt_field "$existing" outcome) + FM_WAKE_APPEND_CREATED=0 + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$task state=$outcome fingerprint=$FP" || status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 + return "$status" +} + read_incarnation() { # local meta=$1 id=$2 token tasktmp window worktree seed rc if token=$(meta_value_unique "$meta" spawn_incarnation); then @@ -415,7 +476,7 @@ child_cleanup() { reconcile_child() { local id=$1 meta="$STATE/$1.meta" kind backend now activity age line outcome source - local snapshot token key route_rc state_tmp state_rc + local snapshot token key route_rc state_tmp state_rc existing_rc valid_task_id "$id" || return 0 [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 kind=$(meta_value "$meta" kind) @@ -471,14 +532,27 @@ reconcile_child() { SNAPSHOT=$snapshot SOURCE=$source KIND=${kind:-ship} - if is_secondmate_home; then - route_rc=$? - [ "$route_rc" = 0 ] || return 0 + is_secondmate_home + route_rc=$? + case "$route_rc" in + 0|1) ;; + *) return 0 ;; + esac + if receipt_existing_core; then + if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then + republish_existing_receipt_wake || return 1 + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" + fi + fi + return 0 + else + existing_rc=$? + [ "$existing_rc" = 1 ] || return 1 + fi + if [ "$route_rc" = 0 ]; then fm_pending_reply_secondmate_route_validate "$FM_HOME" || return 0 KIND=secondmate - else - [ "$?" = 1 ] || return 0 - KIND=${kind:-ship} fi key="inactive-outcome:$FP" publish_receipt_and_wake || return 1 @@ -527,22 +601,8 @@ ack_receipt() { # parent_status=$(receipt_field "$rec" parent_status) corr=$(receipt_field "$rec" parent_corr) [ -n "$parent_task_id" ] || return 2 - fm_pending_reply_secondmate_receipt_validate \ - "$FM_HOME" "$parent_task_id" "$parent_home" "$parent_status" "$corr" || return 2 - if [ "$FM_PENDING_ROUTE_PHASE" = resolved ] || [ "$FM_PENDING_ROUTE_PHASE" = retired ]; then - fm_pending_reply_secondmate_route_clear "$FM_HOME" "$corr" || return 2 - else - line="$outcome [corr=$corr]: inactive terminal outcome replayed: task=$id fingerprint=$fp" - [ ! -L "$parent_status" ] || return 2 - if [ -e "$parent_status" ]; then - [ -f "$parent_status" ] || return 2 - else - : > "$parent_status" || return 2 - fi - if ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then - printf '%s\n' "$line" >> "$parent_status" || return 2 - fi - fi + secondmate_ack_report "$FM_HOME" "$parent_task_id" "$parent_home" \ + "$parent_status" "$corr" "$outcome" "$id" "$fp" || return 2 target=$(receipt_path "$fp" reported) else target=$(receipt_path "$fp" presented) @@ -560,6 +620,35 @@ ack_receipt() { # return 0 } +secondmate_ack_report() { # + local secondmate_home=$1 parent_task_id=$2 parent_home=$3 parent_status=$4 corr=$5 outcome=$6 task_id=$7 fp=$8 + local parent_state token rc=0 line + parent_state="$parent_home/state" + fm_pending_reply_txn_lock_acquire "$parent_state" "$corr" token || return 2 + if ! fm_pending_reply_secondmate_receipt_validate \ + "$secondmate_home" "$parent_task_id" "$parent_home" "$parent_status" "$corr"; then + rc=2 + elif [ "$FM_PENDING_ROUTE_PHASE" = resolved ] || [ "$FM_PENDING_ROUTE_PHASE" = retired ]; then + fm_pending_reply_secondmate_route_clear "$secondmate_home" "$corr" || rc=2 + else + line="$outcome [corr=$corr]: inactive terminal outcome replayed: task=$task_id fingerprint=$fp" + [ ! -L "$parent_status" ] || rc=2 + if [ "$rc" = 0 ] && [ -e "$parent_status" ]; then + [ -f "$parent_status" ] || rc=2 + elif [ "$rc" = 0 ]; then + : > "$parent_status" || rc=2 + fi + if [ "$rc" = 0 ] && ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then + printf '%s\n' "$line" >> "$parent_status" || rc=2 + fi + if [ "$rc" = 0 ]; then + fm_pending_reply_secondmate_route_clear "$secondmate_home" "$corr" || rc=2 + fi + fi + fm_pending_reply_txn_lock_release "$parent_state" "$corr" "$token" || rc=2 + return "$rc" +} + scan_locked() { local startup=${1:-0} marker_mtime now age cursor meta id started=1 cursor_seen=1 local scan_started remaining rc complete=1 scan_failed=0 find_tmp @@ -579,7 +668,13 @@ scan_locked() { [ -n "$cursor" ] && cursor_seen=0 find_tmp=$(mktemp "$STATE/.inactive-outcome-find.XXXXXX") || return 1 [ -f "$find_tmp" ] && [ ! -L "$find_tmp" ] || { rm -f "$find_tmp"; return 1; } - if ! find "$STATE" \( -type d ! -path "$STATE" -prune \) -o \ + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + if [ "$remaining" -le 0 ]; then + rm -f "$find_tmp" + return 1 + fi + if ! run_bounded_child "$remaining" find "$STATE" \( -type d ! -path "$STATE" -prune \) -o \ \( -type f -name '*.meta' -print0 \) > "$find_tmp"; then rm -f "$find_tmp" return 1 @@ -659,6 +754,10 @@ case "${1:-}" in [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_reserve "$2" "$3" ;; + presenting) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_presenting "$2" "$3" + ;; presented) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_presented "$2" "$3" diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index a71c6402f65..618c76e5f87 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -559,6 +559,10 @@ fm_pending_reply_secondmate_route_write() { # < case "$existing_home" in /*) ;; *) route_status=1 ;; esac case "$existing_status" in /*) ;; *) route_status=1 ;; esac printf '%s' "$existing_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || route_status=1 + if [ "$route_status" = 0 ]; then + [ -d "$existing_home" ] && [ ! -L "$existing_home" ] || route_status=1 + [ -d "$existing_home/state" ] && [ ! -L "$existing_home/state" ] || route_status=1 + fi if [ "$route_status" = 0 ]; then existing_state=$(cd "$existing_home/state" 2>/dev/null && pwd -P) || route_status=1 fi @@ -566,6 +570,10 @@ fm_pending_reply_secondmate_route_write() { # < && [ "$existing_status" != "$existing_state/$existing_id.status" ]; then route_status=1 fi + if [ "$route_status" = 0 ] \ + && [ -L "$existing_state/$existing_id.status" ]; then + route_status=1 + fi if [ "$route_status" = 0 ]; then if [ -f "$(fm_pending_reply_active_path "$existing_state" "$existing_corr")" ]; then existing_record=$(fm_pending_reply_active_path "$existing_state" "$existing_corr") @@ -646,6 +654,8 @@ fm_pending_reply_secondmate_route_validate() { # && [ "$seen_parent_home" = 1 ] && [ "$seen_parent_status" = 1 ] \ && [ "$seen_corr" = 1 ] || return 1 [ "$schema" = fm-jt-parent-route.v1 ] || return 1 + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 home_marker="$secondmate_home/.fm-secondmate-home" [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 marker_id=$(cat "$home_marker" 2>/dev/null || true) @@ -695,10 +705,6 @@ fm_pending_reply_secondmate_route_validate() { # awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; *) return 1 ;; esac - if [ "$phase" = resolved ] || [ "$phase" = retired ]; then - fm_pending_reply_secondmate_route_clear "$secondmate_home" "$corr" || return 1 - return 1 - fi # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_PARENT_HOME=$parent_abs # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 8a3806dd3af..bd7ece44440 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -85,7 +85,7 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" claim "$_key" "$drain_row" || claim_status=$? case "$claim_status" in 0) - if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$_key" "$drain_row"; then + if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presenting "$_key" "$drain_row"; then DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 @@ -95,6 +95,11 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 fi + if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$_key" "$drain_row"; then + DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" + awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + exit 1 + fi ;; 1) ;; *) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 5a2990efa22..7c26baae88c 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -583,6 +583,14 @@ while :; do # alive. Supervision scripts warn when this goes stale with tasks in flight. touch "$STATE/.last-watcher-beat" + if ! wake_drain_out=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1); then + printf '%s\n' "$wake_drain_out" >&2 + exit 1 + fi + if [ -n "$wake_drain_out" ]; then + printf '%s\n' "$wake_drain_out" + fi + # Parent-owned secondmate pending-reply reconciliation: resolve correlated # parent reports, observe backend busy/idle turn completion, send one recovery # repost after grace, and escalate once if the recovery turn is also missed. diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 8d67a135753..2f5795538e8 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -346,6 +346,28 @@ SH pass "find enumeration failures propagate and preserve retry state" } +test_find_enumeration_respects_scan_budget() { + local dir root home fakebin state + new_case find-budget + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" find-budget-x1 find-budget-inc + cat > "$fakebin/find" <<'SH' +#!/usr/bin/env bash +sleep 2 +exit 0 +SH + chmod +x "$fakebin/find" + export FM_INACTIVE_OUTCOME_BUDGET_SECS=1 + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "slow find enumeration exceeded the scan budget without failing" + fi + [ ! -e "$state/.inactive-outcome-reconcile" ] || fail "budget-exhausted enumeration advanced the cadence marker" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "budget-exhausted enumeration created a receipt" + unset FM_INACTIVE_OUTCOME_BUDGET_SECS + pass "inactive enumeration is bounded by the per-scan budget" +} + test_ack_recomputes_fingerprint_from_receipt_fields() { local dir root home fakebin state rec fingerprint new_case fingerprint-binding @@ -389,6 +411,30 @@ test_reserved_claim_recovers_to_a_new_wake_row() { pass "reserved inactive claims recover when the wake row is recreated" } +test_presenting_claim_recovers_before_output() { + local dir root home fakebin state fingerprint row + new_case presenting-claim-recovery + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'presenting-x1|presenting-inc|done|state: done · source: pane · presenting recovery') + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=presenting-x1 \ + incarnation=presenting-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · presenting recovery' kind=ship + row='2 2 check inactive-outcome:'"$fingerprint"$'\trecreated row' + printf 'schema=fm-inactive-outcome-claim.v1\nfingerprint=%s\nrow=1\t1\tcheck\tinactive-outcome:%s\told row\nstate=presenting\ncreated_epoch=1\n' \ + "$fingerprint" "$fingerprint" > "$state/terminal-outcomes/.$fingerprint.claim" + printf '%s\n' "$row" > "$state/.wake-queue" + drain "$root" "$home" "$fakebin" >"$dir/presenting.out" \ + || fail "drain did not recover a presenting claim" + grep -F 'recreated row' "$dir/presenting.out" >/dev/null \ + || fail "recovered presenting claim did not present the recreated wake" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] || fail "recovered presenting claim did not present its receipt" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "recovered presenting claim was not retired" + pass "presenting inactive claims recover before output" +} + test_scan_failure_retries_without_advancing_cadence() { local dir root home fakebin state wake_dir wake_removed new_case scan-failure @@ -590,12 +636,13 @@ test_session_start_drains_before_inactive_scan() { } test_watcher_runs_inactive_cadence() { - local dir root home fakebin state out status + local dir root home fakebin state out status wake_line inactive_line new_case watcher-wiring dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" cp -a "$ROOT/bin/." "$root/bin/" write_meta "$state" watcher-x1 watcher-inc + printf '1\t1\tsignal\ttask-before\tqueued before watcher scan\n' > "$state/.wake-queue" export FM_FAKE_CREW_STATE_WATCHER_X1='state: failed · source: pane · watcher wiring' cat > "$fakebin/tmux" <<'SH' #!/usr/bin/env bash @@ -623,8 +670,14 @@ SH [ "$status" = 0 ] || fail "watcher cadence failed while surfacing the inactive outcome wake" printf '%s\n' "$out" | grep -F 'check: inactive terminal outcome replay queued' >/dev/null \ || fail "watcher did not surface the inactive reconciliation result" + wake_line=$(printf '%s\n' "$out" | grep -n '^1[[:space:]]\+1[[:space:]]\+signal[[:space:]]\+task-before' | head -1 | cut -d: -f1) + inactive_line=$(printf '%s\n' "$out" | grep -n 'check: inactive terminal outcome replay queued' | head -1 | cut -d: -f1) + [ -n "$wake_line" ] && [ -n "$inactive_line" ] && [ "$wake_line" -lt "$inactive_line" ] \ + || fail "watcher did not drain the existing wake before inactive reconciliation" + [ "$(awk -F '\t' '$4 == "task-before" { n++ } END { print n + 0 }' "$state/.wake-queue")" = 0 ] \ + || fail "watcher left the existing wake queued" [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher cadence did not create the inactive receipt" - [ "$(queue_count "$state")" = 1 ] || fail "watcher cadence did not retain the inactive outcome wake" + [ "$(queue_count "$state")" = 1 ] || fail "watcher cadence did not retain exactly one inactive outcome wake" unset FM_FAKE_CREW_STATE_WATCHER_X1 pass "watcher cadence runs inactive reconciliation and surfaces its wake" } @@ -779,7 +832,8 @@ test_status_log_terminal_is_not_replayed() { } test_valid_secondmate_route_reports_parent_once() { - local dir root home fakebin state child_home child_state parent_status corr rec outside parent_record send_out + local dir root home fakebin state child_home child_state parent_status corr rec outside send_out + local history_corr history_record history_status new_case secondmate-route-valid dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -836,22 +890,57 @@ SH fail "valid secondmate route drain failed after symlink removal" fi [ "$(receipt_count "$child_state" reported)" = 1 ] || fail "valid secondmate route was not reported" - [ -f "$child_state/.fm-jt-parent-route" ] || fail "reported secondmate route was cleared before its parent lifecycle resolved" + [ ! -e "$child_state/.fm-jt-parent-route" ] || fail "reported secondmate route was not cleared after its parent report" grep -F "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status" >/dev/null \ || fail "valid secondmate route did not append the correlated parent status" drain "$root" "$child_home" "$fakebin" >/dev/null [ "$(grep -Fc "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status")" = 1 ] \ || fail "secondmate parent report was duplicated" - parent_record="$state/pending-replies/$corr" - sed -i 's/^phase=.*/phase=resolved/' "$parent_record" - scan "$root" "$child_home" "$fakebin" --startup >/dev/null || fail "resolved history lifecycle scan failed" - [ ! -e "$child_state/.fm-jt-parent-route" ] || fail "resolved parent history left a stale route marker" - unset FM_FAKE_CREW_STATE_CHILD_X1 + + printf 'sm-history\n' > "$child_home/.fm-secondmate-home" + write_meta "$state" sm-history history-parent-inc secondmate tmux firstmate:fm-sm-history + printf 'home=%s\n' "$child_home" >> "$state/sm-history.meta" + history_status="$state/sm-history.status" + prepare_primary_proof "$root" "$home" "$fakebin" + send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ + fm-sm-history "history request" 2>&1) || fail "public fm-send history route setup failed: $send_out" + history_corr= + for history_record in "$state"/pending-replies/*; do + [ -f "$history_record" ] || continue + [ "$(basename "$history_record")" = "$corr" ] || history_corr=$(basename "$history_record") + done + [ -n "$history_corr" ] || fail "public fm-send did not create a distinct history correlation" + history_record="$state/pending-replies/$history_corr" + sed -i 's/^phase=.*/phase=resolved/' "$history_record" + (cd "$root" && env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ + bash -c '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_archive_terminal "$2" "$3"' \ + _ "$ROOT" "$state" "$history_corr") \ + || fail "parent terminal lifecycle did not archive the history record" + [ -f "$state/pending-reply-history/$history_corr" ] || fail "parent history record was not archived" + write_meta "$child_state" child-history-x1 child-history-inc + export FM_FAKE_CREW_STATE_CHILD_HISTORY_X1='state: done · source: pane · history child quiet' + scan "$root" "$child_home" "$fakebin" --startup \ + || fail "pending-reply-history route scan failed" + [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "pending-reply-history route did not create a pending receipt" + rec=$(find "$child_state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + [ "$(receipt_value "$rec" parent_corr)" = "$history_corr" ] \ + || fail "history route receipt used the wrong parent correlation" + drain "$root" "$child_home" "$fakebin" >/dev/null \ + || fail "pending-reply-history route drain failed" + [ "$(receipt_count "$child_state" reported)" = 2 ] || fail "history route receipt was not reported" + [ ! -e "$child_state/.fm-jt-parent-route" ] || fail "history route marker was not cleared after presentation" + ! grep -F 'inactive terminal outcome replayed: task=child-history-x1' "$history_status" >/dev/null 2>&1 \ + || fail "resolved history route appended a duplicate parent status" + unset FM_FAKE_CREW_STATE_CHILD_X1 FM_FAKE_CREW_STATE_CHILD_HISTORY_X1 pass "valid secondmate outcomes use the parent status correlation exactly once" } test_concurrent_secondmate_routes_are_rejected() { - local dir root home fakebin state child_home child_state marker corr_one corr_two outside + local dir root home fakebin state child_home child_state marker corr_one corr_two outside existing_real existing_link new_case secondmate-route-concurrent dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -881,6 +970,20 @@ test_concurrent_secondmate_routes_are_rejected() { fi rm -f "$child_home/.fm-secondmate-home" mv "$dir/secondmate-marker" "$child_home/.fm-secondmate-home" + existing_real="$dir/existing-real-home" + existing_link="$dir/existing-home-link" + mkdir -p "$existing_real/state/pending-replies" + fm_write_meta "$existing_real/state/pending-replies/$corr_one" \ + schema=fm-pending-reply.v1 corr_id="$corr_one" task_id=sm-concurrent \ + parent_home="$existing_real" parent_status="$existing_real/state/sm-concurrent.status" \ + delivered_epoch=1 phase=resolved + ln -s "$existing_real" "$existing_link" + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-concurrent\nparent_home=%s\nparent_status=%s\ncorr_id=%s\n' \ + "$existing_link" "$existing_real/state/sm-concurrent.status" "$corr_one" > "$marker" + if route_write "$corr_two"; then + fail "existing secondmate home symlink was accepted during route replacement" + fi + rm -f "$marker" route_write "$corr_one" || fail "initial secondmate route was not written" marker_before=$(cat "$marker") outside="$dir/temp-target" @@ -979,8 +1082,10 @@ test_portable_timeout_runner_is_used test_portable_timeout_preserves_signal_failure test_leading_zero_cadence_is_normalized test_find_failure_propagates_without_advancing_scan +test_find_enumeration_respects_scan_budget test_ack_recomputes_fingerprint_from_receipt_fields test_reserved_claim_recovers_to_a_new_wake_row +test_presenting_claim_recovers_before_output test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint From 52de8ceed22564e82f492b0c068e724330135b8c Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 03:15:00 +0000 Subject: [PATCH 008/163] no-mistakes(review): Hardened presentation recovery and secondmate path validation --- bin/fm-inactive-reconcile.sh | 63 ++++++++++++++++++---- bin/fm-pending-reply-lib.sh | 25 +++++++-- bin/fm-wake-drain.sh | 22 +++++--- tests/fm-inactive-outcome.test.sh | 88 +++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 22 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index b2e12a24b93..4b9c676825e 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -219,7 +219,7 @@ claim_validate() { # } claim_reserve() { # - local key=$1 row=$2 fp claim tmp state existing old_row line + local key=$1 row=$2 fp claim tmp state existing old_row line output_started drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -243,6 +243,13 @@ claim_reserve() { # [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } fi + if [ "$state" = presenting ]; then + output_started=$(claim_field "$claim" output_started 2>/dev/null || true) + if [ "$output_started" = 1 ]; then + claim_mark_presented "$key" "$row" || return 2 + return 1 + fi + fi [ "$state" = presented ] && return 1 return 0 fi @@ -268,7 +275,7 @@ claim_reserve() { # } claim_mark_presenting() { # - local key=$1 row=$2 fp claim state tmp line + local key=$1 row=$2 fp claim state tmp line seen_pid=0 seen_output=0 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -276,15 +283,43 @@ claim_mark_presenting() { # [ ! -L "$claim" ] || return 2 state=$(claim_validate "$claim" "$fp" "$row") || return 2 case "$state" in - presenting) return 0 ;; - reserved) ;; + presenting|reserved) ;; *) return 2 ;; esac tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 chmod 600 "$tmp" 2>/dev/null || true while IFS= read -r line || [ -n "$line" ]; do - case "$line" in state=*) printf 'state=presenting\n' ;; *) printf '%s\n' "$line" ;; esac + case "$line" in + state=*) printf 'state=presenting\n' ;; + presentation_pid=*) printf 'presentation_pid=%s\n' "${BASHPID:-$$}"; seen_pid=1 ;; + output_started=*) printf 'output_started=0\n'; seen_output=1 ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ "$seen_pid" = 1 ] || printf 'presentation_pid=%s\n' "${BASHPID:-$$}" >> "$tmp" + [ "$seen_output" = 1 ] || printf 'output_started=0\n' >> "$tmp" + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_output_started() { # + local key=$1 row=$2 fp claim state tmp line seen_output=0 + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + *) printf '%s\n' "$line" ;; + esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ "$seen_output" = 1 ] || printf 'output_started=1\n' >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } @@ -437,18 +472,18 @@ republish_existing_receipt_wake() { } read_incarnation() { # - local meta=$1 id=$2 token tasktmp window worktree seed rc + local meta=$1 id=$2 token tasktmp window worktree seed rc token_present=0 if token=$(meta_value_unique "$meta" spawn_incarnation); then - : + token_present=1 else rc=$? [ "$rc" = 1 ] || return 1 token= fi - case "$token" in - ''|legacy-unknown|*[!A-Za-z0-9._:-]*) token= ;; - esac - if [ -n "$token" ]; then + if [ "$token_present" = 1 ]; then + case "$token" in + ''|legacy-unknown|*[!A-Za-z0-9._:-]*) return 1 ;; + esac printf '%s' "$token" return 0 fi @@ -623,6 +658,8 @@ ack_receipt() { # secondmate_ack_report() { # local secondmate_home=$1 parent_task_id=$2 parent_home=$3 parent_status=$4 corr=$5 outcome=$6 task_id=$7 fp=$8 local parent_state token rc=0 line + fm_pending_reply_secondmate_receipt_validate \ + "$secondmate_home" "$parent_task_id" "$parent_home" "$parent_status" "$corr" || return 2 parent_state="$parent_home/state" fm_pending_reply_txn_lock_acquire "$parent_state" "$corr" token || return 2 if ! fm_pending_reply_secondmate_receipt_validate \ @@ -758,6 +795,10 @@ case "${1:-}" in [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_presenting "$2" "$3" ;; + output-started) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_output_started "$2" "$3" + ;; presented) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_presented "$2" "$3" diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 618c76e5f87..687859f23ae 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -498,7 +498,8 @@ fm_pending_reply_secondmate_route_write() { # < local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 local marker home_marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 local existing_schema existing_id existing_home existing_status existing_corr - local existing_state existing_record existing_phase + local existing_state existing_record existing_phase existing_active_dir existing_history_dir + local existing_active_record existing_history_record marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 @@ -575,10 +576,22 @@ fm_pending_reply_secondmate_route_write() { # < route_status=1 fi if [ "$route_status" = 0 ]; then - if [ -f "$(fm_pending_reply_active_path "$existing_state" "$existing_corr")" ]; then - existing_record=$(fm_pending_reply_active_path "$existing_state" "$existing_corr") - elif [ -f "$(fm_pending_reply_history_dir "$existing_state")/$existing_corr" ]; then - existing_record="$(fm_pending_reply_history_dir "$existing_state")/$existing_corr" + existing_active_dir=$(fm_pending_reply_dir "$existing_state") + [ -d "$existing_active_dir" ] && [ ! -L "$existing_active_dir" ] || route_status=1 + existing_history_dir=$(fm_pending_reply_history_dir "$existing_state") + if [ -e "$existing_history_dir" ] || [ -L "$existing_history_dir" ]; then + [ -d "$existing_history_dir" ] && [ ! -L "$existing_history_dir" ] || route_status=1 + fi + fi + if [ "$route_status" = 0 ]; then + existing_active_record="$existing_active_dir/$existing_corr" + existing_history_record="$existing_history_dir/$existing_corr" + if [ -e "$existing_active_record" ] || [ -L "$existing_active_record" ]; then + [ -f "$existing_active_record" ] && [ ! -L "$existing_active_record" ] || route_status=1 + [ "$route_status" = 0 ] && existing_record="$existing_active_record" + elif [ -e "$existing_history_record" ] || [ -L "$existing_history_record" ]; then + [ -f "$existing_history_record" ] && [ ! -L "$existing_history_record" ] || route_status=1 + [ "$route_status" = 0 ] && existing_record="$existing_history_record" else route_status=1 fi @@ -604,6 +617,8 @@ fm_pending_reply_secondmate_route_write() { # < fm_pending_reply_secondmate_route_clear() { # local secondmate_home=$1 corr=$2 marker route_lock existing_corr status=0 + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -e "$marker" ] || [ -L "$marker" ] || return 0 [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index bd7ece44440..7565914e542 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -15,6 +15,21 @@ DRAIN_RESTORE= DRAIN_PID=${BASHPID:-$$} DRAIN_LOCK_HELD=false +present_inactive_row() { + local key=$1 row=$2 status=0 + trap - INT TERM HUP + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" output-started "$key" "$row" || status=1 + if [ "$status" = 0 ] && ! printf '%s\n' "$row"; then + status=1 + fi + if [ "$status" = 0 ] && ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$key" "$row"; then + status=1 + fi + trap 'exit 130' INT + trap 'exit 143' TERM + return "$status" +} + # Defense in depth for the watcher re-arm chain: this script runs at the top of # every wake-handling and recovery turn, so assert watcher liveness here too. A # lapsed supervision chain then surfaces on a plain drain-and-handle turn, not @@ -90,12 +105,7 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 fi - if ! printf '%s\n' "$drain_row"; then - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" - awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 - exit 1 - fi - if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$_key" "$drain_row"; then + if ! present_inactive_row "$_key" "$drain_row"; then DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 exit 1 diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 2f5795538e8..174dce62cce 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -435,6 +435,29 @@ test_presenting_claim_recovers_before_output() { pass "presenting inactive claims recover before output" } +test_output_started_claim_is_not_reprinted() { + local dir root home fakebin state fingerprint row + new_case output-started-claim + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'output-started-x1|output-started-inc|done|state: done · source: pane · output started') + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=output-started-x1 \ + incarnation=output-started-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · output started' kind=ship + row='2 2 check inactive-outcome:'"$fingerprint"$'\tpost-output row' + printf 'schema=fm-inactive-outcome-claim.v1\nfingerprint=%s\nrow=1\t1\tcheck\tinactive-outcome:%s\told row\nstate=presenting\noutput_started=1\ncreated_epoch=1\n' \ + "$fingerprint" "$fingerprint" > "$state/terminal-outcomes/.$fingerprint.claim" + printf '%s\n' "$row" > "$state/.wake-queue" + drain "$root" "$home" "$fakebin" >"$dir/output-started.out" \ + || fail "drain did not recover an output-started claim" + [ ! -s "$dir/output-started.out" ] || fail "output-started claim was printed a second time" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] || fail "output-started claim did not acknowledge its receipt" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "output-started claim was not retired" + pass "output-started inactive claims do not reprint after a drain crash" +} + test_scan_failure_retries_without_advancing_cadence() { local dir root home fakebin state wake_dir wake_removed new_case scan-failure @@ -700,6 +723,21 @@ test_legacy_metadata_uses_stable_fallback() { pass "legacy metadata receives a stable fallback incarnation" } +test_empty_spawn_incarnation_is_rejected() { + local dir root home fakebin state + new_case empty-spawn-incarnation + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" empty-inc-x1 empty-inc + sed -i 's/^spawn_incarnation=.*/spawn_incarnation=/' "$state/empty-inc-x1.meta" + export FM_FAKE_CREW_STATE_EMPTY_INC_X1='state: done · source: pane · malformed incarnation' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "empty incarnation scan failed" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "explicit empty incarnation used the legacy fallback" + [ "$(queue_count "$state")" = 0 ] || fail "explicit empty incarnation created a wake" + unset FM_FAKE_CREW_STATE_EMPTY_INC_X1 + pass "explicit empty spawn incarnations fail closed" +} + test_relaunch_and_teardown_races_recheck_under_spawn_lock() { local dir root home fakebin state holder scanner ready release new_case races @@ -833,6 +871,7 @@ test_status_log_terminal_is_not_replayed() { test_valid_secondmate_route_reports_parent_once() { local dir root home fakebin state child_home child_state parent_status corr rec outside send_out + local outside_parent outside_parent_link local history_corr history_record history_status new_case secondmate-route-valid dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN @@ -876,6 +915,17 @@ SH [ "$(receipt_value "$rec" parent_corr)" = "$corr" ] || fail "secondmate receipt did not persist its parent correlation" [ "$(receipt_value "$rec" parent_home)" = "$home" ] || fail "secondmate receipt did not persist its parent home" [ "$(receipt_value "$rec" parent_status)" = "$parent_status" ] || fail "secondmate receipt did not persist its parent status path" + outside_parent="$dir/outside-parent" + outside_parent_link="$dir/outside-parent-link" + mkdir -p "$outside_parent/state" + ln -s "$outside_parent" "$outside_parent_link" + sed -i "s|^parent_home=.*|parent_home=$outside_parent_link|" "$rec" + if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then + fail "secondmate acknowledgement accepted a symlinked parent home" + fi + [ ! -e "$outside_parent/state/pending-replies/.txn-$corr.lock" ] \ + || fail "secondmate acknowledgement acquired a transaction lock before path validation" + sed -i "s|^parent_home=.*|parent_home=$home|" "$rec" outside="$dir/outside-status" printf 'outside\n' > "$outside" rm -f "$parent_status" @@ -941,6 +991,7 @@ SH test_concurrent_secondmate_routes_are_rejected() { local dir root home fakebin state child_home child_state marker corr_one corr_two outside existing_real existing_link + local existing_record record_backup existing_history history_backup clear_real clear_link new_case secondmate-route-concurrent dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -984,6 +1035,29 @@ test_concurrent_secondmate_routes_are_rejected() { fail "existing secondmate home symlink was accepted during route replacement" fi rm -f "$marker" + existing_record="$existing_real/state/pending-replies/$corr_one" + record_backup="$dir/record-backup" + mv "$existing_record" "$record_backup" + ln -s "$record_backup" "$existing_record" + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-concurrent\nparent_home=%s\nparent_status=%s\ncorr_id=%s\n' \ + "$existing_real" "$existing_real/state/sm-concurrent.status" "$corr_one" > "$marker" + if route_write "$corr_two"; then + fail "symlinked active secondmate record was accepted during route replacement" + fi + rm -f "$marker" "$existing_record" + mv "$record_backup" "$existing_record" + existing_history="$existing_real/state/pending-reply-history/$corr_one" + history_backup="$dir/history-backup" + mkdir -p "$existing_real/state/pending-reply-history" + mv "$existing_record" "$history_backup" + ln -s "$history_backup" "$existing_history" + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-concurrent\nparent_home=%s\nparent_status=%s\ncorr_id=%s\n' \ + "$existing_real" "$existing_real/state/sm-concurrent.status" "$corr_one" > "$marker" + if route_write "$corr_two"; then + fail "symlinked history secondmate record was accepted during route replacement" + fi + rm -f "$marker" "$existing_history" + mv "$history_backup" "$existing_record" route_write "$corr_one" || fail "initial secondmate route was not written" marker_before=$(cat "$marker") outside="$dir/temp-target" @@ -999,6 +1073,18 @@ test_concurrent_secondmate_routes_are_rejected() { fail "concurrent secondmate route was silently replaced" fi [ "$(cat "$marker")" = "$marker_before" ] || fail "concurrent route rejection changed the active marker" + clear_real="$dir/clear-real" + clear_link="$dir/clear-link" + mkdir -p "$clear_real/state" + printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-concurrent\nparent_home=%s\nparent_status=%s\ncorr_id=%s\n' \ + "$home" "$state/sm-concurrent.status" "$corr_one" > "$clear_real/state/.fm-jt-parent-route" + ln -s "$clear_real" "$clear_link" + if env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ + bash -c '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_clear "$2" "$3"' \ + _ "$ROOT" "$clear_link" "$corr_one"; then + fail "route cleanup followed a symlinked secondmate home" + fi + [ -e "$clear_real/state/.fm-jt-parent-route" ] || fail "unsafe route cleanup removed the target marker" pass "concurrent secondmate routes fail closed without overwriting" } @@ -1086,6 +1172,7 @@ test_find_enumeration_respects_scan_budget test_ack_recomputes_fingerprint_from_receipt_fields test_reserved_claim_recovers_to_a_new_wake_row test_presenting_claim_recovers_before_output +test_output_started_claim_is_not_reprinted test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint @@ -1093,6 +1180,7 @@ test_spawn_publishes_incarnation_token test_session_start_drains_before_inactive_scan test_watcher_runs_inactive_cadence test_legacy_metadata_uses_stable_fallback +test_empty_spawn_incarnation_is_rejected test_relaunch_and_teardown_races_recheck_under_spawn_lock test_parent_home_secondmate_records_are_skipped test_herdr_identity_and_default_captain_refusal From 009849c2376ee48158bd15cbe8cac583820586d2 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 04:05:09 +0000 Subject: [PATCH 009/163] no-mistakes(review): Hardened inactive replay and secondmate route recovery --- .github/workflows/ci.yml | 1 + bin/fm-inactive-reconcile.sh | 116 +++++++++++++++--- bin/fm-pending-reply-lib.sh | 28 +++-- bin/fm-send.sh | 48 ++++---- bin/fm-session-start.sh | 21 +++- bin/fm-wake-drain.sh | 44 ++++--- bin/fm-wake-lib.sh | 17 ++- tests/fm-inactive-outcome.test.sh | 194 +++++++++++++++++++++++------- 8 files changed, 344 insertions(+), 125 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39d32d0859a..f4cc8286262 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,7 @@ jobs: /bin/bash --version | head -1 /bin/bash -n bin/fm-slot-owner-lib.sh /bin/bash tests/fm-worker-isolation.test.sh + /bin/bash tests/fm-inactive-outcome.test.sh invariants: name: Repo invariants diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 4b9c676825e..524d3cc2cab 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -451,7 +451,33 @@ receipt_existing_core() { [ "$(receipt_field "$existing" terminal_source)" = "$SOURCE" ] || return 2 [ "$(receipt_field "$existing" terminal_snapshot)" = "$SNAPSHOT" ] || return 2 existing_kind=$(receipt_field "$existing" kind) - case "$existing_kind" in ship|scout) parent_id=$(receipt_field "$existing" parent_task_id); parent_home=$(receipt_field "$existing" parent_home); parent_status=$(receipt_field "$existing" parent_status); parent_corr=$(receipt_field "$existing" parent_corr); [ -z "$parent_id" ] && [ -z "$parent_home" ] && [ -z "$parent_status" ] && [ -z "$parent_corr" ] || return 2 ;; secondmate) parent_id=$(receipt_field "$existing" parent_task_id); parent_home=$(receipt_field "$existing" parent_home); parent_status=$(receipt_field "$existing" parent_status); parent_corr=$(receipt_field "$existing" parent_corr); [ -n "$parent_id" ] && case "$parent_home" in /*) ;; *) return 2 ;; esac && case "$parent_status" in /*) ;; *) return 2 ;; esac && printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 2 ;; *) return 2 ;; esac + case "$existing_kind" in + ship|scout) + parent_id=$(receipt_field "$existing" parent_task_id) + parent_home=$(receipt_field "$existing" parent_home) + parent_status=$(receipt_field "$existing" parent_status) + parent_corr=$(receipt_field "$existing" parent_corr) + [ -z "$parent_id" ] && [ -z "$parent_home" ] && [ -z "$parent_status" ] && [ -z "$parent_corr" ] || return 2 + ;; + secondmate) + parent_id=$(receipt_field "$existing" parent_task_id) + parent_home=$(receipt_field "$existing" parent_home) + parent_status=$(receipt_field "$existing" parent_status) + parent_corr=$(receipt_field "$existing" parent_corr) + if [ "$suffix" = pending ]; then + [ "$parent_id" = "${FM_PENDING_ROUTE_SECOND_MATE_ID:-}" ] || return 2 + [ "$parent_home" = "${FM_PENDING_ROUTE_PARENT_HOME:-}" ] || return 2 + [ "$parent_status" = "${FM_PENDING_ROUTE_PARENT_STATUS:-}" ] || return 2 + [ "$parent_corr" = "${FM_PENDING_ROUTE_CORR:-}" ] || return 2 + else + [ -n "$parent_id" ] || return 2 + fi + case "$parent_home" in /*) ;; *) return 2 ;; esac + case "$parent_status" in /*) ;; *) return 2 ;; esac + printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 2 + ;; + *) return 2 ;; + esac RECEIPT_EXISTING_SUFFIX=$suffix return 0 done @@ -459,14 +485,65 @@ receipt_existing_core() { } republish_existing_receipt_wake() { - local existing task outcome status=0 - existing=$(receipt_path "$FP" pending) - task=$(receipt_field "$existing" task_id) - outcome=$(receipt_field "$existing" outcome) + local existing task outcome status=0 existing_rc FM_WAKE_APPEND_CREATED=0 fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 - fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ - "inactive terminal outcome: task=$task state=$outcome fingerprint=$FP" || status=$? + if receipt_existing_core; then + if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then + existing=$(receipt_path "$FP" pending) + task=$(receipt_field "$existing" task_id) + outcome=$(receipt_field "$existing" outcome) + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$task state=$outcome fingerprint=$FP" || status=$? + fi + else + existing_rc=$? + [ "$existing_rc" = 1 ] || status=1 + fi + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 + return "$status" +} + +publish_secondmate_receipt_and_wake() { + local route_lock status=0 existing_rc + FM_WAKE_APPEND_CREATED=0 + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 + if [ -L "$FM_HOME" ] || [ ! -d "$FM_HOME" ] || [ -L "$FM_HOME/state" ] || [ ! -d "$FM_HOME/state" ]; then + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || true + return 0 + fi + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$FM_HOME") + if ! fm_lock_acquire_wait "$route_lock"; then + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || true + return 1 + fi + if ! fm_pending_reply_secondmate_route_validate "$FM_HOME"; then + fm_lock_release "$route_lock" || true + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || true + return 0 + fi + KIND=secondmate + if receipt_existing_core; then + if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$ID state=$OUTCOME fingerprint=$FP" || status=$? + fi + else + existing_rc=$? + if [ "$existing_rc" = 1 ]; then + if receipt_write; then + if [ -f "$(receipt_path "$FP" pending)" ]; then + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$ID state=$OUTCOME fingerprint=$FP" || status=$? + fi + else + status=$? + fi + else + status=1 + fi + fi + fm_lock_release "$route_lock" || status=1 fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 return "$status" } @@ -519,7 +596,7 @@ reconcile_child() { case "$kind" in ''|ship|scout) ;; *) return 0 ;; esac herdr_identity_allowed "$meta" || return 0 CHILD_LOCK="$STATE/.spawn-$id.lock" - FM_LOCK_WAIT_SECS=${FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS:-30} + FM_LOCK_WAIT_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS:-30}" 30 0 300) fm_lock_acquire_wait "$CHILD_LOCK" || return 0 CHILD_LOCK_HELD=1 trap child_cleanup EXIT INT TERM @@ -573,6 +650,13 @@ reconcile_child() { 0|1) ;; *) return 0 ;; esac + if [ "$route_rc" = 0 ]; then + publish_secondmate_receipt_and_wake || return 1 + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" + fi + return 0 + fi if receipt_existing_core; then if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then republish_existing_receipt_wake || return 1 @@ -585,10 +669,6 @@ reconcile_child() { existing_rc=$? [ "$existing_rc" = 1 ] || return 1 fi - if [ "$route_rc" = 0 ]; then - fm_pending_reply_secondmate_route_validate "$FM_HOME" || return 0 - KIND=secondmate - fi key="inactive-outcome:$FP" publish_receipt_and_wake || return 1 if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then @@ -764,10 +844,18 @@ scan_locked() { } scan() { - local startup=${1:-0} + local startup=${1:-0} rc + scan_abort() { + local status=$1 + fm_lock_release "$SCAN_LOCK" || true + trap - EXIT INT TERM + exit "$status" + } inactive_state_preflight || return 1 fm_lock_acquire_wait "$SCAN_LOCK" || return 1 - trap 'fm_lock_release "$SCAN_LOCK" || true' EXIT INT TERM + trap 'fm_lock_release "$SCAN_LOCK" || true' EXIT + trap 'scan_abort 130' INT + trap 'scan_abort 143' TERM scan_locked "$startup" rc=$? fm_lock_release "$SCAN_LOCK" || true diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 687859f23ae..108fa88726d 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -528,6 +528,11 @@ fm_pending_reply_secondmate_route_write() { # < fm_lock_release "$route_lock" || true return 1 } + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { + rm -f "$tmp" + fm_lock_release "$route_lock" || true + return 1 + } { printf 'schema=fm-jt-parent-route.v1\n' printf 'secondmate_id=%s\n' "$secondmate_id" @@ -616,7 +621,7 @@ fm_pending_reply_secondmate_route_write() { # < } fm_pending_reply_secondmate_route_clear() { # - local secondmate_home=$1 corr=$2 marker route_lock existing_corr status=0 + local secondmate_home=$1 corr=$2 marker route_lock status=0 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") @@ -625,14 +630,11 @@ fm_pending_reply_secondmate_route_clear() { # printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") fm_lock_acquire_wait "$route_lock" || return 1 - if [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" != 5 ]; then - status=1 - else - existing_corr=$(fm_pending_reply_get "$marker" corr_id) - [ "$existing_corr" = "$corr" ] || status=1 - fi - if [ "$status" = 0 ]; then + if fm_pending_reply_secondmate_route_validate "$secondmate_home" \ + && [ "$FM_PENDING_ROUTE_CORR" = "$corr" ]; then rm -f "$marker" || status=1 + else + status=1 fi fm_lock_release "$route_lock" || status=1 return "$status" @@ -699,7 +701,8 @@ fm_pending_reply_secondmate_route_validate() { # if [ -e "$active_rec" ] || [ -L "$active_rec" ]; then [ -f "$active_rec" ] && [ ! -L "$active_rec" ] || return 1 rec=$active_rec - elif [ -f "$history_rec" ] && [ ! -L "$history_rec" ]; then + elif [ -e "$history_rec" ] || [ -L "$history_rec" ]; then + [ -f "$history_rec" ] && [ ! -L "$history_rec" ] || return 1 rec=$history_rec else return 1 @@ -740,6 +743,7 @@ fm_pending_reply_secondmate_receipt_validate() { # /dev/null || true) + case "$secondmate_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac [ "$marker_id" = "$secondmate_id" ] || return 1 case "$parent_home" in /*) ;; *) return 1 ;; esac case "$parent_status" in /*) ;; *) return 1 ;; esac @@ -761,9 +765,11 @@ fm_pending_reply_secondmate_receipt_validate() { # &2 + return 1 + fi + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ + && ! fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR"; then + echo "error: failed to discard the undelivered pending-reply record" >&2 + return 1 + fi + return 0 +} + case "$RAW_TARGET" in fm-*) meta="$STATE/${RAW_TARGET#fm-}.meta" @@ -145,18 +158,14 @@ else fm_pending_reply_embed_corr "$MESSAGE" "$PENDING_REPLY_CORR" MESSAGE if [ "$PENDING_REPLY_CREATED" = 1 ] \ && ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then - clear_new_pending_route - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + discard_new_pending_reply || exit 1 echo "error: failed to durably prepare pending-reply delivery for $TARGET_TASK_ID" >&2 exit 1 fi TARGET_HOME=$(fm_meta_get "$meta" home) if ! fm_pending_reply_secondmate_route_write \ "$TARGET_HOME" "$FM_HOME" "$STATE" "$TARGET_TASK_ID" "$PENDING_REPLY_CORR"; then - clear_new_pending_route - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: failed to bind the secondmate pending-reply route for $TARGET_TASK_ID" >&2 exit 1 fi @@ -191,10 +200,7 @@ else # Type once, submit, verify. Lenient: only a positively-confirmed swallow # (text still in the composer) is an error; an unreadable pane is assumed sent. if ! verdict=$(fm_backend_send_text_submit "$TARGET_BACKEND" "$T" "$MESSAGE" "$retries" "$sleep_s" "$settle"); then - clear_new_pending_route - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: text not sent to $T ($TARGET_BACKEND send failed)" >&2 exit 1 fi @@ -205,37 +211,25 @@ else sleep "$settle" final_after_pending=1 if ! verdict=$(fm_backend_submit_enter "$TARGET_BACKEND" "$T" 1 "$sleep_s" "$MESSAGE"); then - clear_new_pending_route - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: final Enter submission to $T failed" >&2 exit 1 fi fi case "$verdict" in pending) - clear_new_pending_route - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: text not submitted to $T (Enter swallowed; text left in composer)" >&2 exit 1 ;; send-failed) - clear_new_pending_route - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: text not sent to $T (tmux send-keys failed)" >&2 exit 1 ;; unknown) if [ "$final_after_pending" = 1 ]; then - clear_new_pending_route - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: final Enter submission to $T could not be confirmed" >&2 exit 1 fi diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 7b65fba9ef1..c4ffa29a2dc 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -362,17 +362,26 @@ if [ "$READ_ONLY" -eq 1 ]; then GUARD_OUT=$(FM_GUARD_READ_ONLY=1 "$SCRIPT_DIR/fm-guard.sh" 2>&1) [ -n "$GUARD_OUT" ] && printf '%s\n' "$GUARD_OUT" else - DRAIN_OUT=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) - if [ -n "$DRAIN_OUT" ]; then + DRAIN_STATUS=0 + DRAIN_OUT=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) || DRAIN_STATUS=$? + if [ "$DRAIN_STATUS" -ne 0 ]; then + printf 'error: wake drain failed (status %s); inactive reconciliation skipped\n%s\n' \ + "$DRAIN_STATUS" "$DRAIN_OUT" >&2 + elif [ -n "$DRAIN_OUT" ]; then printf '%s\n' "$DRAIN_OUT" else printf '(no queued wakes)\n' fi - if ! INACTIVE_OUT=$("$SCRIPT_DIR/fm-inactive-reconcile.sh" scan --startup 2>&1); then - printf '%s\n' "$INACTIVE_OUT" >&2 - exit 1 + if [ "$DRAIN_STATUS" -eq 0 ]; then + INACTIVE_STATUS=0 + INACTIVE_OUT=$("$SCRIPT_DIR/fm-inactive-reconcile.sh" scan --startup 2>&1) || INACTIVE_STATUS=$? + if [ "$INACTIVE_STATUS" -ne 0 ]; then + printf 'error: inactive outcome reconciliation failed (status %s)\n%s\n' \ + "$INACTIVE_STATUS" "$INACTIVE_OUT" >&2 + elif [ -n "$INACTIVE_OUT" ]; then + printf '%s\n' "$INACTIVE_OUT" + fi fi - [ -z "$INACTIVE_OUT" ] || printf '%s\n' "$INACTIVE_OUT" fi # --- 4. supervision operating instructions ---------------------------------- diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 7565914e542..77aef847d53 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -18,8 +18,10 @@ DRAIN_LOCK_HELD=false present_inactive_row() { local key=$1 row=$2 status=0 trap - INT TERM HUP - FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" output-started "$key" "$row" || status=1 - if [ "$status" = 0 ] && ! printf '%s\n' "$row"; then + if ! printf '%s\n' "$row"; then + status=1 + fi + if [ "$status" = 0 ] && ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" output-started "$key" "$row"; then status=1 fi if [ "$status" = 0 ] && ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$key" "$row"; then @@ -45,18 +47,31 @@ assert_watcher_liveness() { "$SCRIPT_DIR/fm-guard.sh" || true } +restore_unprocessed_rows() { + local start=$1 restored + restored=$(mktemp "$STATE/.wake-queue.unprocessed.XXXXXX") || return 1 + [ -f "$restored" ] && [ ! -L "$restored" ] || { rm -f "$restored"; return 1; } + awk -v start="$start" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$restored" || { + rm -f "$restored" + return 1 + } + DRAIN_RESTORE=$restored +} + # shellcheck disable=SC2317,SC2329 # Invoked by trap handlers below. cleanup() { - local status=$? + local status=$? restore_status=0 if [ "$status" -ne 0 ] && [ "$DRAIN_LOCK_HELD" = true ]; then if [ -n "$DRAIN_RESTORE" ] && [ -e "$DRAIN_RESTORE" ]; then - fm_wake_restore_queue "$DRAIN_RESTORE" || true + fm_wake_restore_queue "$DRAIN_RESTORE" || restore_status=1 elif [ -n "$DRAIN_TMP" ] && [ -e "$DRAIN_TMP" ]; then - fm_wake_restore_queue "$DRAIN_TMP" || true + fm_wake_restore_queue "$DRAIN_TMP" || restore_status=1 fi fi - [ -z "$DRAIN_TMP" ] || rm -f "$DRAIN_TMP" || true - [ -z "$DRAIN_RESTORE" ] || rm -f "$DRAIN_RESTORE" || true + if [ "$restore_status" = 0 ]; then + [ -z "$DRAIN_TMP" ] || rm -f "$DRAIN_TMP" || true + [ -z "$DRAIN_RESTORE" ] || rm -f "$DRAIN_RESTORE" || true + fi [ -z "$DRAIN_DEDUPED" ] || rm -f "$DRAIN_DEDUPED" || true if [ "$DRAIN_LOCK_HELD" = true ]; then fm_lock_release "$FM_WAKE_QUEUE_LOCK" @@ -101,20 +116,17 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do case "$claim_status" in 0) if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presenting "$_key" "$drain_row"; then - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" - awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + restore_unprocessed_rows "$drain_line" || exit 1 exit 1 fi if ! present_inactive_row "$_key" "$drain_row"; then - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" - awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + restore_unprocessed_rows "$drain_line" || exit 1 exit 1 fi ;; 1) ;; *) - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" - awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + restore_unprocessed_rows "$drain_line" || exit 1 exit "$claim_status" ;; esac @@ -123,16 +135,14 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do # 1 means the receipt was already acknowledged or is not ours. Any # other failure keeps the drained row durable for a later turn. if [ "$ack_status" != 1 ]; then - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" - awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + restore_unprocessed_rows "$drain_line" || exit 1 exit "$ack_status" fi } ;; *) if ! printf '%s\n' "$drain_row"; then - DRAIN_RESTORE="$STATE/.wake-queue.unprocessed.$DRAIN_PID" - awk -v start="$drain_line" 'NR >= start { print }' "$DRAIN_DEDUPED" > "$DRAIN_RESTORE" || exit 1 + restore_unprocessed_rows "$drain_line" || exit 1 exit 1 fi ;; diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index e4d7a903867..de3dfd1f9ba 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -695,13 +695,22 @@ fm_wake_append_if_absent() { # } fm_wake_restore_queue() { - local drained=$1 restore - restore="$STATE/.wake-queue.restore.$(fm_current_pid)" + local drained=$1 restore status=0 + [ -f "$drained" ] && [ ! -L "$drained" ] || return 1 + [ ! -L "$FM_WAKE_QUEUE" ] || return 1 + restore=$(mktemp "$STATE/.wake-queue.restore.XXXXXX") || return 1 + [ -f "$restore" ] && [ ! -L "$restore" ] || { rm -f "$restore"; return 1; } if [ -e "$FM_WAKE_QUEUE" ]; then - cat "$drained" "$FM_WAKE_QUEUE" > "$restore" && mv "$restore" "$FM_WAKE_QUEUE" + [ -f "$FM_WAKE_QUEUE" ] || { rm -f "$restore"; return 1; } + cat "$drained" "$FM_WAKE_QUEUE" > "$restore" || status=1 else - mv "$drained" "$FM_WAKE_QUEUE" + cat "$drained" > "$restore" || status=1 fi + if [ "$status" = 0 ]; then + [ ! -L "$FM_WAKE_QUEUE" ] && mv -f "$restore" "$FM_WAKE_QUEUE" || status=1 + fi + [ "$status" = 0 ] || rm -f "$restore" + return "$status" } fm_wake_print_deduped() { diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 174dce62cce..6b22ce09b76 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -91,6 +91,46 @@ SH CASE_THREAD="inactive-${name//[^A-Za-z0-9]/-}" } +set_old_mtime() { + if touch -d '2 minutes ago' "$@" 2>/dev/null; then + return 0 + fi + touch -t "$(date -v-2M '+%Y%m%d%H%M.%S')" "$@" +} + +replace_field() { + local file=$1 key=$2 value=$3 tmp + tmp=$(mktemp "$file.edit.XXXXXX") || return 1 + awk -F= -v wanted="$key" -v replacement="$value" ' + $1 == wanted { print wanted "=" replacement; found=1; next } + { print } + END { if (!found) print wanted "=" replacement } + ' "$file" > "$tmp" || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$file" +} + +direct_file_count() { + local dir=$1 pattern=$2 file base count=0 + for file in "$dir"/*; do + [ -f "$file" ] || continue + base=${file##*/} + case "$base" in $pattern) count=$((count + 1)) ;; esac + done + printf '%s' "$count" +} + +direct_first_file() { + local dir=$1 pattern=$2 file base + for file in "$dir"/*; do + [ -f "$file" ] || continue + base=${file##*/} + case "$base" in + $pattern) printf '%s' "$file"; return 0 ;; + esac + done + return 1 +} + prepare_primary_proof() { local root=$1 home=$2 fakebin=$3 state="$home/state" token mkdir -p "$state" "$home/projects" @@ -168,7 +208,7 @@ write_meta() { mkdir -p "$state/work-$id" printf 'working: fixture\n' > "$state/$id.status" : > "$state/$id.turn-ended" - touch -d '2 minutes ago' "$file" "$state/$id.status" "$state/$id.turn-ended" + set_old_mtime "$file" "$state/$id.status" "$state/$id.turn-ended" } write_legacy_meta() { @@ -180,7 +220,7 @@ write_legacy_meta() { mkdir -p "$state/work-$id" printf 'working: fixture\n' > "$state/$id.status" : > "$state/$id.turn-ended" - touch -d '2 minutes ago' "$state/$id.meta" "$state/$id.status" "$state/$id.turn-ended" + set_old_mtime "$state/$id.meta" "$state/$id.status" "$state/$id.turn-ended" } receipt_value() { @@ -201,7 +241,7 @@ receipt_fingerprint() { receipt_count() { local state=$1 suffix=$2 - find "$state/terminal-outcomes" -maxdepth 1 -type f -name "*.$suffix" 2>/dev/null | wc -l | tr -d ' ' + direct_file_count "$state/terminal-outcomes" "*.$suffix" } queue_count() { @@ -212,7 +252,7 @@ queue_count() { } test_done_and_failed_are_replayed_once() { - local dir root home fakebin state rec task fingerprint + local dir root home fakebin state rec task fingerprint drain_output new_case done-failed dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -247,7 +287,7 @@ test_done_and_failed_are_replayed_once() { [ "$(receipt_value "$rec" terminal_snapshot)" != "" ] || fail "receipt lost terminal snapshot" [ "$(receipt_value "$rec" parent_home)" = "" ] || fail "firstmate receipt invented a parent route" done - rec=$(find "$state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + rec=$(direct_first_file "$state/terminal-outcomes" '*.pending') fingerprint=$(basename "$rec" .pending) if ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ @@ -257,12 +297,20 @@ test_done_and_failed_are_replayed_once() { fail "direct inactive acknowledgement bypassed the wake drain" fi [ -f "$rec" ] || fail "direct inactive acknowledgement removed its pending receipt" - touch -d '2 minutes ago' "$state/.inactive-outcome-reconcile" + set_old_mtime "$state/.inactive-outcome-reconcile" scan "$root" "$home" "$fakebin" >/dev/null || fail "normal watcher cadence scan failed" scan "$root" "$home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$state" pending)" = 2 ] || fail "rescan duplicated inactive outcome receipts" [ "$(queue_count "$state")" = 2 ] || fail "rescan duplicated inactive outcome wakes" - drain "$root" "$home" "$fakebin" >/dev/null + drain_output="$dir/drain.out" + drain "$root" "$home" "$fakebin" >"$drain_output" \ + || fail "drain did not present done and failed outcomes" + grep -F 'inactive-outcome:' "$drain_output" >/dev/null \ + || fail "drain did not emit the inactive outcome wake" + grep -F 'task=done-x1' "$drain_output" >/dev/null \ + || fail "drain output omitted the done outcome" + grep -F 'task=failed-x1' "$drain_output" >/dev/null \ + || fail "drain output omitted the failed outcome" [ "$(receipt_count "$state" pending)" = 0 ] || fail "drain did not acknowledge pending receipts" [ "$(receipt_count "$state" presented)" = 2 ] || fail "drain did not preserve two presented receipts" scan "$root" "$home" "$fakebin" --startup >/dev/null @@ -320,10 +368,10 @@ test_leading_zero_cadence_is_normalized() { state="$home/state" write_meta "$state" zero-x1 zero-inc export FM_FAKE_CREW_STATE_ZERO_X1='state: done · source: pane · leading zero' - export FM_INACTIVE_OUTCOME_SECS=0080 FM_INACTIVE_OUTCOME_BUDGET_SECS=0010 + export FM_INACTIVE_OUTCOME_SECS=0080 FM_INACTIVE_OUTCOME_BUDGET_SECS=0010 FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS=008 scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "leading-zero cadence aborted the scan" [ "$(receipt_count "$state" pending)" = 1 ] || fail "leading-zero cadence did not reconcile the child" - unset FM_FAKE_CREW_STATE_ZERO_X1 FM_INACTIVE_OUTCOME_SECS FM_INACTIVE_OUTCOME_BUDGET_SECS + unset FM_FAKE_CREW_STATE_ZERO_X1 FM_INACTIVE_OUTCOME_SECS FM_INACTIVE_OUTCOME_BUDGET_SECS FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS pass "leading-zero cadence values are normalized before arithmetic" } @@ -369,23 +417,31 @@ SH } test_ack_recomputes_fingerprint_from_receipt_fields() { - local dir root home fakebin state rec fingerprint - new_case fingerprint-binding - dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN - state="$home/state" - write_meta "$state" fingerprint-x1 fingerprint-inc - export FM_FAKE_CREW_STATE_FINGERPRINT_X1='state: done · source: pane · original snapshot' - scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "fingerprint fixture scan failed" - rec=$(find "$state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) - fingerprint=$(basename "$rec" .pending) - sed -i 's/^terminal_snapshot=.*/terminal_snapshot=tampered snapshot/' "$rec" - if drain "$root" "$home" "$fakebin" >/dev/null 2>&1; then - fail "drain accepted a receipt whose snapshot no longer matched its fingerprint" - fi - [ -f "$rec" ] || fail "fingerprint mismatch removed the pending receipt" - [ "$(queue_count "$state")" = 1 ] || fail "fingerprint mismatch did not preserve the wake for retry" - [ "$(receipt_value "$rec" fingerprint)" = "$fingerprint" ] || fail "fingerprint fixture changed its filename binding" - unset FM_FAKE_CREW_STATE_FINGERPRINT_X1 + local dir root home fakebin state rec fingerprint field tampered + for field in task_id incarnation outcome terminal_snapshot; do + new_case "fingerprint-binding-$field" + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" fingerprint-x1 fingerprint-inc + export FM_FAKE_CREW_STATE_FINGERPRINT_X1='state: done · source: pane · original snapshot' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "fingerprint fixture scan failed for $field" + rec=$(direct_first_file "$state/terminal-outcomes" '*.pending') + fingerprint=$(basename "$rec" .pending) + case "$field" in + task_id) tampered=tampered-x1 ;; + incarnation) tampered=tampered-inc ;; + outcome) tampered=failed ;; + terminal_snapshot) tampered='tampered snapshot' ;; + esac + replace_field "$rec" "$field" "$tampered" + if drain "$root" "$home" "$fakebin" >/dev/null 2>&1; then + fail "drain accepted a receipt whose $field no longer matched its fingerprint" + fi + [ -f "$rec" ] || fail "$field fingerprint mismatch removed the pending receipt" + [ "$(queue_count "$state")" = 1 ] || fail "$field fingerprint mismatch did not preserve the wake for retry" + [ "$(receipt_value "$rec" fingerprint)" = "$fingerprint" ] || fail "$field fixture changed its filename binding" + unset FM_FAKE_CREW_STATE_FINGERPRINT_X1 + done pass "drain recomputes the receipt fingerprint from bound fields" } @@ -540,8 +596,8 @@ test_reused_task_id_gets_new_fingerprint() { write_meta "$state" reused-x1 incarnation-old export FM_FAKE_CREW_STATE_REUSED_X1='state: done · source: pane · first run quiet' scan "$root" "$home" "$fakebin" --startup >/dev/null - sed -i 's/^spawn_incarnation=.*/spawn_incarnation=incarnation-new/' "$state/reused-x1.meta" - touch -d '2 minutes ago' "$state/reused-x1.meta" + replace_field "$state/reused-x1.meta" spawn_incarnation incarnation-new + set_old_mtime "$state/reused-x1.meta" scan "$root" "$home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$state" pending)" = 2 ] || fail "reused task id did not create a new incarnation receipt" [ "$(queue_count "$state")" = 2 ] || fail "reused task id did not create a new fingerprinted wake" @@ -654,6 +710,25 @@ test_session_start_drains_before_inactive_scan() { || fail "session-start left the pre-existing wake queued" [ "$(queue_count "$state")" = 1 ] || fail "session-start did not queue exactly one inactive wake after draining" [ "$(receipt_count "$state" pending)" = 1 ] || fail "session-start did not run inactive reconciliation" + mv "$root/bin/fm-wake-drain.sh" "$root/bin/fm-wake-drain.real" + cat > "$root/bin/fm-wake-drain.sh" <<'SH' +#!/usr/bin/env bash +exit 23 +SH + chmod +x "$root/bin/fm-wake-drain.sh" + out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_INACTIVE_OUTCOME_SECS=60 \ + FM_INACTIVE_OUTCOME_BUDGET_SECS=10 FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux \ + "$root/bin/fm-session-start.sh" 2>&1) + status=$? + [ "$status" = 0 ] || fail "session-start changed its always-zero reporting contract after drain failure" + printf '%s\n' "$out" | grep -F 'inactive reconciliation skipped' >/dev/null \ + || fail "session-start did not report that inactive reconciliation was skipped after drain failure" + [ "$(receipt_count "$state" pending)" = 1 ] || fail "session-start scanned after a failed wake drain" + [ "$(queue_count "$state")" = 1 ] || fail "session-start changed the queue after a failed wake drain" unset FM_FAKE_CREW_STATE_SESSION_X1 pass "session-start drains existing wakes before inactive reconciliation" } @@ -713,7 +788,7 @@ test_legacy_metadata_uses_stable_fallback() { write_legacy_meta "$state" legacy-x1 export FM_FAKE_CREW_STATE_LEGACY_X1='state: done · source: pane · legacy quiet' scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "legacy metadata scan failed" - rec=$(find "$state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + rec=$(direct_first_file "$state/terminal-outcomes" '*.pending') [ -n "$rec" ] || fail "legacy metadata did not create a receipt" incarnation=$(receipt_value "$rec" incarnation) case "$incarnation" in legacy-*) ;; *) fail "legacy metadata lacked a documented fallback incarnation" ;; esac @@ -729,7 +804,7 @@ test_empty_spawn_incarnation_is_rejected() { dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" write_meta "$state" empty-inc-x1 empty-inc - sed -i 's/^spawn_incarnation=.*/spawn_incarnation=/' "$state/empty-inc-x1.meta" + replace_field "$state/empty-inc-x1.meta" spawn_incarnation '' export FM_FAKE_CREW_STATE_EMPTY_INC_X1='state: done · source: pane · malformed incarnation' scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "empty incarnation scan failed" [ "$(receipt_count "$state" pending)" = 0 ] || fail "explicit empty incarnation used the legacy fallback" @@ -759,13 +834,13 @@ test_relaunch_and_teardown_races_recheck_under_spawn_lock() { scan "$root" "$home" "$fakebin" --startup >"$dir/relaunch.scan.out" 2>&1 & scanner=$! sleep 1 - sed -i 's/^spawn_incarnation=.*/spawn_incarnation=new-inc/' "$state/relaunch-x1.meta" + replace_field "$state/relaunch-x1.meta" spawn_incarnation new-inc : > "$release" wait "$holder" || fail "spawn-lock relaunch fixture failed" wait "$scanner" || fail "relaunch reconciliation fixture failed" - [ "$(find "$state/terminal-outcomes" -type f -name '*.pending' 2>/dev/null | wc -l | tr -d ' ')" = 0 ] \ + [ "$(direct_file_count "$state/terminal-outcomes" '*.pending')" = 0 ] \ || fail "fresh relaunch was replayed before its quiet period" - touch -d '2 minutes ago' "$state/relaunch-x1.meta" + set_old_mtime "$state/relaunch-x1.meta" scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "aged relaunch reconciliation failed" grep -F 'incarnation=new-inc' "$state"/terminal-outcomes/*.pending >/dev/null || fail "relaunch race used stale incarnation" @@ -788,7 +863,7 @@ test_relaunch_and_teardown_races_recheck_under_spawn_lock() { : > "$release" wait "$holder" || fail "spawn-lock teardown fixture failed" wait "$scanner" || fail "teardown reconciliation fixture failed" - [ "$(find "$state/terminal-outcomes" -type f -name '*teardown-x1*.pending' 2>/dev/null | wc -l | tr -d ' ')" = 0 ] || fail "teardown race created a receipt after meta removal" + [ "$(direct_file_count "$state/terminal-outcomes" '*teardown-x1*.pending')" = 0 ] || fail "teardown race created a receipt after meta removal" unset FM_FAKE_CREW_STATE_RELAUNCH_X1 FM_FAKE_CREW_STATE_TEARDOWN_X1 pass "relaunch and teardown races recheck metadata under the spawn lock" } @@ -821,7 +896,7 @@ test_herdr_identity_and_default_captain_refusal() { printf 'herdr_session=default\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-default.meta" write_meta "$state" herdr-captain captain-inc ship herdr CAPTAIN:pane printf 'herdr_session=CAPTAIN\nherdr_workspace_id=ws\nherdr_tab_id=tab\nherdr_pane_id=pane\n' >> "$state/herdr-captain.meta" - touch -d '2 minutes ago' "$state/herdr-good.meta" "$state/herdr-unique.meta" \ + set_old_mtime "$state/herdr-good.meta" "$state/herdr-unique.meta" \ "$state/herdr-default.meta" "$state/herdr-captain.meta" export FM_FAKE_CREW_STATE_HERDR_GOOD='state: done · source: pane · dedicated session quiet' export FM_FAKE_CREW_STATE_HERDR_UNIQUE='state: done · source: pane · unique session quiet' @@ -872,7 +947,7 @@ test_status_log_terminal_is_not_replayed() { test_valid_secondmate_route_reports_parent_once() { local dir root home fakebin state child_home child_state parent_status corr rec outside send_out local outside_parent outside_parent_link - local history_corr history_record history_status + local history_corr history_record history_status active_record active_backup new_case secondmate-route-valid dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -905,27 +980,39 @@ SH CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ fm-sm-valid "parent request" 2>&1) || fail "public fm-send route setup failed: $send_out" - corr=$(basename "$(find "$state/pending-replies" -maxdepth 1 -type f | head -1)") + corr=$(basename "$(direct_first_file "$state/pending-replies" '*')") [ -n "$corr" ] || fail "public fm-send did not create a correlation record" export FM_FAKE_CREW_STATE_CHILD_X1='state: failed · source: pane · child quiet' scan "$root" "$child_home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "valid secondmate route did not create a pending receipt" - rec=$(find "$child_state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + rec=$(direct_first_file "$child_state/terminal-outcomes" '*.pending') [ "$(receipt_value "$rec" parent_task_id)" = sm-valid ] || fail "secondmate receipt did not persist its parent task identity" [ "$(receipt_value "$rec" parent_corr)" = "$corr" ] || fail "secondmate receipt did not persist its parent correlation" [ "$(receipt_value "$rec" parent_home)" = "$home" ] || fail "secondmate receipt did not persist its parent home" [ "$(receipt_value "$rec" parent_status)" = "$parent_status" ] || fail "secondmate receipt did not persist its parent status path" + active_record="$state/pending-replies/$corr" + active_backup="$dir/active-record-backup" + mkdir -p "$state/pending-reply-history" + cp "$active_record" "$state/pending-reply-history/$corr" + mv "$active_record" "$active_backup" + ln -s "$active_backup" "$active_record" + if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then + fail "secondmate receipt validation fell through a symlinked active record" + fi + [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "symlinked active record lost the pending receipt" + rm -f "$active_record" "$state/pending-reply-history/$corr" + mv "$active_backup" "$active_record" outside_parent="$dir/outside-parent" outside_parent_link="$dir/outside-parent-link" mkdir -p "$outside_parent/state" ln -s "$outside_parent" "$outside_parent_link" - sed -i "s|^parent_home=.*|parent_home=$outside_parent_link|" "$rec" + replace_field "$rec" parent_home "$outside_parent_link" if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then fail "secondmate acknowledgement accepted a symlinked parent home" fi [ ! -e "$outside_parent/state/pending-replies/.txn-$corr.lock" ] \ || fail "secondmate acknowledgement acquired a transaction lock before path validation" - sed -i "s|^parent_home=.*|parent_home=$home|" "$rec" + replace_field "$rec" parent_home "$home" outside="$dir/outside-status" printf 'outside\n' > "$outside" rm -f "$parent_status" @@ -965,7 +1052,7 @@ SH done [ -n "$history_corr" ] || fail "public fm-send did not create a distinct history correlation" history_record="$state/pending-replies/$history_corr" - sed -i 's/^phase=.*/phase=resolved/' "$history_record" + replace_field "$history_record" phase resolved (cd "$root" && env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" FM_STATE_OVERRIDE="$state" \ bash -c '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_archive_terminal "$2" "$3"' \ _ "$ROOT" "$state" "$history_corr") \ @@ -976,7 +1063,7 @@ SH scan "$root" "$child_home" "$fakebin" --startup \ || fail "pending-reply-history route scan failed" [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "pending-reply-history route did not create a pending receipt" - rec=$(find "$child_state/terminal-outcomes" -maxdepth 1 -type f -name '*.pending' | head -1) + rec=$(direct_first_file "$child_state/terminal-outcomes" '*.pending') [ "$(receipt_value "$rec" parent_corr)" = "$history_corr" ] \ || fail "history route receipt used the wrong parent correlation" drain "$root" "$child_home" "$fakebin" >/dev/null \ @@ -1009,7 +1096,7 @@ test_concurrent_secondmate_routes_are_rejected() { schema=fm-pending-reply.v1 corr_id="$corr_two" task_id=sm-concurrent \ parent_home="$home" parent_status="$state/sm-concurrent.status" delivered_epoch=1 phase=awaiting_report route_write() { - env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + env PATH="$fakebin:$PATH" FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ FM_STATE_OVERRIDE="$state" bash -c \ '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ _ "$ROOT" "$child_home" "$home" "$state" sm-concurrent "$1" @@ -1062,10 +1149,25 @@ test_concurrent_secondmate_routes_are_rejected() { marker_before=$(cat "$marker") outside="$dir/temp-target" printf 'protected\n' > "$outside" + cat > "$fakebin/mktemp" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-}" in + */.fm-jt-parent-route.XXXXXX) + target="${1%.XXXXXX}blocked" + rm -f "$target" + ln -s "$FM_ROUTE_TEMP_TARGET" "$target" + printf '%s\n' "$target" + exit 0 + ;; +esac +exec /usr/bin/mktemp "$@" +SH + chmod +x "$fakebin/mktemp" if env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" bash -c \ - '. "$1/bin/fm-pending-reply-lib.sh"; ln -s "$7" "$2/state/.fm-jt-parent-route.tmp.${BASHPID}"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ - _ "$ROOT" "$child_home" "$home" "$state" sm-concurrent "$corr_two" "$outside"; then + FM_STATE_OVERRIDE="$state" PATH="$fakebin:$PATH" FM_ROUTE_TEMP_TARGET="$outside" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-concurrent "$corr_two"; then fail "second route replaced an active route unexpectedly" fi [ "$(cat "$outside")" = protected ] || fail "route publication followed a pre-created temporary symlink" From 9cf6b9bd798077fa9fd2fd1ff27f03379e38ebb7 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 04:34:56 +0000 Subject: [PATCH 010/163] no-mistakes(review): Hardened replay recovery, scan retry, and route correlation --- bin/fm-inactive-reconcile.sh | 25 +++++--- bin/fm-pending-reply-lib.sh | 70 +++++++++++++++++--- bin/fm-wake-drain.sh | 36 +++++++++-- tests/fm-inactive-outcome.test.sh | 103 ++++++++++++++++++++++++++++-- 4 files changed, 206 insertions(+), 28 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 524d3cc2cab..0cfc1314245 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -113,7 +113,7 @@ hash_text() { elif command -v sha256sum >/dev/null 2>&1; then printf '%s' "$1" | sha256sum | awk '{print $1}' else - printf '%s' "$1" | cksum | awk '{print $1}' + return 1 fi } @@ -195,7 +195,11 @@ claim_field() { # drain_claim_owner() { local row=$1 owner parent_pid drain_file drain_dir state_dir owner=$(fm_lock_link_owner "$FM_WAKE_QUEUE_LOCK" 2>/dev/null || true) - parent_pid=${PPID:-} + if [ "${FM_WAKE_DRAIN_DELEGATED:-0}" = 1 ]; then + parent_pid=${FM_WAKE_DRAIN_PARENT_PID:-} + else + parent_pid=${PPID:-} + fi drain_file=${FM_WAKE_DRAIN_FILE:-} [ -n "$owner" ] && [ -n "$parent_pid" ] && [ -n "$row" ] && [ -n "$drain_file" ] || return 1 [ "$(cat "$owner/pid" 2>/dev/null || true)" = "$parent_pid" ] || return 1 @@ -436,7 +440,7 @@ publish_receipt_and_wake() { receipt_existing_core() { local suffix existing expected_fp existing_kind parent_id parent_home parent_status parent_corr RECEIPT_EXISTING_SUFFIX= - expected_fp=$(hash_text "$ID|$INC|$OUTCOME|$SNAPSHOT") + expected_fp=$(hash_text "$ID|$INC|$OUTCOME|$SNAPSHOT|$KIND") || return 1 [ "$expected_fp" = "$FP" ] || return 1 for suffix in pending presented reported; do existing=$(receipt_path "$FP" "$suffix") @@ -549,7 +553,7 @@ publish_secondmate_receipt_and_wake() { } read_incarnation() { # - local meta=$1 id=$2 token tasktmp window worktree seed rc token_present=0 + local meta=$1 id=$2 token tasktmp window worktree seed digest rc token_present=0 if token=$(meta_value_unique "$meta" spawn_incarnation); then token_present=1 else @@ -575,7 +579,8 @@ read_incarnation() { # else seed="legacy|window=$window|worktree=$worktree" fi - printf 'legacy-%s' "$(hash_text "$seed" | cut -c1-32)" + digest=$(hash_text "$seed") || return 1 + printf 'legacy-%s' "${digest:0:32}" } child_cleanup() { @@ -597,7 +602,7 @@ reconcile_child() { herdr_identity_allowed "$meta" || return 0 CHILD_LOCK="$STATE/.spawn-$id.lock" FM_LOCK_WAIT_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS:-30}" 30 0 300) - fm_lock_acquire_wait "$CHILD_LOCK" || return 0 + fm_lock_acquire_wait "$CHILD_LOCK" || return 75 CHILD_LOCK_HELD=1 trap child_cleanup EXIT INT TERM # Teardown/relaunch can replace or remove metadata only after the same lock is @@ -638,7 +643,6 @@ reconcile_child() { [ -n "$source" ] && [ "$source" != none ] || return 0 snapshot=$(single_line "$line") INC=$(read_incarnation "$meta" "$id") || return 0 - FP=$(hash_text "$id|$INC|$outcome|$snapshot") ID=$id OUTCOME=$outcome SNAPSHOT=$snapshot @@ -650,6 +654,8 @@ reconcile_child() { 0|1) ;; *) return 0 ;; esac + [ "$route_rc" = 0 ] && KIND=secondmate + FP=$(hash_text "$id|$INC|$outcome|$snapshot|$KIND") || return 1 if [ "$route_rc" = 0 ]; then publish_secondmate_receipt_and_wake || return 1 if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then @@ -708,7 +714,7 @@ ack_receipt() { # parent_task_id=$(receipt_field "$rec" parent_task_id) outcome=$(receipt_field "$rec" outcome) snapshot=$(receipt_field "$rec" terminal_snapshot) - expected_fp=$(hash_text "$id|$incarnation|$outcome|$snapshot") + expected_fp=$(hash_text "$id|$incarnation|$outcome|$snapshot|$kind") || return 2 [ "$expected_fp" = "$fp" ] || return 2 case "$kind" in ship|scout|secondmate) ;; *) return 2 ;; esac if [ "$kind" = secondmate ]; then @@ -833,7 +839,8 @@ scan_locked() { done < "$find_tmp" rm -f "$find_tmp" || return 1 [ "$scan_failed" = 0 ] || return "$rc" - if [ "$complete" = 1 ] && [ "$cursor_seen" = 0 ]; then + [ "$complete" = 1 ] || return 1 + if [ "$cursor_seen" = 0 ]; then return 1 fi if [ "$complete" = 1 ]; then diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 108fa88726d..c3228aad899 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -490,15 +490,36 @@ fm_pending_reply_secondmate_route_path() { # printf '%s/state/.fm-jt-parent-route' "$1" } +fm_pending_reply_secondmate_route_history_path() { # + printf '%s/state/.fm-jt-parent-route-history.%s' "$1" "$2" +} + fm_pending_reply_secondmate_route_lock_path() { # printf '%s/state/.fm-jt-parent-route.lock' "$1" } +fm_pending_reply_secondmate_route_shape() { # + awk -F= ' + BEGIN { schema=secondmate_id=parent_home=parent_status=corr_id=0; valid=1 } + NF < 2 { valid=0; next } + $1 == "schema" { schema++; next } + $1 == "secondmate_id" { secondmate_id++; next } + $1 == "parent_home" { parent_home++; next } + $1 == "parent_status" { parent_status++; next } + $1 == "corr_id" { corr_id++; next } + { valid=0 } + END { + if (schema != 1 || secondmate_id != 1 || parent_home != 1 || parent_status != 1 || corr_id != 1) valid=0 + exit !valid + } + ' "$1" 2>/dev/null +} + fm_pending_reply_secondmate_route_write() { # local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 local marker home_marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 local existing_schema existing_id existing_home existing_status existing_corr - local existing_state existing_record existing_phase existing_active_dir existing_history_dir + local existing_state existing_record existing_phase existing_active_dir existing_history_dir history_marker local existing_active_record existing_history_record marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 @@ -608,6 +629,16 @@ fm_pending_reply_secondmate_route_write() { # < *) route_status=1 ;; esac fi + if [ "$route_status" = 0 ] && [ "$existing_corr" != "$corr" ]; then + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$existing_corr") + [ ! -L "$history_marker" ] || route_status=1 + if [ "$route_status" = 0 ] && [ -e "$history_marker" ]; then + [ -f "$history_marker" ] || route_status=1 + [ "$route_status" = 0 ] && cmp -s "$marker" "$history_marker" || route_status=1 + elif [ "$route_status" = 0 ]; then + ln "$marker" "$history_marker" || route_status=1 + fi + fi else route_status=1 fi @@ -624,15 +655,16 @@ fm_pending_reply_secondmate_route_clear() { # local secondmate_home=$1 corr=$2 marker route_lock status=0 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 - marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") - [ -e "$marker" ] || [ -L "$marker" ] || return 0 - [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then + marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + fi route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") fm_lock_acquire_wait "$route_lock" || return 1 - if fm_pending_reply_secondmate_route_validate "$secondmate_home" \ - && [ "$FM_PENDING_ROUTE_CORR" = "$corr" ]; then - rm -f "$marker" || status=1 + if fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr"; then + rm -f "$FM_PENDING_ROUTE_MARKER" || status=1 else status=1 fi @@ -640,8 +672,8 @@ fm_pending_reply_secondmate_route_clear() { # return "$status" } -fm_pending_reply_secondmate_route_validate() { # - local secondmate_home=$1 marker line key value schema marker_id secondmate_id +fm_pending_reply_secondmate_route_validate() { # [] + local secondmate_home=$1 wanted_corr=${2:-} marker line key value schema marker_id secondmate_id current_corr history_marker local parent_home parent_status corr parent_abs state_abs expected_status rec active_rec history_rec history_dir local phase delivered record_home record_status record_task record_corr home_marker local seen_schema=0 seen_secondmate_id=0 seen_parent_home=0 seen_parent_status=0 seen_corr=0 @@ -650,7 +682,26 @@ fm_pending_reply_secondmate_route_validate() { # FM_PENDING_ROUTE_CORR= FM_PENDING_ROUTE_SECOND_MATE_ID= FM_PENDING_ROUTE_PHASE= + FM_PENDING_ROUTE_MARKER= marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + if [ -n "$wanted_corr" ]; then + printf '%s' "$wanted_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + if [ -e "$marker" ] || [ -L "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ] || return 1 + fm_pending_reply_secondmate_route_shape "$marker" || return 1 + current_corr=$(fm_pending_reply_get "$marker" corr_id) + if [ "$current_corr" != "$wanted_corr" ]; then + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$wanted_corr") + [ -f "$history_marker" ] && [ ! -L "$history_marker" ] || return 1 + marker=$history_marker + fi + else + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$wanted_corr") + [ -f "$history_marker" ] && [ ! -L "$history_marker" ] || return 1 + marker=$history_marker + fi + fi [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 schema='' secondmate_id='' parent_home='' parent_status='' corr='' while IFS= read -r line || [ -n "$line" ]; do @@ -731,6 +782,7 @@ fm_pending_reply_secondmate_route_validate() { # FM_PENDING_ROUTE_CORR=$corr # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_SECOND_MATE_ID=$secondmate_id + FM_PENDING_ROUTE_MARKER=$marker return 0 } diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 77aef847d53..677829f2f98 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -16,13 +16,41 @@ DRAIN_PID=${BASHPID:-$$} DRAIN_LOCK_HELD=false present_inactive_row() { - local key=$1 row=$2 status=0 + local key=$1 row=$2 status=0 go worker worker_status=0 trap - INT TERM HUP - if ! printf '%s\n' "$row"; then + go=$(mktemp "$STATE/.wake-presentation.XXXXXX") || return 1 + [ -f "$go" ] && [ ! -L "$go" ] || { rm -f "$go"; return 1; } + rm -f "$go" + ( + while [ ! -e "$go" ]; do + if ! kill -0 "$DRAIN_PID" 2>/dev/null; then + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ + FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-started "$key" "$row" || exit 1 + break + fi + sleep 0.01 + done + printf '%s\n' "$row" + ) & + worker=$! + if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" output-started "$key" "$row"; then status=1 fi - if [ "$status" = 0 ] && ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" output-started "$key" "$row"; then - status=1 + if [ "$status" = 0 ]; then + : > "$go" || status=1 + fi + if [ "$status" = 0 ]; then + wait "$worker" || worker_status=$? + [ "$worker_status" = 0 ] || status=1 + else + kill "$worker" 2>/dev/null || true + wait "$worker" 2>/dev/null || true + fi + rm -f "$go" + if [ "$status" -ne 0 ]; then + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + presenting "$key" "$row" >/dev/null 2>&1 || true fi if [ "$status" = 0 ] && ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$key" "$row"; then status=1 diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 6b22ce09b76..b7b1d302bd9 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -229,13 +229,13 @@ receipt_value() { } receipt_fingerprint() { - local value=$1 + local value=$1 kind=${2:-ship} if command -v shasum >/dev/null 2>&1; then - printf '%s' "$value" | shasum -a 256 | awk '{print $1}' + printf '%s' "$value|$kind" | shasum -a 256 | awk '{print $1}' elif command -v sha256sum >/dev/null 2>&1; then - printf '%s' "$value" | sha256sum | awk '{print $1}' + printf '%s' "$value|$kind" | sha256sum | awk '{print $1}' else - printf '%s' "$value" | cksum | awk '{print $1}' + return 1 fi } @@ -361,6 +361,28 @@ test_portable_timeout_preserves_signal_failure() { pass "portable timeout preserves signal failures" } +test_portable_timeout_expires_child() { + local dir root home fakebin state + new_case portable-timeout-expiry + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" expiry-x1 expiry-inc + cat > "$fakebin/fm-crew-state.sh" <<'SH' +#!/usr/bin/env bash +sleep 2 +printf 'state: done · source: pane · should time out\n' +SH + chmod +x "$fakebin/fm-crew-state.sh" + export FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT=1 FM_INACTIVE_OUTCOME_BUDGET_SECS=1 + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "portable timeout fallback allowed an expired child" + fi + [ ! -e "$state/.inactive-outcome-reconcile" ] || fail "portable timeout expiry advanced the cadence marker" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "portable timeout expiry created a receipt" + unset FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT FM_INACTIVE_OUTCOME_BUDGET_SECS + pass "portable timeout fallback propagates child expiry" +} + test_leading_zero_cadence_is_normalized() { local dir root home fakebin state new_case leading-zero-cadence @@ -418,7 +440,7 @@ SH test_ack_recomputes_fingerprint_from_receipt_fields() { local dir root home fakebin state rec fingerprint field tampered - for field in task_id incarnation outcome terminal_snapshot; do + for field in task_id incarnation outcome terminal_snapshot kind; do new_case "fingerprint-binding-$field" dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -432,6 +454,7 @@ test_ack_recomputes_fingerprint_from_receipt_fields() { incarnation) tampered=tampered-inc ;; outcome) tampered=failed ;; terminal_snapshot) tampered='tampered snapshot' ;; + kind) tampered=secondmate ;; esac replace_field "$rec" "$field" "$tampered" if drain "$root" "$home" "$fakebin" >/dev/null 2>&1; then @@ -1076,6 +1099,72 @@ SH pass "valid secondmate outcomes use the parent status correlation exactly once" } +test_secondmate_route_replacement_preserves_old_receipt() { + local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker + new_case secondmate-route-replacement + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + cp -a "$ROOT/bin/." "$root/bin/" + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$state/pending-replies" + printf 'sm-replace\n' > "$child_home/.fm-secondmate-home" + write_meta "$state" sm-replace parent-replace-inc secondmate tmux firstmate:fm-sm-replace + printf 'home=%s\n' "$child_home" >> "$state/sm-replace.meta" + write_meta "$child_state" child-replace-x1 child-replace-inc + parent_status="$state/sm-replace.status" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u +case "$*" in + *"#{cursor_y}"*) printf '1\n' ;; + *capture-pane*) : ;; + *) : ;; +esac +exit 0 +SH + chmod +x "$fakebin/tmux" + prepare_primary_proof "$root" "$home" "$fakebin" + prepare_watcher_protocol "$root" "$home" "$state" + send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ + fm-sm-replace "first request" 2>&1) || fail "initial public route setup failed: $send_out" + corr_a=$(basename "$(direct_first_file "$state/pending-replies" '*')") + [ -n "$corr_a" ] || fail "initial route did not create a correlation" + export FM_FAKE_CREW_STATE_CHILD_REPLACE_X1='state: done · source: pane · old route receipt' + scan "$root" "$child_home" "$fakebin" --startup >/dev/null || fail "old route scan failed" + rec=$(direct_first_file "$child_state/terminal-outcomes" '*.pending') + [ -n "$rec" ] || fail "old route did not create a pending receipt" + replace_field "$state/pending-replies/$corr_a" phase resolved + prepare_primary_proof "$root" "$home" "$fakebin" + send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ + fm-sm-replace "replacement request" 2>&1) || fail "replacement public route setup failed: $send_out" + corr_b= + for marker in "$state"/pending-replies/*; do + [ -f "$marker" ] || continue + [ "$(basename "$marker")" = "$corr_a" ] || corr_b=$(basename "$marker") + done + [ -n "$corr_b" ] || fail "replacement route did not create a new correlation" + [ "$(receipt_value "$child_state/.fm-jt-parent-route" corr_id)" = "$corr_b" ] \ + || fail "replacement route did not publish the new correlation" + drain "$root" "$child_home" "$fakebin" >/dev/null \ + || fail "old receipt did not drain after route replacement" + [ "$(receipt_count "$child_state" reported)" = 1 ] || fail "old route receipt was not reported" + [ "$(receipt_value "$child_state/.fm-jt-parent-route" corr_id)" = "$corr_b" ] \ + || fail "old receipt cleanup removed the replacement route" + [ ! -e "$child_state/.fm-jt-parent-route-history.$corr_a" ] \ + || fail "old route history was not retired after acknowledgement" + unset FM_FAKE_CREW_STATE_CHILD_REPLACE_X1 + pass "secondmate route replacement preserves correlation-scoped old receipts" +} + test_concurrent_secondmate_routes_are_rejected() { local dir root home fakebin state child_home child_state marker corr_one corr_two outside existing_real existing_link local existing_record record_backup existing_history history_backup clear_real clear_link @@ -1196,7 +1285,7 @@ test_drain_restores_only_unprocessed_rows() { dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" first=$(receipt_fingerprint 'first-x1|first-inc|done|done') - second=$(receipt_fingerprint 'second-x1|second-inc|failed|failed') + second=$(receipt_fingerprint 'second-x1|second-inc|failed|failed' secondmate) mkdir -p "$state/terminal-outcomes" fm_write_meta "$state/terminal-outcomes/$first.pending" \ schema=fm-jt-terminal-outcome.v1 fingerprint="$first" task_id=first-x1 \ @@ -1268,6 +1357,7 @@ test_malformed_or_missing_secondmate_route_fails_closed() { test_done_and_failed_are_replayed_once test_portable_timeout_runner_is_used test_portable_timeout_preserves_signal_failure +test_portable_timeout_expires_child test_leading_zero_cadence_is_normalized test_find_failure_propagates_without_advancing_scan test_find_enumeration_respects_scan_budget @@ -1289,6 +1379,7 @@ test_herdr_identity_and_default_captain_refusal test_occupancy_unknown_is_not_terminal test_status_log_terminal_is_not_replayed test_valid_secondmate_route_reports_parent_once +test_secondmate_route_replacement_preserves_old_receipt test_concurrent_secondmate_routes_are_rejected test_drain_restores_only_unprocessed_rows test_malformed_or_missing_secondmate_route_fails_closed From 0ca206d73457add61e32fa4d585656adc60d3b9f Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 06:07:29 +0000 Subject: [PATCH 011/163] no-mistakes(review): Fixed undelivered cleanup and history-route replay --- bin/fm-inactive-reconcile.sh | 31 +++++++++++++++++++++--- bin/fm-pending-reply-lib.sh | 25 ++++++++++++++----- bin/fm-send.sh | 2 +- tests/fm-inactive-outcome.test.sh | 40 +++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 10 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 0cfc1314245..35c34f7b4c3 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -509,7 +509,7 @@ republish_existing_receipt_wake() { } publish_secondmate_receipt_and_wake() { - local route_lock status=0 existing_rc + local route_lock status=0 existing_rc pending pending_corr pending_parent_id pending_parent_home pending_parent_status FM_WAKE_APPEND_CREATED=0 fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 if [ -L "$FM_HOME" ] || [ ! -d "$FM_HOME" ] || [ -L "$FM_HOME/state" ] || [ ! -d "$FM_HOME/state" ]; then @@ -521,12 +521,37 @@ publish_secondmate_receipt_and_wake() { fm_lock_release "$FM_WAKE_QUEUE_LOCK" || true return 1 fi - if ! fm_pending_reply_secondmate_route_validate "$FM_HOME"; then + KIND=secondmate + pending=$(receipt_path "$FP" pending) + if [ -e "$pending" ] || [ -L "$pending" ]; then + [ -f "$pending" ] && [ ! -L "$pending" ] || status=1 + if [ "$status" = 0 ]; then + pending_corr=$(receipt_field "$pending" parent_corr) + printf '%s' "$pending_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || status=1 + fi + if [ "$status" = 0 ] && ! fm_pending_reply_secondmate_route_validate "$FM_HOME" "$pending_corr"; then + fm_lock_release "$route_lock" || true + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || true + return 0 + fi + if [ "$status" = 0 ]; then + pending_parent_id=$(receipt_field "$pending" parent_task_id) + pending_parent_home=$(receipt_field "$pending" parent_home) + pending_parent_status=$(receipt_field "$pending" parent_status) + [ "$pending_parent_id" = "$FM_PENDING_ROUTE_SECOND_MATE_ID" ] || status=1 + [ "$pending_parent_home" = "$FM_PENDING_ROUTE_PARENT_HOME" ] || status=1 + [ "$pending_parent_status" = "$FM_PENDING_ROUTE_PARENT_STATUS" ] || status=1 + fi + elif ! fm_pending_reply_secondmate_route_validate "$FM_HOME"; then fm_lock_release "$route_lock" || true fm_lock_release "$FM_WAKE_QUEUE_LOCK" || true return 0 fi - KIND=secondmate + [ "$status" = 0 ] || { + fm_lock_release "$route_lock" || true + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || true + return 1 + } if receipt_existing_core; then if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index c3228aad899..49eb76fff04 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -651,8 +651,8 @@ fm_pending_reply_secondmate_route_write() { # < return "$route_status" } -fm_pending_reply_secondmate_route_clear() { # - local secondmate_home=$1 corr=$2 marker route_lock status=0 +fm_pending_reply_secondmate_route_clear_with_mode() { # + local secondmate_home=$1 corr=$2 allow_undelivered=${3:-0} marker route_lock status=0 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 @@ -663,7 +663,7 @@ fm_pending_reply_secondmate_route_clear() { # fi route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") fm_lock_acquire_wait "$route_lock" || return 1 - if fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr"; then + if fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" "$allow_undelivered"; then rm -f "$FM_PENDING_ROUTE_MARKER" || status=1 else status=1 @@ -672,8 +672,16 @@ fm_pending_reply_secondmate_route_clear() { # return "$status" } -fm_pending_reply_secondmate_route_validate() { # [] - local secondmate_home=$1 wanted_corr=${2:-} marker line key value schema marker_id secondmate_id current_corr history_marker +fm_pending_reply_secondmate_route_clear() { # + fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 0 +} + +fm_pending_reply_secondmate_route_clear_undelivered() { # + fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 1 +} + +fm_pending_reply_secondmate_route_validate() { # [] [] + local secondmate_home=$1 wanted_corr=${2:-} allow_undelivered=${3:-0} marker line key value schema marker_id secondmate_id current_corr history_marker local parent_home parent_status corr parent_abs state_abs expected_status rec active_rec history_rec history_dir local phase delivered record_home record_status record_task record_corr home_marker local seen_schema=0 seen_secondmate_id=0 seen_parent_home=0 seen_parent_status=0 seen_corr=0 @@ -768,12 +776,17 @@ fm_pending_reply_secondmate_route_validate() { # [] [ "$record_status" = "$expected_status" ] || return 1 [ "$record_corr" = "$corr" ] || return 1 delivered=$(fm_pending_reply_get "$rec" delivered_epoch) - [ -n "$delivered" ] || return 1 phase=$(fm_pending_reply_get "$rec" phase) case "$phase" in awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; *) return 1 ;; esac + if [ "$allow_undelivered" = 1 ]; then + [ -z "$delivered" ] || return 1 + [ "$phase" = awaiting_report ] || return 1 + else + [ -n "$delivered" ] || return 1 + fi # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_PARENT_HOME=$parent_abs # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 6b552d012c6..658cea747d7 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -93,7 +93,7 @@ TARGET_HOME= clear_new_pending_route() { if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ && [ -n "$TARGET_HOME" ]; then - fm_pending_reply_secondmate_route_clear "$TARGET_HOME" "$PENDING_REPLY_CORR" + fm_pending_reply_secondmate_route_clear_undelivered "$TARGET_HOME" "$PENDING_REPLY_CORR" fi } diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index b7b1d302bd9..4c1c6dd12c2 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -1154,6 +1154,11 @@ SH [ -n "$corr_b" ] || fail "replacement route did not create a new correlation" [ "$(receipt_value "$child_state/.fm-jt-parent-route" corr_id)" = "$corr_b" ] \ || fail "replacement route did not publish the new correlation" + : > "$child_state/.wake-queue" + scan "$root" "$child_home" "$fakebin" --startup \ + || fail "pending old-route receipt was not reconciled after route replacement" + [ "$(queue_count "$child_state")" = 1 ] \ + || fail "pending old-route receipt did not get a matching history wake" drain "$root" "$child_home" "$fakebin" >/dev/null \ || fail "old receipt did not drain after route replacement" [ "$(receipt_count "$child_state" reported)" = 1 ] || fail "old route receipt was not reported" @@ -1165,6 +1170,40 @@ SH pass "secondmate route replacement preserves correlation-scoped old receipts" } +test_undelivered_secondmate_route_cleanup_is_idempotent() { + local dir root home fakebin state child_home child_state marker corr + new_case secondmate-undelivered-cleanup + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + marker="$child_state/.fm-jt-parent-route" + corr=0123456789abcdef + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$state/pending-replies" + printf 'sm-cleanup\n' > "$child_home/.fm-secondmate-home" + fm_write_meta "$state/pending-replies/$corr" \ + schema=fm-pending-reply.v1 corr_id="$corr" task_id=sm-cleanup \ + parent_home="$home" parent_status="$state/sm-cleanup.status" phase=awaiting_report delivered_epoch= + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-cleanup "$corr" \ + || fail "undelivered route fixture was not written" + [ -f "$marker" ] || fail "undelivered route fixture is missing" + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_clear_undelivered "$2" "$3"' \ + _ "$ROOT" "$child_home" "$corr" \ + || fail "undelivered route cleanup rejected an awaiting report" + [ ! -e "$marker" ] || fail "undelivered route cleanup left a stale marker" + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_clear_undelivered "$2" "$3"' \ + _ "$ROOT" "$child_home" "$corr" \ + || fail "undelivered route cleanup was not idempotent" + pass "undelivered secondmate route cleanup is safe and idempotent" +} + test_concurrent_secondmate_routes_are_rejected() { local dir root home fakebin state child_home child_state marker corr_one corr_two outside existing_real existing_link local existing_record record_backup existing_history history_backup clear_real clear_link @@ -1380,6 +1419,7 @@ test_occupancy_unknown_is_not_terminal test_status_log_terminal_is_not_replayed test_valid_secondmate_route_reports_parent_once test_secondmate_route_replacement_preserves_old_receipt +test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected test_drain_restores_only_unprocessed_rows test_malformed_or_missing_secondmate_route_fails_closed From 6d62061b2c2fbed117aeedc95bb4814f8523999a Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 06:33:47 +0000 Subject: [PATCH 012/163] no-mistakes(review): Fix secondmate acknowledgement route binding --- bin/fm-inactive-reconcile.sh | 53 +++++++++++++++++++++++++------ tests/fm-inactive-outcome.test.sh | 19 ++++++++++- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 35c34f7b4c3..cb302261166 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -768,7 +768,7 @@ ack_receipt() { # secondmate_ack_report() { # local secondmate_home=$1 parent_task_id=$2 parent_home=$3 parent_status=$4 corr=$5 outcome=$6 task_id=$7 fp=$8 - local parent_state token rc=0 line + local parent_state token route_lock route_marker route_history line phase rc=0 route_lock_held=0 marker_present=0 report_recorded=0 fm_pending_reply_secondmate_receipt_validate \ "$secondmate_home" "$parent_task_id" "$parent_home" "$parent_status" "$corr" || return 2 parent_state="$parent_home/state" @@ -776,23 +776,58 @@ secondmate_ack_report() { # "$parent_status" || rc=2 fi - if [ "$rc" = 0 ] && ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then - printf '%s\n' "$line" >> "$parent_status" || rc=2 + if [ "$rc" = 0 ] && [ -f "$parent_status" ] \ + && grep -Fqx "$line" "$parent_status" 2>/dev/null; then + report_recorded=1 + fi + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + if [ "$rc" = 0 ] && fm_lock_acquire_wait "$route_lock"; then + route_lock_held=1 + elif [ "$rc" = 0 ]; then + rc=2 fi - if [ "$rc" = 0 ]; then - fm_pending_reply_secondmate_route_clear "$secondmate_home" "$corr" || rc=2 + if [ "$route_lock_held" = 1 ]; then + route_marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + route_history=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ -e "$route_marker" ] || [ -L "$route_marker" ] \ + || [ -e "$route_history" ] || [ -L "$route_history" ]; then + marker_present=1 + fi + if [ "$marker_present" = 1 ]; then + fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" || rc=2 + elif [ "$phase" != resolved ] \ + && [ "$phase" != retired ] \ + && [ "$report_recorded" != 1 ]; then + rc=2 + fi + if [ "$rc" = 0 ] \ + && [ "$phase" != resolved ] \ + && [ "$phase" != retired ] \ + && [ "$report_recorded" != 1 ]; then + if [ ! -e "$parent_status" ]; then + : > "$parent_status" || rc=2 + fi + if [ "$rc" = 0 ] && ! grep -Fqx "$line" "$parent_status" 2>/dev/null; then + printf '%s\n' "$line" >> "$parent_status" || rc=2 + fi + fi + fm_lock_release "$route_lock" || rc=2 + route_lock_held=0 fi fi + if [ "$route_lock_held" = 1 ]; then + fm_lock_release "$route_lock" || rc=2 + fi + if [ "$rc" = 0 ]; then + fm_pending_reply_secondmate_route_clear "$secondmate_home" "$corr" || rc=2 + fi fm_pending_reply_txn_lock_release "$parent_state" "$corr" "$token" || rc=2 return "$rc" } diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 4c1c6dd12c2..1c1a1e4ec4d 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -968,7 +968,7 @@ test_status_log_terminal_is_not_replayed() { } test_valid_secondmate_route_reports_parent_once() { - local dir root home fakebin state child_home child_state parent_status corr rec outside send_out + local dir root home fakebin state child_home child_state parent_status corr rec outside send_out route_backup local outside_parent outside_parent_link local history_corr history_record history_status active_record active_backup new_case secondmate-route-valid @@ -1013,6 +1013,23 @@ SH [ "$(receipt_value "$rec" parent_corr)" = "$corr" ] || fail "secondmate receipt did not persist its parent correlation" [ "$(receipt_value "$rec" parent_home)" = "$home" ] || fail "secondmate receipt did not persist its parent home" [ "$(receipt_value "$rec" parent_status)" = "$parent_status" ] || fail "secondmate receipt did not persist its parent status path" + route_backup="$dir/route-backup" + mv "$child_state/.fm-jt-parent-route" "$route_backup" + if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then + fail "secondmate acknowledgement accepted a missing route marker" + fi + [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "missing route marker consumed the pending receipt" + ! grep -Fq "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status" 2>/dev/null \ + || fail "missing route marker wrote parent status" + mv "$route_backup" "$child_state/.fm-jt-parent-route" + replace_field "$child_state/.fm-jt-parent-route" corr_id 1123456789abcdef + if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then + fail "secondmate acknowledgement accepted a mismatched route marker" + fi + [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "mismatched route marker consumed the pending receipt" + ! grep -Fq "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status" 2>/dev/null \ + || fail "mismatched route marker wrote parent status" + replace_field "$child_state/.fm-jt-parent-route" corr_id "$corr" active_record="$state/pending-replies/$corr" active_backup="$dir/active-record-backup" mkdir -p "$state/pending-reply-history" From 1df3edfde1d950472dda255ed7b4b7808da855a1 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 06:56:51 +0000 Subject: [PATCH 013/163] no-mistakes(review): Fix wake drain, receipt retry, history binding --- bin/fm-inactive-reconcile.sh | 13 ++++-- bin/fm-pending-reply-lib.sh | 1 + bin/fm-watch.sh | 2 +- tests/fm-inactive-outcome.test.sh | 69 ++++++++++++++++++++++++++----- 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index cb302261166..8c15ac38b4c 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -709,7 +709,7 @@ reconcile_child() { } ack_receipt() { # - local key=$1 row=${2:-} fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing claim_state + local key=$1 row=${2:-} fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing existing_kind existing_corr claim_state [ -n "$row" ] || return 2 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 0 ;; esac @@ -724,6 +724,11 @@ ack_receipt() { # if [ -e "$existing" ]; then [ -f "$existing" ] || return 2 [ "$(receipt_field "$existing" fingerprint)" = "$fp" ] || return 2 + existing_kind=$(receipt_field "$existing" kind) + if [ "$existing_kind" = secondmate ]; then + existing_corr=$(receipt_field "$existing" parent_corr) + fm_pending_reply_secondmate_route_clear "$FM_HOME" "$existing_corr" || return 2 + fi claim_remove "$key" "$row" || return 2 return 1 fi @@ -762,6 +767,9 @@ ack_receipt() { # return 1 fi mv "$rec" "$target" || return 2 + if [ "$kind" = secondmate ]; then + fm_pending_reply_secondmate_route_clear "$FM_HOME" "$corr" || return 2 + fi claim_remove "$key" "$row" || return 2 return 0 } @@ -825,9 +833,6 @@ secondmate_ack_report() { # [] && [ "$seen_parent_home" = 1 ] && [ "$seen_parent_status" = 1 ] \ && [ "$seen_corr" = 1 ] || return 1 [ "$schema" = fm-jt-parent-route.v1 ] || return 1 + [ -z "$wanted_corr" ] || [ "$corr" = "$wanted_corr" ] || return 1 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 home_marker="$secondmate_home/.fm-secondmate-home" diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 7c26baae88c..e4ed70cb7d9 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -588,7 +588,7 @@ while :; do exit 1 fi if [ -n "$wake_drain_out" ]; then - printf '%s\n' "$wake_drain_out" + wake "$wake_drain_out" fi # Parent-owned secondmate pending-reply reconciliation: resolve correlated diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 1c1a1e4ec4d..ff85784f241 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -757,7 +757,7 @@ SH } test_watcher_runs_inactive_cadence() { - local dir root home fakebin state out status wake_line inactive_line + local dir root home fakebin state out second_out status new_case watcher-wiring dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -788,15 +788,26 @@ SH FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ "$root/bin/fm-watch.sh" 2>&1) status=$? - [ "$status" = 0 ] || fail "watcher cadence failed while surfacing the inactive outcome wake" - printf '%s\n' "$out" | grep -F 'check: inactive terminal outcome replay queued' >/dev/null \ - || fail "watcher did not surface the inactive reconciliation result" - wake_line=$(printf '%s\n' "$out" | grep -n '^1[[:space:]]\+1[[:space:]]\+signal[[:space:]]\+task-before' | head -1 | cut -d: -f1) - inactive_line=$(printf '%s\n' "$out" | grep -n 'check: inactive terminal outcome replay queued' | head -1 | cut -d: -f1) - [ -n "$wake_line" ] && [ -n "$inactive_line" ] && [ "$wake_line" -lt "$inactive_line" ] \ - || fail "watcher did not drain the existing wake before inactive reconciliation" + [ "$status" = 0 ] || fail "watcher cadence failed while surfacing the queued wake" + printf '%s\n' "$out" | grep -F $'1\t1\tsignal\ttask-before\tqueued before watcher scan' >/dev/null \ + || fail "watcher did not surface the existing wake first" [ "$(awk -F '\t' '$4 == "task-before" { n++ } END { print n + 0 }' "$state/.wake-queue")" = 0 ] \ || fail "watcher left the existing wake queued" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "watcher scanned inactive outcomes before draining the queued wake" + + second_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ + FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 FM_FAKE_PANE_PATH="$home" \ + FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ + "$root/bin/fm-watch.sh" 2>&1) + status=$? + [ "$status" = 0 ] || fail "watcher cadence failed while surfacing the inactive outcome wake" + printf '%s\n' "$second_out" | grep -F 'check: inactive terminal outcome replay queued' >/dev/null \ + || fail "watcher did not surface the inactive reconciliation result" [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher cadence did not create the inactive receipt" [ "$(queue_count "$state")" = 1 ] || fail "watcher cadence did not retain exactly one inactive outcome wake" unset FM_FAKE_CREW_STATE_WATCHER_X1 @@ -969,7 +980,7 @@ test_status_log_terminal_is_not_replayed() { test_valid_secondmate_route_reports_parent_once() { local dir root home fakebin state child_home child_state parent_status corr rec outside send_out route_backup - local outside_parent outside_parent_link + local outside_parent outside_parent_link fail_move_once local history_corr history_record history_status active_record active_backup new_case secondmate-route-valid dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN @@ -1062,6 +1073,31 @@ SH fi [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "symlinked parent status lost the pending receipt" rm -f "$parent_status" + fail_move_once="$dir/fail-mv-once" + : > "$fail_move_once" + cat > "$fakebin/mv" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + case "$arg" in + *.pending) + if [ -e "${FM_TEST_FAIL_MOVE_ONCE:?}" ]; then + rm -f "$FM_TEST_FAIL_MOVE_ONCE" + exit 42 + fi + ;; + esac +done +exec /bin/mv "$@" +SH + chmod +x "$fakebin/mv" + export FM_TEST_FAIL_MOVE_ONCE="$fail_move_once" + if drain "$root" "$child_home" "$fakebin" >/dev/null 2>&1; then + fail "secondmate acknowledgement hid a receipt move failure" + fi + [ "$(receipt_count "$child_state" pending)" = 1 ] || fail "receipt move failure consumed the pending receipt" + [ -f "$child_state/.fm-jt-parent-route" ] || fail "receipt move failure cleared the route before transition" + rm -f "$fakebin/mv" if ! drain "$root" "$child_home" "$fakebin" >"$dir/second-drain.out" 2>&1; then cat "$dir/second-drain.out" >&2 fail "valid secondmate route drain failed after symlink removal" @@ -1073,6 +1109,7 @@ SH drain "$root" "$child_home" "$fakebin" >/dev/null [ "$(grep -Fc "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status")" = 1 ] \ || fail "secondmate parent report was duplicated" + unset FM_TEST_FAIL_MOVE_ONCE printf 'sm-history\n' > "$child_home/.fm-secondmate-home" write_meta "$state" sm-history history-parent-inc secondmate tmux firstmate:fm-sm-history @@ -1117,7 +1154,7 @@ SH } test_secondmate_route_replacement_preserves_old_receipt() { - local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker + local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker history_backup new_case secondmate-route-replacement dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -1171,7 +1208,19 @@ SH [ -n "$corr_b" ] || fail "replacement route did not create a new correlation" [ "$(receipt_value "$child_state/.fm-jt-parent-route" corr_id)" = "$corr_b" ] \ || fail "replacement route did not publish the new correlation" + history_backup="$dir/history-route-backup" + [ -f "$child_state/.fm-jt-parent-route-history.$corr_a" ] \ + || fail "replacement route did not retain the old correlation history marker" + cp "$child_state/.fm-jt-parent-route-history.$corr_a" "$history_backup" + replace_field "$child_state/.fm-jt-parent-route-history.$corr_a" corr_id "$corr_b" : > "$child_state/.wake-queue" + scan "$root" "$child_home" "$fakebin" --startup \ + || fail "mismatched history marker scan failed" + [ "$(queue_count "$child_state")" = 0 ] \ + || fail "mismatched history marker created a wake for the wrong route" + [ "$(receipt_count "$child_state" pending)" = 1 ] \ + || fail "mismatched history marker consumed the old receipt" + cp "$history_backup" "$child_state/.fm-jt-parent-route-history.$corr_a" scan "$root" "$child_home" "$fakebin" --startup \ || fail "pending old-route receipt was not reconciled after route replacement" [ "$(queue_count "$child_state")" = 1 ] \ From d27be5aa9525ff2cf5c7d5e545cb600c2478174b Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 07:24:21 +0000 Subject: [PATCH 014/163] no-mistakes(review): Validate active routes and clamp oversized cadence --- bin/fm-inactive-reconcile.sh | 14 ++++++++++--- bin/fm-pending-reply-lib.sh | 3 ++- tests/fm-inactive-outcome.test.sh | 34 ++++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 8c15ac38b4c..b9559f56315 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -60,12 +60,20 @@ CHILD_LOCK_HELD=0 CHILD_LOCK= bounded_secs() { - local value=$1 fallback=$2 minimum=$3 maximum=$4 + local value=$1 fallback=$2 minimum=$3 maximum=$4 value_len minimum_len maximum_len LC_ALL=C case "$value" in ''|*[!0-9]*) value=$fallback ;; esac while [ "${value#0}" != "$value" ]; do value=${value#0}; done [ -n "$value" ] || value=0 - [ "$value" -lt "$minimum" ] && value=$minimum - [ "$value" -gt "$maximum" ] && value=$maximum + value_len=${#value} + minimum_len=${#minimum} + maximum_len=${#maximum} + if [ "$value_len" -lt "$minimum_len" ] \ + || { [ "$value_len" -eq "$minimum_len" ] && [[ "$value" < "$minimum" ]]; }; then + value=$minimum + elif [ "$value_len" -gt "$maximum_len" ] \ + || { [ "$value_len" -eq "$maximum_len" ] && [[ "$value" > "$maximum" ]]; }; then + value=$maximum + fi printf '%s' "$value" } diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 89f7541534a..adecddc2df5 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -698,7 +698,8 @@ fm_pending_reply_secondmate_route_validate() { # [] [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ] || return 1 fm_pending_reply_secondmate_route_shape "$marker" || return 1 - current_corr=$(fm_pending_reply_get "$marker" corr_id) + fm_pending_reply_secondmate_route_validate "$secondmate_home" "" "$allow_undelivered" || return 1 + current_corr=$FM_PENDING_ROUTE_CORR if [ "$current_corr" != "$wanted_corr" ]; then history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$wanted_corr") [ -f "$history_marker" ] && [ ! -L "$history_marker" ] || return 1 diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index ff85784f241..4b1d753b336 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -397,6 +397,26 @@ test_leading_zero_cadence_is_normalized() { pass "leading-zero cadence values are normalized before arithmetic" } +test_oversized_cadence_is_clamped() { + local dir root home fakebin state + new_case oversized-cadence + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" oversized-x1 oversized-inc + export FM_FAKE_CREW_STATE_OVERSIZED_X1='state: done · source: pane · oversized cadence' + export FM_INACTIVE_OUTCOME_SECS=999999999999999999999999999999999999 + touch -t 200001010000 "$state/oversized-x1.meta" "$state/oversized-x1.status" \ + "$state/oversized-x1.turn-ended" || fail "could not age the oversized cadence fixture" + touch -t 200001010000 "$state/.inactive-outcome-reconcile" \ + || fail "could not age the cadence marker" + scan "$root" "$home" "$fakebin" >/dev/null \ + || fail "oversized cadence value prevented reconciliation" + [ "$(receipt_count "$state" pending)" = 1 ] \ + || fail "oversized cadence value was not clamped before arithmetic" + unset FM_FAKE_CREW_STATE_OVERSIZED_X1 FM_INACTIVE_OUTCOME_SECS + pass "oversized decimal cadence values clamp before arithmetic" +} + test_find_failure_propagates_without_advancing_scan() { local dir root home fakebin state new_case find-failure @@ -1154,7 +1174,7 @@ SH } test_secondmate_route_replacement_preserves_old_receipt() { - local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker history_backup + local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker history_backup active_route_backup new_case secondmate-route-replacement dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -1208,6 +1228,17 @@ SH [ -n "$corr_b" ] || fail "replacement route did not create a new correlation" [ "$(receipt_value "$child_state/.fm-jt-parent-route" corr_id)" = "$corr_b" ] \ || fail "replacement route did not publish the new correlation" + active_route_backup="$dir/active-route-backup" + cp "$child_state/.fm-jt-parent-route" "$active_route_backup" + replace_field "$child_state/.fm-jt-parent-route" parent_home "$dir/not-a-parent-home" + : > "$child_state/.wake-queue" + scan "$root" "$child_home" "$fakebin" --startup \ + || fail "malformed active route scan failed" + [ "$(queue_count "$child_state")" = 0 ] \ + || fail "malformed active route fell through to history" + [ "$(receipt_count "$child_state" pending)" = 1 ] \ + || fail "malformed active route consumed the old receipt" + cp "$active_route_backup" "$child_state/.fm-jt-parent-route" history_backup="$dir/history-route-backup" [ -f "$child_state/.fm-jt-parent-route-history.$corr_a" ] \ || fail "replacement route did not retain the old correlation history marker" @@ -1464,6 +1495,7 @@ test_portable_timeout_runner_is_used test_portable_timeout_preserves_signal_failure test_portable_timeout_expires_child test_leading_zero_cadence_is_normalized +test_oversized_cadence_is_clamped test_find_failure_propagates_without_advancing_scan test_find_enumeration_respects_scan_budget test_ack_recomputes_fingerprint_from_receipt_fields From afef7581122e8a023cf242ee3d0df62792608998 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 07:57:30 +0000 Subject: [PATCH 015/163] no-mistakes(review): Make cleanup correlation-aware and history replay resilient --- bin/fm-pending-reply-lib.sh | 60 ++++++++++++++++++++++++++----- bin/fm-send.sh | 15 ++++---- tests/fm-inactive-outcome.test.sh | 49 +++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 14 deletions(-) diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index adecddc2df5..0ba19b63c89 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -677,7 +677,43 @@ fm_pending_reply_secondmate_route_clear() { # } fm_pending_reply_secondmate_route_clear_undelivered() { # - fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 1 + local secondmate_home=$1 corr=$2 marker route_lock current_corr status=0 + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + fm_lock_acquire_wait "$route_lock" || return 1 + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + if [ -e "$marker" ] || [ -L "$marker" ]; then + if [ ! -f "$marker" ] || [ -L "$marker" ] \ + || [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" != 5 ] \ + || ! fm_pending_reply_secondmate_route_shape "$marker"; then + fm_lock_release "$route_lock" || true + return 0 + fi + current_corr=$(fm_pending_reply_get "$marker" corr_id) + if [ "$current_corr" != "$corr" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + else + marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + if [ ! -f "$marker" ] || [ -L "$marker" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + fi + if fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" 1; then + rm -f "$FM_PENDING_ROUTE_MARKER" || status=1 + else + status=1 + fi + fm_lock_release "$route_lock" || status=1 + return "$status" } fm_pending_reply_secondmate_route_validate() { # [] [] @@ -698,7 +734,7 @@ fm_pending_reply_secondmate_route_validate() { # [] [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ] || return 1 fm_pending_reply_secondmate_route_shape "$marker" || return 1 - fm_pending_reply_secondmate_route_validate "$secondmate_home" "" "$allow_undelivered" || return 1 + fm_pending_reply_secondmate_route_validate "$secondmate_home" "" 2 || return 1 current_corr=$FM_PENDING_ROUTE_CORR if [ "$current_corr" != "$wanted_corr" ]; then history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$wanted_corr") @@ -783,12 +819,20 @@ fm_pending_reply_secondmate_route_validate() { # [] awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; *) return 1 ;; esac - if [ "$allow_undelivered" = 1 ]; then - [ -z "$delivered" ] || return 1 - [ "$phase" = awaiting_report ] || return 1 - else - [ -n "$delivered" ] || return 1 - fi + case "$allow_undelivered" in + 1) + [ -z "$delivered" ] || return 1 + [ "$phase" = awaiting_report ] || return 1 + ;; + 2) + if [ -z "$delivered" ]; then + case "$phase" in awaiting_report|delivery_unknown) ;; *) return 1 ;; esac + fi + ;; + *) + [ -n "$delivered" ] || return 1 + ;; + esac # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_PARENT_HOME=$parent_abs # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 658cea747d7..ab26c7a05cb 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -98,16 +98,19 @@ clear_new_pending_route() { } discard_new_pending_reply() { - if ! clear_new_pending_route; then - echo "error: failed to clear the secondmate pending-reply route; parent record was preserved" >&2 - return 1 - fi + local route_status=0 discard_status=0 + clear_new_pending_route || route_status=1 if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ && ! fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR"; then + discard_status=1 + fi + if [ "$route_status" = 1 ]; then + echo "error: failed to clear the secondmate pending-reply route; undelivered record cleanup continued" >&2 + fi + if [ "$discard_status" = 1 ]; then echo "error: failed to discard the undelivered pending-reply record" >&2 - return 1 fi - return 0 + [ "$route_status" = 0 ] && [ "$discard_status" = 0 ] } case "$RAW_TARGET" in diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 4b1d753b336..034f3fd055b 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -1228,6 +1228,8 @@ SH [ -n "$corr_b" ] || fail "replacement route did not create a new correlation" [ "$(receipt_value "$child_state/.fm-jt-parent-route" corr_id)" = "$corr_b" ] \ || fail "replacement route did not publish the new correlation" + replace_field "$state/pending-replies/$corr_b" phase awaiting_report + replace_field "$state/pending-replies/$corr_b" delivered_epoch '' active_route_backup="$dir/active-route-backup" cp "$child_state/.fm-jt-parent-route" "$active_route_backup" replace_field "$child_state/.fm-jt-parent-route" parent_home "$dir/not-a-parent-home" @@ -1415,6 +1417,52 @@ SH pass "concurrent secondmate routes fail closed without overwriting" } +test_failed_concurrent_send_discards_only_new_record() { + local dir root home fakebin state child_home child_state marker old_corr send_out + new_case failed-concurrent-send + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + cp -a "$ROOT/bin/." "$root/bin/" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + marker="$child_state/.fm-jt-parent-route" + old_corr=0123456789abcdef + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$child_home/projects" \ + "$state/pending-replies" + printf 'sm-race\n' > "$child_home/.fm-secondmate-home" + write_meta "$state" sm-race parent-race-inc secondmate tmux firstmate:fm-sm-race + printf 'home=%s\n' "$child_home" >> "$state/sm-race.meta" + fm_write_meta "$state/pending-replies/$old_corr" \ + schema=fm-pending-reply.v1 corr_id="$old_corr" task_id=sm-race \ + parent_home="$home" parent_status="$state/sm-race.status" \ + delivered_epoch=1 phase=awaiting_report + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-race "$old_corr" \ + || fail "existing concurrent route was not written" + prepare_primary_proof "$root" "$home" "$fakebin" + prepare_watcher_protocol "$root" "$home" "$state" + send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK \ + -u FM_AGENT_OWNER_HOME -u FM_ROOT -u STATE -u FM_PENDING_REPLY_EXISTING_CORR \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ + fm-sm-race "competing request" 2>&1) && fail "concurrent route bind unexpectedly succeeded" + [ "$(direct_file_count "$state/pending-replies" '*')" = 1 ] \ + || fail "failed concurrent send left a new pending-reply record" + [ -f "$state/pending-replies/$old_corr" ] \ + || fail "failed concurrent send discarded the unrelated active record" + [ "$(receipt_value "$state/pending-replies/$old_corr" phase)" = awaiting_report ] \ + || fail "failed concurrent send changed the unrelated route phase" + [ "$(receipt_value "$state/pending-replies/$old_corr" delivered_epoch)" = 1 ] \ + || fail "failed concurrent send changed the unrelated delivery state" + [ "$(receipt_value "$child_state/.fm-jt-parent-route" corr_id)" = "$old_corr" ] \ + || fail "failed concurrent send replaced the unrelated active route" + pass "failed concurrent send discards only its new undelivered record" +} + test_drain_restores_only_unprocessed_rows() { local dir root home fakebin state first second new_case drain-rollback @@ -1519,5 +1567,6 @@ test_valid_secondmate_route_reports_parent_once test_secondmate_route_replacement_preserves_old_receipt test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected +test_failed_concurrent_send_discards_only_new_record test_drain_restores_only_unprocessed_rows test_malformed_or_missing_secondmate_route_fails_closed From 237ab3b9db5dd65667e835d45d6dfbde7ac69b65 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 08:24:23 +0000 Subject: [PATCH 016/163] no-mistakes(review): Fixed delivery-unknown fallback and deferred wake acknowledgement --- bin/fm-inactive-reconcile.sh | 53 ++++++++++++++++++++++++------- bin/fm-pending-reply-lib.sh | 3 ++ bin/fm-session-start.sh | 10 +++++- bin/fm-wake-drain.sh | 20 ++++++------ bin/fm-watch.sh | 15 +++++++-- tests/fm-inactive-outcome.test.sh | 43 +++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 23 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index b9559f56315..3c3818a2d71 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -231,7 +231,7 @@ claim_validate() { # } claim_reserve() { # - local key=$1 row=$2 fp claim tmp state existing old_row line output_started + local key=$1 row=$2 fp claim tmp state existing old_row line output_started defer_ack drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -262,7 +262,24 @@ claim_reserve() { # return 1 fi fi - [ "$state" = presented ] && return 1 + if [ "$state" = presented ]; then + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$defer_ack" = 1 ]; then + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=reserved\n' ;; + defer_ack=*) printf 'defer_ack=0\n' ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } + return 0 + fi + return 1 + fi return 0 fi mkdir -p "$OUTCOME_DIR" || return 2 @@ -337,7 +354,8 @@ claim_mark_output_started() { # } claim_mark_presented() { # - local key=$1 row=$2 fp claim tmp line + local key=$1 row=$2 fp claim tmp line defer_ack=0 seen_defer=0 + [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -347,15 +365,20 @@ claim_mark_presented() { # tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 chmod 600 "$tmp" 2>/dev/null || true while IFS= read -r line || [ -n "$line" ]; do - case "$line" in state=*) printf 'state=presented\n' ;; *) printf '%s\n' "$line" ;; esac + case "$line" in + state=*) printf 'state=presented\n' ;; + defer_ack=*) printf 'defer_ack=%s\n' "$defer_ack"; seen_defer=1 ;; + *) printf '%s\n' "$line" ;; + esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ "$seen_defer" = 1 ] || printf 'defer_ack=%s\n' "$defer_ack" >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } claim_remove() { # - local key=$1 row=$2 fp claim - drain_claim_owner "$row" || return 2 + local key=$1 row=$2 owner_required=${3:-1} fp claim + case "$owner_required" in 0) ;; 1) drain_claim_owner "$row" || return 2 ;; *) return 2 ;; esac case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac claim=$(claim_path "$fp") @@ -717,13 +740,17 @@ reconcile_child() { } ack_receipt() { # - local key=$1 row=${2:-} fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing existing_kind existing_corr claim_state + local key=$1 row=${2:-} owner_required=${3:-1} fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing existing_kind existing_corr claim_state [ -n "$row" ] || return 2 - drain_claim_owner "$row" || return 2 + case "$owner_required" in 0|1) ;; *) return 2 ;; esac + [ "$owner_required" = 0 ] || drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 0 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac claim_state=$(claim_validate "$(claim_path "$fp")" "$fp" "$row") || return 2 [ "$claim_state" = presented ] || return 2 + if [ "$owner_required" = 0 ]; then + [ "$(claim_field "$(claim_path "$fp")" defer_ack 2>/dev/null || true)" = 1 ] || return 2 + fi rec=$(receipt_path "$fp" pending) [ ! -L "$rec" ] || return 2 if [ ! -e "$rec" ]; then @@ -737,7 +764,7 @@ ack_receipt() { # existing_corr=$(receipt_field "$existing" parent_corr) fm_pending_reply_secondmate_route_clear "$FM_HOME" "$existing_corr" || return 2 fi - claim_remove "$key" "$row" || return 2 + claim_remove "$key" "$row" "$owner_required" || return 2 return 1 fi done @@ -771,14 +798,14 @@ ack_receipt() { # [ -f "$target" ] || return 2 [ "$(receipt_field "$target" fingerprint)" = "$fp" ] || return 2 rm -f "$rec" || return 2 - claim_remove "$key" "$row" || return 2 + claim_remove "$key" "$row" "$owner_required" || return 2 return 1 fi mv "$rec" "$target" || return 2 if [ "$kind" = secondmate ]; then fm_pending_reply_secondmate_route_clear "$FM_HOME" "$corr" || return 2 fi - claim_remove "$key" "$row" || return 2 + claim_remove "$key" "$row" "$owner_required" || return 2 return 0 } @@ -955,6 +982,10 @@ case "${1:-}" in [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 ack_receipt "$2" "$3" ;; + confirm) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + ack_receipt "$2" "$3" 0 + ;; claim) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_reserve "$2" "$3" diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 0ba19b63c89..ad301bfcc51 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -817,6 +817,9 @@ fm_pending_reply_secondmate_route_validate() { # [] phase=$(fm_pending_reply_get "$rec" phase) case "$phase" in awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; + delivery_unknown) + [ "$allow_undelivered" = 2 ] || return 1 + ;; *) return 1 ;; esac case "$allow_undelivered" in diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index c4ffa29a2dc..b3ad1c23a69 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -363,12 +363,20 @@ if [ "$READ_ONLY" -eq 1 ]; then [ -n "$GUARD_OUT" ] && printf '%s\n' "$GUARD_OUT" else DRAIN_STATUS=0 - DRAIN_OUT=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) || DRAIN_STATUS=$? + DRAIN_OUT=$(FM_WAKE_DRAIN_DEFER_ACK=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) || DRAIN_STATUS=$? if [ "$DRAIN_STATUS" -ne 0 ]; then printf 'error: wake drain failed (status %s); inactive reconciliation skipped\n%s\n' \ "$DRAIN_STATUS" "$DRAIN_OUT" >&2 elif [ -n "$DRAIN_OUT" ]; then printf '%s\n' "$DRAIN_OUT" + while IFS= read -r drain_row || [ -n "$drain_row" ]; do + IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" + case "$_key" in + inactive-outcome:*) + "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$drain_row" >/dev/null 2>&1 || true + ;; + esac + done <<< "$DRAIN_OUT" else printf '(no queued wakes)\n' fi diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 677829f2f98..f7bdf70011f 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -158,15 +158,17 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do exit "$claim_status" ;; esac - FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" "$drain_row" || { - ack_status=$? - # 1 means the receipt was already acknowledged or is not ours. Any - # other failure keeps the drained row durable for a later turn. - if [ "$ack_status" != 1 ]; then - restore_unprocessed_rows "$drain_line" || exit 1 - exit "$ack_status" - fi - } + if [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" "$drain_row" || { + ack_status=$? + # 1 means the receipt was already acknowledged or is not ours. Any + # other failure keeps the drained row durable for a later turn. + if [ "$ack_status" != 1 ]; then + restore_unprocessed_rows "$drain_line" || exit 1 + exit "$ack_status" + fi + } + fi ;; *) if ! printf '%s\n' "$drain_row"; then diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index e4ed70cb7d9..cd106271e0e 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -339,11 +339,22 @@ event_wait_herdr() { # mean an idle fleet, so the heartbeat interval backs off exponentially # (base * 2^streak, capped at HEARTBEAT_MAX); any real wake resets the cadence. wake() { + local wake_output=$1 row _epoch _seq _kind _key _payload confirm_status case "$1" in heartbeat*) echo $(( $(cat "$STATE/.heartbeat-streak" 2>/dev/null || echo 0) + 1 )) > "$STATE/.heartbeat-streak" ;; *) echo 0 > "$STATE/.heartbeat-streak" ;; esac - echo "$1" + echo "$wake_output" + while IFS= read -r row || [ -n "$row" ]; do + IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$row" + case "$_key" in + inactive-outcome:*) + confirm_status=0 + "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$row" >/dev/null 2>&1 || confirm_status=$? + [ "$confirm_status" = 0 ] || [ "$confirm_status" = 1 ] || exit "$confirm_status" + ;; + esac + done <<< "$wake_output" exit 0 } @@ -583,7 +594,7 @@ while :; do # alive. Supervision scripts warn when this goes stale with tasks in flight. touch "$STATE/.last-watcher-beat" - if ! wake_drain_out=$("$SCRIPT_DIR/fm-wake-drain.sh" 2>&1); then + if ! wake_drain_out=$(FM_WAKE_DRAIN_DEFER_ACK=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1); then printf '%s\n' "$wake_drain_out" >&2 exit 1 fi diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 034f3fd055b..7070e9c7686 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -557,6 +557,41 @@ test_output_started_claim_is_not_reprinted() { pass "output-started inactive claims do not reprint after a drain crash" } +test_deferred_ack_retries_after_caller_crash() { + local dir root home fakebin state fingerprint row drain_output rec + new_case deferred-ack + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" deferred-x1 deferred-inc + export FM_FAKE_CREW_STATE_DEFERRED_X1='state: done · source: pane · deferred presentation' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "deferred receipt setup failed" + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) + row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") + [ -n "$row" ] || fail "deferred receipt did not queue its wake" + drain_output="$dir/deferred.out" + if ! FM_WAKE_DRAIN_DEFER_ACK=1 drain "$root" "$home" "$fakebin" >"$drain_output"; then + fail "deferred wake drain failed" + fi + [ -f "$state/terminal-outcomes/$fingerprint.pending" ] || fail "deferred drain consumed the receipt before caller confirmation" + [ ! -e "$state/terminal-outcomes/$fingerprint.presented" ] || fail "deferred drain finalized the receipt before caller confirmation" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" state)" = presented ] \ + || fail "deferred drain did not retain the presentation claim" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" defer_ack)" = 1 ] \ + || fail "deferred drain did not mark the claim for caller confirmation" + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "pending deferred receipt was not republished after caller crash" + [ "$(queue_count "$state")" = 1 ] || fail "pending deferred receipt did not get a retry wake" + drain_output="$dir/retry.out" + drain "$root" "$home" "$fakebin" >"$drain_output" \ + || fail "retry drain did not recover the deferred presentation" + [ -f "$state/terminal-outcomes/$fingerprint.presented" ] || fail "retry drain did not acknowledge the recovered receipt" + [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] || fail "retry drain left the receipt pending" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "retry drain left the deferred claim" + grep -F 'task=deferred-x1' "$drain_output" >/dev/null \ + || fail "retry drain did not re-present the deferred row" + unset FM_FAKE_CREW_STATE_DEFERRED_X1 + pass "deferred inactive receipts remain retryable until caller confirmation" +} + test_scan_failure_retries_without_advancing_cadence() { local dir root home fakebin state wake_dir wake_removed new_case scan-failure @@ -1254,6 +1289,13 @@ SH [ "$(receipt_count "$child_state" pending)" = 1 ] \ || fail "mismatched history marker consumed the old receipt" cp "$history_backup" "$child_state/.fm-jt-parent-route-history.$corr_a" + replace_field "$state/pending-replies/$corr_b" phase delivery_unknown + : > "$child_state/.wake-queue" + scan "$root" "$child_home" "$fakebin" --startup \ + || fail "delivery-unknown active route scan failed" + [ "$(queue_count "$child_state")" = 1 ] \ + || fail "delivery-unknown active route blocked the matching history receipt" + replace_field "$state/pending-replies/$corr_b" phase awaiting_report scan "$root" "$child_home" "$fakebin" --startup \ || fail "pending old-route receipt was not reconciled after route replacement" [ "$(queue_count "$child_state")" = 1 ] \ @@ -1550,6 +1592,7 @@ test_ack_recomputes_fingerprint_from_receipt_fields test_reserved_claim_recovers_to_a_new_wake_row test_presenting_claim_recovers_before_output test_output_started_claim_is_not_reprinted +test_deferred_ack_retries_after_caller_crash test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint From c09be49ff03fe16cc6ce51a5d85ccad2a907bd4d Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 08:53:46 +0000 Subject: [PATCH 017/163] no-mistakes(review): Suppress finalized rows and recover orphaned pending receipts --- bin/fm-inactive-reconcile.sh | 123 +++++++++++++++++++++++++++++- bin/fm-wake-drain.sh | 4 +- tests/fm-inactive-outcome.test.sh | 23 ++++++ 3 files changed, 146 insertions(+), 4 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 3c3818a2d71..8278496a098 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -200,6 +200,22 @@ claim_field() { # meta_value_unique "$1" "$2" } +claim_receipt_state() { + local fp=$1 suffix path found= + for suffix in pending presented reported; do + path=$(receipt_path "$fp" "$suffix") + [ ! -L "$path" ] || return 2 + [ -e "$path" ] || continue + [ -f "$path" ] || return 2 + [ "$(receipt_field "$path" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 + [ "$(receipt_field "$path" fingerprint)" = "$fp" ] || return 2 + [ -z "$found" ] || return 2 + found=$suffix + done + [ -n "$found" ] || return 1 + printf '%s' "$found" +} + drain_claim_owner() { local row=$1 owner parent_pid drain_file drain_dir state_dir owner=$(fm_lock_link_owner "$FM_WAKE_QUEUE_LOCK" 2>/dev/null || true) @@ -231,11 +247,30 @@ claim_validate() { # } claim_reserve() { # - local key=$1 row=$2 fp claim tmp state existing old_row line output_started defer_ack + local key=$1 row=$2 fp claim tmp state existing old_row line output_started defer_ack receipt_state receipt_rc=0 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac inactive_state_preflight || return 2 + receipt_state=$(claim_receipt_state "$fp" 2>/dev/null) || receipt_rc=$? + [ "$receipt_rc" = 0 ] || return 2 + case "$receipt_state" in + presented|reported) + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + if [ -e "$claim" ]; then + [ -f "$claim" ] || return 2 + [ "$(claim_field "$claim" schema)" = fm-inactive-outcome-claim.v1 ] || return 2 + [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 2 + [ -n "$(claim_field "$claim" row)" ] || return 2 + case "$(claim_field "$claim" state)" in reserved|presenting|presented) ;; *) return 2 ;; esac + rm -f "$claim" || return 2 + fi + return 3 + ;; + pending) ;; + *) return 2 ;; + esac claim=$(claim_path "$fp") [ ! -L "$claim" ] || return 2 if [ -e "$claim" ]; then @@ -608,6 +643,79 @@ publish_secondmate_receipt_and_wake() { return "$status" } +prepare_pending_receipt() { + local pending=$1 expected_fp schema task_id incarnation outcome terminal_source terminal_snapshot kind + local parent_task_id parent_home parent_status parent_corr + [ -f "$pending" ] && [ ! -L "$pending" ] || return 1 + FP=${pending##*/} + FP=${FP%.pending} + case "$FP" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + schema=$(receipt_field "$pending" schema) || return 1 + task_id=$(receipt_field "$pending" task_id) || return 1 + incarnation=$(receipt_field "$pending" incarnation) || return 1 + outcome=$(receipt_field "$pending" outcome) || return 1 + terminal_source=$(receipt_field "$pending" terminal_source) || return 1 + terminal_snapshot=$(receipt_field "$pending" terminal_snapshot) || return 1 + kind=$(receipt_field "$pending" kind) || return 1 + parent_task_id=$(receipt_field "$pending" parent_task_id) || return 1 + parent_home=$(receipt_field "$pending" parent_home) || return 1 + parent_status=$(receipt_field "$pending" parent_status) || return 1 + parent_corr=$(receipt_field "$pending" parent_corr) || return 1 + [ "$schema" = fm-jt-terminal-outcome.v1 ] || return 1 + [ -n "$task_id" ] && [ -n "$incarnation" ] && [ -n "$terminal_source" ] \ + && [ -n "$terminal_snapshot" ] || return 1 + case "$outcome" in done|failed) ;; *) return 1 ;; esac + case "$kind" in ship|scout) + [ -z "$parent_task_id" ] && [ -z "$parent_home" ] \ + && [ -z "$parent_status" ] && [ -z "$parent_corr" ] || return 1 + ;; + secondmate) + [ -n "$parent_task_id" ] && [ -n "$parent_home" ] && [ -n "$parent_status" ] || return 1 + printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + ;; + *) return 1 ;; + esac + expected_fp=$(hash_text "$task_id|$incarnation|$outcome|$terminal_snapshot|$kind") || return 1 + [ "$expected_fp" = "$FP" ] || return 1 + ID=$task_id + INC=$incarnation + OUTCOME=$outcome + SOURCE=$terminal_source + SNAPSHOT=$terminal_snapshot + KIND=$kind + return 0 +} + +republish_pending_receipt() { + local pending=$1 + prepare_pending_receipt "$pending" || return 1 + case "$KIND" in + ship|scout) + republish_existing_receipt_wake + ;; + secondmate) + publish_secondmate_receipt_and_wake + ;; + esac +} + +republish_pending_receipts() { + local scan_started=$1 pending now remaining status=0 + [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 + for pending in "$OUTCOME_DIR"/*.pending; do + [ -e "$pending" ] || [ -L "$pending" ] || continue + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + [ "$remaining" -gt 0 ] || return 1 + if ! FM_LOCK_WAIT_SECS="$remaining" republish_pending_receipt "$pending"; then + status=1 + elif [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$ID" "$OUTCOME" "$FP" + fi + done + return "$status" +} + read_incarnation() { # local meta=$1 id=$2 token tasktmp window worktree seed digest rc token_present=0 if token=$(meta_value_unique "$meta" spawn_incarnation); then @@ -809,6 +917,14 @@ ack_receipt() { # return 0 } +confirm_receipt() { + local status=0 + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 2 + ack_receipt "$1" "$2" 0 || status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=2 + return "$status" +} + secondmate_ack_report() { # local secondmate_home=$1 parent_task_id=$2 parent_home=$3 parent_status=$4 corr=$5 outcome=$6 task_id=$7 fp=$8 local parent_state token route_lock route_marker route_history line phase rc=0 route_lock_held=0 marker_present=0 report_recorded=0 @@ -940,6 +1056,9 @@ scan_locked() { rm -f "$find_tmp" || return 1 [ "$scan_failed" = 0 ] || return "$rc" [ "$complete" = 1 ] || return 1 + if ! republish_pending_receipts "$scan_started"; then + return 1 + fi if [ "$cursor_seen" = 0 ]; then return 1 fi @@ -984,7 +1103,7 @@ case "${1:-}" in ;; confirm) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 - ack_receipt "$2" "$3" 0 + confirm_receipt "$2" "$3" ;; claim) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index f7bdf70011f..7fe4bddb42a 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -152,13 +152,13 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do exit 1 fi ;; - 1) ;; + 1|3) ;; *) restore_unprocessed_rows "$drain_line" || exit 1 exit "$claim_status" ;; esac - if [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then + if [ "$claim_status" != 3 ] && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" "$drain_row" || { ack_status=$? # 1 means the receipt was already acknowledged or is not ours. Any diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 7070e9c7686..4aa772a907e 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -557,6 +557,27 @@ test_output_started_claim_is_not_reprinted() { pass "output-started inactive claims do not reprint after a drain crash" } +test_finalized_receipt_rows_are_suppressed() { + local dir root home fakebin state fingerprint row + new_case finalized-row + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" finalized-x1 finalized-inc + export FM_FAKE_CREW_STATE_FINALIZED_X1='state: done · source: pane · finalized row' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "finalized receipt setup failed" + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) + row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") + drain "$root" "$home" "$fakebin" >/dev/null || fail "initial finalized receipt drain failed" + printf '%s\n' "$row" > "$state/.wake-queue" + drain "$root" "$home" "$fakebin" >"$dir/stale.out" \ + || fail "stale finalized receipt row was not safely suppressed" + [ ! -s "$dir/stale.out" ] || fail "stale finalized receipt row was printed again" + [ "$(receipt_count "$state" presented)" = 1 ] || fail "stale finalized receipt changed receipt state" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "stale finalized receipt left a claim" + unset FM_FAKE_CREW_STATE_FINALIZED_X1 + pass "finalized receipt rows are suppressed after drain rollback" +} + test_deferred_ack_retries_after_caller_crash() { local dir root home fakebin state fingerprint row drain_output rec new_case deferred-ack @@ -578,6 +599,7 @@ test_deferred_ack_retries_after_caller_crash() { || fail "deferred drain did not retain the presentation claim" [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" defer_ack)" = 1 ] \ || fail "deferred drain did not mark the claim for caller confirmation" + rm -f "$state/deferred-x1.meta" "$state/deferred-x1.status" "$state/deferred-x1.turn-ended" scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "pending deferred receipt was not republished after caller crash" [ "$(queue_count "$state")" = 1 ] || fail "pending deferred receipt did not get a retry wake" drain_output="$dir/retry.out" @@ -1592,6 +1614,7 @@ test_ack_recomputes_fingerprint_from_receipt_fields test_reserved_claim_recovers_to_a_new_wake_row test_presenting_claim_recovers_before_output test_output_started_claim_is_not_reprinted +test_finalized_receipt_rows_are_suppressed test_deferred_ack_retries_after_caller_crash test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories From c6e50d354b2bfd68e249f1ba34f21c772f739589 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 09:37:09 +0000 Subject: [PATCH 018/163] no-mistakes(review): Make inactive outcome presentation crash-safe --- bin/fm-inactive-reconcile.sh | 98 +++++++++++++++++++++++++++---- bin/fm-session-start.sh | 3 +- bin/fm-wake-drain.sh | 21 +++---- bin/fm-watch.sh | 3 +- tests/fm-inactive-outcome.test.sh | 39 +++++++++++- 5 files changed, 136 insertions(+), 28 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 8278496a098..73d742c571f 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -216,6 +216,13 @@ claim_receipt_state() { printf '%s' "$found" } +claim_defer_generation_live() { + local generation=$1 + case "$generation" in ''|*[!0-9]*) return 1 ;; esac + [ "$generation" != 0 ] || return 1 + kill -0 "$generation" 2>/dev/null +} + drain_claim_owner() { local row=$1 owner parent_pid drain_file drain_dir state_dir owner=$(fm_lock_link_owner "$FM_WAKE_QUEUE_LOCK" 2>/dev/null || true) @@ -247,7 +254,8 @@ claim_validate() { # } claim_reserve() { # - local key=$1 row=$2 fp claim tmp state existing old_row line output_started defer_ack receipt_state receipt_rc=0 + local key=$1 row=$2 fp claim tmp state existing old_row line output_complete defer_ack + local defer_generation receipt_state receipt_rc=0 recorded_report=0 report_rc=1 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -271,6 +279,12 @@ claim_reserve() { # pending) ;; *) return 2 ;; esac + pending_secondmate_report_recorded "$fp" >/dev/null || report_rc=$? + case "$report_rc" in + 0) recorded_report=1 ;; + 1) ;; + *) return 2 ;; + esac claim=$(claim_path "$fp") [ ! -L "$claim" ] || return 2 if [ -e "$claim" ]; then @@ -281,6 +295,15 @@ claim_reserve() { # case "$state" in presented|presenting|reserved) ;; *) return 2 ;; esac old_row=$(claim_field "$claim" row) [ -n "$old_row" ] || return 2 + if [ "$state" = presented ]; then + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$defer_ack" = 1 ]; then + defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) + if claim_defer_generation_live "$defer_generation"; then + return 4 + fi + fi + fi if [ "$old_row" != "$row" ]; then tmp=$(mktemp "$OUTCOME_DIR/.claim-row.XXXXXX") || return 2 chmod 600 "$tmp" 2>/dev/null || true @@ -290,11 +313,22 @@ claim_reserve() { # [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } fi + if [ "$recorded_report" = 1 ]; then + case "$state" in + presented) return 5 ;; + presenting|reserved) + claim_mark_presenting "$key" "$row" || return 2 + claim_mark_output_complete "$key" "$row" || return 2 + claim_mark_presented "$key" "$row" || return 2 + return 5 + ;; + esac + fi if [ "$state" = presenting ]; then - output_started=$(claim_field "$claim" output_started 2>/dev/null || true) - if [ "$output_started" = 1 ]; then + output_complete=$(claim_field "$claim" output_complete 2>/dev/null || true) + if [ "$output_complete" = 1 ]; then claim_mark_presented "$key" "$row" || return 2 - return 1 + return 5 fi fi if [ "$state" = presented ]; then @@ -306,6 +340,7 @@ claim_reserve() { # case "$line" in state=*) printf 'state=reserved\n' ;; defer_ack=*) printf 'defer_ack=0\n' ;; + defer_generation=*) printf 'defer_generation=\n' ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } @@ -329,6 +364,12 @@ claim_reserve() { # } > "$tmp" || { rm -f "$tmp"; return 2; } if ln "$tmp" "$claim" 2>/dev/null; then rm -f "$tmp" + if [ "$recorded_report" = 1 ]; then + claim_mark_presenting "$key" "$row" || return 2 + claim_mark_output_complete "$key" "$row" || return 2 + claim_mark_presented "$key" "$row" || return 2 + return 5 + fi return 0 fi rm -f "$tmp" @@ -339,7 +380,8 @@ claim_reserve() { # } claim_mark_presenting() { # - local key=$1 row=$2 fp claim state tmp line seen_pid=0 seen_output=0 + local key=$1 row=$2 fp claim state tmp line seen_pid=0 seen_output=0 seen_complete=0 + local seen_defer_ack=0 seen_defer_generation=0 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -357,17 +399,23 @@ claim_mark_presenting() { # state=*) printf 'state=presenting\n' ;; presentation_pid=*) printf 'presentation_pid=%s\n' "${BASHPID:-$$}"; seen_pid=1 ;; output_started=*) printf 'output_started=0\n'; seen_output=1 ;; + output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; + defer_ack=*) printf 'defer_ack=0\n'; seen_defer_ack=1 ;; + defer_generation=*) printf 'defer_generation=\n'; seen_defer_generation=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_pid" = 1 ] || printf 'presentation_pid=%s\n' "${BASHPID:-$$}" >> "$tmp" [ "$seen_output" = 1 ] || printf 'output_started=0\n' >> "$tmp" + [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' >> "$tmp" + [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=0\n' >> "$tmp" + [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=\n' >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } -claim_mark_output_started() { # - local key=$1 row=$2 fp claim state tmp line seen_output=0 +claim_mark_output_complete() { # + local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_complete=0 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -380,17 +428,24 @@ claim_mark_output_started() { # while IFS= read -r line || [ -n "$line" ]; do case "$line" in output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_complete=*) printf 'output_complete=1\n'; seen_complete=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_output" = 1 ] || printf 'output_started=1\n' >> "$tmp" + [ "$seen_complete" = 1 ] || printf 'output_complete=1\n' >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } claim_mark_presented() { # - local key=$1 row=$2 fp claim tmp line defer_ack=0 seen_defer=0 + local key=$1 row=$2 fp claim tmp line defer_ack=0 defer_generation= + local seen_defer_ack=0 seen_defer_generation=0 [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 + if [ "$defer_ack" = 1 ]; then + defer_generation=${FM_WAKE_DRAIN_GENERATION:-} + case "$defer_generation" in ''|*[!0-9]*) [ -z "$defer_generation" ] || return 2 ;; esac + fi drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -402,11 +457,13 @@ claim_mark_presented() { # while IFS= read -r line || [ -n "$line" ]; do case "$line" in state=*) printf 'state=presented\n' ;; - defer_ack=*) printf 'defer_ack=%s\n' "$defer_ack"; seen_defer=1 ;; + defer_ack=*) printf 'defer_ack=%s\n' "$defer_ack"; seen_defer_ack=1 ;; + defer_generation=*) printf 'defer_generation=%s\n' "$defer_generation"; seen_defer_generation=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } - [ "$seen_defer" = 1 ] || printf 'defer_ack=%s\n' "$defer_ack" >> "$tmp" + [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=%s\n' "$defer_ack" >> "$tmp" + [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=%s\n' "$defer_generation" >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } @@ -686,6 +743,23 @@ prepare_pending_receipt() { return 0 } +pending_secondmate_report_recorded() { + local fp=$1 pending kind parent_status parent_corr line + pending=$(receipt_path "$fp" pending) + kind=$(receipt_field "$pending" kind) || return 2 + case "$kind" in + ship|scout) return 1 ;; + secondmate) ;; + *) return 2 ;; + esac + prepare_pending_receipt "$pending" || return 2 + parent_status=$(receipt_field "$pending" parent_status) || return 2 + parent_corr=$(receipt_field "$pending" parent_corr) || return 2 + [ -f "$parent_status" ] && [ ! -L "$parent_status" ] || return 1 + line="$OUTCOME [corr=$parent_corr]: inactive terminal outcome replayed: task=$ID fingerprint=$fp" + grep -Fqx "$line" "$parent_status" 2>/dev/null +} + republish_pending_receipt() { local pending=$1 prepare_pending_receipt "$pending" || return 1 @@ -1113,9 +1187,9 @@ case "${1:-}" in [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_presenting "$2" "$3" ;; - output-started) + output-started|output-complete) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 - claim_mark_output_started "$2" "$3" + claim_mark_output_complete "$2" "$3" ;; presented) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index b3ad1c23a69..9d5d0ad8578 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -363,7 +363,8 @@ if [ "$READ_ONLY" -eq 1 ]; then [ -n "$GUARD_OUT" ] && printf '%s\n' "$GUARD_OUT" else DRAIN_STATUS=0 - DRAIN_OUT=$(FM_WAKE_DRAIN_DEFER_ACK=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) || DRAIN_STATUS=$? + DRAIN_OUT=$(FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" \ + "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) || DRAIN_STATUS=$? if [ "$DRAIN_STATUS" -ne 0 ]; then printf 'error: wake drain failed (status %s); inactive reconciliation skipped\n%s\n' \ "$DRAIN_STATUS" "$DRAIN_OUT" >&2 diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 7fe4bddb42a..8b2fdea44f1 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -24,22 +24,17 @@ present_inactive_row() { ( while [ ! -e "$go" ]; do if ! kill -0 "$DRAIN_PID" 2>/dev/null; then - FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ - FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - output-started "$key" "$row" || exit 1 break fi sleep 0.01 done - printf '%s\n' "$row" + printf '%s\n' "$row" || exit 1 + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ + FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-complete "$key" "$row" || exit 1 ) & worker=$! - if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" output-started "$key" "$row"; then - status=1 - fi - if [ "$status" = 0 ]; then - : > "$go" || status=1 - fi + : > "$go" || status=1 if [ "$status" = 0 ]; then wait "$worker" || worker_status=$? [ "$worker_status" = 0 ] || status=1 @@ -152,13 +147,15 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do exit 1 fi ;; - 1|3) ;; + 1|5) ;; + 3|4) ;; *) restore_unprocessed_rows "$drain_line" || exit 1 exit "$claim_status" ;; esac - if [ "$claim_status" != 3 ] && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then + if [ "$claim_status" != 3 ] && [ "$claim_status" != 4 ] \ + && { [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ] || [ "$claim_status" = 5 ]; }; then FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" ack "$_key" "$drain_row" || { ack_status=$? # 1 means the receipt was already acknowledged or is not ours. Any diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index cd106271e0e..c3440c4ca57 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -594,7 +594,8 @@ while :; do # alive. Supervision scripts warn when this goes stale with tasks in flight. touch "$STATE/.last-watcher-beat" - if ! wake_drain_out=$(FM_WAKE_DRAIN_DEFER_ACK=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1); then + if ! wake_drain_out=$(FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$WATCHER_PID" \ + "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1); then printf '%s\n' "$wake_drain_out" >&2 exit 1 fi diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 4aa772a907e..6a27b5cb016 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -546,7 +546,7 @@ test_output_started_claim_is_not_reprinted() { incarnation=output-started-inc outcome=done terminal_source=pane \ terminal_snapshot='state: done · source: pane · output started' kind=ship row='2 2 check inactive-outcome:'"$fingerprint"$'\tpost-output row' - printf 'schema=fm-inactive-outcome-claim.v1\nfingerprint=%s\nrow=1\t1\tcheck\tinactive-outcome:%s\told row\nstate=presenting\noutput_started=1\ncreated_epoch=1\n' \ + printf 'schema=fm-inactive-outcome-claim.v1\nfingerprint=%s\nrow=1\t1\tcheck\tinactive-outcome:%s\told row\nstate=presenting\noutput_started=1\noutput_complete=1\ncreated_epoch=1\n' \ "$fingerprint" "$fingerprint" > "$state/terminal-outcomes/.$fingerprint.claim" printf '%s\n' "$row" > "$state/.wake-queue" drain "$root" "$home" "$fakebin" >"$dir/output-started.out" \ @@ -557,6 +557,30 @@ test_output_started_claim_is_not_reprinted() { pass "output-started inactive claims do not reprint after a drain crash" } +test_pre_output_claim_retries_after_crash() { + local dir root home fakebin state fingerprint row + new_case pre-output-claim + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'pre-output-x1|pre-output-inc|done|state: done · source: pane · pre-output') + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=pre-output-x1 \ + incarnation=pre-output-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · pre-output' kind=ship + row=$'2\t2\tcheck\tinactive-outcome:'"$fingerprint"$'\tpre-output row' + printf 'schema=fm-inactive-outcome-claim.v1\nfingerprint=%s\nrow=%s\nstate=presenting\noutput_started=1\ncreated_epoch=1\n' \ + "$fingerprint" "$row" > "$state/terminal-outcomes/.$fingerprint.claim" + printf '%s\n' "$row" > "$state/.wake-queue" + drain "$root" "$home" "$fakebin" >"$dir/pre-output.out" \ + || fail "pre-output claim did not recover" + grep -F 'pre-output row' "$dir/pre-output.out" >/dev/null \ + || fail "pre-output claim was suppressed before successful emission" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] \ + || fail "pre-output retry did not finalize the receipt" + pass "pre-output claims retry until emission completes" +} + test_finalized_receipt_rows_are_suppressed() { local dir root home fakebin state fingerprint row new_case finalized-row @@ -590,7 +614,8 @@ test_deferred_ack_retries_after_caller_crash() { row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") [ -n "$row" ] || fail "deferred receipt did not queue its wake" drain_output="$dir/deferred.out" - if ! FM_WAKE_DRAIN_DEFER_ACK=1 drain "$root" "$home" "$fakebin" >"$drain_output"; then + if ! FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" \ + drain "$root" "$home" "$fakebin" >"$drain_output"; then fail "deferred wake drain failed" fi [ -f "$state/terminal-outcomes/$fingerprint.pending" ] || fail "deferred drain consumed the receipt before caller confirmation" @@ -599,6 +624,13 @@ test_deferred_ack_retries_after_caller_crash() { || fail "deferred drain did not retain the presentation claim" [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" defer_ack)" = 1 ] \ || fail "deferred drain did not mark the claim for caller confirmation" + printf '%s\n' "$row" > "$state/.wake-queue" + drain "$root" "$home" "$fakebin" >"$dir/live-deferred.out" \ + || fail "live deferred claim drain failed" + [ ! -s "$dir/live-deferred.out" ] || fail "live deferred claim was presented twice" + [ -f "$state/terminal-outcomes/$fingerprint.pending" ] \ + || fail "live deferred claim was acknowledged before caller confirmation" + replace_field "$state/terminal-outcomes/.$fingerprint.claim" defer_generation 99999999 rm -f "$state/deferred-x1.meta" "$state/deferred-x1.status" "$state/deferred-x1.turn-ended" scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "pending deferred receipt was not republished after caller crash" [ "$(queue_count "$state")" = 1 ] || fail "pending deferred receipt did not get a retry wake" @@ -1179,6 +1211,8 @@ SH cat "$dir/second-drain.out" >&2 fail "valid secondmate route drain failed after symlink removal" fi + ! grep -F 'inactive-outcome:' "$dir/second-drain.out" >/dev/null \ + || fail "already-recorded parent report was presented again" [ "$(receipt_count "$child_state" reported)" = 1 ] || fail "valid secondmate route was not reported" [ ! -e "$child_state/.fm-jt-parent-route" ] || fail "reported secondmate route was not cleared after its parent report" grep -F "failed [corr=$corr]: inactive terminal outcome replayed: task=child-x1" "$parent_status" >/dev/null \ @@ -1614,6 +1648,7 @@ test_ack_recomputes_fingerprint_from_receipt_fields test_reserved_claim_recovers_to_a_new_wake_row test_presenting_claim_recovers_before_output test_output_started_claim_is_not_reprinted +test_pre_output_claim_retries_after_crash test_finalized_receipt_rows_are_suppressed test_deferred_ack_retries_after_caller_crash test_scan_failure_retries_without_advancing_cadence From 2c345f9bed86ed8b911a9905c9863778600943d0 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 10:13:40 +0000 Subject: [PATCH 019/163] no-mistakes(review): Harden inactive receipt recovery and route cleanup --- bin/fm-inactive-reconcile.sh | 87 +++++++++++++++++--- bin/fm-pending-reply-lib.sh | 131 ++++++++++++++++++++---------- bin/fm-send.sh | 4 +- tests/fm-inactive-outcome.test.sh | 127 ++++++++++++++++++++++++++++- 4 files changed, 291 insertions(+), 58 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 73d742c571f..0bad9f24b70 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -217,10 +217,12 @@ claim_receipt_state() { } claim_defer_generation_live() { - local generation=$1 + local generation=$1 stored_start=${2:-} case "$generation" in ''|*[!0-9]*) return 1 ;; esac [ "$generation" != 0 ] || return 1 - kill -0 "$generation" 2>/dev/null + [ -n "$stored_start" ] || return 1 + kill -0 "$generation" 2>/dev/null || return 1 + fm_pid_start_matches_stored "$generation" "$stored_start" } drain_claim_owner() { @@ -255,7 +257,7 @@ claim_validate() { # claim_reserve() { # local key=$1 row=$2 fp claim tmp state existing old_row line output_complete defer_ack - local defer_generation receipt_state receipt_rc=0 recorded_report=0 report_rc=1 + local defer_generation defer_generation_start receipt_state receipt_rc=0 recorded_report=0 report_rc=1 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -299,7 +301,8 @@ claim_reserve() { # defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) if [ "$defer_ack" = 1 ]; then defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) - if claim_defer_generation_live "$defer_generation"; then + defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + if claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then return 4 fi fi @@ -341,6 +344,7 @@ claim_reserve() { # state=*) printf 'state=reserved\n' ;; defer_ack=*) printf 'defer_ack=0\n' ;; defer_generation=*) printf 'defer_generation=\n' ;; + defer_generation_start=*) printf 'defer_generation_start=\n' ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } @@ -348,7 +352,7 @@ claim_reserve() { # mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } return 0 fi - return 1 + return 5 fi return 0 fi @@ -381,7 +385,7 @@ claim_reserve() { # claim_mark_presenting() { # local key=$1 row=$2 fp claim state tmp line seen_pid=0 seen_output=0 seen_complete=0 - local seen_defer_ack=0 seen_defer_generation=0 + local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -402,6 +406,7 @@ claim_mark_presenting() { # output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; defer_ack=*) printf 'defer_ack=0\n'; seen_defer_ack=1 ;; defer_generation=*) printf 'defer_generation=\n'; seen_defer_generation=1 ;; + defer_generation_start=*) printf 'defer_generation_start=\n'; seen_defer_generation_start=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } @@ -410,6 +415,7 @@ claim_mark_presenting() { # [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' >> "$tmp" [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=0\n' >> "$tmp" [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=\n' >> "$tmp" + [ "$seen_defer_generation_start" = 1 ] || printf 'defer_generation_start=\n' >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } @@ -439,12 +445,13 @@ claim_mark_output_complete() { # } claim_mark_presented() { # - local key=$1 row=$2 fp claim tmp line defer_ack=0 defer_generation= - local seen_defer_ack=0 seen_defer_generation=0 + local key=$1 row=$2 fp claim tmp line defer_ack=0 defer_generation= defer_generation_start= + local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 if [ "$defer_ack" = 1 ]; then defer_generation=${FM_WAKE_DRAIN_GENERATION:-} - case "$defer_generation" in ''|*[!0-9]*) [ -z "$defer_generation" ] || return 2 ;; esac + case "$defer_generation" in ''|*[!0-9]*|0) return 2 ;; esac + defer_generation_start=$(fm_pid_start "$defer_generation") || return 2 fi drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac @@ -459,11 +466,13 @@ claim_mark_presented() { # state=*) printf 'state=presented\n' ;; defer_ack=*) printf 'defer_ack=%s\n' "$defer_ack"; seen_defer_ack=1 ;; defer_generation=*) printf 'defer_generation=%s\n' "$defer_generation"; seen_defer_generation=1 ;; + defer_generation_start=*) printf 'defer_generation_start=%s\n' "$defer_generation_start"; seen_defer_generation_start=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=%s\n' "$defer_ack" >> "$tmp" [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=%s\n' "$defer_generation" >> "$tmp" + [ "$seen_defer_generation_start" = 1 ] || printf 'defer_generation_start=%s\n' "$defer_generation_start" >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } @@ -760,6 +769,59 @@ pending_secondmate_report_recorded() { grep -Fqx "$line" "$parent_status" 2>/dev/null } +reported_secondmate_receipt_valid() { + local reported=$1 fp id incarnation outcome snapshot kind parent_task_id parent_home parent_status corr expected_fp + [ -f "$reported" ] && [ ! -L "$reported" ] || return 1 + fp=${reported##*/} + fp=${fp%.reported} + case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + [ "$(receipt_field "$reported" schema)" = fm-jt-terminal-outcome.v1 ] || return 1 + [ "$(receipt_field "$reported" fingerprint)" = "$fp" ] || return 1 + id=$(receipt_field "$reported" task_id) || return 1 + incarnation=$(receipt_field "$reported" incarnation) || return 1 + outcome=$(receipt_field "$reported" outcome) || return 1 + snapshot=$(receipt_field "$reported" terminal_snapshot) || return 1 + kind=$(receipt_field "$reported" kind) || return 1 + [ "$kind" = secondmate ] || return 1 + parent_task_id=$(receipt_field "$reported" parent_task_id) || return 1 + parent_home=$(receipt_field "$reported" parent_home) || return 1 + parent_status=$(receipt_field "$reported" parent_status) || return 1 + corr=$(receipt_field "$reported" parent_corr) || return 1 + [ -n "$id" ] && [ -n "$incarnation" ] && [ -n "$snapshot" ] || return 1 + case "$outcome" in done|failed) ;; *) return 1 ;; esac + case "$parent_task_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$parent_home" in /*) ;; *) return 1 ;; esac + case "$parent_status" in /*) ;; *) return 1 ;; esac + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + expected_fp=$(hash_text "$id|$incarnation|$outcome|$snapshot|$kind") || return 1 + [ "$expected_fp" = "$fp" ] +} + +repair_reported_secondmate_routes() { + local reported kind corr status=0 + [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 + for reported in "$OUTCOME_DIR"/*.reported; do + [ -e "$reported" ] || [ -L "$reported" ] || continue + [ -f "$reported" ] && [ ! -L "$reported" ] || { status=1; continue; } + kind=$(receipt_field "$reported" kind 2>/dev/null || true) + case "$kind" in + ship|scout) continue ;; + secondmate) + if ! reported_secondmate_receipt_valid "$reported"; then + status=1 + continue + fi + corr=$(receipt_field "$reported" parent_corr) + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" || status=1 + ;; + *) status=1 ;; + esac + done + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 + return "$status" +} + republish_pending_receipt() { local pending=$1 prepare_pending_receipt "$pending" || return 1 @@ -944,7 +1006,7 @@ ack_receipt() { # existing_kind=$(receipt_field "$existing" kind) if [ "$existing_kind" = secondmate ]; then existing_corr=$(receipt_field "$existing" parent_corr) - fm_pending_reply_secondmate_route_clear "$FM_HOME" "$existing_corr" || return 2 + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$existing_corr" || return 2 fi claim_remove "$key" "$row" "$owner_required" || return 2 return 1 @@ -985,7 +1047,7 @@ ack_receipt() { # fi mv "$rec" "$target" || return 2 if [ "$kind" = secondmate ]; then - fm_pending_reply_secondmate_route_clear "$FM_HOME" "$corr" || return 2 + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" || return 2 fi claim_remove "$key" "$row" "$owner_required" || return 2 return 0 @@ -1130,6 +1192,9 @@ scan_locked() { rm -f "$find_tmp" || return 1 [ "$scan_failed" = 0 ] || return "$rc" [ "$complete" = 1 ] || return 1 + if ! repair_reported_secondmate_routes; then + return 1 + fi if ! republish_pending_receipts "$scan_started"; then return 1 fi diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index ad301bfcc51..344a3a2d161 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -652,20 +652,43 @@ fm_pending_reply_secondmate_route_write() { # < } fm_pending_reply_secondmate_route_clear_with_mode() { # - local secondmate_home=$1 corr=$2 allow_undelivered=${3:-0} marker route_lock status=0 + local secondmate_home=$1 corr=$2 allow_undelivered=${3:-0} marker route_lock current_corr history_marker status=0 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") - if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then - marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") - [ -e "$marker" ] || [ -L "$marker" ] || return 0 - fi route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") fm_lock_acquire_wait "$route_lock" || return 1 - if fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" "$allow_undelivered"; then - rm -f "$FM_PENDING_ROUTE_MARKER" || status=1 + if [ -e "$marker" ] || [ -L "$marker" ]; then + if [ ! -f "$marker" ] || [ -L "$marker" ] \ + || [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" != 5 ] \ + || ! fm_pending_reply_secondmate_route_shape "$marker"; then + status=1 + else + current_corr=$(fm_pending_reply_get "$marker" corr_id) + if [ "$current_corr" != "$corr" ]; then + if ! fm_pending_reply_secondmate_route_validate "$secondmate_home" "" 2; then + status=1 + else + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ ! -e "$history_marker" ] && [ ! -L "$history_marker" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + marker=$history_marker + fi + fi + fi else + marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + fi + if [ "$status" = 0 ] && fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" "$allow_undelivered"; then + rm -f "$FM_PENDING_ROUTE_MARKER" || status=1 + elif [ "$status" = 0 ]; then status=1 fi fm_lock_release "$route_lock" || status=1 @@ -676,6 +699,10 @@ fm_pending_reply_secondmate_route_clear() { # fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 0 } +fm_pending_reply_secondmate_route_clear_reported() { # + fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 4 +} + fm_pending_reply_secondmate_route_clear_undelivered() { # local secondmate_home=$1 corr=$2 marker route_lock current_corr status=0 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 @@ -707,7 +734,7 @@ fm_pending_reply_secondmate_route_clear_undelivered() { # [] [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ] || return 1 fm_pending_reply_secondmate_route_shape "$marker" || return 1 - fm_pending_reply_secondmate_route_validate "$secondmate_home" "" 2 || return 1 + if [ "$allow_undelivered" = 3 ] || [ "$allow_undelivered" = 4 ]; then + fm_pending_reply_secondmate_route_validate "$secondmate_home" "" "$allow_undelivered" || return 1 + else + fm_pending_reply_secondmate_route_validate "$secondmate_home" "" 2 || return 1 + fi current_corr=$FM_PENDING_ROUTE_CORR if [ "$current_corr" != "$wanted_corr" ]; then history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$wanted_corr") @@ -795,47 +826,63 @@ fm_pending_reply_secondmate_route_validate() { # [] fi active_rec="$state_abs/pending-replies/$corr" history_rec="$history_dir/$corr" + rec= if [ -e "$active_rec" ] || [ -L "$active_rec" ]; then [ -f "$active_rec" ] && [ ! -L "$active_rec" ] || return 1 rec=$active_rec elif [ -e "$history_rec" ] || [ -L "$history_rec" ]; then [ -f "$history_rec" ] && [ ! -L "$history_rec" ] || return 1 rec=$history_rec - else + elif [ "$allow_undelivered" != 3 ] && [ "$allow_undelivered" != 4 ]; then + return 1 + fi + if [ -n "$rec" ]; then + [ "$(fm_pending_reply_get "$rec" schema)" = fm-pending-reply.v1 ] || return 1 + record_task=$(fm_pending_reply_get "$rec" task_id) + record_home=$(fm_pending_reply_get "$rec" parent_home) + record_status=$(fm_pending_reply_get "$rec" parent_status) + record_corr=$(fm_pending_reply_get "$rec" corr_id) + [ "$record_task" = "$secondmate_id" ] || return 1 + [ "$record_home" = "$parent_abs" ] || return 1 + [ "$record_status" = "$expected_status" ] || return 1 + [ "$record_corr" = "$corr" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in + awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; + delivery_unknown) + [ "$allow_undelivered" = 2 ] \ + || { [ "$allow_undelivered" = 4 ] && [ -z "$wanted_corr" ]; } \ + || return 1 + ;; + *) return 1 ;; + esac + case "$allow_undelivered" in + 1) + [ -z "$delivered" ] || return 1 + [ "$phase" = awaiting_report ] || return 1 + ;; + 2) + if [ -z "$delivered" ]; then + case "$phase" in awaiting_report|delivery_unknown) ;; *) return 1 ;; esac + fi + ;; + 3) + [ -z "$delivered" ] || return 1 + [ "$phase" = awaiting_report ] || return 1 + ;; + 4) + if [ -n "$wanted_corr" ]; then + [ -n "$delivered" ] || return 1 + fi + ;; + *) + [ -n "$delivered" ] || return 1 + ;; + esac + elif [ "$allow_undelivered" != 3 ] && [ "$allow_undelivered" != 4 ]; then return 1 fi - [ "$(fm_pending_reply_get "$rec" schema)" = fm-pending-reply.v1 ] || return 1 - record_task=$(fm_pending_reply_get "$rec" task_id) - record_home=$(fm_pending_reply_get "$rec" parent_home) - record_status=$(fm_pending_reply_get "$rec" parent_status) - record_corr=$(fm_pending_reply_get "$rec" corr_id) - [ "$record_task" = "$secondmate_id" ] || return 1 - [ "$record_home" = "$parent_abs" ] || return 1 - [ "$record_status" = "$expected_status" ] || return 1 - [ "$record_corr" = "$corr" ] || return 1 - delivered=$(fm_pending_reply_get "$rec" delivered_epoch) - phase=$(fm_pending_reply_get "$rec" phase) - case "$phase" in - awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; - delivery_unknown) - [ "$allow_undelivered" = 2 ] || return 1 - ;; - *) return 1 ;; - esac - case "$allow_undelivered" in - 1) - [ -z "$delivered" ] || return 1 - [ "$phase" = awaiting_report ] || return 1 - ;; - 2) - if [ -z "$delivered" ]; then - case "$phase" in awaiting_report|delivery_unknown) ;; *) return 1 ;; esac - fi - ;; - *) - [ -n "$delivered" ] || return 1 - ;; - esac # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_PARENT_HOME=$parent_abs # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh diff --git a/bin/fm-send.sh b/bin/fm-send.sh index ab26c7a05cb..8893b688a46 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -99,11 +99,13 @@ clear_new_pending_route() { discard_new_pending_reply() { local route_status=0 discard_status=0 - clear_new_pending_route || route_status=1 if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ && ! fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR"; then discard_status=1 fi + if [ "$discard_status" = 0 ]; then + clear_new_pending_route || route_status=1 + fi if [ "$route_status" = 1 ]; then echo "error: failed to clear the secondmate pending-reply route; undelivered record cleanup continued" >&2 fi diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 6a27b5cb016..4f7d8b63c96 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -557,6 +557,33 @@ test_output_started_claim_is_not_reprinted() { pass "output-started inactive claims do not reprint after a drain crash" } +test_presented_claim_is_acknowledged_in_deferred_drain() { + local dir root home fakebin state fingerprint row + new_case presented-claim-ack + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'presented-x1|presented-inc|done|state: done · source: pane · presented claim') + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=presented-x1 \ + incarnation=presented-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · presented claim' kind=ship + row=$'2\t2\tcheck\tinactive-outcome:'"$fingerprint"$'\tpresented claim row' + printf 'schema=fm-inactive-outcome-claim.v1\nfingerprint=%s\nrow=%s\nstate=presented\noutput_started=1\noutput_complete=1\ndefer_ack=0\ncreated_epoch=1\n' \ + "$fingerprint" "$row" > "$state/terminal-outcomes/.$fingerprint.claim" + printf '%s\n' "$row" > "$state/.wake-queue" + if ! FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" \ + drain "$root" "$home" "$fakebin" >"$dir/presented-claim.out"; then + fail "deferred drain did not acknowledge a completed presentation" + fi + [ ! -s "$dir/presented-claim.out" ] || fail "deferred drain re-presented a completed claim" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] \ + || fail "deferred drain did not finalize the completed presentation" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] \ + || fail "deferred drain left the completed presentation claim" + pass "deferred drains recover completed presentations without duplication" +} + test_pre_output_claim_retries_after_crash() { local dir root home fakebin state fingerprint row new_case pre-output-claim @@ -630,7 +657,8 @@ test_deferred_ack_retries_after_caller_crash() { [ ! -s "$dir/live-deferred.out" ] || fail "live deferred claim was presented twice" [ -f "$state/terminal-outcomes/$fingerprint.pending" ] \ || fail "live deferred claim was acknowledged before caller confirmation" - replace_field "$state/terminal-outcomes/.$fingerprint.claim" defer_generation 99999999 + replace_field "$state/terminal-outcomes/.$fingerprint.claim" defer_generation "$$" + replace_field "$state/terminal-outcomes/.$fingerprint.claim" defer_generation_start proc:0 rm -f "$state/deferred-x1.meta" "$state/deferred-x1.status" "$state/deferred-x1.turn-ended" scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "pending deferred receipt was not republished after caller crash" [ "$(queue_count "$state")" = 1 ] || fail "pending deferred receipt did not get a retry wake" @@ -1097,7 +1125,7 @@ test_valid_secondmate_route_reports_parent_once() { cp -a "$ROOT/bin/." "$root/bin/" child_home="$dir/secondmate-home" child_state="$child_home/state" - mkdir -p "$child_state" "$child_home/data" "$child_home/config" \ + mkdir -p "$child_state" "$child_state/terminal-outcomes" "$child_home/data" "$child_home/config" \ "$state/pending-replies" "$state/pending-reply-history" printf 'sm-valid\n' > "$child_home/.fm-secondmate-home" write_meta "$state" sm-valid parent-inc secondmate tmux firstmate:fm-sm-valid @@ -1264,6 +1292,63 @@ SH pass "valid secondmate outcomes use the parent status correlation exactly once" } +test_reported_secondmate_route_repair_after_crash() { + local dir root home fakebin state child_home child_state parent_status corr fingerprint flag + new_case secondmate-reported-route-repair + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + parent_status="$state/sm-reported.status" + corr=0123456789abcdef + mkdir -p "$child_state" "$child_state/terminal-outcomes" "$child_home/data" "$child_home/config" \ + "$state/pending-replies" "$state/pending-reply-history" + printf 'sm-reported\n' > "$child_home/.fm-secondmate-home" + fm_write_meta "$state/pending-replies/$corr" \ + schema=fm-pending-reply.v1 corr_id="$corr" task_id=sm-reported \ + parent_home="$home" parent_status="$parent_status" delivered_epoch=1 phase=awaiting_report + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-reported "$corr" \ + || fail "reported route fixture was not written" + fingerprint=$(receipt_fingerprint 'child-reported-x1|child-reported-inc|failed|state: failed · source: pane · reported crash' secondmate) + fm_write_meta "$child_state/terminal-outcomes/$fingerprint.reported" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=child-reported-x1 \ + incarnation=child-reported-inc outcome=failed terminal_source=pane \ + terminal_snapshot='state: failed · source: pane · reported crash' kind=secondmate \ + parent_task_id=sm-reported parent_home="$home" parent_status="$parent_status" parent_corr="$corr" + flag="$dir/fail-route-clear-once" + : > "$flag" + cat > "$fakebin/rm" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + if [ "$arg" = "${FM_TEST_ROUTE_RM_PATH:-}" ] && [ -e "${FM_TEST_ROUTE_RM_ONCE:-}" ]; then + /bin/rm -f "$FM_TEST_ROUTE_RM_ONCE" + exit 42 + fi +done +exec /bin/rm "$@" +SH + chmod +x "$fakebin/rm" + export FM_TEST_ROUTE_RM_PATH="$child_state/.fm-jt-parent-route" FM_TEST_ROUTE_RM_ONCE="$flag" + if scan "$root" "$child_home" "$fakebin" --startup >/dev/null 2>&1; then + fail "reported route cleanup failure was hidden" + fi + [ -f "$child_state/terminal-outcomes/$fingerprint.reported" ] \ + || fail "reported receipt was lost during route cleanup failure" + [ -e "$child_state/.fm-jt-parent-route" ] \ + || fail "route was removed before reported cleanup completed" + rm -f "$fakebin/rm" + unset FM_TEST_ROUTE_RM_PATH FM_TEST_ROUTE_RM_ONCE + scan "$root" "$child_home" "$fakebin" --startup >/dev/null \ + || fail "reported route cleanup did not retry" + [ ! -e "$child_state/.fm-jt-parent-route" ] \ + || fail "reported route remained after retry" + pass "reported secondmate routes recover after receipt finalization crashes" +} + test_secondmate_route_replacement_preserves_old_receipt() { local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker history_backup active_route_backup new_case secondmate-route-replacement @@ -1368,7 +1453,7 @@ SH } test_undelivered_secondmate_route_cleanup_is_idempotent() { - local dir root home fakebin state child_home child_state marker corr + local dir root home fakebin state child_home child_state marker corr rec fail_rm_once new_case secondmate-undelivered-cleanup dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -1387,11 +1472,43 @@ test_undelivered_secondmate_route_cleanup_is_idempotent() { _ "$ROOT" "$child_home" "$home" "$state" sm-cleanup "$corr" \ || fail "undelivered route fixture was not written" [ -f "$marker" ] || fail "undelivered route fixture is missing" + rec="$state/pending-replies/$corr" + fail_rm_once="$dir/fail-undelivered-rm-once" + : > "$fail_rm_once" + cat > "$fakebin/rm" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + if [ "$arg" = "${FM_TEST_UNDELIVERED_RECORD:-}" ] && [ -e "${FM_TEST_UNDELIVERED_RM_ONCE:-}" ]; then + /bin/rm -f "$FM_TEST_UNDELIVERED_RM_ONCE" + exit 42 + fi +done +exec /bin/rm "$@" +SH + chmod +x "$fakebin/rm" + export FM_TEST_UNDELIVERED_RECORD="$rec" FM_TEST_UNDELIVERED_RM_ONCE="$fail_rm_once" + if env PATH="$fakebin:$PATH" FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_discard_undelivered "$2" "$3"' \ + _ "$ROOT" "$state" "$corr"; then + fail "undelivered record removal failure was hidden" + fi + [ -e "$rec" ] || fail "undelivered record was removed after a failed cleanup" + [ -e "$marker" ] || fail "undelivered route was cleared before record removal" + rm -f "$fakebin/rm" + unset FM_TEST_UNDELIVERED_RECORD FM_TEST_UNDELIVERED_RM_ONCE + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_discard_undelivered "$2" "$3"' \ + _ "$ROOT" "$state" "$corr" \ + || fail "undelivered record cleanup did not retry" + [ ! -e "$rec" ] || fail "undelivered record remained after retry" env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ FM_STATE_OVERRIDE="$state" bash -c \ '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_clear_undelivered "$2" "$3"' \ _ "$ROOT" "$child_home" "$corr" \ - || fail "undelivered route cleanup rejected an awaiting report" + || fail "undelivered route cleanup rejected an already-removed record" [ ! -e "$marker" ] || fail "undelivered route cleanup left a stale marker" env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ FM_STATE_OVERRIDE="$state" bash -c \ @@ -1650,6 +1767,7 @@ test_presenting_claim_recovers_before_output test_output_started_claim_is_not_reprinted test_pre_output_claim_retries_after_crash test_finalized_receipt_rows_are_suppressed +test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories @@ -1665,6 +1783,7 @@ test_herdr_identity_and_default_captain_refusal test_occupancy_unknown_is_not_terminal test_status_log_terminal_is_not_replayed test_valid_secondmate_route_reports_parent_once +test_reported_secondmate_route_repair_after_crash test_secondmate_route_replacement_preserves_old_receipt test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected From 99c17890a730ca42c5c78660d4c161fa453aa88e Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 10:49:33 +0000 Subject: [PATCH 020/163] no-mistakes(review): Harden inactive presentation and secondmate route validation --- bin/fm-pending-reply-lib.sh | 54 +++------------- bin/fm-session-start.sh | 23 ++++--- bin/fm-wake-drain.sh | 30 +++++++-- bin/fm-watch.sh | 2 +- tests/fm-inactive-outcome.test.sh | 100 +++++++++++++++++++++++++++++- 5 files changed, 148 insertions(+), 61 deletions(-) diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 344a3a2d161..953fb9a281a 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -519,8 +519,7 @@ fm_pending_reply_secondmate_route_write() { # < local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 local marker home_marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 local existing_schema existing_id existing_home existing_status existing_corr - local existing_state existing_record existing_phase existing_active_dir existing_history_dir history_marker - local existing_active_record existing_history_record + local history_marker marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 @@ -583,52 +582,16 @@ fm_pending_reply_secondmate_route_write() { # < || [ "$existing_id" != "$secondmate_id" ]; then route_status=1 fi - case "$existing_home" in /*) ;; *) route_status=1 ;; esac - case "$existing_status" in /*) ;; *) route_status=1 ;; esac - printf '%s' "$existing_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || route_status=1 if [ "$route_status" = 0 ]; then - [ -d "$existing_home" ] && [ ! -L "$existing_home" ] || route_status=1 - [ -d "$existing_home/state" ] && [ ! -L "$existing_home/state" ] || route_status=1 - fi - if [ "$route_status" = 0 ]; then - existing_state=$(cd "$existing_home/state" 2>/dev/null && pwd -P) || route_status=1 - fi - if [ "$route_status" = 0 ] \ - && [ "$existing_status" != "$existing_state/$existing_id.status" ]; then - route_status=1 - fi - if [ "$route_status" = 0 ] \ - && [ -L "$existing_state/$existing_id.status" ]; then - route_status=1 - fi - if [ "$route_status" = 0 ]; then - existing_active_dir=$(fm_pending_reply_dir "$existing_state") - [ -d "$existing_active_dir" ] && [ ! -L "$existing_active_dir" ] || route_status=1 - existing_history_dir=$(fm_pending_reply_history_dir "$existing_state") - if [ -e "$existing_history_dir" ] || [ -L "$existing_history_dir" ]; then - [ -d "$existing_history_dir" ] && [ ! -L "$existing_history_dir" ] || route_status=1 - fi - fi - if [ "$route_status" = 0 ]; then - existing_active_record="$existing_active_dir/$existing_corr" - existing_history_record="$existing_history_dir/$existing_corr" - if [ -e "$existing_active_record" ] || [ -L "$existing_active_record" ]; then - [ -f "$existing_active_record" ] && [ ! -L "$existing_active_record" ] || route_status=1 - [ "$route_status" = 0 ] && existing_record="$existing_active_record" - elif [ -e "$existing_history_record" ] || [ -L "$existing_history_record" ]; then - [ -f "$existing_history_record" ] && [ ! -L "$existing_history_record" ] || route_status=1 - [ "$route_status" = 0 ] && existing_record="$existing_history_record" - else - route_status=1 + fm_pending_reply_secondmate_route_validate "$secondmate_home" "$existing_corr" \ + || route_status=1 + if [ "$route_status" = 0 ]; then + case "$FM_PENDING_ROUTE_PHASE" in + resolved|retired) ;; + *) route_status=1 ;; + esac fi fi - if [ "$route_status" = 0 ]; then - existing_phase=$(fm_pending_reply_get "$existing_record" phase) - case "$existing_phase" in - resolved|retired) ;; - *) route_status=1 ;; - esac - fi if [ "$route_status" = 0 ] && [ "$existing_corr" != "$corr" ]; then history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$existing_corr") [ ! -L "$history_marker" ] || route_status=1 @@ -891,6 +854,7 @@ fm_pending_reply_secondmate_route_validate() { # [] FM_PENDING_ROUTE_CORR=$corr # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh FM_PENDING_ROUTE_SECOND_MATE_ID=$secondmate_id + FM_PENDING_ROUTE_PHASE=${phase:-} FM_PENDING_ROUTE_MARKER=$marker return 0 } diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 9d5d0ad8578..26f24002232 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -369,15 +369,20 @@ else printf 'error: wake drain failed (status %s); inactive reconciliation skipped\n%s\n' \ "$DRAIN_STATUS" "$DRAIN_OUT" >&2 elif [ -n "$DRAIN_OUT" ]; then - printf '%s\n' "$DRAIN_OUT" - while IFS= read -r drain_row || [ -n "$drain_row" ]; do - IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" - case "$_key" in - inactive-outcome:*) - "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$drain_row" >/dev/null 2>&1 || true - ;; - esac - done <<< "$DRAIN_OUT" + if printf '%s\n' "$DRAIN_OUT"; then + while IFS= read -r drain_row || [ -n "$drain_row" ]; do + IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" + case "$_key" in + inactive-outcome:*) + if ! "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$drain_row" >/dev/null 2>&1; then + printf 'warning: inactive outcome confirmation deferred for %s\n' "$_key" >&2 + fi + ;; + esac + done <<< "$DRAIN_OUT" + else + printf 'error: wake presentation failed; inactive outcome confirmations deferred\n' >&2 + fi else printf '(no queued wakes)\n' fi diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 8b2fdea44f1..3ce324a3d82 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -16,11 +16,14 @@ DRAIN_PID=${BASHPID:-$$} DRAIN_LOCK_HELD=false present_inactive_row() { - local key=$1 row=$2 status=0 go worker worker_status=0 + local key=$1 row=$2 status=0 go emitted worker worker_status=0 presentation_marked=0 trap - INT TERM HUP go=$(mktemp "$STATE/.wake-presentation.XXXXXX") || return 1 [ -f "$go" ] && [ ! -L "$go" ] || { rm -f "$go"; return 1; } rm -f "$go" + emitted=$(mktemp "$STATE/.wake-emitted.XXXXXX") || { rm -f "$go"; return 1; } + [ -f "$emitted" ] && [ ! -L "$emitted" ] || { rm -f "$go" "$emitted"; return 1; } + rm -f "$emitted" ( while [ ! -e "$go" ]; do if ! kill -0 "$DRAIN_PID" 2>/dev/null; then @@ -29,6 +32,7 @@ present_inactive_row() { sleep 0.01 done printf '%s\n' "$row" || exit 1 + : > "$emitted" || exit 1 FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ output-complete "$key" "$row" || exit 1 @@ -44,12 +48,28 @@ present_inactive_row() { fi rm -f "$go" if [ "$status" -ne 0 ]; then - FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - presenting "$key" "$row" >/dev/null 2>&1 || true + if [ -e "$emitted" ]; then + if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + presented "$key" "$row" >/dev/null 2>&1; then + presentation_marked=1 + status=0 + fi + else + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + presenting "$key" "$row" >/dev/null 2>&1 || true + fi fi - if [ "$status" = 0 ] && ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$key" "$row"; then - status=1 + if [ "$status" = 0 ] && [ "$presentation_marked" = 0 ]; then + if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$key" "$row"; then + status=1 + if [ -e "$emitted" ] \ + && FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + presented "$key" "$row" >/dev/null 2>&1; then + status=0 + fi + fi fi + rm -f "$emitted" trap 'exit 130' INT trap 'exit 143' TERM return "$status" diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index c3440c4ca57..45fadb1955d 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -344,7 +344,7 @@ wake() { heartbeat*) echo $(( $(cat "$STATE/.heartbeat-streak" 2>/dev/null || echo 0) + 1 )) > "$STATE/.heartbeat-streak" ;; *) echo 0 > "$STATE/.heartbeat-streak" ;; esac - echo "$wake_output" + printf '%s\n' "$wake_output" || exit 1 while IFS= read -r row || [ -n "$row" ]; do IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$row" case "$_key" in diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 4f7d8b63c96..e649d2fcc73 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -608,6 +608,51 @@ test_pre_output_claim_retries_after_crash() { pass "pre-output claims retry until emission completes" } +test_output_completion_failure_does_not_reprint() { + local dir root home fakebin state fingerprint row + new_case output-completion-failure + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'output-failure-x1|output-failure-inc|done|state: done · source: pane · output failure' ) + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=output-failure-x1 \ + incarnation=output-failure-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · output failure' kind=ship + row=$'2\t2\tcheck\tinactive-outcome:'"$fingerprint"$'\toutput completion failure row' + printf '%s\n' "$row" > "$state/.wake-queue" + cat > "$fakebin/mv" <<'SH' +#!/usr/bin/env bash +set -u +target="${!#}" +case "$target" in + *.claim) + count=$(cat "${FM_FAIL_CLAIM_MOVE:?}" 2>/dev/null || printf '0') + count=$((count + 1)) + printf '%s\n' "$count" > "$FM_FAIL_CLAIM_MOVE" + if [ "$count" = 2 ]; then + exit 91 + fi + ;; +esac +exec /usr/bin/mv "$@" +SH + chmod +x "$fakebin/mv" + export FM_FAIL_CLAIM_MOVE="$dir/fail-claim-move" + drain "$root" "$home" "$fakebin" >"$dir/output-failure.out" \ + || fail "output completion failure was not recovered" + [ "$(grep -Fxc "$row" "$dir/output-failure.out")" = 1 ] \ + || fail "output completion failure reprinted or lost the emitted row" + [ "$(receipt_count "$state" presented)" = 1 ] \ + || fail "output completion failure did not finalize the receipt" + [ "$(receipt_count "$state" pending)" = 0 ] \ + || fail "output completion failure left the receipt pending" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] \ + || fail "output completion failure left a stale claim" + unset FM_FAIL_CLAIM_MOVE + pass "post-output failures finalize without duplicate presentation" +} + test_finalized_receipt_rows_are_suppressed() { local dir root home fakebin state fingerprint row new_case finalized-row @@ -947,8 +992,24 @@ SH || fail "watcher did not surface the inactive reconciliation result" [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher cadence did not create the inactive receipt" [ "$(queue_count "$state")" = 1 ] || fail "watcher cadence did not retain exactly one inactive outcome wake" + set +e + ( cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_INACTIVE_OUTCOME_SECS=60 \ + FM_INACTIVE_OUTCOME_BUDGET_SECS=10 FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_FAKE_PANE_PATH="$home" FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + FM_WATCHER_HEARTBEAT=999999 "$root/bin/fm-watch.sh" ) | head -n 0 + status=${PIPESTATUS[0]} + set -u + [ "$status" -ne 0 ] || fail "watcher treated a broken presentation pipe as success" + [ "$(receipt_count "$state" pending)" = 1 ] \ + || fail "watcher consumed the inactive receipt after presentation failure" + [ -e "$state/terminal-outcomes/.$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending).claim" ] \ + || fail "watcher did not retain a retryable presentation claim" unset FM_FAKE_CREW_STATE_WATCHER_X1 - pass "watcher cadence runs inactive reconciliation and surfaces its wake" + pass "watcher cadence gates acknowledgement on successful output" } test_legacy_metadata_uses_stable_fallback() { @@ -1632,6 +1693,41 @@ SH pass "concurrent secondmate routes fail closed without overwriting" } +test_route_replacement_rejects_malformed_parent_record() { + local dir root home fakebin state child_home child_state marker corr_one corr_two + new_case malformed-route-replacement + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + marker="$child_state/.fm-jt-parent-route" + corr_one=0123456789abcdef + corr_two=1123456789abcdef + mkdir -p "$child_state" "$child_home/data" "$child_home/config" \ + "$state/pending-replies" + printf 'sm-malformed\n' > "$child_home/.fm-secondmate-home" + fm_write_meta "$state/pending-replies/$corr_one" \ + schema=fm-pending-reply.v1 corr_id="$corr_one" task_id=sm-malformed \ + parent_home="$home" parent_status="$state/sm-malformed.status" \ + delivered_epoch=1 phase=resolved + route_write() { + env PATH="$fakebin:$PATH" FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" \ + FM_HOME="$home" FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-malformed "$1" + } + route_write "$corr_one" || fail "malformed replacement fixture route was not written" + replace_field "$state/pending-replies/$corr_one" task_id wrong-task + if route_write "$corr_two"; then + fail "route replacement accepted a mismatched existing parent record" + fi + [ "$(receipt_value "$marker" corr_id)" = "$corr_one" ] \ + || fail "malformed parent record replacement changed the active route" + [ ! -e "$child_state/.fm-jt-parent-route-history.$corr_one" ] \ + || fail "malformed parent record replacement archived an invalid route" + pass "route replacement validates the complete existing parent record" +} + test_failed_concurrent_send_discards_only_new_record() { local dir root home fakebin state child_home child_state marker old_corr send_out new_case failed-concurrent-send @@ -1766,6 +1862,7 @@ test_reserved_claim_recovers_to_a_new_wake_row test_presenting_claim_recovers_before_output test_output_started_claim_is_not_reprinted test_pre_output_claim_retries_after_crash +test_output_completion_failure_does_not_reprint test_finalized_receipt_rows_are_suppressed test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash @@ -1787,6 +1884,7 @@ test_reported_secondmate_route_repair_after_crash test_secondmate_route_replacement_preserves_old_receipt test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected +test_route_replacement_rejects_malformed_parent_record test_failed_concurrent_send_discards_only_new_record test_drain_restores_only_unprocessed_rows test_malformed_or_missing_secondmate_route_fails_closed From 4ca554c01327fb5c64a186e080466bd46725d756 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 11:26:35 +0000 Subject: [PATCH 021/163] no-mistakes(review): Harden post-output claims and durable route cleanup --- bin/fm-inactive-reconcile.sh | 20 ++-- bin/fm-pending-reply-lib.sh | 155 ++++++++++++++++++++++++++---- bin/fm-send.sh | 9 +- tests/fm-inactive-outcome.test.sh | 117 +++++++++++++++++++++- 4 files changed, 267 insertions(+), 34 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 0bad9f24b70..d75d5bc0ff4 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -297,6 +297,15 @@ claim_reserve() { # case "$state" in presented|presenting|reserved) ;; *) return 2 ;; esac old_row=$(claim_field "$claim" row) [ -n "$old_row" ] || return 2 + if [ "$old_row" != "$row" ]; then + tmp=$(mktemp "$OUTCOME_DIR/.claim-row.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in row=*) printf 'row=%s\n' "$row" ;; *) printf '%s\n' "$line" ;; esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } + fi if [ "$state" = presented ]; then defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) if [ "$defer_ack" = 1 ]; then @@ -305,17 +314,10 @@ claim_reserve() { # if claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then return 4 fi + [ "$(claim_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + return 5 fi fi - if [ "$old_row" != "$row" ]; then - tmp=$(mktemp "$OUTCOME_DIR/.claim-row.XXXXXX") || return 2 - chmod 600 "$tmp" 2>/dev/null || true - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in row=*) printf 'row=%s\n' "$row" ;; *) printf '%s\n' "$line" ;; esac - done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } - [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } - mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } - fi if [ "$recorded_report" = 1 ]; then case "$state" in presented) return 5 ;; diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 953fb9a281a..4f690b5fbf6 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -481,6 +481,75 @@ fm_pending_reply_get() { # grep "^${key}=" "$rec" 2>/dev/null | tail -1 | cut -d= -f2- || true } +fm_pending_reply_record_validate() { # + local rec=$1 state=$2 wanted_corr=$3 wanted_task=$4 + local record_corr record_task parent_home parent_status parent_abs state_abs expected_status delivered phase + [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["corr_id"]=1; allowed["task_id"]=1 + allowed["parent_home"]=1; allowed["parent_status"]=1 + allowed["parent_status_scan_signature"]=1; allowed["request_summary"]=1 + allowed["created_epoch"]=1; allowed["delivered_epoch"]=1; allowed["phase"]=1 + allowed["turn_seen_busy"]=1; allowed["request_turn_completed_epoch"]=1 + allowed["recovery_attempted_epoch"]=1; allowed["recovery_sender_pid"]=1 + allowed["recovery_sender_identity"]=1; allowed["recovery_sent_epoch"]=1 + allowed["recovery_delivery_outcome"]=1; allowed["recovery_turn_seen_busy"]=1 + allowed["recovery_turn_completed_epoch"]=1; allowed["escalated_epoch"]=1 + allowed["resolved_epoch"]=1; allowed["resolved_via"]=1; allowed["retired_epoch"]=1 + allowed["retired_via"]=1; allowed["retired_from"]=1 + allowed["retirement_staged_epoch"]=1; allowed["retirement_history_state"]=1 + allowed["retirement_staged_from"]=1; allowed["retirement_source_state"]=1 + allowed["wrong_home_hits"]=1; allowed["wrong_home_sightings"]=1 + allowed["wrong_home_scan_signature"]=1; allowed["grace_secs"]=1 + required["schema"]=1; required["corr_id"]=1; required["task_id"]=1 + required["parent_home"]=1; required["parent_status"]=1 + required["delivered_epoch"]=1; required["phase"]=1 + valid=1 + } + /^[^=]+=/{ + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + next + } + { valid=0 } + END { + for (key in required) if (!(key in seen)) valid=0 + exit !valid + } + ' "$rec" 2>/dev/null || return 1 + printf '%s' "$wanted_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + state_abs=$(cd "$state" 2>/dev/null && pwd -P) || return 1 + record_corr=$(fm_pending_reply_get "$rec" corr_id) + record_task=$(fm_pending_reply_get "$rec" task_id) + [ "$record_corr" = "$wanted_corr" ] || return 1 + [ "$record_task" = "$wanted_task" ] || return 1 + parent_home=$(fm_pending_reply_get "$rec" parent_home) + parent_status=$(fm_pending_reply_get "$rec" parent_status) + case "$parent_home:$parent_status" in /*:/*) ;; *) return 1 ;; esac + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + [ -d "$parent_abs/state" ] && [ ! -L "$parent_abs/state" ] || return 1 + [ "$parent_abs/state" = "$state_abs" ] || return 1 + [ -d "$state_abs/pending-replies" ] && [ ! -L "$state_abs/pending-replies" ] || return 1 + expected_status="$state_abs/$wanted_task.status" + [ "$parent_status" = "$expected_status" ] || return 1 + [ ! -L "$expected_status" ] || return 1 + if [ -e "$expected_status" ]; then + [ -f "$expected_status" ] || return 1 + fi + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + case "$delivered" in *[!0-9]*) return 1 ;; esac + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in + awaiting_report|delivery_unknown|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; + *) return 1 ;; + esac + return 0 +} + # JT secondmate outcome routing uses the existing parent-owned pending-reply # record and its corr= acknowledgement grammar. This small marker only binds a # secondmate home to that already-owned record; it is not a second parent @@ -518,7 +587,7 @@ fm_pending_reply_secondmate_route_shape() { # fm_pending_reply_secondmate_route_write() { # local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 local marker home_marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 - local existing_schema existing_id existing_home existing_status existing_corr + local existing_schema existing_id existing_home existing_status existing_corr existing_rec existing_delivered route_mode local history_marker marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 @@ -574,9 +643,21 @@ fm_pending_reply_secondmate_route_write() { # < && [ "$existing_home" = "$parent_abs" ] \ && [ "$existing_status" = "$status_path" ] \ && [ "$existing_corr" = "$corr" ]; then - rm -f "$tmp" - fm_lock_release "$route_lock" || return 1 - return 0 + existing_rec=$(fm_pending_reply_active_path "$state_abs" "$existing_corr") + if ! fm_pending_reply_record_validate "$existing_rec" "$state_abs" "$existing_corr" "$secondmate_id"; then + route_status=1 + else + existing_delivered=$(fm_pending_reply_get "$existing_rec" delivered_epoch) + route_mode=3 + [ -n "$existing_delivered" ] && route_mode=4 + fm_pending_reply_secondmate_route_validate "$secondmate_home" "$existing_corr" "$route_mode" \ + || route_status=1 + fi + if [ "$route_status" = 0 ]; then + rm -f "$tmp" + fm_lock_release "$route_lock" || return 1 + return 0 + fi fi if [ "$existing_schema" != fm-jt-parent-route.v1 ] \ || [ "$existing_id" != "$secondmate_id" ]; then @@ -800,15 +881,7 @@ fm_pending_reply_secondmate_route_validate() { # [] return 1 fi if [ -n "$rec" ]; then - [ "$(fm_pending_reply_get "$rec" schema)" = fm-pending-reply.v1 ] || return 1 - record_task=$(fm_pending_reply_get "$rec" task_id) - record_home=$(fm_pending_reply_get "$rec" parent_home) - record_status=$(fm_pending_reply_get "$rec" parent_status) - record_corr=$(fm_pending_reply_get "$rec" corr_id) - [ "$record_task" = "$secondmate_id" ] || return 1 - [ "$record_home" = "$parent_abs" ] || return 1 - [ "$record_status" = "$expected_status" ] || return 1 - [ "$record_corr" = "$corr" ] || return 1 + fm_pending_reply_record_validate "$rec" "$state_abs" "$corr" "$secondmate_id" || return 1 delivered=$(fm_pending_reply_get "$rec" delivered_epoch) phase=$(fm_pending_reply_get "$rec" phase) case "$phase" in @@ -920,11 +993,12 @@ fm_pending_reply_secondmate_receipt_validate() { # - local state=$1 corr=$2 task_id=$3 rec phase + local state=$1 corr=$2 task_id=$3 rec phase delivered printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 rec=$(fm_pending_reply_active_path "$state" "$corr") - [ -f "$rec" ] || return 1 - [ "$(fm_pending_reply_get "$rec" task_id)" = "$task_id" ] || return 1 + fm_pending_reply_record_validate "$rec" "$state" "$corr" "$task_id" || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -n "$delivered" ] || return 1 phase=$(fm_pending_reply_get "$rec" phase) case "$phase" in awaiting_report|recovery_sending|recovery_sent) return 0 ;; @@ -1189,14 +1263,57 @@ fm_pending_reply_reconcile_delivery() { # # Drop an undelivered expectation after a failed send so transport failure does # not masquerade as a missed report later. fm_pending_reply_discard_undelivered() { # - local state=$1 corr=$2 rec delivered marker + local state=$1 corr=$2 keep_backup=${3:-0} rec delivered marker backup + case "$keep_backup" in 0|1) ;; *) return 1 ;; esac rec=$(fm_pending_reply_path "$state" "$corr") - [ -f "$rec" ] || return 0 + backup="${rec}.cleanup" + [ ! -L "$rec" ] && [ ! -L "$backup" ] || return 1 + if [ ! -e "$rec" ]; then + if [ "$keep_backup" = 0 ] && [ -e "$backup" ]; then + rm -f "$backup" || return 1 + fi + return 0 + fi + [ -f "$rec" ] || return 1 + if [ -e "$backup" ]; then + [ -f "$backup" ] || return 1 + cmp -s "$rec" "$backup" || return 1 + rm -f "$backup" || return 1 + fi delivered=$(fm_pending_reply_get "$rec" delivered_epoch) [ -z "$delivered" ] || return 1 marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") rm -f "$marker" 2>/dev/null || true - rm -f "$rec" + if [ "$keep_backup" = 1 ]; then + ln "$rec" "$backup" || return 1 + fi + rm -f "$rec" || return 1 + [ "$keep_backup" = 1 ] || rm -f "$backup" +} + +fm_pending_reply_restore_undelivered() { # + local state=$1 corr=$2 rec backup + rec=$(fm_pending_reply_path "$state" "$corr") + backup="${rec}.cleanup" + [ ! -L "$rec" ] && [ ! -L "$backup" ] || return 1 + if [ -e "$rec" ]; then + [ -f "$rec" ] && [ -f "$backup" ] || return 1 + cmp -s "$rec" "$backup" || return 1 + rm -f "$backup" + return $? + fi + [ -f "$backup" ] || return 1 + mv "$backup" "$rec" +} + +fm_pending_reply_finish_undelivered() { # + local state=$1 corr=$2 rec backup + rec=$(fm_pending_reply_path "$state" "$corr") + backup="${rec}.cleanup" + [ ! -L "$backup" ] || return 1 + [ -e "$backup" ] || return 0 + [ -f "$backup" ] || return 1 + rm -f "$backup" } # 0 if a status line is a correlated acknowledgement for . diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 8893b688a46..70cd6af5bdc 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -100,11 +100,16 @@ clear_new_pending_route() { discard_new_pending_reply() { local route_status=0 discard_status=0 if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ - && ! fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR"; then + && ! fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" 1; then discard_status=1 fi if [ "$discard_status" = 0 ]; then - clear_new_pending_route || route_status=1 + if clear_new_pending_route; then + fm_pending_reply_finish_undelivered "$STATE" "$PENDING_REPLY_CORR" || route_status=1 + else + route_status=1 + fm_pending_reply_restore_undelivered "$STATE" "$PENDING_REPLY_CORR" || discard_status=1 + fi fi if [ "$route_status" = 1 ]; then echo "error: failed to clear the secondmate pending-reply route; undelivered record cleanup continued" >&2 diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index e649d2fcc73..57a0da0bdea 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -709,14 +709,13 @@ test_deferred_ack_retries_after_caller_crash() { [ "$(queue_count "$state")" = 1 ] || fail "pending deferred receipt did not get a retry wake" drain_output="$dir/retry.out" drain "$root" "$home" "$fakebin" >"$drain_output" \ - || fail "retry drain did not recover the deferred presentation" + || fail "retry drain did not recover the deferred acknowledgement" [ -f "$state/terminal-outcomes/$fingerprint.presented" ] || fail "retry drain did not acknowledge the recovered receipt" [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] || fail "retry drain left the receipt pending" [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "retry drain left the deferred claim" - grep -F 'task=deferred-x1' "$drain_output" >/dev/null \ - || fail "retry drain did not re-present the deferred row" + [ ! -s "$drain_output" ] || fail "retry drain re-presented output already emitted before the caller crash" unset FM_FAKE_CREW_STATE_DEFERRED_X1 - pass "deferred inactive receipts remain retryable until caller confirmation" + pass "deferred inactive receipts acknowledge stale post-output claims without replay" } test_scan_failure_retries_without_advancing_cadence() { @@ -1728,6 +1727,114 @@ test_route_replacement_rejects_malformed_parent_record() { pass "route replacement validates the complete existing parent record" } +test_recovery_route_reuse_validates_parent_record() { + local dir root home fakebin state child_home child_state marker corr + new_case recovery-route-reuse + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + marker="$child_state/.fm-jt-parent-route" + corr=0123456789abcdef + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$state/pending-replies" + printf 'sm-reuse\n' > "$child_home/.fm-secondmate-home" + fm_write_meta "$state/pending-replies/$corr" \ + schema=fm-pending-reply.v1 corr_id="$corr" task_id=sm-reuse \ + parent_home="$home" parent_status="$state/sm-reuse.status" \ + delivered_epoch=1 phase=recovery_sending + route_write() { + env PATH="$fakebin:$PATH" FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" \ + FM_HOME="$home" FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" "$5" "$6"' \ + _ "$ROOT" "$child_home" "$home" "$state" sm-reuse "$1" + } + route_write "$corr" || fail "recovery route fixture was not written" + marker_before=$(cat "$marker") + replace_field "$state/pending-replies/$corr" task_id wrong-task + if route_write "$corr"; then + fail "recovery route reuse accepted a mismatched parent record" + fi + [ "$(cat "$marker")" = "$marker_before" ] || fail "mismatched recovery route reuse changed the marker" + if env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_corr_reusable "$2" "$3" sm-reuse' \ + _ "$ROOT" "$state" "$corr"; then + fail "recovery resend reused a mismatched parent record" + fi + replace_field "$state/pending-replies/$corr" task_id sm-reuse + printf 'task_id=sm-reuse\n' >> "$state/pending-replies/$corr" + if route_write "$corr"; then + fail "recovery route reuse accepted duplicate parent fields" + fi + if env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_corr_reusable "$2" "$3" sm-reuse' \ + _ "$ROOT" "$state" "$corr"; then + fail "recovery resend reused duplicate parent fields" + fi + pass "recovery route reuse validates the complete parent record" +} + +test_failed_marked_send_restores_record_on_route_cleanup_failure() { + local dir root home fakebin state child_home child_state marker flag rec corr send_out + new_case failed-marked-cleanup + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + marker="$child_state/.fm-jt-parent-route" + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$child_home/projects" + cp -a "$ROOT/bin/." "$root/bin/" + printf 'sm-cleanup\n' > "$child_home/.fm-secondmate-home" + write_meta "$state" sm-cleanup cleanup-inc secondmate tmux firstmate:fm-sm-cleanup + printf 'home=%s\n' "$child_home" >> "$state/sm-cleanup.meta" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u +exit 1 +SH + chmod +x "$fakebin/tmux" + flag="$dir/fail-route-clear-once" + : > "$flag" + cat > "$fakebin/rm" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + if [ "$arg" = "${FM_TEST_CLEANUP_ROUTE:-}" ] && [ -e "${FM_TEST_CLEANUP_ROUTE_ONCE:-}" ]; then + /bin/rm -f "$FM_TEST_CLEANUP_ROUTE_ONCE" + exit 42 + fi +done +exec /bin/rm "$@" +SH + chmod +x "$fakebin/rm" + prepare_primary_proof "$root" "$home" "$fakebin" + prepare_watcher_protocol "$root" "$home" "$state" + export FM_TEST_CLEANUP_ROUTE="$marker" FM_TEST_CLEANUP_ROUTE_ONCE="$flag" + send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK \ + -u FM_AGENT_OWNER_HOME -u FM_ROOT -u STATE -u FM_PENDING_REPLY_EXISTING_CORR \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ + fm-sm-cleanup "cleanup request" 2>&1) && fail "marked send cleanup failure was hidden" + rec=$(direct_first_file "$state/pending-replies" '*') + [ -f "$rec" ] || fail "route cleanup failure orphaned the parent expectation" + corr=$(basename "$rec") + [ "$(receipt_value "$marker" corr_id)" = "$corr" ] || fail "route cleanup failure changed the active route" + [ ! -e "$rec.cleanup" ] || fail "route cleanup failure left a hidden record backup" + rm -f "$fakebin/rm" + unset FM_TEST_CLEANUP_ROUTE FM_TEST_CLEANUP_ROUTE_ONCE + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_discard_undelivered "$2" "$3" 1 && fm_pending_reply_secondmate_route_clear_undelivered "$4" "$3" && fm_pending_reply_finish_undelivered "$2" "$3"' \ + _ "$ROOT" "$state" "$corr" "$child_home" \ + || fail "marked send cleanup did not retry transactionally" + [ ! -e "$rec" ] || fail "marked send cleanup retry left the parent record" + [ ! -e "$marker" ] || fail "marked send cleanup retry left the route" + pass "failed marked sends restore parent records across route cleanup failures" +} + test_failed_concurrent_send_discards_only_new_record() { local dir root home fakebin state child_home child_state marker old_corr send_out new_case failed-concurrent-send @@ -1885,6 +1992,8 @@ test_secondmate_route_replacement_preserves_old_receipt test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected test_route_replacement_rejects_malformed_parent_record +test_recovery_route_reuse_validates_parent_record +test_failed_marked_send_restores_record_on_route_cleanup_failure test_failed_concurrent_send_discards_only_new_record test_drain_restores_only_unprocessed_rows test_malformed_or_missing_secondmate_route_fails_closed From 2ed56764329bb28a9bf134397de76845ff0194eb Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 12:17:55 +0000 Subject: [PATCH 022/163] no-mistakes(review): Harden deferred receipts and secondmate cleanup --- bin/fm-inactive-reconcile.sh | 207 +++++++++++++++++++++++------- bin/fm-pending-reply-lib.sh | 38 +++--- bin/fm-wake-drain.sh | 17 ++- tests/fm-inactive-outcome.test.sh | 48 ++++++- 4 files changed, 237 insertions(+), 73 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index d75d5bc0ff4..7fab8ac61f4 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -281,6 +281,7 @@ claim_reserve() { # pending) ;; *) return 2 ;; esac + prepare_pending_receipt "$(receipt_path "$fp" pending)" || return 2 pending_secondmate_report_recorded "$fp" >/dev/null || report_rc=$? case "$report_rc" in 0) recorded_report=1 ;; @@ -335,6 +336,18 @@ claim_reserve() { # claim_mark_presented "$key" "$row" || return 2 return 5 fi + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$(claim_field "$claim" output_started 2>/dev/null || true)" = 1 ] \ + && [ "$defer_ack" = 1 ]; then + defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) + defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + if claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then + return 4 + fi + claim_mark_output_complete "$key" "$row" || return 2 + claim_mark_presented "$key" "$row" || return 2 + return 5 + fi fi if [ "$state" = presented ]; then defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) @@ -386,8 +399,15 @@ claim_reserve() { # } claim_mark_presenting() { # - local key=$1 row=$2 fp claim state tmp line seen_pid=0 seen_output=0 seen_complete=0 + local key=$1 row=$2 fp claim state tmp line defer_ack=0 defer_generation= defer_generation_start= + local seen_pid=0 seen_output=0 seen_complete=0 local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 + [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 + if [ "$defer_ack" = 1 ]; then + defer_generation=${FM_WAKE_DRAIN_GENERATION:-} + case "$defer_generation" in ''|*[!0-9]*|0) return 2 ;; esac + defer_generation_start=$(fm_pid_start "$defer_generation") || return 2 + fi drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -406,18 +426,18 @@ claim_mark_presenting() { # presentation_pid=*) printf 'presentation_pid=%s\n' "${BASHPID:-$$}"; seen_pid=1 ;; output_started=*) printf 'output_started=0\n'; seen_output=1 ;; output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; - defer_ack=*) printf 'defer_ack=0\n'; seen_defer_ack=1 ;; - defer_generation=*) printf 'defer_generation=\n'; seen_defer_generation=1 ;; - defer_generation_start=*) printf 'defer_generation_start=\n'; seen_defer_generation_start=1 ;; + defer_ack=*) printf 'defer_ack=%s\n' "$defer_ack"; seen_defer_ack=1 ;; + defer_generation=*) printf 'defer_generation=%s\n' "$defer_generation"; seen_defer_generation=1 ;; + defer_generation_start=*) printf 'defer_generation_start=%s\n' "$defer_generation_start"; seen_defer_generation_start=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_pid" = 1 ] || printf 'presentation_pid=%s\n' "${BASHPID:-$$}" >> "$tmp" [ "$seen_output" = 1 ] || printf 'output_started=0\n' >> "$tmp" [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' >> "$tmp" - [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=0\n' >> "$tmp" - [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=\n' >> "$tmp" - [ "$seen_defer_generation_start" = 1 ] || printf 'defer_generation_start=\n' >> "$tmp" + [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=%s\n' "$defer_ack" >> "$tmp" + [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=%s\n' "$defer_generation" >> "$tmp" + [ "$seen_defer_generation_start" = 1 ] || printf 'defer_generation_start=%s\n' "$defer_generation_start" >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } @@ -446,6 +466,30 @@ claim_mark_output_complete() { # mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } +claim_mark_output_started() { # + local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_complete=0 + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ "$seen_output" = 1 ] || printf 'output_started=1\n' >> "$tmp" + [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' >> "$tmp" + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + claim_mark_presented() { # local key=$1 row=$2 fp claim tmp line defer_ack=0 defer_generation= defer_generation_start= local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 @@ -479,6 +523,38 @@ claim_mark_presented() { # mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } } +claim_mark_confirmed() { # + local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_complete=0 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + case "$state" in + presented) + [ "$(claim_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + return 0 + ;; + presenting) ;; + *) return 2 ;; + esac + [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=presented\n' ;; + output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_complete=*) printf 'output_complete=1\n'; seen_complete=1 ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } + [ "$seen_output" = 1 ] || printf 'output_started=1\n' >> "$tmp" + [ "$seen_complete" = 1 ] || printf 'output_complete=1\n' >> "$tmp" + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + claim_remove() { # local key=$1 row=$2 owner_required=${3:-1} fp claim case "$owner_required" in 0) ;; 1) drain_claim_owner "$row" || return 2 ;; *) return 2 ;; esac @@ -712,12 +788,43 @@ publish_secondmate_receipt_and_wake() { } prepare_pending_receipt() { - local pending=$1 expected_fp schema task_id incarnation outcome terminal_source terminal_snapshot kind + local pending=$1 expected_fp schema task_id incarnation outcome terminal_source terminal_snapshot kind key local parent_task_id parent_home parent_status parent_corr [ -f "$pending" ] && [ ! -L "$pending" ] || return 1 FP=${pending##*/} - FP=${FP%.pending} + case "$FP" in + *.pending) FP=${FP%.pending} ;; + *.presented) FP=${FP%.presented} ;; + *.reported) FP=${FP%.reported} ;; + *) return 1 ;; + esac case "$FP" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["fingerprint"]=1; allowed["task_id"]=1 + allowed["incarnation"]=1; allowed["outcome"]=1; allowed["terminal_source"]=1 + allowed["terminal_snapshot"]=1; allowed["kind"]=1; allowed["parent_task_id"]=1 + allowed["parent_home"]=1; allowed["parent_status"]=1; allowed["parent_corr"]=1 + allowed["created_epoch"]=1; valid=1 + } + /^[^=]+=/ { + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + next + } + { valid=0 } + END { + for (key in allowed) { + if (key == "schema" || key == "fingerprint" || key == "task_id" || + key == "incarnation" || key == "outcome" || key == "terminal_source" || + key == "terminal_snapshot" || key == "kind") { + if (!(key in seen)) valid=0 + } + } + exit !valid + } + ' "$pending" 2>/dev/null || return 1 schema=$(receipt_field "$pending" schema) || return 1 task_id=$(receipt_field "$pending" task_id) || return 1 incarnation=$(receipt_field "$pending" incarnation) || return 1 @@ -725,10 +832,10 @@ prepare_pending_receipt() { terminal_source=$(receipt_field "$pending" terminal_source) || return 1 terminal_snapshot=$(receipt_field "$pending" terminal_snapshot) || return 1 kind=$(receipt_field "$pending" kind) || return 1 - parent_task_id=$(receipt_field "$pending" parent_task_id) || return 1 - parent_home=$(receipt_field "$pending" parent_home) || return 1 - parent_status=$(receipt_field "$pending" parent_status) || return 1 - parent_corr=$(receipt_field "$pending" parent_corr) || return 1 + parent_task_id=$(receipt_field "$pending" parent_task_id 2>/dev/null || true) + parent_home=$(receipt_field "$pending" parent_home 2>/dev/null || true) + parent_status=$(receipt_field "$pending" parent_status 2>/dev/null || true) + parent_corr=$(receipt_field "$pending" parent_corr 2>/dev/null || true) [ "$schema" = fm-jt-terminal-outcome.v1 ] || return 1 [ -n "$task_id" ] && [ -n "$incarnation" ] && [ -n "$terminal_source" ] \ && [ -n "$terminal_snapshot" ] || return 1 @@ -772,38 +879,28 @@ pending_secondmate_report_recorded() { } reported_secondmate_receipt_valid() { - local reported=$1 fp id incarnation outcome snapshot kind parent_task_id parent_home parent_status corr expected_fp - [ -f "$reported" ] && [ ! -L "$reported" ] || return 1 - fp=${reported##*/} - fp=${fp%.reported} - case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac - [ "$(receipt_field "$reported" schema)" = fm-jt-terminal-outcome.v1 ] || return 1 - [ "$(receipt_field "$reported" fingerprint)" = "$fp" ] || return 1 - id=$(receipt_field "$reported" task_id) || return 1 - incarnation=$(receipt_field "$reported" incarnation) || return 1 - outcome=$(receipt_field "$reported" outcome) || return 1 - snapshot=$(receipt_field "$reported" terminal_snapshot) || return 1 - kind=$(receipt_field "$reported" kind) || return 1 - [ "$kind" = secondmate ] || return 1 + local reported=$1 parent_task_id parent_home parent_status parent_corr + prepare_pending_receipt "$reported" || return 1 + [ "$KIND" = secondmate ] || return 1 parent_task_id=$(receipt_field "$reported" parent_task_id) || return 1 parent_home=$(receipt_field "$reported" parent_home) || return 1 parent_status=$(receipt_field "$reported" parent_status) || return 1 - corr=$(receipt_field "$reported" parent_corr) || return 1 - [ -n "$id" ] && [ -n "$incarnation" ] && [ -n "$snapshot" ] || return 1 - case "$outcome" in done|failed) ;; *) return 1 ;; esac - case "$parent_task_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac - case "$parent_home" in /*) ;; *) return 1 ;; esac - case "$parent_status" in /*) ;; *) return 1 ;; esac - printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 - expected_fp=$(hash_text "$id|$incarnation|$outcome|$snapshot|$kind") || return 1 - [ "$expected_fp" = "$fp" ] + parent_corr=$(receipt_field "$reported" parent_corr) || return 1 + [ -n "$parent_task_id" ] && [ -n "$parent_home" ] && [ -n "$parent_status" ] || return 1 + printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' } repair_reported_secondmate_routes() { - local reported kind corr status=0 + local scan_started=$1 reported kind corr parent_task_id parent_home parent_status now remaining status=0 [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + [ "$remaining" -gt 0 ] || return 1 + FM_LOCK_WAIT_SECS="$remaining" fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 for reported in "$OUTCOME_DIR"/*.reported; do + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + [ "$remaining" -gt 0 ] || { status=1; break; } [ -e "$reported" ] || [ -L "$reported" ] || continue [ -f "$reported" ] && [ ! -L "$reported" ] || { status=1; continue; } kind=$(receipt_field "$reported" kind 2>/dev/null || true) @@ -815,7 +912,11 @@ repair_reported_secondmate_routes() { continue fi corr=$(receipt_field "$reported" parent_corr) - fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" || status=1 + parent_task_id=$(receipt_field "$reported" parent_task_id) + parent_home=$(receipt_field "$reported" parent_home) + parent_status=$(receipt_field "$reported" parent_status) + FM_LOCK_WAIT_SECS="$remaining" fm_pending_reply_secondmate_route_clear_reported \ + "$FM_HOME" "$corr" "$parent_task_id" "$parent_home" "$parent_status" || status=1 ;; *) status=1 ;; esac @@ -1007,8 +1108,12 @@ ack_receipt() { # [ "$(receipt_field "$existing" fingerprint)" = "$fp" ] || return 2 existing_kind=$(receipt_field "$existing" kind) if [ "$existing_kind" = secondmate ]; then + reported_secondmate_receipt_valid "$existing" || return 2 existing_corr=$(receipt_field "$existing" parent_corr) - fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$existing_corr" || return 2 + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$existing_corr" \ + "$(receipt_field "$existing" parent_task_id)" \ + "$(receipt_field "$existing" parent_home)" \ + "$(receipt_field "$existing" parent_status)" || return 2 fi claim_remove "$key" "$row" "$owner_required" || return 2 return 1 @@ -1017,8 +1122,7 @@ ack_receipt() { # return 2 fi [ -f "$rec" ] || return 2 - [ "$(receipt_field "$rec" fingerprint)" = "$fp" ] || return 2 - [ "$(receipt_field "$rec" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 + prepare_pending_receipt "$rec" || return 2 id=$(receipt_field "$rec" task_id) kind=$(receipt_field "$rec" kind) incarnation=$(receipt_field "$rec" incarnation) @@ -1043,13 +1147,19 @@ ack_receipt() { # if [ -e "$target" ]; then [ -f "$target" ] || return 2 [ "$(receipt_field "$target" fingerprint)" = "$fp" ] || return 2 + if [ "$kind" = secondmate ]; then + reported_secondmate_receipt_valid "$target" || return 2 + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" \ + "$parent_task_id" "$parent_home" "$parent_status" || return 2 + fi rm -f "$rec" || return 2 claim_remove "$key" "$row" "$owner_required" || return 2 return 1 fi mv "$rec" "$target" || return 2 if [ "$kind" = secondmate ]; then - fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" || return 2 + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" \ + "$parent_task_id" "$parent_home" "$parent_status" || return 2 fi claim_remove "$key" "$row" "$owner_required" || return 2 return 0 @@ -1058,7 +1168,10 @@ ack_receipt() { # confirm_receipt() { local status=0 fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 2 - ack_receipt "$1" "$2" 0 || status=$? + claim_mark_confirmed "$1" "$2" || status=$? + if [ "$status" = 0 ]; then + ack_receipt "$1" "$2" 0 || status=$? + fi fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=2 return "$status" } @@ -1194,7 +1307,7 @@ scan_locked() { rm -f "$find_tmp" || return 1 [ "$scan_failed" = 0 ] || return "$rc" [ "$complete" = 1 ] || return 1 - if ! repair_reported_secondmate_routes; then + if ! repair_reported_secondmate_routes "$scan_started"; then return 1 fi if ! republish_pending_receipts "$scan_started"; then @@ -1254,7 +1367,11 @@ case "${1:-}" in [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_presenting "$2" "$3" ;; - output-started|output-complete) + output-started) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_output_started "$2" "$3" + ;; + output-complete) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_output_complete "$2" "$3" ;; diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 4f690b5fbf6..5dad711ffd7 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -695,8 +695,9 @@ fm_pending_reply_secondmate_route_write() { # < return "$route_status" } -fm_pending_reply_secondmate_route_clear_with_mode() { # - local secondmate_home=$1 corr=$2 allow_undelivered=${3:-0} marker route_lock current_corr history_marker status=0 +fm_pending_reply_secondmate_route_clear_with_mode() { # [ ] + local secondmate_home=$1 corr=$2 allow_undelivered=${3:-0} expected_id=${4:-} expected_home=${5:-} expected_status=${6:-} + local marker route_lock current_corr history_marker status=0 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 @@ -731,7 +732,12 @@ fm_pending_reply_secondmate_route_clear_with_mode() { # fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 0 } -fm_pending_reply_secondmate_route_clear_reported() { # - fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 4 +fm_pending_reply_secondmate_route_clear_reported() { # [ ] + fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 4 "${3:-}" "${4:-}" "${5:-}" } fm_pending_reply_secondmate_route_clear_undelivered() { # - local secondmate_home=$1 corr=$2 marker route_lock current_corr status=0 + local secondmate_home=$1 corr=$2 marker route_lock current_corr status=0 marker_removed=0 [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 @@ -779,11 +785,15 @@ fm_pending_reply_secondmate_route_clear_undelivered() { # [] fm_pending_reply_secondmate_receipt_validate() { # local secondmate_home=$1 secondmate_id=$2 parent_home=$3 parent_status=$4 corr=$5 local home_marker marker_id parent_abs state_abs expected_status rec active_rec history_rec history_dir - local record_task record_home record_status record_corr delivered phase + local delivered phase [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 home_marker="$secondmate_home/.fm-secondmate-home" @@ -972,15 +982,7 @@ fm_pending_reply_secondmate_receipt_validate() { # "$emitted" || exit 1 - FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ - FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - output-complete "$key" "$row" || exit 1 + if [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ]; then + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ + FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-started "$key" "$row" || exit 1 + else + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ + FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-complete "$key" "$row" || exit 1 + fi ) & worker=$! : > "$go" || status=1 @@ -48,7 +54,7 @@ present_inactive_row() { fi rm -f "$go" if [ "$status" -ne 0 ]; then - if [ -e "$emitted" ]; then + if [ -e "$emitted" ] && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ presented "$key" "$row" >/dev/null 2>&1; then presentation_marked=1 @@ -59,7 +65,8 @@ present_inactive_row() { presenting "$key" "$row" >/dev/null 2>&1 || true fi fi - if [ "$status" = 0 ] && [ "$presentation_marked" = 0 ]; then + if [ "$status" = 0 ] && [ "$presentation_marked" = 0 ] \ + && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$key" "$row"; then status=1 if [ -e "$emitted" ] \ diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 57a0da0bdea..3eada20b8cc 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -194,7 +194,9 @@ drain() { ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ FM_STATE_OVERRIDE="$home/state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ - CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" "$DRAIN" ) + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + FM_WAKE_DRAIN_DEFER_ACK="${FM_WAKE_DRAIN_DEFER_ACK:-0}" \ + FM_WAKE_DRAIN_GENERATION="${FM_WAKE_DRAIN_GENERATION:-}" "$DRAIN" ) } write_meta() { @@ -686,16 +688,20 @@ test_deferred_ack_retries_after_caller_crash() { row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") [ -n "$row" ] || fail "deferred receipt did not queue its wake" drain_output="$dir/deferred.out" - if ! FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" \ - drain "$root" "$home" "$fakebin" >"$drain_output"; then + export FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" + if ! drain "$root" "$home" "$fakebin" >"$drain_output"; then fail "deferred wake drain failed" fi [ -f "$state/terminal-outcomes/$fingerprint.pending" ] || fail "deferred drain consumed the receipt before caller confirmation" [ ! -e "$state/terminal-outcomes/$fingerprint.presented" ] || fail "deferred drain finalized the receipt before caller confirmation" - [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" state)" = presented ] \ + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" state)" = presenting ] \ || fail "deferred drain did not retain the presentation claim" [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" defer_ack)" = 1 ] \ || fail "deferred drain did not mark the claim for caller confirmation" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_started)" = 1 ] \ + || fail "deferred drain did not retain post-output state" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_complete)" = 0 ] \ + || fail "deferred drain completed output before caller confirmation" printf '%s\n' "$row" > "$state/.wake-queue" drain "$root" "$home" "$fakebin" >"$dir/live-deferred.out" \ || fail "live deferred claim drain failed" @@ -714,10 +720,41 @@ test_deferred_ack_retries_after_caller_crash() { [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] || fail "retry drain left the receipt pending" [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "retry drain left the deferred claim" [ ! -s "$drain_output" ] || fail "retry drain re-presented output already emitted before the caller crash" - unset FM_FAKE_CREW_STATE_DEFERRED_X1 + unset FM_WAKE_DRAIN_DEFER_ACK FM_WAKE_DRAIN_GENERATION FM_FAKE_CREW_STATE_DEFERRED_X1 pass "deferred inactive receipts acknowledge stale post-output claims without replay" } +test_deferred_ack_confirms_after_caller_emission() { + local dir root home fakebin state fingerprint row drain_output + new_case deferred-confirm + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" deferred-confirm-x1 deferred-confirm-inc + export FM_FAKE_CREW_STATE_DEFERRED_CONFIRM_X1='state: done · source: pane · deferred confirmation' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "deferred confirmation setup failed" + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) + row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") + drain_output="$dir/deferred-confirm.out" + export FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" + drain "$root" "$home" "$fakebin" >"$drain_output" \ + || fail "deferred confirmation drain failed" + printf '%s\n' "$(cat "$drain_output")" > "$dir/caller-visible.out" + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + "$RECON" confirm "inactive-outcome:$fingerprint" "$row" ) \ + || fail "caller confirmation did not finalize the receipt" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] \ + || fail "caller confirmation did not move the receipt" + [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] \ + || fail "caller confirmation left the receipt pending" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] \ + || fail "caller confirmation left the claim" + unset FM_WAKE_DRAIN_DEFER_ACK FM_WAKE_DRAIN_GENERATION FM_FAKE_CREW_STATE_DEFERRED_CONFIRM_X1 + pass "deferred inactive receipts finalize after caller emission" +} + test_scan_failure_retries_without_advancing_cadence() { local dir root home fakebin state wake_dir wake_removed new_case scan-failure @@ -1973,6 +2010,7 @@ test_output_completion_failure_does_not_reprint test_finalized_receipt_rows_are_suppressed test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash +test_deferred_ack_confirms_after_caller_emission test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint From 22a76002d989fc585532cb8f6a0e04b874862e59 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 12:53:56 +0000 Subject: [PATCH 023/163] no-mistakes(review): Harden caller-confirmed receipts and transactional route cleanup --- bin/fm-inactive-reconcile.sh | 36 +++++++++--- bin/fm-pending-reply-lib.sh | 9 ++- bin/fm-send.sh | 7 ++- bin/fm-session-start.sh | 5 +- bin/fm-wake-drain.sh | 6 +- bin/fm-watch.sh | 2 + tests/fm-inactive-outcome.test.sh | 98 +++++++++++++++++++++++++++++-- 7 files changed, 143 insertions(+), 20 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 7fab8ac61f4..bb35873f253 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -337,16 +337,13 @@ claim_reserve() { # return 5 fi defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) - if [ "$(claim_field "$claim" output_started 2>/dev/null || true)" = 1 ] \ - && [ "$defer_ack" = 1 ]; then + if [ "$defer_ack" = 1 ]; then defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) if claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then return 4 fi - claim_mark_output_complete "$key" "$row" || return 2 - claim_mark_presented "$key" "$row" || return 2 - return 5 + return 0 fi fi if [ "$state" = presented ]; then @@ -443,14 +440,27 @@ claim_mark_presenting() { # } claim_mark_output_complete() { # - local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_complete=0 - drain_claim_owner "$row" || return 2 + local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} + local fp claim state tmp line seen_output=0 seen_complete=0 + case "$owner_required" in + 1) drain_claim_owner "$row" || return 2 ;; + 0) + case "$expected_generation" in ''|*[!0-9]*|0) return 2 ;; esac + ;; + *) return 2 ;; + esac case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac claim=$(claim_path "$fp") [ ! -L "$claim" ] || return 2 state=$(claim_validate "$claim" "$fp" "$row") || return 2 [ "$state" = presenting ] || return 2 + if [ "$owner_required" = 0 ]; then + [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_field "$claim" defer_generation 2>/dev/null || true)" = "$expected_generation" ] || return 2 + fm_pid_start_matches_stored "$expected_generation" \ + "$(claim_field "$claim" defer_generation_start 2>/dev/null || true)" || return 2 + fi tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 chmod 600 "$tmp" 2>/dev/null || true while IFS= read -r line || [ -n "$line" ]; do @@ -1176,6 +1186,14 @@ confirm_receipt() { return "$status" } +caller_output_complete() { + local status=0 + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 2 + claim_mark_output_complete "$1" "$2" 0 "$3" || status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=2 + return "$status" +} + secondmate_ack_report() { # local secondmate_home=$1 parent_task_id=$2 parent_home=$3 parent_status=$4 corr=$5 outcome=$6 task_id=$7 fp=$8 local parent_state token route_lock route_marker route_history line phase rc=0 route_lock_held=0 marker_present=0 report_recorded=0 @@ -1375,6 +1393,10 @@ case "${1:-}" in [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_output_complete "$2" "$3" ;; + caller-output-complete) + [ -n "${2:-}" ] && [ -n "${3:-}" ] && [ -n "${4:-}" ] || exit 2 + caller_output_complete "$2" "$3" "$4" + ;; presented) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 claim_mark_presented "$2" "$3" diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 5dad711ffd7..6ad9e75704a 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -81,6 +81,7 @@ _FM_PENDING_REPLY_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd 2>/dev/n FM_PENDING_REPLY_SCHEMA='fm-pending-reply.v1' FM_PENDING_REPLY_CORR_RE='(^|[^[:alnum:]_])corr=([A-Fa-f0-9]{16})($|[^[:alnum:]_])' FM_PENDING_REPLY_GRACE_DEFAULT=120 +FM_PENDING_REPLY_ROUTE_COMMITTED=0 fm_pending_reply_now() { if [ -n "${FM_PENDING_REPLY_NOW:-}" ]; then @@ -589,6 +590,7 @@ fm_pending_reply_secondmate_route_write() { # < local marker home_marker route_lock tmp parent_abs state_abs status_path marker_id route_status=0 local existing_schema existing_id existing_home existing_status existing_corr existing_rec existing_delivered route_mode local history_marker + FM_PENDING_REPLY_ROUTE_COMMITTED=0 marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 @@ -654,6 +656,7 @@ fm_pending_reply_secondmate_route_write() { # < || route_status=1 fi if [ "$route_status" = 0 ]; then + FM_PENDING_REPLY_ROUTE_COMMITTED=1 rm -f "$tmp" fm_lock_release "$route_lock" || return 1 return 0 @@ -688,7 +691,11 @@ fm_pending_reply_secondmate_route_write() { # < fi fi if [ "$route_status" = 0 ]; then - mv -f "$tmp" "$marker" || route_status=1 + if mv -f "$tmp" "$marker"; then + FM_PENDING_REPLY_ROUTE_COMMITTED=1 + else + route_status=1 + fi fi [ "$route_status" = 0 ] || rm -f "$tmp" fm_lock_release "$route_lock" || route_status=1 diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 70cd6af5bdc..352d607babc 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -87,10 +87,12 @@ shift MARK_FROM_FIRSTMATE=0 PENDING_REPLY_CORR= PENDING_REPLY_CREATED=0 +PENDING_ROUTE_COMMITTED=0 TARGET_TASK_ID= TARGET_HOME= clear_new_pending_route() { + [ "$PENDING_ROUTE_COMMITTED" = 1 ] || return 0 if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ && [ -n "$TARGET_HOME" ]; then fm_pending_reply_secondmate_route_clear_undelivered "$TARGET_HOME" "$PENDING_REPLY_CORR" @@ -173,8 +175,11 @@ else exit 1 fi TARGET_HOME=$(fm_meta_get "$meta" home) - if ! fm_pending_reply_secondmate_route_write \ + if fm_pending_reply_secondmate_route_write \ "$TARGET_HOME" "$FM_HOME" "$STATE" "$TARGET_TASK_ID" "$PENDING_REPLY_CORR"; then + PENDING_ROUTE_COMMITTED=${FM_PENDING_REPLY_ROUTE_COMMITTED:-0} + else + PENDING_ROUTE_COMMITTED=${FM_PENDING_REPLY_ROUTE_COMMITTED:-0} discard_new_pending_reply || exit 1 echo "error: failed to bind the secondmate pending-reply route for $TARGET_TASK_ID" >&2 exit 1 diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 26f24002232..63445863093 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -374,7 +374,10 @@ else IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" case "$_key" in inactive-outcome:*) - if ! "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$drain_row" >/dev/null 2>&1; then + if ! "$SCRIPT_DIR/fm-inactive-reconcile.sh" caller-output-complete \ + "$_key" "$drain_row" "$$" >/dev/null 2>&1; then + printf 'warning: inactive outcome output confirmation deferred for %s\n' "$_key" >&2 + elif ! "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$drain_row" >/dev/null 2>&1; then printf 'warning: inactive outcome confirmation deferred for %s\n' "$_key" >&2 fi ;; diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 6baeb34b27e..03733aeb314 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -33,11 +33,7 @@ present_inactive_row() { done printf '%s\n' "$row" || exit 1 : > "$emitted" || exit 1 - if [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ]; then - FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ - FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - output-started "$key" "$row" || exit 1 - else + if [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ output-complete "$key" "$row" || exit 1 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 45fadb1955d..52993852f91 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -349,6 +349,8 @@ wake() { IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$row" case "$_key" in inactive-outcome:*) + "$SCRIPT_DIR/fm-inactive-reconcile.sh" caller-output-complete \ + "$_key" "$row" "$WATCHER_PID" >/dev/null 2>&1 || exit 1 confirm_status=0 "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$row" >/dev/null 2>&1 || confirm_status=$? [ "$confirm_status" = 0 ] || [ "$confirm_status" = 1 ] || exit "$confirm_status" diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 3eada20b8cc..54be6dd0565 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -698,8 +698,8 @@ test_deferred_ack_retries_after_caller_crash() { || fail "deferred drain did not retain the presentation claim" [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" defer_ack)" = 1 ] \ || fail "deferred drain did not mark the claim for caller confirmation" - [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_started)" = 1 ] \ - || fail "deferred drain did not retain post-output state" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_started)" = 0 ] \ + || fail "deferred drain advanced output before caller emission" [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_complete)" = 0 ] \ || fail "deferred drain completed output before caller confirmation" printf '%s\n' "$row" > "$state/.wake-queue" @@ -713,15 +713,28 @@ test_deferred_ack_retries_after_caller_crash() { rm -f "$state/deferred-x1.meta" "$state/deferred-x1.status" "$state/deferred-x1.turn-ended" scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "pending deferred receipt was not republished after caller crash" [ "$(queue_count "$state")" = 1 ] || fail "pending deferred receipt did not get a retry wake" + row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") drain_output="$dir/retry.out" drain "$root" "$home" "$fakebin" >"$drain_output" \ - || fail "retry drain did not recover the deferred acknowledgement" + || fail "retry drain did not recover the deferred presentation" + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + "$RECON" caller-output-complete "inactive-outcome:$fingerprint" "$row" "$$" ) \ + || fail "caller output confirmation did not recover the retried presentation" + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + "$RECON" confirm "inactive-outcome:$fingerprint" "$row" ) \ + || fail "caller confirmation did not finalize the retried presentation" [ -f "$state/terminal-outcomes/$fingerprint.presented" ] || fail "retry drain did not acknowledge the recovered receipt" [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] || fail "retry drain left the receipt pending" [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "retry drain left the deferred claim" - [ ! -s "$drain_output" ] || fail "retry drain re-presented output already emitted before the caller crash" + [ "$(grep -Fxc "$row" "$drain_output")" = 1 ] || fail "retry drain did not re-present output never emitted by the caller" unset FM_WAKE_DRAIN_DEFER_ACK FM_WAKE_DRAIN_GENERATION FM_FAKE_CREW_STATE_DEFERRED_X1 - pass "deferred inactive receipts acknowledge stale post-output claims without replay" + pass "deferred inactive receipts retry until caller-visible emission" } test_deferred_ack_confirms_after_caller_emission() { @@ -739,6 +752,12 @@ test_deferred_ack_confirms_after_caller_emission() { drain "$root" "$home" "$fakebin" >"$drain_output" \ || fail "deferred confirmation drain failed" printf '%s\n' "$(cat "$drain_output")" > "$dir/caller-visible.out" + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + "$RECON" caller-output-complete "inactive-outcome:$fingerprint" "$row" "$$" ) \ + || fail "caller output confirmation did not finalize the receipt" ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ @@ -755,6 +774,38 @@ test_deferred_ack_confirms_after_caller_emission() { pass "deferred inactive receipts finalize after caller emission" } +test_deferred_ack_recovers_after_output_confirmation() { + local dir root home fakebin state fingerprint row drain_output + new_case deferred-output-recovery + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" deferred-output-recovery-x1 deferred-output-recovery-inc + export FM_FAKE_CREW_STATE_DEFERRED_OUTPUT_RECOVERY_X1='state: done · source: pane · deferred output recovery' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "deferred output recovery setup failed" + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) + row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") + export FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" + drain "$root" "$home" "$fakebin" >"$dir/deferred-output-recovery.out" \ + || fail "deferred output recovery drain failed" + ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ + "$RECON" caller-output-complete "inactive-outcome:$fingerprint" "$row" "$$" ) \ + || fail "caller output confirmation failed" + printf '%s\n' "$row" > "$state/.wake-queue" + drain_output="$dir/deferred-output-recovery-retry.out" + drain "$root" "$home" "$fakebin" >"$drain_output" \ + || fail "deferred output recovery retry failed" + [ ! -s "$drain_output" ] || fail "confirmed output was presented again after caller crash" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] \ + || fail "confirmed output was not acknowledged during recovery" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] \ + || fail "confirmed output left a recovery claim" + unset FM_WAKE_DRAIN_DEFER_ACK FM_WAKE_DRAIN_GENERATION FM_FAKE_CREW_STATE_DEFERRED_OUTPUT_RECOVERY_X1 + pass "deferred inactive receipts recover confirmed output without replay" +} + test_scan_failure_retries_without_advancing_cadence() { local dir root home fakebin state wake_dir wake_removed new_case scan-failure @@ -1918,6 +1969,41 @@ test_failed_concurrent_send_discards_only_new_record() { pass "failed concurrent send discards only its new undelivered record" } +test_failed_marked_send_discards_never_bound_record() { + local dir root home fakebin state child_home child_state send_out + new_case failed-never-bound-send + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/unsafe-secondmate-home" + child_state="$child_home/state" + mkdir -p "$state/pending-replies" "$child_home/state-real" "$child_home/data" \ + "$child_home/config" "$child_home/projects" + ln -s "$child_home/state-real" "$child_state" + printf 'sm-never-bound\n' > "$child_home/.fm-secondmate-home" + write_meta "$state" sm-never-bound never-bound-inc secondmate tmux firstmate:fm-sm-never-bound + printf 'home=%s\n' "$child_home" >> "$state/sm-never-bound.meta" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u +exit 1 +SH + chmod +x "$fakebin/tmux" + prepare_primary_proof "$root" "$home" "$fakebin" + prepare_watcher_protocol "$root" "$home" "$state" + send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK \ + -u FM_AGENT_OWNER_HOME -u FM_ROOT -u STATE -u FM_PENDING_REPLY_EXISTING_CORR \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ + fm-sm-never-bound "never bound request" 2>&1) && fail "never-bound marked send unexpectedly succeeded" + [ "$(direct_file_count "$state/pending-replies" '*')" = 0 ] \ + || fail "never-bound send left an awaiting_report record without a route" + [ ! -e "$child_state/.fm-jt-parent-route" ] \ + || fail "never-bound send installed a route through an unsafe state path" + pass "failed marked sends discard expectations without a committed route" +} + test_drain_restores_only_unprocessed_rows() { local dir root home fakebin state first second new_case drain-rollback @@ -2011,6 +2097,7 @@ test_finalized_receipt_rows_are_suppressed test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash test_deferred_ack_confirms_after_caller_emission +test_deferred_ack_recovers_after_output_confirmation test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint @@ -2033,5 +2120,6 @@ test_route_replacement_rejects_malformed_parent_record test_recovery_route_reuse_validates_parent_record test_failed_marked_send_restores_record_on_route_cleanup_failure test_failed_concurrent_send_discards_only_new_record +test_failed_marked_send_discards_never_bound_record test_drain_restores_only_unprocessed_rows test_malformed_or_missing_secondmate_route_fails_closed From ca0c69c6b906704914340e136f7585ffa41b2a2e Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 13:29:47 +0000 Subject: [PATCH 024/163] no-mistakes(review): Harden inactive replay ownership and cleanup recovery --- bin/fm-inactive-reconcile.sh | 119 ++++++++++++++++++-------- bin/fm-pending-reply-lib.sh | 94 +++++++++++++++++++-- bin/fm-send.sh | 9 ++ bin/fm-session-start.sh | 31 ++----- bin/fm-wake-drain.sh | 14 +++- bin/fm-watch.sh | 15 ++-- tests/fm-inactive-outcome.test.sh | 135 ++++++++++++++++++++++++------ 7 files changed, 318 insertions(+), 99 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index bb35873f253..74d6a9d38fb 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -255,6 +255,16 @@ claim_validate() { # case "$state" in reserved|presenting|presented) printf '%s' "$state" ;; *) return 1 ;; esac } +claim_validate_caller_owner() { # + local claim=$1 caller_pid=$2 caller_start + [ -f "$claim" ] && [ ! -L "$claim" ] || return 1 + case "$caller_pid" in ''|*[!0-9]*|0) return 1 ;; esac + [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 1 + [ "$(claim_field "$claim" defer_generation 2>/dev/null || true)" = "$caller_pid" ] || return 1 + caller_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + fm_pid_start_matches_stored "$caller_pid" "$caller_start" +} + claim_reserve() { # local key=$1 row=$2 fp claim tmp state existing old_row line output_complete defer_ack local defer_generation defer_generation_start receipt_state receipt_rc=0 recorded_report=0 report_rc=1 @@ -399,7 +409,8 @@ claim_mark_presenting() { # local key=$1 row=$2 fp claim state tmp line defer_ack=0 defer_generation= defer_generation_start= local seen_pid=0 seen_output=0 seen_complete=0 local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 - [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 + [ "${FM_WAKE_DRAIN_DIRECT:-0}" != 1 ] \ + && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 if [ "$defer_ack" = 1 ]; then defer_generation=${FM_WAKE_DRAIN_GENERATION:-} case "$defer_generation" in ''|*[!0-9]*|0) return 2 ;; esac @@ -456,10 +467,7 @@ claim_mark_output_complete() { # state=$(claim_validate "$claim" "$fp" "$row") || return 2 [ "$state" = presenting ] || return 2 if [ "$owner_required" = 0 ]; then - [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 2 - [ "$(claim_field "$claim" defer_generation 2>/dev/null || true)" = "$expected_generation" ] || return 2 - fm_pid_start_matches_stored "$expected_generation" \ - "$(claim_field "$claim" defer_generation_start 2>/dev/null || true)" || return 2 + claim_validate_caller_owner "$claim" "$expected_generation" || return 2 fi tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 chmod 600 "$tmp" 2>/dev/null || true @@ -503,7 +511,8 @@ claim_mark_output_started() { # claim_mark_presented() { # local key=$1 row=$2 fp claim tmp line defer_ack=0 defer_generation= defer_generation_start= local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 - [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 + [ "${FM_WAKE_DRAIN_DIRECT:-0}" != 1 ] \ + && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 if [ "$defer_ack" = 1 ]; then defer_generation=${FM_WAKE_DRAIN_GENERATION:-} case "$defer_generation" in ''|*[!0-9]*|0) return 2 ;; esac @@ -534,12 +543,21 @@ claim_mark_presented() { # } claim_mark_confirmed() { # - local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_complete=0 + local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} + local fp claim state tmp line seen_output=0 seen_complete=0 + case "$owner_required" in + 1) drain_claim_owner "$row" || return 2 ;; + 0) ;; + *) return 2 ;; + esac case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac claim=$(claim_path "$fp") [ ! -L "$claim" ] || return 2 state=$(claim_validate "$claim" "$fp" "$row") || return 2 + if [ "$owner_required" = 0 ]; then + claim_validate_caller_owner "$claim" "$expected_generation" || return 2 + fi case "$state" in presented) [ "$(claim_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 @@ -566,11 +584,15 @@ claim_mark_confirmed() { # } claim_remove() { # - local key=$1 row=$2 owner_required=${3:-1} fp claim - case "$owner_required" in 0) ;; 1) drain_claim_owner "$row" || return 2 ;; *) return 2 ;; esac + local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} fp claim case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac claim=$(claim_path "$fp") + case "$owner_required" in + 0) claim_validate_caller_owner "$claim" "$expected_generation" || return 2 ;; + 1) drain_claim_owner "$row" || return 2 ;; + *) return 2 ;; + esac [ ! -L "$claim" ] || return 2 [ -e "$claim" ] || return 0 claim_validate "$claim" "$fp" "$row" >/dev/null || return 2 @@ -1097,7 +1119,8 @@ reconcile_child() { } ack_receipt() { # - local key=$1 row=${2:-} owner_required=${3:-1} fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing existing_kind existing_corr claim_state + local key=$1 row=${2:-} owner_required=${3:-1} expected_generation=${4:-} + local fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing existing_kind existing_corr claim_state [ -n "$row" ] || return 2 case "$owner_required" in 0|1) ;; *) return 2 ;; esac [ "$owner_required" = 0 ] || drain_claim_owner "$row" || return 2 @@ -1106,7 +1129,7 @@ ack_receipt() { # claim_state=$(claim_validate "$(claim_path "$fp")" "$fp" "$row") || return 2 [ "$claim_state" = presented ] || return 2 if [ "$owner_required" = 0 ]; then - [ "$(claim_field "$(claim_path "$fp")" defer_ack 2>/dev/null || true)" = 1 ] || return 2 + claim_validate_caller_owner "$(claim_path "$fp")" "$expected_generation" || return 2 fi rec=$(receipt_path "$fp" pending) [ ! -L "$rec" ] || return 2 @@ -1125,7 +1148,7 @@ ack_receipt() { # "$(receipt_field "$existing" parent_home)" \ "$(receipt_field "$existing" parent_status)" || return 2 fi - claim_remove "$key" "$row" "$owner_required" || return 2 + claim_remove "$key" "$row" "$owner_required" "$expected_generation" || return 2 return 1 fi done @@ -1163,7 +1186,7 @@ ack_receipt() { # "$parent_task_id" "$parent_home" "$parent_status" || return 2 fi rm -f "$rec" || return 2 - claim_remove "$key" "$row" "$owner_required" || return 2 + claim_remove "$key" "$row" "$owner_required" "$expected_generation" || return 2 return 1 fi mv "$rec" "$target" || return 2 @@ -1171,25 +1194,25 @@ ack_receipt() { # fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" \ "$parent_task_id" "$parent_home" "$parent_status" || return 2 fi - claim_remove "$key" "$row" "$owner_required" || return 2 + claim_remove "$key" "$row" "$owner_required" "$expected_generation" || return 2 return 0 } confirm_receipt() { - local status=0 + local status=0 caller_pid=${PPID:-} fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 2 - claim_mark_confirmed "$1" "$2" || status=$? + claim_mark_confirmed "$1" "$2" 0 "$caller_pid" || status=$? if [ "$status" = 0 ]; then - ack_receipt "$1" "$2" 0 || status=$? + ack_receipt "$1" "$2" 0 "$caller_pid" || status=$? fi fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=2 return "$status" } caller_output_complete() { - local status=0 + local status=0 caller_pid=${PPID:-} fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 2 - claim_mark_output_complete "$1" "$2" 0 "$3" || status=$? + claim_mark_output_complete "$1" "$2" 0 "$caller_pid" || status=$? fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=2 return "$status" } @@ -1259,7 +1282,7 @@ secondmate_ack_report() { # /dev/null || true) now=$(date +%s) @@ -1268,26 +1291,46 @@ scan_locked() { [ "$age" -ge "$RECONCILE_SECS" ] || return 0 fi scan_started=$now + run_maintenance() { + if ! republish_pending_receipts "$scan_started"; then + maintenance_status=1 + fi + if ! repair_reported_secondmate_routes "$scan_started"; then + maintenance_status=1 + fi + } cursor=$(cat "$SCAN_CURSOR" 2>/dev/null || true) if [ -n "$cursor" ] && { [ ! -f "$STATE/$cursor.meta" ] || [ -L "$STATE/$cursor.meta" ]; }; then cursor= fi if [ -n "$cursor" ]; then started=0; fi [ -n "$cursor" ] && cursor_seen=0 - find_tmp=$(mktemp "$STATE/.inactive-outcome-find.XXXXXX") || return 1 - [ -f "$find_tmp" ] && [ ! -L "$find_tmp" ] || { rm -f "$find_tmp"; return 1; } + find_tmp=$(mktemp "$STATE/.inactive-outcome-find.XXXXXX") || { + run_maintenance + return 1 + } + [ -f "$find_tmp" ] && [ ! -L "$find_tmp" ] || { + rm -f "$find_tmp" + run_maintenance + return 1 + } now=$(date +%s) remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) if [ "$remaining" -le 0 ]; then rm -f "$find_tmp" + run_maintenance return 1 fi - if ! run_bounded_child "$remaining" find "$STATE" \( -type d ! -path "$STATE" -prune \) -o \ + if run_bounded_child "$remaining" find "$STATE" \( -type d ! -path "$STATE" -prune \) -o \ \( -type f -name '*.meta' -print0 \) > "$find_tmp"; then - rm -f "$find_tmp" - return 1 + : + else + rc=$? + [ "$rc" -ne 0 ] || rc=1 + complete=0 + scan_failed=1 fi - while IFS= read -r -d '' meta; do + while [ "$scan_failed" = 0 ] && IFS= read -r -d '' meta; do now=$(date +%s) remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) if [ "$remaining" -le 0 ]; then @@ -1322,15 +1365,21 @@ scan_locked() { break fi done < "$find_tmp" - rm -f "$find_tmp" || return 1 - [ "$scan_failed" = 0 ] || return "$rc" - [ "$complete" = 1 ] || return 1 - if ! repair_reported_secondmate_routes "$scan_started"; then - return 1 + if rm -f "$find_tmp"; then + : + else + rc=$? + [ "$rc" -ne 0 ] || rc=1 + complete=0 + scan_failed=1 fi - if ! republish_pending_receipts "$scan_started"; then - return 1 + run_maintenance + if [ "$scan_failed" = 1 ]; then + [ "$rc" -ne 0 ] || rc=1 + return "$rc" fi + [ "$complete" = 1 ] || return 1 + [ "$maintenance_status" = 0 ] || return 1 if [ "$cursor_seen" = 0 ]; then return 1 fi @@ -1394,8 +1443,8 @@ case "${1:-}" in claim_mark_output_complete "$2" "$3" ;; caller-output-complete) - [ -n "${2:-}" ] && [ -n "${3:-}" ] && [ -n "${4:-}" ] || exit 2 - caller_output_complete "$2" "$3" "$4" + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + caller_output_complete "$2" "$3" ;; presented) [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 6ad9e75704a..715eca5e300 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -1269,6 +1269,82 @@ fm_pending_reply_reconcile_delivery() { # return 1 } +fm_pending_reply_undelivered_cleanup_meta_path() { # + printf '%s.cleanup-meta' "$(fm_pending_reply_active_path "$1" "$2")" +} + +fm_pending_reply_undelivered_cleanup_meta_valid() { # + local meta=$1 corr=$2 secondmate_home=$3 + [ -f "$meta" ] && [ ! -L "$meta" ] || return 1 + awk -F= ' + BEGIN { schema=corr=home=0; valid=1 } + NF != 2 { valid=0; next } + $1 == "schema" { schema++; next } + $1 == "corr_id" { corr++; next } + $1 == "secondmate_home" { home++; next } + { valid=0 } + END { if (schema != 1 || corr != 1 || home != 1) valid=0; exit !valid } + ' "$meta" 2>/dev/null || return 1 + [ "$(fm_pending_reply_get "$meta" schema)" = fm-undelivered-cleanup.v1 ] || return 1 + [ "$(fm_pending_reply_get "$meta" corr_id)" = "$corr" ] || return 1 + [ "$(fm_pending_reply_get "$meta" secondmate_home)" = "$secondmate_home" ] || return 1 +} + +fm_pending_reply_schedule_undelivered_cleanup() { # + local state=$1 corr=$2 secondmate_home=$3 dir meta tmp + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + case "$secondmate_home" in + /*) + case "$secondmate_home" in *$'\n'*|*'='*) return 1 ;; esac + ;; + *) return 1 ;; + esac + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + dir=$(fm_pending_reply_dir "$state") + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + meta=$(fm_pending_reply_undelivered_cleanup_meta_path "$state" "$corr") + if [ -e "$meta" ] || [ -L "$meta" ]; then + fm_pending_reply_undelivered_cleanup_meta_valid "$meta" "$corr" "$secondmate_home" + return $? + fi + tmp=$(mktemp "$dir/.cleanup-meta.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + { + printf 'schema=fm-undelivered-cleanup.v1\n' + printf 'corr_id=%s\n' "$corr" + printf 'secondmate_home=%s\n' "$secondmate_home" + } > "$tmp" || { rm -f "$tmp"; return 1; } + chmod 600 "$tmp" 2>/dev/null || true + if ln "$tmp" "$meta" 2>/dev/null; then + rm -f "$tmp" + return 0 + fi + rm -f "$tmp" + fm_pending_reply_undelivered_cleanup_meta_valid "$meta" "$corr" "$secondmate_home" +} + +fm_pending_reply_retry_undelivered_cleanup() { # + local state=$1 meta=$2 corr secondmate_home token rc=0 + corr=$(fm_pending_reply_get "$meta" corr_id) + secondmate_home=$(fm_pending_reply_get "$meta" secondmate_home) + fm_pending_reply_undelivered_cleanup_meta_valid "$meta" "$corr" "$secondmate_home" || return 1 + fm_pending_reply_txn_lock_acquire "$state" "$corr" token || return 1 + if ! fm_pending_reply_discard_undelivered "$state" "$corr" 1; then + fm_pending_reply_restore_undelivered "$state" "$corr" || true + rc=1 + elif ! fm_pending_reply_secondmate_route_clear_undelivered "$secondmate_home" "$corr"; then + fm_pending_reply_restore_undelivered "$state" "$corr" || true + rc=1 + elif ! fm_pending_reply_finish_undelivered "$state" "$corr"; then + rc=1 + fi + fm_pending_reply_txn_lock_release "$state" "$corr" "$token" || rc=1 + if [ "$rc" = 0 ]; then + rm -f "$meta" || rc=1 + fi + return "$rc" +} + # Drop an undelivered expectation after a failed send so transport failure does # not masquerade as a missed report later. fm_pending_reply_discard_undelivered() { # @@ -1316,13 +1392,17 @@ fm_pending_reply_restore_undelivered() { # } fm_pending_reply_finish_undelivered() { # - local state=$1 corr=$2 rec backup + local state=$1 corr=$2 rec backup meta rec=$(fm_pending_reply_path "$state" "$corr") backup="${rec}.cleanup" + meta=$(fm_pending_reply_undelivered_cleanup_meta_path "$state" "$corr") [ ! -L "$backup" ] || return 1 - [ -e "$backup" ] || return 0 - [ -f "$backup" ] || return 1 - rm -f "$backup" + [ ! -L "$meta" ] || return 1 + if [ -e "$backup" ]; then + [ -f "$backup" ] || return 1 + rm -f "$backup" || return 1 + fi + [ ! -e "$meta" ] || rm -f "$meta" } # 0 if a status line is a correlated acknowledgement for . @@ -2040,11 +2120,15 @@ fm_pending_reply_tick_one() { # [secondmate- # Never scrapes secondmate conversation; uses only parent status, backend busy # state, and optional secondmate-home wrong-home path checks. fm_pending_reply_tick() { # - local state=$1 dir rec corr task_id phase delivered meta backend target label busy sm_home + local state=$1 dir rec corr task_id phase delivered meta cleanup_meta backend target label busy sm_home local observation observation_task found i local -a observation_tasks=() observation_values=() dir=$(fm_pending_reply_dir "$state") [ -d "$dir" ] || return 0 + for cleanup_meta in "$dir"/*.cleanup-meta; do + [ -e "$cleanup_meta" ] || [ -L "$cleanup_meta" ] || continue + fm_pending_reply_retry_undelivered_cleanup "$state" "$cleanup_meta" || true + done for rec in "$dir"/*; do [ -f "$rec" ] || continue case "$(basename "$rec")" in diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 352d607babc..e713d79f010 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -101,6 +101,13 @@ clear_new_pending_route() { discard_new_pending_reply() { local route_status=0 discard_status=0 + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ "$PENDING_ROUTE_COMMITTED" = 1 ] \ + && [ -n "$PENDING_REPLY_CORR" ] \ + && [ -n "$TARGET_HOME" ] \ + && ! fm_pending_reply_schedule_undelivered_cleanup \ + "$STATE" "$PENDING_REPLY_CORR" "$TARGET_HOME"; then + discard_status=1 + fi if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ && ! fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" 1; then discard_status=1 @@ -112,6 +119,8 @@ discard_new_pending_reply() { route_status=1 fm_pending_reply_restore_undelivered "$STATE" "$PENDING_REPLY_CORR" || discard_status=1 fi + elif [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_restore_undelivered "$STATE" "$PENDING_REPLY_CORR" || true fi if [ "$route_status" = 1 ]; then echo "error: failed to clear the secondmate pending-reply route; undelivered record cleanup continued" >&2 diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 63445863093..8d61d5a7415 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -363,33 +363,14 @@ if [ "$READ_ONLY" -eq 1 ]; then [ -n "$GUARD_OUT" ] && printf '%s\n' "$GUARD_OUT" else DRAIN_STATUS=0 - DRAIN_OUT=$(FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" \ - "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1) || DRAIN_STATUS=$? - if [ "$DRAIN_STATUS" -ne 0 ]; then - printf 'error: wake drain failed (status %s); inactive reconciliation skipped\n%s\n' \ - "$DRAIN_STATUS" "$DRAIN_OUT" >&2 - elif [ -n "$DRAIN_OUT" ]; then - if printf '%s\n' "$DRAIN_OUT"; then - while IFS= read -r drain_row || [ -n "$drain_row" ]; do - IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$drain_row" - case "$_key" in - inactive-outcome:*) - if ! "$SCRIPT_DIR/fm-inactive-reconcile.sh" caller-output-complete \ - "$_key" "$drain_row" "$$" >/dev/null 2>&1; then - printf 'warning: inactive outcome output confirmation deferred for %s\n' "$_key" >&2 - elif ! "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$drain_row" >/dev/null 2>&1; then - printf 'warning: inactive outcome confirmation deferred for %s\n' "$_key" >&2 - fi - ;; - esac - done <<< "$DRAIN_OUT" - else - printf 'error: wake presentation failed; inactive outcome confirmations deferred\n' >&2 - fi - else + FM_WAKE_DRAIN_DIRECT=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1 || DRAIN_STATUS=$? + if [ "$DRAIN_STATUS" -ne 0 ] && [ "$DRAIN_STATUS" -ne 3 ]; then + printf 'error: wake drain failed (status %s); inactive reconciliation skipped\n' \ + "$DRAIN_STATUS" >&2 + elif [ "$DRAIN_STATUS" = 0 ]; then printf '(no queued wakes)\n' fi - if [ "$DRAIN_STATUS" -eq 0 ]; then + if [ "$DRAIN_STATUS" = 0 ] || [ "$DRAIN_STATUS" = 3 ]; then INACTIVE_STATUS=0 INACTIVE_OUT=$("$SCRIPT_DIR/fm-inactive-reconcile.sh" scan --startup 2>&1) || INACTIVE_STATUS=$? if [ "$INACTIVE_STATUS" -ne 0 ]; then diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 03733aeb314..4a61408050b 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -14,9 +14,16 @@ DRAIN_DEDUPED= DRAIN_RESTORE= DRAIN_PID=${BASHPID:-$$} DRAIN_LOCK_HELD=false +DRAIN_ACTIONABLE=0 present_inactive_row() { local key=$1 row=$2 status=0 go emitted worker worker_status=0 presentation_marked=0 + if [ "${FM_WAKE_DRAIN_DIRECT:-0}" = 1 ]; then + printf '%s\n' "$row" || return 1 + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-complete "$key" "$row" || return 1 + return 0 + fi trap - INT TERM HUP go=$(mktemp "$STATE/.wake-presentation.XXXXXX") || return 1 [ -f "$go" ] && [ ! -L "$go" ] || { rm -f "$go"; return 1; } @@ -107,7 +114,7 @@ restore_unprocessed_rows() { # shellcheck disable=SC2317,SC2329 # Invoked by trap handlers below. cleanup() { local status=$? restore_status=0 - if [ "$status" -ne 0 ] && [ "$DRAIN_LOCK_HELD" = true ]; then + if [ "$status" -ne 0 ] && [ "$status" -ne 3 ] && [ "$DRAIN_LOCK_HELD" = true ]; then if [ -n "$DRAIN_RESTORE" ] && [ -e "$DRAIN_RESTORE" ]; then fm_wake_restore_queue "$DRAIN_RESTORE" || restore_status=1 elif [ -n "$DRAIN_TMP" ] && [ -e "$DRAIN_TMP" ]; then @@ -161,6 +168,7 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" claim "$_key" "$drain_row" || claim_status=$? case "$claim_status" in 0) + DRAIN_ACTIONABLE=1 if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presenting "$_key" "$drain_row"; then restore_unprocessed_rows "$drain_line" || exit 1 exit 1 @@ -195,6 +203,7 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do restore_unprocessed_rows "$drain_line" || exit 1 exit 1 fi + DRAIN_ACTIONABLE=1 ;; esac done < "$DRAIN_DEDUPED" @@ -203,4 +212,7 @@ DRAIN_TMP= rm -f "$DRAIN_DEDUPED" DRAIN_DEDUPED= assert_watcher_liveness +if [ "${FM_WAKE_DRAIN_DIRECT:-0}" = 1 ] && [ "$DRAIN_ACTIONABLE" = 1 ]; then + exit 3 +fi exit 0 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 52993852f91..de59ba9aa75 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -596,14 +596,13 @@ while :; do # alive. Supervision scripts warn when this goes stale with tasks in flight. touch "$STATE/.last-watcher-beat" - if ! wake_drain_out=$(FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$WATCHER_PID" \ - "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1); then - printf '%s\n' "$wake_drain_out" >&2 - exit 1 - fi - if [ -n "$wake_drain_out" ]; then - wake "$wake_drain_out" - fi + drain_status=0 + FM_WAKE_DRAIN_DIRECT=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1 || drain_status=$? + case "$drain_status" in + 0) ;; + 3) wake "check: wake queue drained" ;; + *) exit "$drain_status" ;; + esac # Parent-owned secondmate pending-reply reconciliation: resolve correlated # parent reports, observe backend busy/idle turn completion, send one recovery diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 54be6dd0565..985f27c39c1 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -199,6 +199,19 @@ drain() { FM_WAKE_DRAIN_GENERATION="${FM_WAKE_DRAIN_GENERATION:-}" "$DRAIN" ) } +recon_from_root() { + local root=$1 fakebin=$2 home=$3 state=$4 previous=$PWD status + shift 4 + cd "$root" || return 1 + env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" "$RECON" "$@" + status=$? + cd "$previous" || return 1 + return "$status" +} + write_meta() { local state=$1 id=$2 token=$3 kind=${4:-ship} backend=${5:-tmux} window window=${6:-tmux:fm-$id} @@ -717,17 +730,11 @@ test_deferred_ack_retries_after_caller_crash() { drain_output="$dir/retry.out" drain "$root" "$home" "$fakebin" >"$drain_output" \ || fail "retry drain did not recover the deferred presentation" - ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ - -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ - CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ - "$RECON" caller-output-complete "inactive-outcome:$fingerprint" "$row" "$$" ) \ + recon_from_root "$root" "$fakebin" "$home" "$state" \ + caller-output-complete "inactive-outcome:$fingerprint" "$row" \ || fail "caller output confirmation did not recover the retried presentation" - ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ - -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ - CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ - "$RECON" confirm "inactive-outcome:$fingerprint" "$row" ) \ + recon_from_root "$root" "$fakebin" "$home" "$state" \ + confirm "inactive-outcome:$fingerprint" "$row" \ || fail "caller confirmation did not finalize the retried presentation" [ -f "$state/terminal-outcomes/$fingerprint.presented" ] || fail "retry drain did not acknowledge the recovered receipt" [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] || fail "retry drain left the receipt pending" @@ -752,17 +759,19 @@ test_deferred_ack_confirms_after_caller_emission() { drain "$root" "$home" "$fakebin" >"$drain_output" \ || fail "deferred confirmation drain failed" printf '%s\n' "$(cat "$drain_output")" > "$dir/caller-visible.out" - ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ - -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ - CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ - "$RECON" caller-output-complete "inactive-outcome:$fingerprint" "$row" "$$" ) \ + if env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ + -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" bash -c \ + 'cd "$4"; "$1" caller-output-complete "$2" "$3"; status=$?; exit "$status"' _ "$RECON" \ + "inactive-outcome:$fingerprint" "$row" "$root"; then + fail "foreign caller finalized a deferred receipt" + fi + recon_from_root "$root" "$fakebin" "$home" "$state" \ + caller-output-complete "inactive-outcome:$fingerprint" "$row" \ || fail "caller output confirmation did not finalize the receipt" - ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ - -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ - CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ - "$RECON" confirm "inactive-outcome:$fingerprint" "$row" ) \ + recon_from_root "$root" "$fakebin" "$home" "$state" \ + confirm "inactive-outcome:$fingerprint" "$row" \ || fail "caller confirmation did not finalize the receipt" [ -e "$state/terminal-outcomes/$fingerprint.presented" ] \ || fail "caller confirmation did not move the receipt" @@ -787,11 +796,8 @@ test_deferred_ack_recovers_after_output_confirmation() { export FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" drain "$root" "$home" "$fakebin" >"$dir/deferred-output-recovery.out" \ || fail "deferred output recovery drain failed" - ( cd "$root" && env -u FM_AGENT_ROLE -u FM_AGENT_TASK -u FM_AGENT_OWNER_HOME \ - -u FM_ROOT -u STATE PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ - CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ - "$RECON" caller-output-complete "inactive-outcome:$fingerprint" "$row" "$$" ) \ + recon_from_root "$root" "$fakebin" "$home" "$state" \ + caller-output-complete "inactive-outcome:$fingerprint" "$row" \ || fail "caller output confirmation failed" printf '%s\n' "$row" > "$state/.wake-queue" drain_output="$dir/deferred-output-recovery-retry.out" @@ -806,6 +812,24 @@ test_deferred_ack_recovers_after_output_confirmation() { pass "deferred inactive receipts recover confirmed output without replay" } +test_pending_receipts_replay_after_child_scan_failure() { + local dir root home fakebin state + new_case pending-replay-after-failure + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" pending-replay-x1 pending-replay-inc + export FM_FAKE_CREW_STATE_PENDING_REPLAY_X1='state: done · source: pane · pending replay' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "pending replay setup failed" + : > "$state/.wake-queue" + export FM_FAKE_CREW_STATE_EXIT=19 + if scan "$root" "$home" "$fakebin" --startup >/dev/null 2>&1; then + fail "child scan failure was reported as success during pending replay" + fi + [ "$(queue_count "$state")" = 1 ] || fail "pending receipt was not replayed after child scan failure" + unset FM_FAKE_CREW_STATE_PENDING_REPLAY_X1 FM_FAKE_CREW_STATE_EXIT + pass "pending receipts replay independently after child scan failure" +} + test_scan_failure_retries_without_advancing_cadence() { local dir root home fakebin state wake_dir wake_removed new_case scan-failure @@ -1863,6 +1887,65 @@ test_recovery_route_reuse_validates_parent_record() { pass "recovery route reuse validates the complete parent record" } +test_failed_record_removal_is_retryable() { + local dir root home fakebin state child_home child_state marker flag rec corr + new_case failed-record-cleanup + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + child_home="$dir/secondmate-home" + child_state="$child_home/state" + marker="$child_state/.fm-jt-parent-route" + mkdir -p "$child_state" "$child_home/data" "$child_home/config" "$child_home/projects" + printf 'sm-record-cleanup\n' > "$child_home/.fm-secondmate-home" + prepare_primary_proof "$root" "$home" "$fakebin" + prepare_watcher_protocol "$root" "$home" "$state" + corr=$(env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_create "$2" "$3" sm-record-cleanup "cleanup request"' \ + _ "$ROOT" "$home" "$state") || fail "cleanup record was not created" + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_write "$2" "$3" "$4" sm-record-cleanup "$5"' \ + _ "$ROOT" "$child_home" "$home" "$state" "$corr" \ + || fail "cleanup route was not written" + rec="$state/pending-replies/$corr" + flag="$dir/fail-record-remove-once" + : > "$flag" + cat > "$fakebin/rm" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + if [ "$arg" = "${FM_TEST_CLEANUP_RECORD:-}" ] && [ -e "${FM_TEST_CLEANUP_RECORD_ONCE:-}" ]; then + /bin/rm -f "$FM_TEST_CLEANUP_RECORD_ONCE" + exit 42 + fi +done +exec /bin/rm "$@" +SH + chmod +x "$fakebin/rm" + export FM_TEST_CLEANUP_RECORD="$rec" FM_TEST_CLEANUP_RECORD_ONCE="$flag" + if env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" PATH="$fakebin:$PATH" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_schedule_undelivered_cleanup "$2" "$3" "$4" && fm_pending_reply_discard_undelivered "$2" "$3" 1' \ + _ "$ROOT" "$state" "$corr" "$child_home"; then + fail "record removal failure was hidden" + fi + [ -f "$rec" ] || fail "record removal failure lost the parent record" + [ -e "$marker" ] || fail "record removal failure cleared the route" + [ -f "$rec.cleanup-meta" ] || fail "record removal failure was not durable" + rm -f "$fakebin/rm" + unset FM_TEST_CLEANUP_RECORD FM_TEST_CLEANUP_RECORD_ONCE + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_retry_undelivered_cleanup "$2" "$3"' \ + _ "$ROOT" "$state" "$rec.cleanup-meta" \ + || fail "record removal cleanup did not retry" + [ ! -e "$rec" ] || fail "record removal cleanup left the parent record" + [ ! -e "$marker" ] || fail "record removal cleanup left the route" + [ ! -e "$rec.cleanup-meta" ] || fail "record removal cleanup left its retry marker" + pass "failed record removal is durably retryable with its route" +} + test_failed_marked_send_restores_record_on_route_cleanup_failure() { local dir root home fakebin state child_home child_state marker flag rec corr send_out new_case failed-marked-cleanup @@ -2098,6 +2181,7 @@ test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash test_deferred_ack_confirms_after_caller_emission test_deferred_ack_recovers_after_output_confirmation +test_pending_receipts_replay_after_child_scan_failure test_scan_failure_retries_without_advancing_cadence test_state_paths_reject_symlinks_and_non_directories test_reused_task_id_gets_new_fingerprint @@ -2118,6 +2202,7 @@ test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected test_route_replacement_rejects_malformed_parent_record test_recovery_route_reuse_validates_parent_record +test_failed_record_removal_is_retryable test_failed_marked_send_restores_record_on_route_cleanup_failure test_failed_concurrent_send_discards_only_new_record test_failed_marked_send_discards_never_bound_record From 6a9685ca2aec357ebe5cc79a343a069806342ed9 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 14:11:59 +0000 Subject: [PATCH 025/163] no-mistakes(review): Harden replay claims and transactional cleanup --- bin/fm-inactive-reconcile.sh | 54 ++++++++++----- bin/fm-pending-reply-lib.sh | 4 +- bin/fm-send.sh | 2 +- bin/fm-wake-drain.sh | 34 +++++++--- tests/fm-inactive-outcome.test.sh | 109 ++++++++++++++++++++++++------ 5 files changed, 152 insertions(+), 51 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 74d6a9d38fb..1b45e853f78 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -207,8 +207,7 @@ claim_receipt_state() { [ ! -L "$path" ] || return 2 [ -e "$path" ] || continue [ -f "$path" ] || return 2 - [ "$(receipt_field "$path" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 - [ "$(receipt_field "$path" fingerprint)" = "$fp" ] || return 2 + prepare_pending_receipt "$path" || return 2 [ -z "$found" ] || return 2 found=$suffix done @@ -265,6 +264,17 @@ claim_validate_caller_owner() { # fm_pid_start_matches_stored "$caller_pid" "$caller_start" } +claim_rewrite_row() { + local claim=$1 row=$2 tmp line + tmp=$(mktemp "$OUTCOME_DIR/.claim-row.XXXXXX") || return 1 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in row=*) printf 'row=%s\n' "$row" ;; *) printf '%s\n' "$line" ;; esac + done < "$claim" > "$tmp" || { rm -f "$tmp"; return 1; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 1; } +} + claim_reserve() { # local key=$1 row=$2 fp claim tmp state existing old_row line output_complete defer_ack local defer_generation defer_generation_start receipt_state receipt_rc=0 recorded_report=0 report_rc=1 @@ -308,15 +318,6 @@ claim_reserve() { # case "$state" in presented|presenting|reserved) ;; *) return 2 ;; esac old_row=$(claim_field "$claim" row) [ -n "$old_row" ] || return 2 - if [ "$old_row" != "$row" ]; then - tmp=$(mktemp "$OUTCOME_DIR/.claim-row.XXXXXX") || return 2 - chmod 600 "$tmp" 2>/dev/null || true - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in row=*) printf 'row=%s\n' "$row" ;; *) printf '%s\n' "$line" ;; esac - done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } - [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } - mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } - fi if [ "$state" = presented ]; then defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) if [ "$defer_ack" = 1 ]; then @@ -326,9 +327,24 @@ claim_reserve() { # return 4 fi [ "$(claim_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 - return 5 fi fi + if [ "$state" = presenting ]; then + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$defer_ack" = 1 ]; then + defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) + defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + if claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then + return 4 + fi + fi + fi + if [ "$old_row" != "$row" ]; then + claim_rewrite_row "$claim" "$row" || return 2 + fi + if [ "$state" = presented ] && [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ]; then + return 5 + fi if [ "$recorded_report" = 1 ]; then case "$state" in presented) return 5 ;; @@ -342,10 +358,6 @@ claim_reserve() { # fi if [ "$state" = presenting ]; then output_complete=$(claim_field "$claim" output_complete 2>/dev/null || true) - if [ "$output_complete" = 1 ]; then - claim_mark_presented "$key" "$row" || return 2 - return 5 - fi defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) if [ "$defer_ack" = 1 ]; then defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) @@ -353,6 +365,12 @@ claim_reserve() { # if claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then return 4 fi + fi + if [ "$output_complete" = 1 ]; then + claim_mark_presented "$key" "$row" || return 2 + return 5 + fi + if [ "$defer_ack" = 1 ]; then return 0 fi fi @@ -1138,8 +1156,8 @@ ack_receipt() { # [ ! -L "$existing" ] || return 2 if [ -e "$existing" ]; then [ -f "$existing" ] || return 2 - [ "$(receipt_field "$existing" fingerprint)" = "$fp" ] || return 2 - existing_kind=$(receipt_field "$existing" kind) + prepare_pending_receipt "$existing" || return 2 + existing_kind=$KIND if [ "$existing_kind" = secondmate ]; then reported_secondmate_receipt_valid "$existing" || return 2 existing_corr=$(receipt_field "$existing" parent_corr) diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 715eca5e300..ed4ca9f5734 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -773,7 +773,7 @@ fm_pending_reply_secondmate_route_clear_undelivered() { # /dev/null || true)" != 5 ] \ || ! fm_pending_reply_secondmate_route_shape "$marker"; then fm_lock_release "$route_lock" || true - return 0 + return 1 fi current_corr=$(fm_pending_reply_get "$marker" corr_id) if [ "$current_corr" != "$corr" ]; then @@ -788,7 +788,7 @@ fm_pending_reply_secondmate_route_clear_undelivered() { # /dev/null 2>&1; then + return 0 + fi + return 3 fi trap - INT TERM HUP go=$(mktemp "$STATE/.wake-presentation.XXXXXX") || return 1 @@ -40,11 +46,9 @@ present_inactive_row() { done printf '%s\n' "$row" || exit 1 : > "$emitted" || exit 1 - if [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then - FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ - FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - output-complete "$key" "$row" || exit 1 - fi + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ + FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-complete "$key" "$row" || exit 1 ) & worker=$! : > "$go" || status=1 @@ -57,11 +61,13 @@ present_inactive_row() { fi rm -f "$go" if [ "$status" -ne 0 ]; then - if [ -e "$emitted" ] && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then + if [ -e "$emitted" ]; then if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ presented "$key" "$row" >/dev/null 2>&1; then presentation_marked=1 status=0 + else + status=3 fi else FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ @@ -173,8 +179,14 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do restore_unprocessed_rows "$drain_line" || exit 1 exit 1 fi - if ! present_inactive_row "$_key" "$drain_row"; then - restore_unprocessed_rows "$drain_line" || exit 1 + present_status=0 + present_inactive_row "$_key" "$drain_row" || present_status=$? + if [ "$present_status" -ne 0 ]; then + if [ "$present_status" = 3 ]; then + restore_unprocessed_rows "$((drain_line + 1))" || exit 1 + else + restore_unprocessed_rows "$drain_line" || exit 1 + fi exit 1 fi ;; diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 985f27c39c1..976488e64c8 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -187,7 +187,7 @@ scan() { } drain() { - local root=$1 home=$2 fakebin=$3 + local root=$1 home=$2 fakebin=$3 status if [ -d "$home/state" ] && [ ! -L "$home/state" ]; then prepare_primary_proof "$root" "$home" "$fakebin" fi @@ -197,6 +197,8 @@ drain() { CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" \ FM_WAKE_DRAIN_DEFER_ACK="${FM_WAKE_DRAIN_DEFER_ACK:-0}" \ FM_WAKE_DRAIN_GENERATION="${FM_WAKE_DRAIN_GENERATION:-}" "$DRAIN" ) + status=$? + [ "$status" = 0 ] || [ "$status" = 3 ] || return "$status" } recon_from_root() { @@ -654,6 +656,7 @@ exec /usr/bin/mv "$@" SH chmod +x "$fakebin/mv" export FM_FAIL_CLAIM_MOVE="$dir/fail-claim-move" + export FM_WAKE_DRAIN_DIRECT=1 drain "$root" "$home" "$fakebin" >"$dir/output-failure.out" \ || fail "output completion failure was not recovered" [ "$(grep -Fxc "$row" "$dir/output-failure.out")" = 1 ] \ @@ -664,7 +667,7 @@ SH || fail "output completion failure left the receipt pending" [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] \ || fail "output completion failure left a stale claim" - unset FM_FAIL_CLAIM_MOVE + unset FM_FAIL_CLAIM_MOVE FM_WAKE_DRAIN_DIRECT pass "post-output failures finalize without duplicate presentation" } @@ -689,8 +692,30 @@ test_finalized_receipt_rows_are_suppressed() { pass "finalized receipt rows are suppressed after drain rollback" } +test_malformed_finalized_receipt_fails_closed() { + local dir root home fakebin state fingerprint row + new_case malformed-finalized-receipt + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" malformed-finalized-x1 malformed-finalized-inc + export FM_FAKE_CREW_STATE_MALFORMED_FINALIZED_X1='state: done · source: pane · malformed finalized' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "malformed finalized receipt setup failed" + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) + row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") + drain "$root" "$home" "$fakebin" >/dev/null || fail "initial finalized receipt drain failed" + replace_field "$state/terminal-outcomes/$fingerprint.presented" task_id tampered-finalized + printf '%s\n' "$row" > "$state/.wake-queue" + if drain "$root" "$home" "$fakebin" >"$dir/malformed-finalized.out" 2>&1; then + fail "malformed finalized receipt was suppressed" + fi + [ "$(queue_count "$state")" = 1 ] || fail "malformed finalized receipt wake was consumed" + [ "$(receipt_count "$state" presented)" = 1 ] || fail "malformed finalized receipt state disappeared" + unset FM_FAKE_CREW_STATE_MALFORMED_FINALIZED_X1 + pass "malformed finalized receipts fail closed before wake suppression" +} + test_deferred_ack_retries_after_caller_crash() { - local dir root home fakebin state fingerprint row drain_output rec + local dir root home fakebin state fingerprint row replayed_row drain_output rec new_case deferred-ack dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -711,14 +736,17 @@ test_deferred_ack_retries_after_caller_crash() { || fail "deferred drain did not retain the presentation claim" [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" defer_ack)" = 1 ] \ || fail "deferred drain did not mark the claim for caller confirmation" - [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_started)" = 0 ] \ - || fail "deferred drain advanced output before caller emission" - [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_complete)" = 0 ] \ - || fail "deferred drain completed output before caller confirmation" - printf '%s\n' "$row" > "$state/.wake-queue" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_started)" = 1 ] \ + || fail "deferred drain did not persist the emitted handoff" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_complete)" = 1 ] \ + || fail "deferred drain did not persist output completion" + replayed_row=$'2\t2\tcheck\tinactive-outcome:'"$fingerprint"$'\treplayed deferred row' + printf '%s\n' "$replayed_row" > "$state/.wake-queue" drain "$root" "$home" "$fakebin" >"$dir/live-deferred.out" \ || fail "live deferred claim drain failed" [ ! -s "$dir/live-deferred.out" ] || fail "live deferred claim was presented twice" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" row)" = "$row" ] \ + || fail "live deferred claim row was replaced by a replayed wake" [ -f "$state/terminal-outcomes/$fingerprint.pending" ] \ || fail "live deferred claim was acknowledged before caller confirmation" replace_field "$state/terminal-outcomes/.$fingerprint.claim" defer_generation "$$" @@ -730,16 +758,10 @@ test_deferred_ack_retries_after_caller_crash() { drain_output="$dir/retry.out" drain "$root" "$home" "$fakebin" >"$drain_output" \ || fail "retry drain did not recover the deferred presentation" - recon_from_root "$root" "$fakebin" "$home" "$state" \ - caller-output-complete "inactive-outcome:$fingerprint" "$row" \ - || fail "caller output confirmation did not recover the retried presentation" - recon_from_root "$root" "$fakebin" "$home" "$state" \ - confirm "inactive-outcome:$fingerprint" "$row" \ - || fail "caller confirmation did not finalize the retried presentation" [ -f "$state/terminal-outcomes/$fingerprint.presented" ] || fail "retry drain did not acknowledge the recovered receipt" [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] || fail "retry drain left the receipt pending" [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "retry drain left the deferred claim" - [ "$(grep -Fxc "$row" "$drain_output")" = 1 ] || fail "retry drain did not re-present output never emitted by the caller" + [ ! -s "$drain_output" ] || fail "retry drain re-presented an already emitted row" unset FM_WAKE_DRAIN_DEFER_ACK FM_WAKE_DRAIN_GENERATION FM_FAKE_CREW_STATE_DEFERRED_X1 pass "deferred inactive receipts retry until caller-visible emission" } @@ -799,6 +821,7 @@ test_deferred_ack_recovers_after_output_confirmation() { recon_from_root "$root" "$fakebin" "$home" "$state" \ caller-output-complete "inactive-outcome:$fingerprint" "$row" \ || fail "caller output confirmation failed" + replace_field "$state/terminal-outcomes/.$fingerprint.claim" defer_generation_start proc:0 printf '%s\n' "$row" > "$state/.wake-queue" drain_output="$dir/deferred-output-recovery-retry.out" drain "$root" "$home" "$fakebin" >"$drain_output" \ @@ -1625,7 +1648,7 @@ SH } test_undelivered_secondmate_route_cleanup_is_idempotent() { - local dir root home fakebin state child_home child_state marker corr rec fail_rm_once + local dir root home fakebin state child_home child_state marker corr rec fail_rm_once valid_marker new_case secondmate-undelivered-cleanup dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -1668,6 +1691,16 @@ SH fi [ -e "$rec" ] || fail "undelivered record was removed after a failed cleanup" [ -e "$marker" ] || fail "undelivered route was cleared before record removal" + valid_marker=$(cat "$marker") + printf 'malformed=route\n' > "$marker" + if env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_secondmate_route_clear_undelivered "$2" "$3"' \ + _ "$ROOT" "$child_home" "$corr"; then + fail "malformed undelivered route cleanup was treated as success" + fi + [ -e "$rec" ] || fail "malformed undelivered route cleanup removed the parent record" + printf '%s\n' "$valid_marker" > "$marker" rm -f "$fakebin/rm" unset FM_TEST_UNDELIVERED_RECORD FM_TEST_UNDELIVERED_RM_ONCE env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ @@ -2053,7 +2086,7 @@ test_failed_concurrent_send_discards_only_new_record() { } test_failed_marked_send_discards_never_bound_record() { - local dir root home fakebin state child_home child_state send_out + local dir root home fakebin state child_home child_state send_out flag rec new_case failed-never-bound-send dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -2073,6 +2106,31 @@ SH chmod +x "$fakebin/tmux" prepare_primary_proof "$root" "$home" "$fakebin" prepare_watcher_protocol "$root" "$home" "$state" + flag="$dir/never-bound-record-remove" + : > "$flag" + cat > "$fakebin/rm" <<'SH' +#!/usr/bin/env bash +set -u +for arg in "$@"; do + case "$arg" in + */pending-replies/*) + case "$(basename "$arg")" in + .*) ;; + *) + if [ -e "${FM_TEST_NEVER_BOUND_ONCE:-}" ]; then + /bin/rm -f "$FM_TEST_NEVER_BOUND_ONCE" + exit 42 + fi + ;; + esac + ;; + esac +done +exec /bin/rm "$@" +SH + chmod +x "$fakebin/rm" + export FM_TEST_NEVER_BOUND_ONCE="$flag" + cp -a "$ROOT/bin/." "$root/bin/" send_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u FM_AGENT_ROLE -u FM_AGENT_TASK \ -u FM_AGENT_OWNER_HOME -u FM_ROOT -u STATE -u FM_PENDING_REPLY_EXISTING_CORR \ PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ @@ -2080,10 +2138,22 @@ SH CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 "$root/bin/fm-send.sh" \ fm-sm-never-bound "never bound request" 2>&1) && fail "never-bound marked send unexpectedly succeeded" - [ "$(direct_file_count "$state/pending-replies" '*')" = 0 ] \ - || fail "never-bound send left an awaiting_report record without a route" + rec=$(find "$state/pending-replies" -maxdepth 1 -type f -name '????????????????' -print -quit) + [ -f "$rec" ] || fail "never-bound cleanup did not retain the failed record for retry" + [ -f "$rec.cleanup-meta" ] || fail "never-bound cleanup did not persist repair metadata" [ ! -e "$child_state/.fm-jt-parent-route" ] \ || fail "never-bound send installed a route through an unsafe state path" + rm -f "$fakebin/rm" + unset FM_TEST_NEVER_BOUND_ONCE + rm -f "$child_state" + mkdir -p "$child_state" + env FM_SESSION_LOCK_BOOTSTRAP=1 FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" bash -c \ + '. "$1/bin/fm-pending-reply-lib.sh"; fm_pending_reply_retry_undelivered_cleanup "$2" "$3"' \ + _ "$ROOT" "$state" "$rec.cleanup-meta" \ + || fail "never-bound cleanup did not retry after the unsafe state was repaired" + [ ! -e "$rec" ] || fail "never-bound retry left the parent record" + [ ! -e "$rec.cleanup-meta" ] || fail "never-bound retry left repair metadata" pass "failed marked sends discard expectations without a committed route" } @@ -2177,6 +2247,7 @@ test_output_started_claim_is_not_reprinted test_pre_output_claim_retries_after_crash test_output_completion_failure_does_not_reprint test_finalized_receipt_rows_are_suppressed +test_malformed_finalized_receipt_fails_closed test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash test_deferred_ack_confirms_after_caller_emission From 29585d63104915a6ba4fbe58983f10e8e2dbf460 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 17:07:26 +0000 Subject: [PATCH 026/163] no-mistakes(review): Fix direct finalization and generation-bound watcher acknowledgement --- bin/fm-wake-drain.sh | 6 ++- bin/fm-watch.sh | 9 +++-- tests/fm-inactive-outcome.test.sh | 67 ++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 8797df02a63..41937e43654 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -22,7 +22,11 @@ present_inactive_row() { printf '%s\n' "$row" || return 1 if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ output-complete "$key" "$row"; then - return 0 + if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + presented "$key" "$row" >/dev/null 2>&1; then + return 0 + fi + return 3 fi if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ presented "$key" "$row" >/dev/null 2>&1; then diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index de59ba9aa75..0ea28cb930b 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -596,11 +596,14 @@ while :; do # alive. Supervision scripts warn when this goes stale with tasks in flight. touch "$STATE/.last-watcher-beat" + drain_output= drain_status=0 - FM_WAKE_DRAIN_DIRECT=1 "$SCRIPT_DIR/fm-wake-drain.sh" 2>&1 || drain_status=$? + drain_output=$(FM_WAKE_DRAIN_DIRECT=0 FM_WAKE_DRAIN_DEFER_ACK=1 \ + FM_WAKE_DRAIN_GENERATION="$WATCHER_PID" "$SCRIPT_DIR/fm-wake-drain.sh") \ + || drain_status=$? case "$drain_status" in - 0) ;; - 3) wake "check: wake queue drained" ;; + 0) [ -n "$drain_output" ] && wake "$drain_output" ;; + 3) exit 3 ;; *) exit "$drain_status" ;; esac diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 976488e64c8..1fb5d2913ea 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -671,6 +671,32 @@ SH pass "post-output failures finalize without duplicate presentation" } +test_direct_drain_finalizes_after_successful_output() { + local dir root home fakebin state fingerprint row + new_case direct-success + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + write_meta "$state" direct-x1 direct-inc + export FM_FAKE_CREW_STATE_DIRECT_X1='state: done · source: pane · direct presentation' + scan "$root" "$home" "$fakebin" --startup >/dev/null || fail "direct receipt setup failed" + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) + row=$(awk -F '\t' -v key="inactive-outcome:$fingerprint" '$4 == key { print; exit }' "$state/.wake-queue") + export FM_WAKE_DRAIN_DIRECT=1 + drain "$root" "$home" "$fakebin" >"$dir/direct.out" \ + || fail "direct drain did not finalize a successful presentation" + grep -F "$row" "$dir/direct.out" >/dev/null || fail "direct drain did not emit the wake row" + [ "$(receipt_count "$state" presented)" = 1 ] || fail "direct drain left the receipt unfinalized" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "direct drain left a pending receipt" + [ "$(queue_count "$state")" = 0 ] || fail "direct drain left the wake queued" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "direct drain left a presentation claim" + printf '%s\n' "$row" > "$state/.wake-queue" + drain "$root" "$home" "$fakebin" >"$dir/direct-replay.out" \ + || fail "direct replay drain failed" + [ ! -s "$dir/direct-replay.out" ] || fail "direct drain replayed a finalized receipt" + unset FM_FAKE_CREW_STATE_DIRECT_X1 FM_WAKE_DRAIN_DIRECT + pass "direct inactive drains finalize successful output once" +} + test_finalized_receipt_rows_are_suppressed() { local dir root home fakebin state fingerprint row new_case finalized-row @@ -1073,7 +1099,7 @@ SH } test_watcher_runs_inactive_cadence() { - local dir root home fakebin state out second_out status + local dir root home fakebin state out second_out third_out status fingerprint new_case watcher-wiring dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -1126,6 +1152,42 @@ SH || fail "watcher did not surface the inactive reconciliation result" [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher cadence did not create the inactive receipt" [ "$(queue_count "$state")" = 1 ] || fail "watcher cadence did not retain exactly one inactive outcome wake" + + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) + third_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ + FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 FM_FAKE_PANE_PATH="$home" \ + FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ + "$root/bin/fm-watch.sh" 2>&1) + status=$? + [ "$status" = 0 ] || fail "watcher cadence failed while draining the inactive outcome wake" + printf '%s\n' "$third_out" | grep -F 'inactive-outcome:' >/dev/null \ + || fail "watcher did not surface the exact inactive outcome wake" + [ "$(receipt_count "$state" pending)" = 0 ] || fail "watcher cadence did not acknowledge the inactive receipt" + [ "$(receipt_count "$state" presented)" = 1 ] || fail "watcher cadence did not finalize the inactive receipt" + [ "$(queue_count "$state")" = 0 ] || fail "watcher cadence did not consume the inactive outcome wake" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] || fail "watcher cadence left a presentation claim" + + write_meta "$state" watcher-failure-x1 watcher-failure-inc + export FM_FAKE_CREW_STATE_WATCHER_FAILURE_X1='state: failed · source: pane · watcher output failure' + rm -f "$state/.inactive-outcome-reconcile" + out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ + FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 FM_FAKE_PANE_PATH="$home" \ + FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ + "$root/bin/fm-watch.sh" 2>&1) + status=$? + [ "$status" = 0 ] || fail "watcher cadence failed while queuing the failure fixture" + [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher failure fixture did not create one pending receipt" + [ "$(queue_count "$state")" = 1 ] || fail "watcher failure fixture did not retain one wake" set +e ( cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ @@ -1142,7 +1204,7 @@ SH || fail "watcher consumed the inactive receipt after presentation failure" [ -e "$state/terminal-outcomes/.$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending).claim" ] \ || fail "watcher did not retain a retryable presentation claim" - unset FM_FAKE_CREW_STATE_WATCHER_X1 + unset FM_FAKE_CREW_STATE_WATCHER_X1 FM_FAKE_CREW_STATE_WATCHER_FAILURE_X1 pass "watcher cadence gates acknowledgement on successful output" } @@ -2246,6 +2308,7 @@ test_presenting_claim_recovers_before_output test_output_started_claim_is_not_reprinted test_pre_output_claim_retries_after_crash test_output_completion_failure_does_not_reprint +test_direct_drain_finalizes_after_successful_output test_finalized_receipt_rows_are_suppressed test_malformed_finalized_receipt_fails_closed test_presented_claim_is_acknowledged_in_deferred_drain From 5928ff6353e62801d2c6b15bbbf66ae49aede99c Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 17:32:30 +0000 Subject: [PATCH 027/163] no-mistakes(review): Retry deferred output completion before acknowledgement --- bin/fm-wake-drain.sh | 6 +++- tests/fm-inactive-outcome.test.sh | 57 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 41937e43654..e24945bc30d 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -67,7 +67,11 @@ present_inactive_row() { if [ "$status" -ne 0 ]; then if [ -e "$emitted" ]; then if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - presented "$key" "$row" >/dev/null 2>&1; then + output-complete "$key" "$row" >/dev/null 2>&1; then + status=0 + elif [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ] \ + && FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + presented "$key" "$row" >/dev/null 2>&1; then presentation_marked=1 status=0 else diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 1fb5d2913ea..6572ef25edd 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -792,6 +792,62 @@ test_deferred_ack_retries_after_caller_crash() { pass "deferred inactive receipts retry until caller-visible emission" } +test_deferred_output_completion_retries_before_confirmation() { + local dir root home fakebin state fingerprint row + new_case deferred-output-complete-retry + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'deferred-retry-x1|deferred-retry-inc|done|state: done · source: pane · deferred output retry') + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=deferred-retry-x1 \ + incarnation=deferred-retry-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · deferred output retry' kind=ship + row=$'2\t2\tcheck\tinactive-outcome:'"$fingerprint"$'\tdeferred output retry row' + printf '%s\n' "$row" > "$state/.wake-queue" + cat > "$fakebin/mv" <<'SH' +#!/usr/bin/env bash +set -u +target="${!#}" +case "$target" in + *.claim) + count=$(cat "${FM_FAIL_CLAIM_MOVE:?}" 2>/dev/null || printf '0') + count=$((count + 1)) + printf '%s\n' "$count" > "$FM_FAIL_CLAIM_MOVE" + if [ "$count" = 2 ]; then + exit 91 + fi + ;; +esac +exec /usr/bin/mv "$@" +SH + chmod +x "$fakebin/mv" + export FM_FAIL_CLAIM_MOVE="$dir/fail-claim-move" + export FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" + drain "$root" "$home" "$fakebin" >"$dir/deferred-retry.out" \ + || fail "deferred output-complete retry drain failed" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" state)" = presenting ] \ + || fail "deferred output-complete retry advanced the claim before confirmation" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_complete)" = 1 ] \ + || fail "deferred output-complete retry did not persist completion" + [ -f "$state/terminal-outcomes/$fingerprint.pending" ] \ + || fail "deferred output-complete retry consumed the receipt early" + [ ! -e "$state/terminal-outcomes/$fingerprint.presented" ] \ + || fail "deferred output-complete retry presented before confirmation" + recon_from_root "$root" "$fakebin" "$home" "$state" \ + caller-output-complete "inactive-outcome:$fingerprint" "$row" \ + || fail "deferred output-complete retry rejected caller completion" + recon_from_root "$root" "$fakebin" "$home" "$state" \ + confirm "inactive-outcome:$fingerprint" "$row" \ + || fail "deferred output-complete retry rejected caller confirmation" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] \ + || fail "deferred output-complete retry did not finalize after confirmation" + [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] \ + || fail "deferred output-complete retry left a claim" + unset FM_FAIL_CLAIM_MOVE FM_WAKE_DRAIN_DEFER_ACK FM_WAKE_DRAIN_GENERATION + pass "deferred output completion retries before caller confirmation" +} + test_deferred_ack_confirms_after_caller_emission() { local dir root home fakebin state fingerprint row drain_output new_case deferred-confirm @@ -2313,6 +2369,7 @@ test_finalized_receipt_rows_are_suppressed test_malformed_finalized_receipt_fails_closed test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash +test_deferred_output_completion_retries_before_confirmation test_deferred_ack_confirms_after_caller_emission test_deferred_ack_recovers_after_output_confirmation test_pending_receipts_replay_after_child_scan_failure From 8724ea7d8165873f9a129e062e94d6ed963b9ba6 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 18:07:50 +0000 Subject: [PATCH 028/163] no-mistakes(review): Hardened replay retry and bounded receipt maintenance --- bin/fm-inactive-reconcile.sh | 121 +++++++++++++++++++----- bin/fm-pending-reply-lib.sh | 2 +- bin/fm-wake-drain.sh | 27 +++--- tests/fm-inactive-outcome.test.sh | 149 ++++++++++++++++++++++++++---- 4 files changed, 243 insertions(+), 56 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 1b45e853f78..535bf39d80d 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -15,6 +15,7 @@ STATE="${FM_STATE_OVERRIDE:-${STATE:-$FM_HOME/state}}" OUTCOME_DIR="$STATE/terminal-outcomes" SCAN_MARKER="$STATE/.inactive-outcome-reconcile" SCAN_CURSOR="$STATE/.inactive-outcome-reconcile.cursor" +REPORTED_ROUTE_CURSOR="$STATE/.reported-secondmate-route-repair.cursor" FM_WAKE_QUEUE="${FM_WAKE_QUEUE:-$STATE/.wake-queue}" inactive_state_path_is_safe() { @@ -40,6 +41,7 @@ inactive_state_preflight() { inactive_state_path_is_safe "$OUTCOME_DIR" dir || return 1 inactive_state_path_is_safe "$SCAN_MARKER" file || return 1 inactive_state_path_is_safe "$SCAN_CURSOR" file || return 1 + inactive_state_path_is_safe "$REPORTED_ROUTE_CURSOR" file || return 1 inactive_state_path_is_safe "$FM_WAKE_QUEUE" file || return 1 } @@ -79,6 +81,7 @@ bounded_secs() { RECONCILE_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_SECS:-900}" 900 60 1800) SCAN_BUDGET_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_BUDGET_SECS:-10}" 10 1 300) +REPORTED_ROUTE_REPAIR_LIMIT=$(bounded_secs "${FM_REPORTED_ROUTE_REPAIR_LIMIT:-32}" 32 1 256) meta_value() { # awk -F= -v wanted="$2" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$1" 2>/dev/null @@ -276,7 +279,7 @@ claim_rewrite_row() { } claim_reserve() { # - local key=$1 row=$2 fp claim tmp state existing old_row line output_complete defer_ack + local key=$1 row=$2 fp claim tmp state existing old_row line output_started output_emitted output_complete defer_ack local defer_generation defer_generation_start receipt_state receipt_rc=0 recorded_report=0 report_rc=1 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac @@ -330,6 +333,8 @@ claim_reserve() { # fi fi if [ "$state" = presenting ]; then + output_started=$(claim_field "$claim" output_started 2>/dev/null || true) + output_emitted=$(claim_field "$claim" output_emitted 2>/dev/null || true) defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) if [ "$defer_ack" = 1 ]; then defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) @@ -370,6 +375,13 @@ claim_reserve() { # claim_mark_presented "$key" "$row" || return 2 return 5 fi + if [ "$output_emitted" = 1 ]; then + if claim_mark_output_complete "$key" "$row"; then + claim_mark_presented "$key" "$row" || return 2 + return 5 + fi + return 6 + fi if [ "$defer_ack" = 1 ]; then return 0 fi @@ -425,7 +437,7 @@ claim_reserve() { # claim_mark_presenting() { # local key=$1 row=$2 fp claim state tmp line defer_ack=0 defer_generation= defer_generation_start= - local seen_pid=0 seen_output=0 seen_complete=0 + local seen_pid=0 seen_output=0 seen_emitted=0 seen_complete=0 local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 [ "${FM_WAKE_DRAIN_DIRECT:-0}" != 1 ] \ && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] && defer_ack=1 @@ -451,6 +463,7 @@ claim_mark_presenting() { # state=*) printf 'state=presenting\n' ;; presentation_pid=*) printf 'presentation_pid=%s\n' "${BASHPID:-$$}"; seen_pid=1 ;; output_started=*) printf 'output_started=0\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=0\n'; seen_emitted=1 ;; output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; defer_ack=*) printf 'defer_ack=%s\n' "$defer_ack"; seen_defer_ack=1 ;; defer_generation=*) printf 'defer_generation=%s\n' "$defer_generation"; seen_defer_generation=1 ;; @@ -460,6 +473,7 @@ claim_mark_presenting() { # done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_pid" = 1 ] || printf 'presentation_pid=%s\n' "${BASHPID:-$$}" >> "$tmp" [ "$seen_output" = 1 ] || printf 'output_started=0\n' >> "$tmp" + [ "$seen_emitted" = 1 ] || printf 'output_emitted=0\n' >> "$tmp" [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' >> "$tmp" [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=%s\n' "$defer_ack" >> "$tmp" [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=%s\n' "$defer_generation" >> "$tmp" @@ -471,6 +485,7 @@ claim_mark_presenting() { # claim_mark_output_complete() { # local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} local fp claim state tmp line seen_output=0 seen_complete=0 + local seen_emitted=0 case "$owner_required" in 1) drain_claim_owner "$row" || return 2 ;; 0) @@ -492,11 +507,13 @@ claim_mark_output_complete() { # while IFS= read -r line || [ -n "$line" ]; do case "$line" in output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=1\n'; seen_emitted=1 ;; output_complete=*) printf 'output_complete=1\n'; seen_complete=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_output" = 1 ] || printf 'output_started=1\n' >> "$tmp" + [ "$seen_emitted" = 1 ] || printf 'output_emitted=1\n' >> "$tmp" [ "$seen_complete" = 1 ] || printf 'output_complete=1\n' >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } @@ -504,6 +521,7 @@ claim_mark_output_complete() { # claim_mark_output_started() { # local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_complete=0 + local seen_emitted=0 drain_claim_owner "$row" || return 2 case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac @@ -516,11 +534,13 @@ claim_mark_output_started() { # while IFS= read -r line || [ -n "$line" ]; do case "$line" in output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=1\n'; seen_emitted=1 ;; output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_output" = 1 ] || printf 'output_started=1\n' >> "$tmp" + [ "$seen_emitted" = 1 ] || printf 'output_emitted=1\n' >> "$tmp" [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } @@ -542,6 +562,7 @@ claim_mark_presented() { # claim=$(claim_path "$fp") [ ! -L "$claim" ] || return 2 [ "$(claim_validate "$claim" "$fp" "$row")" = presenting ] || return 2 + [ "$(claim_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 chmod 600 "$tmp" 2>/dev/null || true while IFS= read -r line || [ -n "$line" ]; do @@ -562,7 +583,7 @@ claim_mark_presented() { # claim_mark_confirmed() { # local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} - local fp claim state tmp line seen_output=0 seen_complete=0 + local fp claim state tmp line seen_output=0 seen_emitted=0 seen_complete=0 case "$owner_required" in 1) drain_claim_owner "$row" || return 2 ;; 0) ;; @@ -591,11 +612,13 @@ claim_mark_confirmed() { # case "$line" in state=*) printf 'state=presented\n' ;; output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=1\n'; seen_emitted=1 ;; output_complete=*) printf 'output_complete=1\n'; seen_complete=1 ;; *) printf '%s\n' "$line" ;; esac done < "$claim" > "$tmp" || { rm -f "$tmp"; return 2; } [ "$seen_output" = 1 ] || printf 'output_started=1\n' >> "$tmp" + [ "$seen_emitted" = 1 ] || printf 'output_emitted=1\n' >> "$tmp" [ "$seen_complete" = 1 ] || printf 'output_complete=1\n' >> "$tmp" [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } mv -f "$tmp" "$claim" || { rm -f "$tmp"; return 2; } @@ -942,35 +965,82 @@ reported_secondmate_receipt_valid() { repair_reported_secondmate_routes() { local scan_started=$1 reported kind corr parent_task_id parent_home parent_status now remaining status=0 + local cursor='' cursor_found=0 started=1 pass base last processed=0 stop_after=0 cursor_tmp [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 now=$(date +%s) remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) [ "$remaining" -gt 0 ] || return 1 FM_LOCK_WAIT_SECS="$remaining" fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 - for reported in "$OUTCOME_DIR"/*.reported; do - now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) - [ "$remaining" -gt 0 ] || { status=1; break; } - [ -e "$reported" ] || [ -L "$reported" ] || continue - [ -f "$reported" ] && [ ! -L "$reported" ] || { status=1; continue; } - kind=$(receipt_field "$reported" kind 2>/dev/null || true) - case "$kind" in - ship|scout) continue ;; - secondmate) - if ! reported_secondmate_receipt_valid "$reported"; then - status=1 - continue + cursor=$(cat "$REPORTED_ROUTE_CURSOR" 2>/dev/null || true) + case "$cursor" in + '') ;; + *.reported) + case "$cursor" in *[!A-Za-z0-9._-]*) cursor=;; esac + ;; + *) cursor=;; + esac + if [ -n "$cursor" ] && { [ ! -f "$OUTCOME_DIR/$cursor" ] || [ -L "$OUTCOME_DIR/$cursor" ]; }; then + cursor= + fi + [ -n "$cursor" ] && started=0 + for pass in 1 2; do + if [ "$pass" = 2 ]; then + [ -n "$cursor" ] && [ "$cursor_found" = 1 ] || break + started=1 + stop_after=0 + fi + for reported in "$OUTCOME_DIR"/*.reported; do + [ -e "$reported" ] || [ -L "$reported" ] || continue + base=${reported##*/} + if [ "$pass" = 1 ] && [ -n "$cursor" ] && [ "$started" = 0 ]; then + if [ "$base" = "$cursor" ]; then + started=1 + cursor_found=1 fi - corr=$(receipt_field "$reported" parent_corr) - parent_task_id=$(receipt_field "$reported" parent_task_id) - parent_home=$(receipt_field "$reported" parent_home) - parent_status=$(receipt_field "$reported" parent_status) - FM_LOCK_WAIT_SECS="$remaining" fm_pending_reply_secondmate_route_clear_reported \ - "$FM_HOME" "$corr" "$parent_task_id" "$parent_home" "$parent_status" || status=1 - ;; - *) status=1 ;; - esac + continue + fi + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + [ "$remaining" -gt 0 ] || { status=1; break 2; } + [ -f "$reported" ] && [ ! -L "$reported" ] || { status=1; continue; } + kind=$(receipt_field "$reported" kind 2>/dev/null || true) + case "$kind" in + ship|scout) : ;; + secondmate) + if ! reported_secondmate_receipt_valid "$reported"; then + status=1 + else + corr=$(receipt_field "$reported" parent_corr) + parent_task_id=$(receipt_field "$reported" parent_task_id) + parent_home=$(receipt_field "$reported" parent_home) + parent_status=$(receipt_field "$reported" parent_status) + FM_LOCK_WAIT_SECS="$remaining" fm_pending_reply_secondmate_route_clear_reported \ + "$FM_HOME" "$corr" "$parent_task_id" "$parent_home" "$parent_status" || status=1 + fi + ;; + *) status=1 ;; + esac + last=$base + processed=$((processed + 1)) + if [ "$pass" = 2 ] && [ "$base" = "$cursor" ]; then + stop_after=1 + fi + [ "$processed" -lt "$REPORTED_ROUTE_REPAIR_LIMIT" ] || break 2 + [ "$stop_after" = 0 ] || break + done done + if [ "$processed" -gt 0 ]; then + if cursor_tmp=$(mktemp "$STATE/.reported-route-repair.cursor.XXXXXX"); then + if [ ! -f "$cursor_tmp" ] || [ -L "$cursor_tmp" ] \ + || ! printf '%s\n' "$last" > "$cursor_tmp" \ + || ! mv -f "$cursor_tmp" "$REPORTED_ROUTE_CURSOR"; then + status=1 + rm -f "$cursor_tmp" + fi + else + status=1 + fi + fi fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=1 return "$status" } @@ -1146,6 +1216,7 @@ ack_receipt() { # case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac claim_state=$(claim_validate "$(claim_path "$fp")" "$fp" "$row") || return 2 [ "$claim_state" = presented ] || return 2 + [ "$(claim_field "$(claim_path "$fp")" output_complete 2>/dev/null || true)" = 1 ] || return 2 if [ "$owner_required" = 0 ]; then claim_validate_caller_owner "$(claim_path "$fp")" "$expected_generation" || return 2 fi diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index ed4ca9f5734..83e12be8ec9 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -2132,7 +2132,7 @@ fm_pending_reply_tick() { # for rec in "$dir"/*; do [ -f "$rec" ] || continue case "$(basename "$rec")" in - .*) continue ;; + .*|*.cleanup|*.cleanup-meta) continue ;; esac corr=$(fm_pending_reply_get "$rec" corr_id) [ -n "$corr" ] || corr=$(basename "$rec") diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index e24945bc30d..8c94000a12e 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -17,9 +17,11 @@ DRAIN_LOCK_HELD=false DRAIN_ACTIONABLE=0 present_inactive_row() { - local key=$1 row=$2 status=0 go emitted worker worker_status=0 presentation_marked=0 + local key=$1 row=$2 status=0 go emitted worker worker_status=0 if [ "${FM_WAKE_DRAIN_DIRECT:-0}" = 1 ]; then printf '%s\n' "$row" || return 1 + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-started "$key" "$row" >/dev/null 2>&1 || true if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ output-complete "$key" "$row"; then if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ @@ -28,10 +30,6 @@ present_inactive_row() { fi return 3 fi - if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - presented "$key" "$row" >/dev/null 2>&1; then - return 0 - fi return 3 fi trap - INT TERM HUP @@ -50,6 +48,9 @@ present_inactive_row() { done printf '%s\n' "$row" || exit 1 : > "$emitted" || exit 1 + FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ + FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ + output-started "$key" "$row" || exit 1 FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" FM_WAKE_DRAIN_DELEGATED=1 \ FM_WAKE_DRAIN_PARENT_PID="$DRAIN_PID" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ output-complete "$key" "$row" || exit 1 @@ -67,12 +68,9 @@ present_inactive_row() { if [ "$status" -ne 0 ]; then if [ -e "$emitted" ]; then if FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - output-complete "$key" "$row" >/dev/null 2>&1; then - status=0 - elif [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ] \ + output-started "$key" "$row" >/dev/null 2>&1 \ && FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" \ - presented "$key" "$row" >/dev/null 2>&1; then - presentation_marked=1 + output-complete "$key" "$row" >/dev/null 2>&1; then status=0 else status=3 @@ -82,8 +80,7 @@ present_inactive_row() { presenting "$key" "$row" >/dev/null 2>&1 || true fi fi - if [ "$status" = 0 ] && [ "$presentation_marked" = 0 ] \ - && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then + if [ "$status" = 0 ] && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" != 1 ]; then if ! FM_WAKE_DRAIN_FILE="$DRAIN_DEDUPED" "$SCRIPT_DIR/fm-inactive-reconcile.sh" presented "$key" "$row"; then status=1 if [ -e "$emitted" ] \ @@ -191,7 +188,7 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do present_inactive_row "$_key" "$drain_row" || present_status=$? if [ "$present_status" -ne 0 ]; then if [ "$present_status" = 3 ]; then - restore_unprocessed_rows "$((drain_line + 1))" || exit 1 + restore_unprocessed_rows "$drain_line" || exit 1 else restore_unprocessed_rows "$drain_line" || exit 1 fi @@ -200,6 +197,10 @@ while IFS= read -r drain_row || [ -n "$drain_row" ]; do ;; 1|5) ;; 3|4) ;; + 6) + restore_unprocessed_rows "$drain_line" || exit 1 + exit 1 + ;; *) restore_unprocessed_rows "$drain_line" || exit 1 exit "$claim_status" diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index 6572ef25edd..f612f49bb83 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -642,13 +642,16 @@ test_output_completion_failure_does_not_reprint() { #!/usr/bin/env bash set -u target="${!#}" +source="${@: -2:1}" case "$target" in *.claim) - count=$(cat "${FM_FAIL_CLAIM_MOVE:?}" 2>/dev/null || printf '0') - count=$((count + 1)) - printf '%s\n' "$count" > "$FM_FAIL_CLAIM_MOVE" - if [ "$count" = 2 ]; then - exit 91 + if grep -Fqx 'output_complete=1' "$source" 2>/dev/null; then + count=$(cat "${FM_FAIL_CLAIM_MOVE:?}" 2>/dev/null || printf '0') + count=$((count + 1)) + printf '%s\n' "$count" > "$FM_FAIL_CLAIM_MOVE" + if [ "$count" = 1 ]; then + exit 91 + fi fi ;; esac @@ -657,18 +660,33 @@ SH chmod +x "$fakebin/mv" export FM_FAIL_CLAIM_MOVE="$dir/fail-claim-move" export FM_WAKE_DRAIN_DIRECT=1 - drain "$root" "$home" "$fakebin" >"$dir/output-failure.out" \ - || fail "output completion failure was not recovered" + if drain "$root" "$home" "$fakebin" >"$dir/output-failure.out"; then + fail "output completion failure was hidden" + fi [ "$(grep -Fxc "$row" "$dir/output-failure.out")" = 1 ] \ - || fail "output completion failure reprinted or lost the emitted row" + || fail "output completion failure lost or duplicated the emitted row" + [ "$(receipt_count "$state" presented)" = 0 ] \ + || fail "output completion failure finalized the receipt" + [ "$(receipt_count "$state" pending)" = 1 ] \ + || fail "output completion failure did not retain the receipt" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_started)" = 1 ] \ + || fail "output completion failure did not retain the emitted marker" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_emitted)" = 1 ] \ + || fail "output completion failure did not bind emission state" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_complete)" = 0 ] \ + || fail "output completion failure incorrectly persisted completion" + drain "$root" "$home" "$fakebin" >"$dir/output-failure-retry.out" \ + || fail "output completion retry did not finalize the receipt" + [ ! -s "$dir/output-failure-retry.out" ] \ + || fail "output completion retry reprinted the emitted row" [ "$(receipt_count "$state" presented)" = 1 ] \ - || fail "output completion failure did not finalize the receipt" + || fail "output completion retry did not finalize the receipt" [ "$(receipt_count "$state" pending)" = 0 ] \ - || fail "output completion failure left the receipt pending" + || fail "output completion retry left the receipt pending" [ ! -e "$state/terminal-outcomes/.$fingerprint.claim" ] \ - || fail "output completion failure left a stale claim" + || fail "output completion retry left a stale claim" unset FM_FAIL_CLAIM_MOVE FM_WAKE_DRAIN_DIRECT - pass "post-output failures finalize without duplicate presentation" + pass "post-output failures retry without duplicate presentation" } test_direct_drain_finalizes_after_successful_output() { @@ -809,13 +827,16 @@ test_deferred_output_completion_retries_before_confirmation() { #!/usr/bin/env bash set -u target="${!#}" +source="${@: -2:1}" case "$target" in *.claim) - count=$(cat "${FM_FAIL_CLAIM_MOVE:?}" 2>/dev/null || printf '0') - count=$((count + 1)) - printf '%s\n' "$count" > "$FM_FAIL_CLAIM_MOVE" - if [ "$count" = 2 ]; then - exit 91 + if grep -Fqx 'output_complete=1' "$source" 2>/dev/null; then + count=$(cat "${FM_FAIL_CLAIM_MOVE:?}" 2>/dev/null || printf '0') + count=$((count + 1)) + printf '%s\n' "$count" > "$FM_FAIL_CLAIM_MOVE" + if [ "$count" = 1 ]; then + exit 91 + fi fi ;; esac @@ -848,6 +869,69 @@ SH pass "deferred output completion retries before caller confirmation" } +test_deferred_output_completion_failure_retains_emitted_row() { + local dir root home fakebin state fingerprint row + new_case deferred-output-complete-failure + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + fingerprint=$(receipt_fingerprint 'deferred-failure-x1|deferred-failure-inc|done|state: done · source: pane · deferred output failure') + mkdir -p "$state/terminal-outcomes" + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id=deferred-failure-x1 \ + incarnation=deferred-failure-inc outcome=done terminal_source=pane \ + terminal_snapshot='state: done · source: pane · deferred output failure' kind=ship + row=$'2\t2\tcheck\tinactive-outcome:'"$fingerprint"$'\tdeferred output failure row' + printf '%s\n' "$row" > "$state/.wake-queue" + cat > "$fakebin/mv" <<'SH' +#!/usr/bin/env bash +set -u +target="${!#}" +source="${@: -2:1}" +case "$target" in + *.claim) + if grep -Fqx 'output_complete=1' "$source" 2>/dev/null; then + count=$(cat "${FM_FAIL_CLAIM_MOVE:?}" 2>/dev/null || printf '0') + count=$((count + 1)) + printf '%s\n' "$count" > "$FM_FAIL_CLAIM_MOVE" + [ "$count" -le 2 ] || exec /usr/bin/mv "$@" + exit 91 + fi + ;; +esac +exec /usr/bin/mv "$@" +SH + chmod +x "$fakebin/mv" + export FM_FAIL_CLAIM_MOVE="$dir/fail-claim-move" + export FM_WAKE_DRAIN_DEFER_ACK=1 FM_WAKE_DRAIN_GENERATION="$$" + if drain "$root" "$home" "$fakebin" >"$dir/deferred-failure.out"; then + fail "deferred output-complete persistence failure was hidden" + fi + [ "$(grep -Fxc "$row" "$dir/deferred-failure.out")" = 1 ] \ + || fail "deferred output-complete failure duplicated or lost the emitted row" + [ "$(queue_count "$state")" = 1 ] \ + || fail "deferred output-complete failure did not retain the wake row" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_started)" = 1 ] \ + || fail "deferred output-complete failure lost the emitted marker" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_emitted)" = 1 ] \ + || fail "deferred output-complete failure lost the emission state" + [ "$(receipt_value "$state/terminal-outcomes/.$fingerprint.claim" output_complete)" = 0 ] \ + || fail "deferred output-complete failure persisted completion" + replace_field "$state/terminal-outcomes/.$fingerprint.claim" defer_generation_start proc:0 + rm -f "$fakebin/mv" + unset FM_FAIL_CLAIM_MOVE FM_WAKE_DRAIN_DEFER_ACK FM_WAKE_DRAIN_GENERATION + drain "$root" "$home" "$fakebin" >"$dir/deferred-failure-retry.out" \ + || fail "deferred output-complete retry did not recover the emitted row" + [ ! -s "$dir/deferred-failure-retry.out" ] \ + || fail "deferred output-complete retry reprinted the emitted row" + [ -e "$state/terminal-outcomes/$fingerprint.presented" ] \ + || fail "deferred output-complete retry did not finalize the receipt" + [ ! -e "$state/terminal-outcomes/$fingerprint.pending" ] \ + || fail "deferred output-complete retry left the receipt pending" + [ "$(queue_count "$state")" = 0 ] \ + || fail "deferred output-complete retry left the wake queued" + pass "deferred output completion failures retain emitted rows for retry" +} + test_deferred_ack_confirms_after_caller_emission() { local dir root home fakebin state fingerprint row drain_output new_case deferred-confirm @@ -1662,6 +1746,35 @@ SH pass "reported secondmate routes recover after receipt finalization crashes" } +test_reported_route_repair_is_bounded() { + local dir root home fakebin state fingerprint first second index + new_case reported-route-repair-limit + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + mkdir -p "$state/terminal-outcomes" + for index in 1 2 3; do + fingerprint=$(receipt_fingerprint "reported-limit-x${index}|reported-limit-inc-${index}|done|reported limit ${index}") + fm_write_meta "$state/terminal-outcomes/$fingerprint.reported" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id="reported-limit-x${index}" \ + incarnation="reported-limit-inc-${index}" outcome=done terminal_source=pane \ + terminal_snapshot="reported limit ${index}" kind=ship + done + export FM_REPORTED_ROUTE_REPAIR_LIMIT=1 + scan "$root" "$home" "$fakebin" --startup >/dev/null \ + || fail "bounded reported-receipt maintenance failed on the first scan" + first=$(cat "$state/.reported-secondmate-route-repair.cursor" 2>/dev/null || true) + [ -n "$first" ] || fail "bounded reported-receipt maintenance did not persist a cursor" + scan "$root" "$home" "$fakebin" --startup >/dev/null \ + || fail "bounded reported-receipt maintenance failed on the second scan" + second=$(cat "$state/.reported-secondmate-route-repair.cursor" 2>/dev/null || true) + [ -n "$second" ] && [ "$second" != "$first" ] \ + || fail "bounded reported-receipt maintenance did not advance incrementally" + [ "$(receipt_count "$state" reported)" = 3 ] \ + || fail "bounded reported-receipt maintenance discarded durable receipts" + unset FM_REPORTED_ROUTE_REPAIR_LIMIT + pass "reported route maintenance advances through bounded receipt batches" +} + test_secondmate_route_replacement_preserves_old_receipt() { local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker history_backup active_route_backup new_case secondmate-route-replacement @@ -2370,6 +2483,7 @@ test_malformed_finalized_receipt_fails_closed test_presented_claim_is_acknowledged_in_deferred_drain test_deferred_ack_retries_after_caller_crash test_deferred_output_completion_retries_before_confirmation +test_deferred_output_completion_failure_retains_emitted_row test_deferred_ack_confirms_after_caller_emission test_deferred_ack_recovers_after_output_confirmation test_pending_receipts_replay_after_child_scan_failure @@ -2388,6 +2502,7 @@ test_occupancy_unknown_is_not_terminal test_status_log_terminal_is_not_replayed test_valid_secondmate_route_reports_parent_once test_reported_secondmate_route_repair_after_crash +test_reported_route_repair_is_bounded test_secondmate_route_replacement_preserves_old_receipt test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected From 804776b7253601a9eef09361a2c4be166c27f323 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 18:48:16 +0000 Subject: [PATCH 029/163] no-mistakes(review): Applied fail-closed replay fixes; focused verification passed --- bin/fm-inactive-reconcile.sh | 131 ++++++++++++++++++++++++-- bin/fm-pending-reply-lib.sh | 1 + bin/fm-watch.sh | 40 +++++++- tests/fm-inactive-outcome.test.sh | 147 ++++++++++++++++++++++++------ 4 files changed, 279 insertions(+), 40 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 535bf39d80d..140d5a0ec65 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -16,6 +16,7 @@ OUTCOME_DIR="$STATE/terminal-outcomes" SCAN_MARKER="$STATE/.inactive-outcome-reconcile" SCAN_CURSOR="$STATE/.inactive-outcome-reconcile.cursor" REPORTED_ROUTE_CURSOR="$STATE/.reported-secondmate-route-repair.cursor" +PENDING_RECEIPT_CURSOR="$STATE/.pending-receipt-republish.cursor" FM_WAKE_QUEUE="${FM_WAKE_QUEUE:-$STATE/.wake-queue}" inactive_state_path_is_safe() { @@ -42,6 +43,7 @@ inactive_state_preflight() { inactive_state_path_is_safe "$SCAN_MARKER" file || return 1 inactive_state_path_is_safe "$SCAN_CURSOR" file || return 1 inactive_state_path_is_safe "$REPORTED_ROUTE_CURSOR" file || return 1 + inactive_state_path_is_safe "$PENDING_RECEIPT_CURSOR" file || return 1 inactive_state_path_is_safe "$FM_WAKE_QUEUE" file || return 1 } @@ -82,6 +84,7 @@ bounded_secs() { RECONCILE_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_SECS:-900}" 900 60 1800) SCAN_BUDGET_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_BUDGET_SECS:-10}" 10 1 300) REPORTED_ROUTE_REPAIR_LIMIT=$(bounded_secs "${FM_REPORTED_ROUTE_REPAIR_LIMIT:-32}" 32 1 256) +PENDING_RECEIPT_REPUBLISH_LIMIT=$(bounded_secs "${FM_PENDING_RECEIPT_REPUBLISH_LIMIT:-32}" 32 1 256) meta_value() { # awk -F= -v wanted="$2" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$1" 2>/dev/null @@ -1060,18 +1063,73 @@ republish_pending_receipt() { republish_pending_receipts() { local scan_started=$1 pending now remaining status=0 + local cursor='' pass started=1 boundary='' stop_after=0 + local base last='' processed=0 cursor_tmp + local LC_ALL=C [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 - for pending in "$OUTCOME_DIR"/*.pending; do - [ -e "$pending" ] || [ -L "$pending" ] || continue - now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) - [ "$remaining" -gt 0 ] || return 1 - if ! FM_LOCK_WAIT_SECS="$remaining" republish_pending_receipt "$pending"; then - status=1 - elif [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then - printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$ID" "$OUTCOME" "$FP" + cursor=$(cat "$PENDING_RECEIPT_CURSOR" 2>/dev/null || true) + case "$cursor" in + '') ;; + *.pending) + base=${cursor%.pending} + case "$base" in ''|*[!A-Fa-f0-9]*) cursor=;; esac + ;; + *) cursor=;; + esac + [ -n "$cursor" ] && started=0 + for pass in 1 2; do + if [ "$pass" = 2 ]; then + [ -n "$cursor" ] || break + started=1 + stop_after=0 fi + for pending in "$OUTCOME_DIR"/*.pending; do + [ -e "$pending" ] || [ -L "$pending" ] || continue + base=${pending##*/} + if [ "$pass" = 1 ] && [ -n "$cursor" ] && [ "$started" = 0 ]; then + if [ "$base" = "$cursor" ]; then + started=1 + continue + fi + if [[ "$base" > "$cursor" ]]; then + started=1 + [ -n "$boundary" ] || boundary=$base + else + continue + fi + fi + if [ "$pass" = 2 ] && [ -n "$boundary" ] && [ "$base" = "$boundary" ]; then + break + fi + now=$(date +%s) + remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + [ "$remaining" -gt 0 ] || { status=1; break 2; } + if ! FM_LOCK_WAIT_SECS="$remaining" republish_pending_receipt "$pending"; then + status=1 + elif [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$ID" "$OUTCOME" "$FP" + fi + last=$base + processed=$((processed + 1)) + if [ "$pass" = 2 ] && [ "$base" = "$cursor" ]; then + stop_after=1 + fi + [ "$processed" -lt "$PENDING_RECEIPT_REPUBLISH_LIMIT" ] || break 2 + [ "$stop_after" = 0 ] || break + done done + if [ "$processed" -gt 0 ]; then + if cursor_tmp=$(mktemp "$STATE/.pending-receipt-republish.cursor.XXXXXX"); then + if [ ! -f "$cursor_tmp" ] || [ -L "$cursor_tmp" ] \ + || ! printf '%s\n' "$last" > "$cursor_tmp" \ + || ! mv -f "$cursor_tmp" "$PENDING_RECEIPT_CURSOR"; then + status=1 + rm -f "$cursor_tmp" + fi + else + status=1 + fi + fi return "$status" } @@ -1106,6 +1164,60 @@ read_incarnation() { # printf 'legacy-%s' "${digest:0:32}" } +terminal_outcome_surfaced() { + local id=$1 meta=$2 outcome=$3 key raw marker marker_snapshot marker_spawn + local current_spawn current_tasktmp current_window current_worktree + key=$(printf '%s' "$id" | tr ':/.' '___') + raw="$STATE/.hb-surfaced-$key" + marker="$STATE/.hb-terminal-surfaced-$key" + [ -f "$raw" ] && [ ! -L "$raw" ] || return 1 + raw=$(cat "$raw" 2>/dev/null || true) + [ -n "$raw" ] || return 1 + case "$raw" in + done:*|failed:*) ;; + *) return 1 ;; + esac + [ "${raw%%:*}" = "$outcome" ] || return 1 + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["snapshot"]=1; allowed["spawn_incarnation"]=1 + allowed["tasktmp"]=1; allowed["window"]=1; allowed["worktree"]=1 + required["schema"]=1; required["snapshot"]=1; required["spawn_incarnation"]=1 + required["tasktmp"]=1; required["window"]=1; required["worktree"]=1 + valid=1 + } + /^[^=]+=/{ + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + values[key]=substr($0, index($0, "=") + 1) + next + } + { valid=0 } + END { + for (key in required) if (!(key in seen)) valid=0 + exit !(valid && values["schema"] == "fm-hb-terminal-surfaced.v1") + } + ' "$marker" 2>/dev/null || return 1 + marker_snapshot=$(meta_value_unique "$marker" snapshot 2>/dev/null) || return 1 + [ "$marker_snapshot" = "$raw" ] || return 1 + marker_spawn=$(meta_value_unique "$marker" spawn_incarnation 2>/dev/null) || return 1 + current_spawn= + if current_spawn=$(meta_value_unique "$meta" spawn_incarnation 2>/dev/null); then + [ "$marker_spawn" = "$current_spawn" ] || return 1 + else + [ -z "$marker_spawn" ] || return 1 + current_tasktmp=$(meta_value "$meta" tasktmp) + current_window=$(meta_value "$meta" window) + current_worktree=$(meta_value "$meta" worktree) + [ "$(meta_value_unique "$marker" tasktmp 2>/dev/null)" = "$current_tasktmp" ] || return 1 + [ "$(meta_value_unique "$marker" window 2>/dev/null)" = "$current_window" ] || return 1 + [ "$(meta_value_unique "$marker" worktree 2>/dev/null)" = "$current_worktree" ] || return 1 + fi + return 0 +} + child_cleanup() { local status=$? if [ "$CHILD_LOCK_HELD" = 1 ]; then @@ -1171,6 +1283,7 @@ reconcile_child() { SNAPSHOT=$snapshot SOURCE=$source KIND=${kind:-ship} + terminal_outcome_surfaced "$id" "$meta" "$outcome" && return 0 is_secondmate_home route_rc=$? case "$route_rc" in diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 83e12be8ec9..56429639ee5 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -520,6 +520,7 @@ fm_pending_reply_record_validate() { # /dev/null || return 1 + [ "$(fm_pending_reply_get "$rec" schema)" = fm-pending-reply.v1 ] || return 1 printf '%s' "$wanted_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 [ -d "$state" ] && [ ! -L "$state" ] || return 1 state_abs=$(cd "$state" 2>/dev/null && pwd -P) || return 1 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 0ea28cb930b..164d7d1b8dc 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -479,6 +479,30 @@ run_check_capture() { # fleet-scan can tell apart a captain-relevant status that already woke firstmate # from one that has not - the latter being a per-wake-path miss it must surface. _hb_surfaced_path() { printf '%s/.hb-surfaced-%s' "$STATE" "$(printf '%s' "$1" | tr ':/.' '___')"; } +_hb_terminal_surfaced_path() { printf '%s/.hb-terminal-surfaced-%s' "$STATE" "$(printf '%s' "$1" | tr ':/.' '___')"; } + +mark_terminal_surfaced() { + local task=$1 last=$2 meta marker tmp spawn_incarnation tasktmp window worktree + case "$(status_line_verb "$last")" in + done|failed) ;; + *) return 0 ;; + esac + meta="$STATE/$task.meta" + [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 + spawn_incarnation=$(awk -F= '$1 == "spawn_incarnation" { print substr($0, index($0, "=") + 1); count++ } END { exit !(count == 1) }' "$meta" 2>/dev/null || true) + tasktmp=$(awk -F= '$1 == "tasktmp" { print substr($0, index($0, "=") + 1); exit }' "$meta" 2>/dev/null || true) + window=$(awk -F= '$1 == "window" { print substr($0, index($0, "=") + 1); exit }' "$meta" 2>/dev/null || true) + worktree=$(awk -F= '$1 == "worktree" { print substr($0, index($0, "=") + 1); exit }' "$meta" 2>/dev/null || true) + marker=$(_hb_terminal_surfaced_path "$task") + tmp=$(mktemp "$STATE/.hb-terminal-surfaced.XXXXXX") || return 1 + if ! printf 'schema=fm-hb-terminal-surfaced.v1\nsnapshot=%s\nspawn_incarnation=%s\ntasktmp=%s\nwindow=%s\nworktree=%s\n' \ + "$last" "$spawn_incarnation" "$tasktmp" "$window" "$worktree" > "$tmp" \ + || ! mv -f "$tmp" "$marker"; then + rm -f "$tmp" + return 1 + fi + return 0 +} # Record a status file's captain-relevant last line as surfaced (no-op for a # non-captain-relevant or empty status). Call AFTER the wake is enqueued, so the @@ -490,6 +514,7 @@ mark_surfaced() { # [ -n "$last" ] || return 0 status_is_captain_relevant "$last" || return 0 printf '%s' "$last" > "$(_hb_surfaced_path "$task")" + mark_terminal_surfaced "$task" "$last" } # Mark every current captain-relevant status as surfaced. Called after the @@ -500,6 +525,7 @@ mark_all_captain_relevant_surfaced() { while IFS=$(printf '\t') read -r f task last; do [ -n "$f" ] || continue printf '%s' "$last" > "$(_hb_surfaced_path "$task")" + mark_terminal_surfaced "$task" "$last" done < <(scan_captain_relevant_statuses "$STATE") } @@ -621,7 +647,19 @@ while :; do exit 1 fi if [ -n "$inactive_out" ]; then - wake "check: inactive terminal outcome replay queued" + inactive_drain_output= + inactive_drain_status=0 + inactive_drain_output=$(FM_WAKE_DRAIN_DIRECT=0 FM_WAKE_DRAIN_DEFER_ACK=1 \ + FM_WAKE_DRAIN_GENERATION="$WATCHER_PID" "$SCRIPT_DIR/fm-wake-drain.sh") \ + || inactive_drain_status=$? + case "$inactive_drain_status" in + 0) + [ -n "$inactive_drain_output" ] || exit 1 + wake "$inactive_drain_output" + ;; + 3) exit 3 ;; + *) exit "$inactive_drain_status" ;; + esac fi # Slow per-task checks (firstmate writes these, e.g. a merged-PR poll). diff --git a/tests/fm-inactive-outcome.test.sh b/tests/fm-inactive-outcome.test.sh index f612f49bb83..28d8d1c7930 100755 --- a/tests/fm-inactive-outcome.test.sh +++ b/tests/fm-inactive-outcome.test.sh @@ -1239,7 +1239,7 @@ SH } test_watcher_runs_inactive_cadence() { - local dir root home fakebin state out second_out third_out status fingerprint + local dir root home fakebin state out second_out status fingerprint new_case watcher-wiring dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN state="$home/state" @@ -1277,6 +1277,7 @@ SH || fail "watcher left the existing wake queued" [ "$(receipt_count "$state" pending)" = 0 ] || fail "watcher scanned inactive outcomes before draining the queued wake" + fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) second_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ @@ -1288,25 +1289,10 @@ SH "$root/bin/fm-watch.sh" 2>&1) status=$? [ "$status" = 0 ] || fail "watcher cadence failed while surfacing the inactive outcome wake" - printf '%s\n' "$second_out" | grep -F 'check: inactive terminal outcome replay queued' >/dev/null \ - || fail "watcher did not surface the inactive reconciliation result" - [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher cadence did not create the inactive receipt" - [ "$(queue_count "$state")" = 1 ] || fail "watcher cadence did not retain exactly one inactive outcome wake" - - fingerprint=$(basename "$(direct_first_file "$state/terminal-outcomes" '*.pending')" .pending) - third_out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ - FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ - FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ - FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ - FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 FM_FAKE_PANE_PATH="$home" \ - FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ - "$root/bin/fm-watch.sh" 2>&1) - status=$? - [ "$status" = 0 ] || fail "watcher cadence failed while draining the inactive outcome wake" - printf '%s\n' "$third_out" | grep -F 'inactive-outcome:' >/dev/null \ - || fail "watcher did not surface the exact inactive outcome wake" + printf '%s\n' "$second_out" | grep -F 'inactive-outcome:' >/dev/null \ + || fail "watcher did not surface the exact inactive outcome wake in the scan turn" + ! printf '%s\n' "$second_out" | grep -F 'check: inactive terminal outcome replay queued' >/dev/null \ + || fail "watcher emitted a second generic inactive wake" [ "$(receipt_count "$state" pending)" = 0 ] || fail "watcher cadence did not acknowledge the inactive receipt" [ "$(receipt_count "$state" presented)" = 1 ] || fail "watcher cadence did not finalize the inactive receipt" [ "$(queue_count "$state")" = 0 ] || fail "watcher cadence did not consume the inactive outcome wake" @@ -1315,19 +1301,36 @@ SH write_meta "$state" watcher-failure-x1 watcher-failure-inc export FM_FAKE_CREW_STATE_WATCHER_FAILURE_X1='state: failed · source: pane · watcher output failure' rm -f "$state/.inactive-outcome-reconcile" - out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ - FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ - FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ - FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ - FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ - FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 FM_FAKE_PANE_PATH="$home" \ - FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ - "$root/bin/fm-watch.sh" 2>&1) + mv "$root/bin/fm-wake-drain.sh" "$root/bin/fm-wake-drain.real" + cat > "$root/bin/fm-wake-drain.sh" </dev/null || echo 0) +count=\$((count + 1)) +printf '%s\n' "\$count" > "\$count_file" +if [ "\$count" = 2 ]; then + exit 3 +fi +exec "$root/bin/fm-wake-drain.real" +SH + chmod +x "$root/bin/fm-wake-drain.sh" + set +e + ( cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_INACTIVE_OUTCOME_SECS=60 FM_INACTIVE_OUTCOME_BUDGET_SECS=10 \ + FM_PRIMARY_ATTESTATION="$CASE_TOKEN" CODEX_THREAD_ID="$CASE_THREAD" \ + FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 FM_FAKE_PANE_PATH="$home" \ + FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCHER_HEARTBEAT=999999 \ + "$root/bin/fm-watch.sh" ) > "$dir/failed-scan.out" 2>&1 status=$? - [ "$status" = 0 ] || fail "watcher cadence failed while queuing the failure fixture" + set -u + [ "$status" = 3 ] || fail "watcher did not fail closed when the same-turn drain failed" [ "$(receipt_count "$state" pending)" = 1 ] || fail "watcher failure fixture did not create one pending receipt" [ "$(queue_count "$state")" = 1 ] || fail "watcher failure fixture did not retain one wake" + mv "$root/bin/fm-wake-drain.real" "$root/bin/fm-wake-drain.sh" set +e ( cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ @@ -1348,6 +1351,55 @@ SH pass "watcher cadence gates acknowledgement on successful output" } +test_surfaced_terminal_is_not_replayed() { + local dir root home fakebin state out status + new_case surfaced-terminal + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + cp -a "$ROOT/bin/." "$root/bin/" + write_meta "$state" surfaced-x1 surfaced-inc + printf 'done: surfaced terminal\n' > "$state/surfaced-x1.status" + : > "$state/surfaced-x1.turn-ended" + touch "$state/surfaced-x1.meta" "$state/surfaced-x1.status" "$state/surfaced-x1.turn-ended" + export FM_FAKE_CREW_STATE_SURFACED_X1='state: done · source: pane · surfaced terminal' + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +set -u +case "$*" in + *"#{pane_current_path}"*) printf '%s\n' "${FM_FAKE_PANE_PATH:-}" ;; + *"#{pane_pid}"*) printf '%s\n' "${FM_FAKE_HARNESS_PID:-$$}" ;; + *"#{window_name}"*) printf '%s\n' firstmate ;; + capture-pane) : ;; +esac +exit 0 +SH + chmod +x "$fakebin/tmux" + prepare_primary_proof "$root" "$home" "$fakebin" + out=$(cd "$root" && env -u NO_MISTAKES_GATE -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT \ + PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_INACTIVE_OUTCOME_SECS=60 \ + FM_INACTIVE_OUTCOME_BUDGET_SECS=10 FM_PRIMARY_ATTESTATION="$CASE_TOKEN" \ + CODEX_THREAD_ID="$CASE_THREAD" FM_FAKE_HARNESS_PID="$$" FM_BACKEND=tmux TMUX=fake,1,0 \ + FM_FAKE_PANE_PATH="$home" FM_POLL=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + FM_WATCHER_HEARTBEAT=999999 "$root/bin/fm-watch.sh" 2>&1) + status=$? + [ "$status" = 0 ] || fail "watcher did not surface the terminal status: $out" + [ -f "$state/.hb-surfaced-surfaced-x1" ] || fail "watcher did not persist the surfaced status" + [ -f "$state/.hb-terminal-surfaced-surfaced-x1" ] || fail "watcher did not persist the incarnation-bound terminal marker" + set_old_mtime "$state/surfaced-x1.meta" "$state/surfaced-x1.status" "$state/surfaced-x1.turn-ended" + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 0 ] || fail "already surfaced terminal outcome was replayed" + [ "$(queue_count "$state")" = 0 ] || fail "already surfaced terminal outcome queued a wake" + replace_field "$state/surfaced-x1.meta" spawn_incarnation resurfaced-inc + set_old_mtime "$state/surfaced-x1.meta" "$state/surfaced-x1.status" "$state/surfaced-x1.turn-ended" + scan "$root" "$home" "$fakebin" --startup >/dev/null + [ "$(receipt_count "$state" pending)" = 1 ] || fail "new incarnation was incorrectly suppressed by an old surface marker" + [ "$(queue_count "$state")" = 1 ] || fail "new incarnation did not queue its inactive wake" + unset FM_FAKE_CREW_STATE_SURFACED_X1 + pass "terminal replay suppression is bound to surfaced status and incarnation" +} + test_legacy_metadata_uses_stable_fallback() { local dir root home fakebin state rec incarnation new_case legacy-fallback @@ -1775,6 +1827,37 @@ test_reported_route_repair_is_bounded() { pass "reported route maintenance advances through bounded receipt batches" } +test_pending_receipt_republish_is_bounded() { + local dir root home fakebin state fingerprint first second index + new_case pending-receipt-republish-limit + dir=$CASE_DIR; root=$CASE_ROOT; home=$CASE_HOME; fakebin=$CASE_FAKEBIN + state="$home/state" + mkdir -p "$state/terminal-outcomes" + for index in 1 2 3; do + fingerprint=$(receipt_fingerprint "pending-limit-x${index}|pending-limit-inc-${index}|done|pending limit ${index}") + fm_write_meta "$state/terminal-outcomes/$fingerprint.pending" \ + schema=fm-jt-terminal-outcome.v1 fingerprint="$fingerprint" task_id="pending-limit-x${index}" \ + incarnation="pending-limit-inc-${index}" outcome=done terminal_source=pane \ + terminal_snapshot="pending limit ${index}" kind=ship + done + export FM_PENDING_RECEIPT_REPUBLISH_LIMIT=1 + scan "$root" "$home" "$fakebin" --startup >/dev/null \ + || fail "bounded pending-receipt maintenance failed on the first scan" + first=$(cat "$state/.pending-receipt-republish.cursor" 2>/dev/null || true) + [ -n "$first" ] || fail "bounded pending-receipt maintenance did not persist a cursor" + [ "$(queue_count "$state")" = 1 ] || fail "bounded pending-receipt maintenance exceeded its first batch" + scan "$root" "$home" "$fakebin" --startup >/dev/null \ + || fail "bounded pending-receipt maintenance failed on the second scan" + second=$(cat "$state/.pending-receipt-republish.cursor" 2>/dev/null || true) + [ -n "$second" ] && [ "$second" != "$first" ] \ + || fail "bounded pending-receipt maintenance did not rotate its cursor" + [ "$(queue_count "$state")" = 2 ] || fail "bounded pending-receipt maintenance did not republish the next receipt" + [ "$(receipt_count "$state" pending)" = 3 ] \ + || fail "bounded pending-receipt maintenance discarded durable receipts" + unset FM_PENDING_RECEIPT_REPUBLISH_LIMIT + pass "pending-receipt maintenance rotates bounded receipt batches" +} + test_secondmate_route_replacement_preserves_old_receipt() { local dir root home fakebin state child_home child_state parent_status corr_a corr_b rec send_out marker history_backup active_route_backup new_case secondmate-route-replacement @@ -2436,6 +2519,8 @@ test_malformed_or_missing_secondmate_route_fails_closed() { scan "$root" "$child_home" "$fakebin" --startup >/dev/null [ "$(receipt_count "$child_state" pending)" = 0 ] || fail "malformed secondmate parent route was not fail-closed" [ ! -e "$parent_status" ] || fail "malformed secondmate route wrote parent status" + parent_status="$home/state/sm-x1.status" + : > "$parent_status" printf 'schema=fm-jt-parent-route.v1\nsecondmate_id=sm-x1\nparent_home=%s\nparent_status=%s\ncorr_id=0123456789abcdef\n' \ "$home" "$parent_status" > "$child_state/.fm-jt-parent-route" fm_write_meta "$home/state/pending-replies/0123456789abcdef" \ @@ -2493,6 +2578,7 @@ test_reused_task_id_gets_new_fingerprint test_spawn_publishes_incarnation_token test_session_start_drains_before_inactive_scan test_watcher_runs_inactive_cadence +test_surfaced_terminal_is_not_replayed test_legacy_metadata_uses_stable_fallback test_empty_spawn_incarnation_is_rejected test_relaunch_and_teardown_races_recheck_under_spawn_lock @@ -2503,6 +2589,7 @@ test_status_log_terminal_is_not_replayed test_valid_secondmate_route_reports_parent_once test_reported_secondmate_route_repair_after_crash test_reported_route_repair_is_bounded +test_pending_receipt_republish_is_bounded test_secondmate_route_replacement_preserves_old_receipt test_undelivered_secondmate_route_cleanup_is_idempotent test_concurrent_secondmate_routes_are_rejected From 09aaf3f3a5539d5cd950f78927ef957e56db51a7 Mon Sep 17 00:00:00 2001 From: tests Date: Fri, 14 Aug 2026 19:43:36 +0000 Subject: [PATCH 030/163] no-mistakes(review): Hardened generation-bound replay handoff --- bin/fm-inactive-reconcile.sh | 28 +++-- bin/fm-session-start.sh | 46 +++++++- bin/fm-wake-drain.sh | 11 +- bin/fm-watch.sh | 167 ++++++++++++++++++++++++++---- tests/fm-inactive-outcome.test.sh | 112 +++++++++++++++++++- 5 files changed, 321 insertions(+), 43 deletions(-) diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index 140d5a0ec65..5df6589f376 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -85,6 +85,11 @@ RECONCILE_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_SECS:-900}" 900 60 1800) SCAN_BUDGET_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_BUDGET_SECS:-10}" 10 1 300) REPORTED_ROUTE_REPAIR_LIMIT=$(bounded_secs "${FM_REPORTED_ROUTE_REPAIR_LIMIT:-32}" 32 1 256) PENDING_RECEIPT_REPUBLISH_LIMIT=$(bounded_secs "${FM_PENDING_RECEIPT_REPUBLISH_LIMIT:-32}" 32 1 256) +MAINTENANCE_RESERVE_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_MAINTENANCE_RESERVE_SECS:-2}" 2 1 60) +[ "$MAINTENANCE_RESERVE_SECS" -le "$SCAN_BUDGET_SECS" ] || MAINTENANCE_RESERVE_SECS=$SCAN_BUDGET_SECS +DIRECT_SCAN_BUDGET_SECS=$((SCAN_BUDGET_SECS - MAINTENANCE_RESERVE_SECS)) +MAINTENANCE_STAGE_SECS=$((MAINTENANCE_RESERVE_SECS / 2)) +[ "$MAINTENANCE_STAGE_SECS" -gt 0 ] || MAINTENANCE_STAGE_SECS=1 meta_value() { # awk -F= -v wanted="$2" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$1" 2>/dev/null @@ -967,11 +972,11 @@ reported_secondmate_receipt_valid() { } repair_reported_secondmate_routes() { - local scan_started=$1 reported kind corr parent_task_id parent_home parent_status now remaining status=0 + local scan_deadline=$1 reported kind corr parent_task_id parent_home parent_status now remaining status=0 local cursor='' cursor_found=0 started=1 pass base last processed=0 stop_after=0 cursor_tmp [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + remaining=$((scan_deadline - now)) [ "$remaining" -gt 0 ] || return 1 FM_LOCK_WAIT_SECS="$remaining" fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 cursor=$(cat "$REPORTED_ROUTE_CURSOR" 2>/dev/null || true) @@ -1003,7 +1008,7 @@ repair_reported_secondmate_routes() { continue fi now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + remaining=$((scan_deadline - now)) [ "$remaining" -gt 0 ] || { status=1; break 2; } [ -f "$reported" ] && [ ! -L "$reported" ] || { status=1; continue; } kind=$(receipt_field "$reported" kind 2>/dev/null || true) @@ -1062,7 +1067,7 @@ republish_pending_receipt() { } republish_pending_receipts() { - local scan_started=$1 pending now remaining status=0 + local scan_deadline=$1 pending now remaining status=0 local cursor='' pass started=1 boundary='' stop_after=0 local base last='' processed=0 cursor_tmp local LC_ALL=C @@ -1102,7 +1107,7 @@ republish_pending_receipts() { break fi now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + remaining=$((scan_deadline - now)) [ "$remaining" -gt 0 ] || { status=1; break 2; } if ! FM_LOCK_WAIT_SECS="$remaining" republish_pending_receipt "$pending"; then status=1 @@ -1484,7 +1489,7 @@ secondmate_ack_report() { # /dev/null || true) now=$(date +%s) @@ -1493,11 +1498,14 @@ scan_locked() { [ "$age" -ge "$RECONCILE_SECS" ] || return 0 fi scan_started=$now + scan_deadline=$((scan_started + DIRECT_SCAN_BUDGET_SECS)) run_maintenance() { - if ! republish_pending_receipts "$scan_started"; then + maintenance_deadline=$(( $(date +%s) + MAINTENANCE_STAGE_SECS )) + if ! republish_pending_receipts "$maintenance_deadline"; then maintenance_status=1 fi - if ! repair_reported_secondmate_routes "$scan_started"; then + maintenance_deadline=$(( $(date +%s) + MAINTENANCE_STAGE_SECS )) + if ! repair_reported_secondmate_routes "$maintenance_deadline"; then maintenance_status=1 fi } @@ -1517,7 +1525,7 @@ scan_locked() { return 1 } now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + remaining=$((scan_deadline - now)) if [ "$remaining" -le 0 ]; then rm -f "$find_tmp" run_maintenance @@ -1534,7 +1542,7 @@ scan_locked() { fi while [ "$scan_failed" = 0 ] && IFS= read -r -d '' meta; do now=$(date +%s) - remaining=$((SCAN_BUDGET_SECS - (now - scan_started))) + remaining=$((scan_deadline - now)) if [ "$remaining" -le 0 ]; then complete=0 break diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 8d61d5a7415..963a8db0c30 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -126,6 +126,24 @@ SUBRULE='----------------------------------------------------------------------- section() { printf '\n%s\n%s\n%s\n' "$RULE" "$1" "$RULE"; } subsection() { printf '\n%s\n%s\n' "$1" "$SUBRULE"; } +session_start_confirm_inactive_rows() { + local wake_output=$1 row _epoch _seq _kind _key _payload confirm_status + while IFS= read -r row || [ -n "$row" ]; do + IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$row" + case "$_key" in + inactive-outcome:*) + "$SCRIPT_DIR/fm-inactive-reconcile.sh" caller-output-complete \ + "$_key" "$row" >/dev/null 2>&1 || return 1 + confirm_status=0 + "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$row" >/dev/null 2>&1 \ + || confirm_status=$? + [ "$confirm_status" = 0 ] || [ "$confirm_status" = 1 ] \ + || return "$confirm_status" + ;; + esac + done <<< "$wake_output" +} + # print_file_or_absent