diff --git a/bin/fm-afk-return.sh b/bin/fm-afk-return.sh index cf5addb24cf..88fb27792c4 100755 --- a/bin/fm-afk-return.sh +++ b/bin/fm-afk-return.sh @@ -228,7 +228,7 @@ main() { . "$SCRIPT_DIR/fm-classify-lib.sh" mkdir -p "$STATE" || return 1 - fm_lock_acquire_wait "$LOCK" + fm_lock_acquire_wait "$LOCK" || return 1 trap 'fm_lock_release "$LOCK"' EXIT write_pending_seed || { fm_lock_release "$LOCK"; trap - EXIT; return 1; } return_reconcile diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 879de6053db..fc423d8b4b1 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -700,14 +700,24 @@ with_remote_route_locks() { # shift 2 case "$id" in ''|*[!A-Za-z0-9._-]*) echo "error: unsafe remote handoff id: $id" >&2; return 1 ;; esac ACTIVE_REGISTRY_LOCK=$(secondmate_registry_lock_path "$STATE") - fm_lock_acquire_wait "$ACTIVE_REGISTRY_LOCK" + if ! fm_lock_acquire_wait "$ACTIVE_REGISTRY_LOCK"; then + echo "error: could not lock the secondmate registry for $id" >&2 + ACTIVE_REGISTRY_LOCK= + release_remote_locks + return 1 + fi if [ "$(secondmate_registry_field "$REG" "$id" remote 2>/dev/null || true)" != 1 ]; then echo "error: pending outbox has no matching remote secondmate route: $id" >&2 release_remote_locks return 1 fi ACTIVE_HANDOFF_LOCK="$STATE/.backlog-handoff-$id.lock" - fm_lock_acquire_wait "$ACTIVE_HANDOFF_LOCK" + if ! fm_lock_acquire_wait "$ACTIVE_HANDOFF_LOCK"; then + echo "error: could not lock the pending handoff outbox for $id" >&2 + ACTIVE_HANDOFF_LOCK= + release_remote_locks + return 1 + fi if "$operation" "$@"; then rc=0; else rc=$?; fi release_remote_locks return "$rc" diff --git a/bin/fm-captain-hold.sh b/bin/fm-captain-hold.sh index cb429d95238..ed56246ce87 100755 --- a/bin/fm-captain-hold.sh +++ b/bin/fm-captain-hold.sh @@ -775,7 +775,7 @@ command_complete() { [ -f "$meta" ] && has_meta=1 if [ "$has_meta" = 1 ]; then CAPTAIN_META_LOCK=$(fm_meta_lock_path "$meta") || fail "could not resolve task metadata lock" - fm_lock_acquire_wait "$CAPTAIN_META_LOCK" + fm_lock_acquire_wait "$CAPTAIN_META_LOCK" || fail "could not lock task metadata" CAPTAIN_META_LOCK_HELD=1 [ -f "$meta" ] || fail "task metadata disappeared while recording completion" fi diff --git a/bin/fm-lock.sh b/bin/fm-lock.sh index 52d7c8aee4b..207b8a099ad 100755 --- a/bin/fm-lock.sh +++ b/bin/fm-lock.sh @@ -73,7 +73,7 @@ if ! fm_lock_try_acquire "$CLAIM_LOCK"; then echo "error: the prior session's bounded startup sweep is finishing; operate read-only until it releases the fleet lock" >&2 exit 1 fi - fm_lock_acquire_wait "$CLAIM_LOCK" + fm_lock_acquire_wait "$CLAIM_LOCK" || exit 1 fi CLAIM_LOCK_HELD=1 diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 9158fce64df..a9cefccf154 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -3014,7 +3014,7 @@ spawn_record_traceparent() { # independent critical section so other metadata interfaces can serialize. if [ "$SPAWN_META_LOCK_HELD" != 1 ]; then SPAWN_META_LOCK=$(fm_meta_lock_path "$meta") || return 1 - fm_lock_acquire_wait "$SPAWN_META_LOCK" + fm_lock_acquire_wait "$SPAWN_META_LOCK" || return 1 SPAWN_META_LOCK_HELD=1 acquired=1 fi diff --git a/bin/fm-startup-network.sh b/bin/fm-startup-network.sh index 380138ae25f..fdf8ee4d1ee 100755 --- a/bin/fm-startup-network.sh +++ b/bin/fm-startup-network.sh @@ -222,7 +222,7 @@ cmd_start() { # return 1 fi - fm_lock_acquire_wait "$PUBLISH_LOCK" + fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1 if [ "$(status_get state)" = running ] && worker_alive \ && worker_covers_request "$locked" "$lock_pid"; then # A worker whose phases cover this request is still going. Starting another @@ -334,7 +334,7 @@ await_delivery() { # limit=$(( $(delivery_budget) * 10 )) while [ "$waited" -lt "$limit" ]; do claim_live=0 - fm_lock_acquire_wait "$PUBLISH_LOCK" + fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1 if [ "$(status_get generation)" != "$generation" ]; then fm_lock_release "$PUBLISH_LOCK" return 0 @@ -369,7 +369,7 @@ EOF sleep 0.1 waited=$((waited + 1)) done - fm_lock_acquire_wait "$PUBLISH_LOCK" + fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1 if [ "$(status_get generation)" != "$generation" ] || [ -f "$DELIVERED_FILE" ]; then fm_lock_release "$PUBLISH_LOCK" return 0 @@ -384,7 +384,7 @@ EOF publish() { # local generation=$1 state=$2 phases=$3 locked=$4 started=$5 rc=$6 out=$7 timings=${8:-} report_published=1 - fm_lock_acquire_wait "$PUBLISH_LOCK" + fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1 if [ "$(status_get generation)" != "$generation" ]; then fm_lock_release "$PUBLISH_LOCK" return 0 @@ -426,7 +426,7 @@ cmd_run() { # budget=$(stage_budget) phases=probe if [ -n "$generation" ]; then - fm_lock_acquire_wait "$PUBLISH_LOCK" + fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1 if [ "$(status_get generation)" = "$generation" ] && [ "$(status_get pid)" = "$$" ]; then internal=1 started=$(status_get started) @@ -449,7 +449,7 @@ cmd_run() { # if [ "$internal" -eq 0 ]; then generation="$(now).$$.manual" - fm_lock_acquire_wait "$PUBLISH_LOCK" + fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1 if [ "$(status_get state)" = running ] && worker_alive; then fm_lock_release "$PUBLISH_LOCK" return 1 @@ -477,7 +477,7 @@ EOF stage_started=$(fm_timing_now_ms) rc=0 if [ "$sweep_locked" -eq 1 ]; then - fm_lock_acquire_wait "$STATE/.lock.acquire" + fm_lock_acquire_wait "$STATE/.lock.acquire" || return 1 lease_held=1 if ! lock_unchanged "$lock_pid"; then sweep_locked=0 @@ -593,7 +593,7 @@ print_state() { cmd_harvest() { # local pid=$1 generation state claim_record claim_generation claim_pid - fm_lock_acquire_wait "$PUBLISH_LOCK" + fm_lock_acquire_wait "$PUBLISH_LOCK" || return 1 generation=$(status_get generation) # Another session's live claim is left alone; the worker reaps a dead one. if [ -f "$CLAIM_FILE" ]; then diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 88fc8edb5d8..f37fa10a829 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -576,7 +576,7 @@ trap 'exit 130' INT trap 'exit 143' TERM if [ -n "$ACK_THROUGH" ]; then - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || exit 1 elif fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$PRESENTATION_LOCK_TIMEOUT"; then : else @@ -626,7 +626,7 @@ if [ -n "$ACK_THROUGH" ]; then echo "wake drain: inactive outcome receipt could not be recorded safely" >&2 exit 1 fi - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || exit 1 DRAIN_LOCK_HELD=true DRAIN_TMP=$(mktemp "$STATE/.wake-queue.ack.XXXXXX") || exit 1 chmod 0600 "$DRAIN_TMP" || exit 1 diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 7964dab4595..d306813a226 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -297,13 +297,13 @@ fm_afk_daemon_owns_supervision() { # shellcheck disable=SC2034 # Read by callers after the function returns. FM_WATCHER_VERDICT_OK=false # shellcheck disable=SC2034 # Read by callers after the function returns. -FM_WATCHER_VERDICT_REASON=stale-beacon +FM_WATCHER_VERDICT_REASON='stale-beacon' fm_watcher_supervision_verdict() { local state=$1 watch=$2 grace=${3:-${FM_GUARD_GRACE:-300}} home=${4:-$FM_HOME} local root=${5:-$FM_ROOT} local beat age fresh=false model FM_WATCHER_VERDICT_OK=false - FM_WATCHER_VERDICT_REASON=stale-beacon + FM_WATCHER_VERDICT_REASON='stale-beacon' beat="$state/.last-watcher-beat" age=$(fm_path_age "$beat") case "$age" in @@ -377,7 +377,7 @@ fm_lock_owner_dir() { fm_lock_prepare_owner() { local ownerdir=$1 mypid back mypid=${BASHPID:-$$} - printf '%s\n' "$mypid" > "$ownerdir/pid" 2>/dev/null || return 1 + printf '%s\n' "$mypid" 2>/dev/null > "$ownerdir/pid" || return 1 back=$(cat "$ownerdir/pid" 2>/dev/null || true) [ "$back" = "$mypid" ] } @@ -449,17 +449,21 @@ fm_lock_claim() { return 0 } +# Status 1 means the lock is held or was lost to a racer and retrying can win it; +# status 2 means this process could not create its own owner record at all +# (missing or unwritable parent directory), which no amount of waiting fixes. +# Every caller in this file propagates that distinction rather than spinning. fm_lock_try_create() { local lockdir=$1 allowed_steal_owner=${2:-} ownerdir FM_LOCK_OWNER_DIR= - ownerdir=$(fm_lock_owner_dir "$lockdir") || return 1 + ownerdir=$(fm_lock_owner_dir "$lockdir") || return 2 if [ -e "$lockdir" ] || [ -L "$lockdir" ]; then fm_lock_discard_owner "$ownerdir" return 1 fi if ! fm_lock_prepare_owner "$ownerdir"; then fm_lock_discard_owner "$ownerdir" - return 1 + return 2 fi if ln -s "$ownerdir" "$lockdir" 2>/dev/null && fm_lock_points_to_owner "$lockdir" "$ownerdir"; then if fm_lock_claim "$lockdir" "$ownerdir" "$allowed_steal_owner"; then @@ -476,6 +480,240 @@ fm_lock_try_create() { return 1 } +# Acquire a stale-recovery mutex without requesting a second mutex. Status 1 +# means contended; status 2 means the parent or owner record could not be +# created. A dead symlink owner is serialized inside its unique owner directory, +# so interrupted recovery remains reclaimable without creating .steal.steal. +fm_lock_try_acquire_steal_mutex() { + local steal=$1 rc pid ownerdir expected_owner reclaim + case "$steal" in + *.steal) : ;; + *) return 2 ;; + esac + rc=0 + fm_lock_try_create "$steal" || rc=$? + [ "$rc" -ne 0 ] || return 0 + [ "$rc" -ne 2 ] || return 2 + ! fm_lock_legacy_nested_steal_blocks "$steal.steal" || return 1 + # Compare against ${BASHPID:-$$} inline, never via a command substitution, + # and keep the Bash 3 subshell guard the primary path uses: a trap that + # abandoned the frame holding this mutex must not deadlock the exit path + # against itself, but a child frame must never reclaim its parent's hold. + pid=$(cat "$steal/pid" 2>/dev/null || true) + if [ -n "$pid" ] && [ "$pid" = "${BASHPID:-$$}" ] \ + && { [ -n "${BASHPID:-}" ] || [ "${BASH_SUBSHELL:-0}" -eq 0 ]; }; then + fm_lock_remove_path "$steal" || true + rc=0 + fm_lock_try_create "$steal" || rc=$? + return "$rc" + fi + fm_pid_alive "$pid" && return 1 + fm_lock_mid_acquire_is_fresh "$steal" "$pid" && return 1 + if [ -L "$steal" ]; then + ownerdir=$(fm_lock_link_owner "$steal" 2>/dev/null) || return 1 + expected_owner=$ownerdir + if [ ! -d "$ownerdir" ]; then + case "${ownerdir##*/}" in + "${steal##*/}".owner.*) : ;; + *) return 1 ;; + esac + if ! mkdir "$ownerdir" 2>/dev/null; then + [ -d "$ownerdir" ] || return 1 + fi + if ! fm_lock_points_to_owner "$steal" "$ownerdir"; then + rmdir "$ownerdir" 2>/dev/null || true + return 1 + fi + fi + elif [ -d "$steal" ]; then + ownerdir=$steal + expected_owner= + else + return 1 + fi + reclaim="$ownerdir/reclaim" + fm_lock_reclaim_marker_claim "$reclaim" || return 1 + if fm_lock_legacy_nested_steal_blocks "$steal.steal"; then + fm_lock_reclaim_marker_release "$reclaim" + return 1 + fi + if [ "$ownerdir" = "$steal" ]; then + if ! fm_lock_legacy_steal_dir_still_stale "$steal" "$pid"; then + fm_lock_reclaim_marker_release "$reclaim" + return 1 + fi + elif ! fm_lock_recheck_stale_owner "$steal" "$expected_owner" "$pid"; then + fm_lock_reclaim_marker_release "$reclaim" + return 1 + fi + fm_lock_reclaim_marker_held "$reclaim" || return 1 + if [ "$ownerdir" = "$steal" ]; then + if ! fm_lock_retire_legacy_steal_dir "$steal"; then + fm_lock_reclaim_marker_release "$reclaim" + return 1 + fi + else + if ! rm -f "$steal" 2>/dev/null; then + fm_lock_reclaim_marker_release "$reclaim" + return 1 + fi + fm_lock_clean_known_files "$ownerdir" + fm_lock_reclaim_marker_release "$reclaim" + rmdir "$ownerdir" 2>/dev/null || true + fi + fm_lock_try_create "$steal" +} + +# The staleness of a legacy directory-shaped steal mutex was already judged on +# the untouched directory before its reclaim marker was created, so the marker's +# own mtime must not be re-read here; only the owner record is re-verified. +fm_lock_legacy_steal_dir_still_stale() { + local steal=$1 expected_pid=$2 actual_pid + [ -d "$steal" ] && [ ! -L "$steal" ] || return 1 + actual_pid=$(cat "$steal/pid" 2>/dev/null || true) + [ "$actual_pid" = "$expected_pid" ] || return 1 + ! fm_pid_alive "$actual_pid" +} + +# Replace a proven-stale legacy directory steal mutex by renaming it into a +# fresh owner-record name and removing it there. The rename is the removal, so +# the directory's own pid record survives every failure path and the mutex stays +# reclaimable; unknown content refuses the whole retirement untouched. +fm_lock_retire_legacy_steal_dir() { + local steal=$1 aside entry + [ -d "$steal" ] && [ ! -L "$steal" ] || return 1 + fm_lock_clean_known_debris "$steal" + for entry in "$steal"/* "$steal"/.[!.]* "$steal"/..?*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + case "${entry##*/}" in + pid|reclaim) ;; + *) return 1 ;; + esac + done + aside=$(fm_lock_owner_dir "$steal") || return 1 + if ! rmdir "$aside" 2>/dev/null; then + fm_lock_discard_owner "$aside" + return 1 + fi + if [ ! -d "$steal" ] || [ -L "$steal" ]; then + rmdir "$aside" 2>/dev/null || true + return 1 + fi + mv "$steal" "$aside" 2>/dev/null || return 1 + rm -f "$aside/pid" "$aside/reclaim/pid" 2>/dev/null || true + rmdir "$aside/reclaim" 2>/dev/null || true + rmdir "$aside" 2>/dev/null || true + return 0 +} + +# The reclaim marker serializes stale steal-mutex recovery. It records the +# reclaimer's pid so a reclaimer killed mid-recovery cannot wedge every later +# reclaimer: a marker whose pid is dead and whose age passed the stale window is +# itself reclaimable, exactly like every other stale record in this file. +fm_lock_reclaim_marker_claim() { + local reclaim=$1 mypid retired abandoned_pid retired_pid took_over=0 + mypid=${BASHPID:-$$} + if ! mkdir "$reclaim" 2>/dev/null; then + fm_lock_reclaim_marker_is_abandoned "$reclaim" || return 1 + abandoned_pid=$FM_LOCK_RECLAIM_OBSERVED_PID + retired="$reclaim.dead.$mypid" + rm -rf "$retired" 2>/dev/null || true + mv "$reclaim" "$retired" 2>/dev/null || return 1 + if ! mkdir "$reclaim" 2>/dev/null; then + rm -rf "$retired" 2>/dev/null || true + return 1 + fi + took_over=1 + fi + if ! { printf '%s\n' "$mypid" 2>/dev/null > "$reclaim/pid"; } \ + || [ "$(cat "$reclaim/pid" 2>/dev/null || true)" != "$mypid" ]; then + rm -f "$reclaim/pid" 2>/dev/null || true + rmdir "$reclaim" 2>/dev/null || true + [ "$took_over" -eq 0 ] || rm -rf "$retired" 2>/dev/null || true + return 1 + fi + if [ "$took_over" -eq 1 ]; then + retired_pid=$(cat "$retired/pid" 2>/dev/null || true) + rm -rf "$retired" 2>/dev/null || true + if [ "$retired_pid" != "$abandoned_pid" ] \ + || { [ "$retired_pid" != "$mypid" ] && fm_pid_alive "$retired_pid"; }; then + if fm_pid_alive "$retired_pid"; then + printf '%s\n' "$retired_pid" 2>/dev/null > "$reclaim/pid" || true + else + fm_lock_reclaim_marker_release "$reclaim" + fi + return 1 + fi + fi + return 0 +} + +FM_LOCK_RECLAIM_OBSERVED_PID= + +fm_lock_reclaim_marker_is_abandoned() { + local reclaim=$1 pid stale + FM_LOCK_RECLAIM_OBSERVED_PID= + [ -d "$reclaim" ] || return 1 + pid=$(cat "$reclaim/pid" 2>/dev/null || true) + if [ -n "$pid" ] && [ "$pid" = "${BASHPID:-$$}" ]; then + FM_LOCK_RECLAIM_OBSERVED_PID=$pid + return 0 + fi + fm_pid_alive "$pid" && return 1 + stale=$FM_LOCK_STALE_AFTER + [ "$stale" -lt 2 ] && stale=2 + [ "$(fm_path_age "$reclaim")" -ge "$stale" ] || return 1 + FM_LOCK_RECLAIM_OBSERVED_PID=$pid + return 0 +} + +fm_lock_reclaim_marker_held() { + [ "$(cat "$1/pid" 2>/dev/null || true)" = "${BASHPID:-$$}" ] +} + +fm_lock_reclaim_marker_release() { + local reclaim=$1 + fm_lock_reclaim_marker_held "$reclaim" || return 1 + rm -f "$reclaim/pid" 2>/dev/null || true + rmdir "$reclaim" 2>/dev/null || true + return 0 +} + +# A leftover .steal.steal can only come from a pre-upgrade recursive reclaimer. +# It still blocks while its owner is live or too fresh to judge; once the owner +# is dead and the residue passed the stale window it is retired here so the +# non-recursive path can proceed without ever creating a nested mutex. +fm_lock_legacy_nested_steal_blocks() { + local residue=$1 pid stale + [ -e "$residue" ] || [ -L "$residue" ] || return 1 + pid=$(cat "$residue/pid" 2>/dev/null || true) + fm_pid_alive "$pid" && return 0 + stale=$FM_LOCK_STALE_AFTER + [ "$stale" -lt 2 ] && stale=2 + [ "$(fm_path_age "$residue")" -lt "$stale" ] && return 0 + fm_lock_remove_path "$residue" >/dev/null 2>&1 || true + [ -e "$residue" ] || [ -L "$residue" ] || return 1 + return 0 +} + +# Retire only debris this lock implementation is known to create inside a lock +# directory: an interrupted reclaim takeover copy, and a stray owner symlink a +# racer left behind. Anything else is left in place so the caller's rmdir still +# fails closed on state this code does not own. +fm_lock_clean_known_debris() { + local lockdir=$1 base entry + base=${lockdir##*/} + for entry in "$lockdir"/reclaim.dead.*; do + [ -d "$entry" ] && [ ! -L "$entry" ] || continue + rm -f "$entry/pid" 2>/dev/null || true + rmdir "$entry" 2>/dev/null || true + done + for entry in "$lockdir/$base".owner.*; do + [ -L "$entry" ] || continue + rm -f "$entry" 2>/dev/null || true + done +} + fm_lock_remove_path() { local lockdir=$1 ownerdir if [ -L "$lockdir" ]; then @@ -828,20 +1066,29 @@ fm_recovery_marker_reopen_announced() { fm_recovery_transition "$1" reopen-announced } +# Returns 0 on acquisition, 1 while the lock is legitimately contended, and 2 +# when acquisition can never succeed here (see fm_lock_try_create). A path that +# already ends in .steal is never stale-recovered through a second mutex: at +# most one primary-lock-to-.steal transition exists, so no .steal.steal is ever +# created and stale recovery cannot recurse. fm_lock_try_acquire() { local lockdir=$1 pid steal cur rc steal_owner primary_owner FM_LOCK_HELD_PID= FM_LOCK_OWNER_DIR= FM_LOCK_RECOVERED_PID= - if fm_lock_try_create "$lockdir"; then - return 0 - fi + rc=0 + fm_lock_try_create "$lockdir" || rc=$? + [ "$rc" -ne 0 ] || return 0 + [ "$rc" -ne 2 ] || return 2 # Compare against ${BASHPID:-$$} inline, never via a command substitution: # $() forks a subshell whose BASHPID is not this frame's pid. + # Bash 3 lacks BASHPID and preserves $$ in subshells, so BASH_SUBSHELL keeps + # a child frame from being mistaken for the parent that owns the lock. pid=$(cat "$lockdir/pid" 2>/dev/null || true) - if [ -n "$pid" ] && [ "$pid" = "${BASHPID:-$$}" ]; then + if [ -n "$pid" ] && [ "$pid" = "${BASHPID:-$$}" ] \ + && { [ -n "${BASHPID:-}" ] || [ "${BASH_SUBSHELL:-0}" -eq 0 ]; }; then # The recorded holder is THIS very process. Single-threaded bash can only # observe that when an interrupting trap abandoned the frame that held the # lock mid-critical-section (e.g. TERM inside a recovery-marker section, @@ -851,11 +1098,11 @@ fm_lock_try_acquire() { # - the hang reproduced by the self-held reclaim regression in # tests/fm-wake-queue.test.sh - so reclaim the abandoned hold instead. fm_lock_remove_path "$lockdir" || true - if fm_lock_try_create "$lockdir"; then - return 0 - fi + rc=0 + fm_lock_try_create "$lockdir" || rc=$? + [ "$rc" -ne 0 ] || return 0 FM_LOCK_HELD_PID=$(cat "$lockdir/pid" 2>/dev/null || true) - return 1 + return "$rc" fi if fm_pid_alive "$pid"; then FM_LOCK_HELD_PID=$pid @@ -866,11 +1113,20 @@ fm_lock_try_acquire() { return 1 fi + case "$lockdir" in + *.steal) + FM_LOCK_HELD_PID=$pid + return 1 + ;; + esac + steal="$lockdir.steal" - if ! fm_lock_try_acquire "$steal"; then + rc=0 + fm_lock_try_acquire_steal_mutex "$steal" || rc=$? + if [ "$rc" -ne 0 ]; then FM_LOCK_HELD_PID=$(cat "$lockdir/pid" 2>/dev/null || true) FM_LOCK_OWNER_DIR= - return 1 + return "$rc" fi steal_owner=${FM_LOCK_OWNER_DIR:-} @@ -914,9 +1170,9 @@ fm_lock_try_acquire() { return 1 fi fm_lock_remove_path "$lockdir" || true - rc=1 - if fm_lock_try_create "$lockdir" "$steal_owner"; then - rc=0 + rc=0 + fm_lock_try_create "$lockdir" "$steal_owner" || rc=$? + if [ "$rc" -eq 0 ]; then # shellcheck disable=SC2034 # Read by sourcing callers after lock acquisition. FM_LOCK_RECOVERED_PID=$cur fi @@ -929,9 +1185,16 @@ fm_lock_try_acquire() { return "$rc" } +# Waits out ordinary contention but is NOT unconditional: a status-2 failure from +# fm_lock_try_acquire is returned to the caller, so callers must check the result +# and must not assume the lock is held after this returns. fm_lock_acquire_wait() { - local lockdir=$1 - while ! fm_lock_try_acquire "$lockdir"; do + local lockdir=$1 rc + while :; do + rc=0 + fm_lock_try_acquire "$lockdir" || rc=$? + [ "$rc" -ne 0 ] || return 0 + [ "$rc" -ne 2 ] || return 2 sleep 0.1 done } @@ -1437,7 +1700,7 @@ fm_autoarm_release_abandoned() { # [grace] steal="$lock.steal" epoch="$state/.claude-autoarm-epoch" fm_autoarm_claim_abandoned "$state" "$grace" || return 1 - fm_lock_try_acquire "$steal" || return 1 + fm_lock_try_acquire_steal_mutex "$steal" || return 1 if ! fm_autoarm_claim_abandoned "$state" "$grace"; then fm_lock_release "$steal" return 1 @@ -1506,7 +1769,7 @@ fm_wake_append() { recovery_marker="$STATE/.watcher-down" status=0 - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 _fm_recovery_marker_publish "$recovery_marker" downtime || status=$? if [ "$status" -eq 0 ]; then seq=$(cat "$seq_file" 2>/dev/null || echo 0) @@ -1535,7 +1798,7 @@ fm_wake_queued_keys() { signal|stale|check|heartbeat) ;; *) printf 'fm_wake_queued_keys: invalid wake kind: %s\n' "$kind" >&2; return 2 ;; esac - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 fm_wake_queued_keys_locked "$kind" fm_lock_release "$FM_WAKE_QUEUE_LOCK" } diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 23042e9c4b7..8743e847d6d 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -1105,7 +1105,7 @@ procevent_surface_queued() { local key reason PROCEVENT_SURFACED= [ -s "$FM_WAKE_QUEUE" ] || return 0 - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 while IFS= read -r key; do case "$key" in procevent:*) ;; *) continue ;; esac [ -e "$(procevent_surfaced_marker "$key")" ] && continue diff --git a/bin/fm-x-lib.sh b/bin/fm-x-lib.sh index f50ddce781d..f23cdb734bc 100644 --- a/bin/fm-x-lib.sh +++ b/bin/fm-x-lib.sh @@ -931,7 +931,7 @@ fmx_meta_link_set() { local meta=$1 rid=$2 ts=$3 followups=${4:-0} platform=${5:-} reply_max=${6:-} tmp lock [ -f "$meta" ] || return 1 lock=$(fm_meta_lock_path "$meta") || return 1 - fm_lock_acquire_wait "$lock" + fm_lock_acquire_wait "$lock" || return 1 [ -f "$meta" ] || { fm_lock_release "$lock"; return 1; } tmp=$(fmx_meta_tmp "$meta") || { fm_lock_release "$lock"; return 1; } if ! { grep -vE '^x_request=|^x_request_ts=|^x_followups=|^x_platform=|^x_reply_max_chars=' "$meta" || true; } > "$tmp"; then @@ -962,7 +962,7 @@ fmx_meta_followups_set() { local meta=$1 n=$2 tmp lock [ -f "$meta" ] || return 1 lock=$(fm_meta_lock_path "$meta") || return 1 - fm_lock_acquire_wait "$lock" + fm_lock_acquire_wait "$lock" || return 1 [ -f "$meta" ] || { fm_lock_release "$lock"; return 1; } tmp=$(fmx_meta_tmp "$meta") || { fm_lock_release "$lock"; return 1; } if ! { grep -vE '^x_followups=' "$meta" || true; } > "$tmp"; then @@ -1020,7 +1020,7 @@ fmx_meta_link_clear() { case "$lock_timeout" in ''|*[!0-9]*|0) lock_timeout=10 ;; esac fm_lock_acquire_wait_bounded "$lock" "$lock_timeout" || return 1 else - fm_lock_acquire_wait "$lock" + fm_lock_acquire_wait "$lock" || return 1 fi [ ! -L "$meta" ] || { fm_lock_release "$lock"; return 1; } [ -f "$meta" ] || { fm_lock_release "$lock"; return 0; } diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 537a236d842..6cf66ef3d2c 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -107,6 +107,7 @@ The same suite covers ordinary same-process session replacement for `/new`, `/re `tests/fm-watch-arm.test.sh` covers durable queue replay, real remote parent-replies ingestion into the authoritative status log, decision-only OPEN DECISIONS recovery, interrupted handling replay, generation-bound acknowledgement, a persistent live successor after recovery, a watcher close inside the handling window that must leave the printed acknowledgement valid, and the self-healing moved-generation acknowledgement that consumes its handled rows and names its remedy. `tests/fm-watch-recovery-loop.test.sh` covers the once-per-generation announcement bound with the real Pi extension against a refused handling handshake, and a handling successor that must surface a real crew event instead of going blind. `tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a SIGSTOP counterfactual that distinguishes a live PID from a stale beacon before classifying termination. +It also covers the portable lock's stale-recovery boundary: at most one primary-lock-to-`.steal` transition exists, so a stale, malformed, or legacy directory-shaped steal mutex is reclaimed without ever creating a nested `.steal.steal` mutex, and a lock whose parent directory or owner record cannot be created fails promptly with a typed status and a bounded process-launch budget instead of spinning or recursing. `tests/fm-subagent-pretool-check.test.sh` proves Claude retains only the non-status Bash seatbelts. `tests/fm-claude-stop-autoarm.test.sh` covers the auto-arm's scope, stale and live session owners, unchanged AFK and need boundaries, single-flight, bounded failure retries, benign live-watcher cycle ends, one-notice failure episodes, and exit-2 translation. It also covers generation-claim single-flight, stuck-claim supersession, superseded-owner silence, notice-marker refusal and retry, ownership-atomic episode reset, and the legacy upgrade shim; [`turnend-guard.md`](turnend-guard.md) owns those behavior contracts. diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index 77fd4fbcca3..7533e553d3c 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -218,6 +218,620 @@ test_lock_single_winner_under_concurrency() { pass "concurrent fm_lock_try_acquire yields exactly one winner" } +test_lock_missing_parent_returns_typed_failure_with_bounded_launches() { + local dir state missing fakebin count pidfile command_name real_command out rc elapsed result wait_result launches attempt_pid + local proctable snapfile snapshot_pid leaked + dir=$(make_case lock-missing-parent) + state="$dir/state" + missing="$dir/absent/demo.lock" + fakebin="$dir/countbin" + count="$dir/helper-launches" + pidfile="$dir/attempt-pid" + proctable="$dir/live-process-table" + snapfile="$dir/snapshot-pid" + mkdir -p "$fakebin" + for command_name in basename cat date dirname ln mkdir mktemp readlink rm rmdir stat uname; do + real_command=$(command -v "$command_name") + cat > "$fakebin/$command_name" </dev/null || true +count=\$((count + 1)) +printf '%s\n' "\$count" > "\$FM_TEST_LAUNCH_COUNT" +if [ "\$count" -gt "\${FM_TEST_LAUNCH_BUDGET:?}" ]; then + kill -TERM "\${FM_TEST_ROOT_PID:?}" 2>/dev/null || true + exit 97 +fi +exec "$real_command" "\$@" +SH + chmod +x "$fakebin/$command_name" + done + + rc=0 + SECONDS=0 + out=$(PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_TEST_LAUNCH_COUNT="$count" \ + FM_TEST_LAUNCH_BUDGET=18 bash -c ' + FM_TEST_ROOT_PID=${BASHPID:-$$} + export FM_TEST_ROOT_PID + . "$1" + printf "0\n" > "$3" + printf "%s\n" "$FM_TEST_ROOT_PID" > "$4" + trap "exit 97" TERM + i=0 + while [ "$i" -lt 5 ]; do + fm_lock_try_acquire "$2" + result=$? + [ "$result" -eq 2 ] || exit 20 + i=$((i + 1)) + done + fm_lock_acquire_wait "$2" + wait_result=$? + [ "$wait_result" -eq 2 ] || exit 21 + read -r launches < "$3" + ps -eo pid=,ppid= > "$5" 2>/dev/null & + snapshot_pid=$! + wait "$snapshot_pid" 2>/dev/null || true + printf "%s\n" "$snapshot_pid" > "$6" + printf "result=%s wait_result=%s launches=%s\n" "$result" "$wait_result" "$launches" + ' _ "$LIB" "$missing" "$count" "$pidfile" "$proctable" "$snapfile" 2>&1) || rc=$? + elapsed=$SECONDS + + [ "$rc" -eq 0 ] || fail "missing-parent lock attempt did not return typed invalid-path status within its launch fuse (rc=$rc): $out" + result=${out#*result=}; result=${result%% *} + wait_result=${out#*wait_result=}; wait_result=${wait_result%% *} + launches=${out#*launches=}; launches=${launches%%[!0-9]*} + [ "$result" -eq 2 ] || fail "missing-parent lock attempt returned '$result' instead of typed invalid-path status 2: $out" + [ "$wait_result" -eq 2 ] || fail "missing-parent lock wait returned '$wait_result' instead of propagating status 2: $out" + [ "$launches" -le 18 ] || fail "five missing-parent failures plus one wait exceeded the helper-launch budget ($launches): $out" + [ "$elapsed" -lt 10 ] || fail "missing-parent lock attempts did not return promptly (${elapsed}s): $out" + attempt_pid=$(cat "$pidfile") + snapshot_pid=$(cat "$snapfile") + [ -s "$proctable" ] || fail "the live process table was never captured while the lock attempt was running" + leaked=$(awk -v parent="$attempt_pid" -v self="$snapshot_pid" \ + '$2 == parent && $1 != self { print $1 }' "$proctable" | tr '\n' ' ') + [ -z "$leaked" ] \ + || fail "missing-parent lock attempt left a spawned descendant alive: $leaked" + pass "missing-parent lock failure is typed, prompt, descendant-free, and launch-bounded" +} + +test_lock_owner_record_failure_returns_typed_failure() { + local dir state lockdir fakebin count real_mktemp out rc + dir=$(make_case lock-owner-record-failure) + state="$dir/state" + lockdir="$state/.owner-failure.lock" + fakebin="$dir/countbin" + count="$dir/mktemp-launches" + real_mktemp=$(command -v mktemp) + mkdir -p "$fakebin" + cat > "$fakebin/mktemp" </dev/null || true +count=\$((count + 1)) +printf '%s\n' "\$count" > "\$FM_TEST_LAUNCH_COUNT" +if [ "\$count" -gt 4 ]; then + kill -TERM "\${FM_TEST_ROOT_PID:?}" 2>/dev/null || true + exit 97 +fi +ownerdir=\$("$real_mktemp" "\$@") || exit \$? +chmod 0500 "\$ownerdir" || exit 1 +printf '%s\n' "\$ownerdir" +SH + chmod +x "$fakebin/mktemp" + + rc=0 + out=$(PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_TEST_LAUNCH_COUNT="$count" bash -c ' + . "$1" + printf "0\n" > "$3" + FM_TEST_ROOT_PID=${BASHPID:-$$} + export FM_TEST_ROOT_PID + trap "exit 97" TERM + fm_lock_try_acquire "$2" + rc=$? + printf "rc=%s\n" "$rc" + [ "$rc" -eq 2 ] + ' _ "$LIB" "$lockdir" "$count" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "owner-record creation failure did not return typed status 2 promptly (rc=$rc): $out" + [ "$out" = "rc=2" ] || fail "owner-record creation failure returned an unexpected result: $out" + pass "owner-record creation failure returns typed invalid-create status promptly" +} + +test_stale_or_malformed_steal_mutex_never_claims_nested_mutex() { + local kind dir state lockdir steal ownerdir fakebin log real_ln dead out rc + for kind in stale malformed; do + dir=$(make_case "lock-no-nested-steal-$kind") + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + fakebin="$dir/logbin" + log="$dir/ln-targets" + real_ln=$(command -v ln) + dead=$(dead_pid) + mkdir "$lockdir" "$fakebin" + printf '%s\n' "$dead" > "$lockdir/pid" + if [ "$kind" = stale ]; then + ownerdir="$state/.stale-steal-owner" + mkdir "$ownerdir" + printf '%s\n' "$dead" > "$ownerdir/pid" + ln -s "$ownerdir" "$steal" + else + ln -s "$steal.owner.gone01" "$steal" + touch -h -t 202001010000 "$steal" + fi + cat > "$fakebin/ln" <> "\${FM_TEST_LN_LOG:?}" +exec "$real_ln" "\$@" +SH + chmod +x "$fakebin/ln" + + rc=0 + out=$(PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_TEST_LN_LOG="$log" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "$kind steal-mutex fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=0" ] \ + || fail "$kind steal mutex was never reclaimed, so the dead-owner lock stayed unacquirable: $out" + ! grep -F "$lockdir.steal.steal" "$log" >/dev/null 2>&1 \ + || fail "$kind steal mutex attempted a nested .steal.steal claim: $(cat "$log")" + done + pass "stale and malformed steal mutexes are reclaimed without ever claiming .steal.steal" +} + +test_abandoned_reclaim_marker_does_not_wedge_stale_steal_recovery() { + local dir state lockdir steal ownerdir dead out rc + dir=$(make_case lock-abandoned-reclaim-marker) + state="$dir/state" + lockdir="$state/.wedged.lock" + steal="$lockdir.steal" + ownerdir="$state/.stale-steal-owner" + dead=$(dead_pid) + mkdir "$lockdir" "$ownerdir" + printf '%s\n' "$dead" > "$lockdir/pid" + printf '%s\n' "$dead" > "$ownerdir/pid" + ln -s "$ownerdir" "$steal" + mkdir "$ownerdir/reclaim" + touch -t 202001010000 "$ownerdir/reclaim" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "abandoned-reclaim-marker fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=0" ] \ + || fail "a reclaim marker left behind by a killed reclaimer permanently blocked stale steal recovery: $out" + [ ! -d "$ownerdir/reclaim" ] || fail "the reclaimed marker was not released" + pass "a reclaim marker abandoned by a killed reclaimer does not wedge stale steal recovery" +} + +test_legacy_nested_steal_residue_is_retired_only_when_stale() { + local dir state lockdir steal residue ownerdir dead out rc + + dir=$(make_case lock-legacy-nested-stale) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + residue="$steal.steal" + ownerdir="$steal.owner.legacy" + dead=$(dead_pid) + mkdir "$lockdir" "$ownerdir" + printf '%s\n' "$dead" > "$lockdir/pid" + printf '%s\n' "$dead" > "$ownerdir/pid" + ln -s "$ownerdir" "$steal" + ln -s "$state/.retired-nested-owner" "$residue" + touch -h -t 202001010000 "$residue" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "stale nested-steal residue fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=0" ] \ + || fail "a pre-upgrade .steal.steal residue permanently blocked stale primary reclaim: $out" + [ ! -e "$residue" ] && [ ! -L "$residue" ] \ + || fail "the retired nested-steal residue was left behind" + + dir=$(make_case lock-legacy-nested-live) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + residue="$steal.steal" + ownerdir="$steal.owner.legacy" + mkdir "$lockdir" "$ownerdir" "$residue" + printf '%s\n' "$dead" > "$lockdir/pid" + printf '%s\n' "$dead" > "$ownerdir/pid" + printf '%s\n' "$$" > "$residue/pid" + ln -s "$ownerdir" "$steal" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "live nested-steal residue fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=1" ] \ + || fail "a live pre-upgrade nested-steal holder was overrun instead of being waited out: $out" + [ -d "$residue" ] || fail "a live nested-steal residue was destroyed by the upgrade path" + [ -L "$steal" ] || fail "the steal mutex was reclaimed while a live nested holder still owned it" + pass "a pre-upgrade .steal.steal residue is retired only once its owner is dead and stale" +} + +test_reclaim_marker_is_not_stolen_from_a_live_owner() { + local dir state ownerdir reclaim out rc + dir=$(make_case lock-reclaim-marker-ownership) + state="$dir/state" + ownerdir="$state/.owner" + reclaim="$ownerdir/reclaim" + mkdir -p "$reclaim" + printf '%s\n' "$$" > "$reclaim/pid" + touch -t 202001010000 "$reclaim" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_reclaim_marker_release "$2" + printf "release=%s " "$?" + fm_lock_reclaim_marker_claim "$2" + printf "claim=%s\n" "$?" + ' _ "$LIB" "$reclaim" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "reclaim-marker ownership fixture shell failed (rc=$rc): $out" + [ "$out" = "release=1 claim=1" ] \ + || fail "another process released or took over a reclaim marker held by a live owner: $out" + [ -d "$reclaim" ] || fail "the live owner's reclaim marker was removed by a non-owner" + [ "$(cat "$reclaim/pid")" = "$$" ] || fail "the live owner's reclaim-marker pid was overwritten" + pass "a reclaim marker held by a live owner is neither released nor taken over by another process" +} + +test_dangling_steal_owner_reclaim_yields_one_winner() { + local dir state lockdir steal foreign marker dead i pids pid wins out rc + + dir=$(make_case lock-dangling-steal-concurrency) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + marker="$dir/wins" + dead=$(dead_pid) + mkdir "$lockdir" + printf '%s\n' "$dead" > "$lockdir/pid" + ln -s "$steal.owner.gone01" "$steal" + touch -h -t 202001010000 "$steal" + : > "$marker" + pids= + i=1 + while [ "$i" -le 40 ]; do + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + if fm_lock_try_acquire "$2"; then + printf "%s\n" "${BASHPID:-$$}" >> "$3" + sleep 1 + fi + ' _ "$LIB" "$lockdir" "$marker" & + pids="$pids $!" + i=$((i + 1)) + done + for pid in $pids; do + wait "$pid" 2>/dev/null || true + done + wins=$(awk 'NF { c++ } END { print c + 0 }' "$marker") + [ "$wins" -eq 1 ] || fail "expected exactly one dangling-steal reclaimer to win, got $wins" + + dir=$(make_case lock-dangling-steal-foreign) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + foreign="$state/.not-an-owner-record" + mkdir "$lockdir" + printf '%s\n' "$dead" > "$lockdir/pid" + ln -s "$foreign" "$steal" + touch -h -t 202001010000 "$steal" + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "foreign steal-target fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=1" ] || fail "a steal mutex pointing outside the owner-record shape was reclaimed: $out" + [ ! -e "$foreign" ] || fail "reclaim created a directory at a path that is not an owner record" + pass "dangling steal-owner reclaim is serialized to one winner and refuses foreign targets" +} + +test_reclaim_marker_takeover_is_bound_to_the_marker_it_inspected() { + local dir state ownerdir reclaim fakebin real_mv dead out rc + dir=$(make_case lock-reclaim-marker-takeover-identity) + state="$dir/state" + ownerdir="$state/.owner" + reclaim="$ownerdir/reclaim" + fakebin="$dir/racebin" + real_mv=$(command -v mv) + dead=$(dead_pid) + mkdir -p "$reclaim" "$fakebin" + printf '%s\n' "$dead" > "$reclaim/pid" + touch -t 202001010000 "$reclaim" + cat > "$fakebin/mv" < "\$1/pid" 2>/dev/null || true +exec "$real_mv" "\$@" +SH + chmod +x "$fakebin/mv" + + rc=0 + out=$(PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_TEST_RACER_PID="$$" bash -c ' + . "$1" + fm_lock_reclaim_marker_claim "$2" + printf "claim=%s\n" "$?" + ' _ "$LIB" "$reclaim" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "reclaim-marker takeover fixture shell failed (rc=$rc): $out" + [ "$out" = "claim=1" ] \ + || fail "a reclaimer retired a marker another racer had already replaced with its own live one: $out" + [ -d "$reclaim" ] || fail "the racer's live reclaim marker was destroyed by the losing takeover" + [ "$(cat "$reclaim/pid" 2>/dev/null || true)" = "$$" ] \ + || fail "the racer's live reclaim-marker pid did not survive the losing takeover" + pass "a reclaim-marker takeover only retires the abandoned marker it actually inspected" +} + +test_legacy_directory_steal_mutex_is_reclaimed_without_recursion() { + local dir state lockdir steal fakebin log real_ln dead out rc + dir=$(make_case lock-legacy-steal-dir) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + fakebin="$dir/logbin" + log="$dir/ln-targets" + real_ln=$(command -v ln) + dead=$(dead_pid) + mkdir "$lockdir" "$steal" "$fakebin" + printf '%s\n' "$dead" > "$lockdir/pid" + printf '%s\n' "$dead" > "$steal/pid" + cat > "$fakebin/ln" <> "\${FM_TEST_LN_LOG:?}" +exec "$real_ln" "\$@" +SH + chmod +x "$fakebin/ln" + + rc=0 + out=$(PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_TEST_LN_LOG="$log" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "legacy directory steal-mutex fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=0" ] \ + || fail "a legacy directory-shaped steal mutex with a dead owner was never reclaimed: $out" + ! grep -F "$steal.steal" "$log" >/dev/null 2>&1 \ + || fail "reclaiming a legacy directory steal mutex created a nested .steal.steal: $(cat "$log")" + pass "a legacy directory-shaped steal mutex is reclaimed without any nested steal transition" +} + +test_reclaim_marker_takeover_never_vacates_the_marker_slot() { + local dir state ownerdir reclaim fakebin real_cat gaps dead out rc + dir=$(make_case lock-reclaim-marker-no-gap) + state="$dir/state" + ownerdir="$state/.owner" + reclaim="$ownerdir/reclaim" + fakebin="$dir/gapbin" + gaps="$dir/gap-log" + real_cat=$(command -v cat) + dead=$(dead_pid) + mkdir -p "$reclaim" "$fakebin" + printf '%s\n' "$dead" > "$reclaim/pid" + touch -t 202001010000 "$reclaim" + : > "$gaps" + cat > "$fakebin/cat" </dev/null; then + printf '%s\n' "\${FM_TEST_BYSTANDER_PID:?}" > "\$FM_TEST_RECLAIM/pid" 2>/dev/null || true + printf 'claimed\n' >> "\${FM_TEST_GAP_LOG:?}" +fi +exec "$real_cat" "\$@" +SH + chmod +x "$fakebin/cat" + + rc=0 + out=$(PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_TEST_RECLAIM="$reclaim" \ + FM_TEST_GAP_LOG="$gaps" FM_TEST_BYSTANDER_PID="$$" bash -c ' + . "$1" + fm_lock_reclaim_marker_claim "$2" + printf "claim=%s\n" "$?" + ' _ "$LIB" "$reclaim" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "reclaim-marker gap fixture shell failed (rc=$rc): $out" + [ ! -s "$gaps" ] \ + || fail "a bystander claimed the reclaim marker while a takeover had vacated the slot: $(cat "$gaps")" + [ "$out" = "claim=0" ] \ + || fail "the takeover of a genuinely abandoned marker did not succeed: $out" + [ "$(cat "$reclaim/pid" 2>/dev/null || true)" != "$$" ] \ + || fail "the bystander's pid ended up owning the reclaim marker" + pass "a reclaim-marker takeover never leaves the marker slot claimable by a bystander" +} + +test_legacy_directory_steal_mutex_survives_known_recovery_debris() { + local dir state lockdir steal dead out rc + dir=$(make_case lock-legacy-steal-dir-debris) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + dead=$(dead_pid) + mkdir "$lockdir" "$steal" "$steal/reclaim.dead.$dead" + printf '%s\n' "$dead" > "$lockdir/pid" + printf '%s\n' "$dead" > "$steal/pid" + printf '%s\n' "$dead" > "$steal/reclaim.dead.$dead/pid" + ln -s "$state/.gone-owner" "$steal/.contend.lock.steal.owner.abc123" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "legacy steal-dir debris fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=0" ] \ + || fail "known reclaim debris left the legacy directory steal mutex permanently unreclaimable: $out" + + dir=$(make_case lock-legacy-steal-dir-unknown) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + mkdir "$lockdir" "$steal" + printf '%s\n' "$dead" > "$lockdir/pid" + printf '%s\n' "$dead" > "$steal/pid" + printf 'unowned\n' > "$steal/not-a-lock-record" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "unknown steal-dir content fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=1" ] \ + || fail "reclaim destroyed a steal directory holding content this lock code does not own: $out" + [ -f "$steal/not-a-lock-record" ] || fail "unowned content inside the steal directory was deleted" + [ "$(cat "$steal/pid" 2>/dev/null || true)" = "$dead" ] \ + || fail "a refused retirement destroyed the steal mutex's own owner record" + + rm -f "$steal/not-a-lock-record" + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "retry-after-cleanup fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=0" ] \ + || fail "a legacy steal mutex stayed unreclaimable after its unowned content was removed: $out" + pass "legacy steal-dir reclaim retires known debris, fails closed on unowned content, and stays retryable" +} + +test_legacy_directory_steal_mutex_without_owner_record_is_reclaimable_when_aged() { + local dir state lockdir steal dead out rc + dir=$(make_case lock-legacy-steal-dir-no-pid) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + dead=$(dead_pid) + mkdir "$lockdir" "$steal" + printf '%s\n' "$dead" > "$lockdir/pid" + touch -t 202001010000 "$steal" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "pid-less legacy steal-dir fixture shell failed (rc=$rc): $out" + [ "$out" = "rc=0" ] \ + || fail "an aged legacy steal directory carrying no owner record was permanently unreclaimable: $out" + pass "an aged legacy steal directory with no owner record is still reclaimable" +} + +test_legacy_directory_reclaim_never_deletes_a_racer_mutex() { + local dir state lockdir steal racer_owner fakebin real_rmdir dead out rc + dir=$(make_case lock-legacy-steal-dir-racer) + state="$dir/state" + lockdir="$state/.contend.lock" + steal="$lockdir.steal" + racer_owner="$state/.racer-owner" + fakebin="$dir/racebin" + real_rmdir=$(command -v rmdir) + dead=$(dead_pid) + mkdir "$lockdir" "$steal" "$fakebin" + printf '%s\n' "$dead" > "$lockdir/pid" + printf '%s\n' "$dead" > "$steal/pid" + cat > "$fakebin/rmdir" < "\$FM_TEST_RACER_OWNER/pid" + ln -s "\$FM_TEST_RACER_OWNER" "\${FM_TEST_STEAL:?}" 2>/dev/null || true + exit \$rc + ;; +esac +exec "$real_rmdir" "\$@" +SH + chmod +x "$fakebin/rmdir" + + rc=0 + out=$(PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" \ + FM_TEST_STEAL="$steal" FM_TEST_RACER_OWNER="$racer_owner" FM_TEST_RACER_PID="$$" bash -c ' + . "$1" + fm_lock_try_acquire "$2" + printf "rc=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "legacy steal-dir racer fixture shell failed (rc=$rc): $out" + [ -L "$steal" ] \ + || fail "the legacy directory reclaim deleted the live steal mutex a racer created: $out" + [ "$(readlink "$steal")" = "$racer_owner" ] \ + || fail "the racer's steal mutex no longer points at its own owner record: $(readlink "$steal")" + [ "$out" = "rc=1" ] \ + || fail "the losing reclaimer reported success after a racer took the steal mutex: $out" + pass "legacy directory reclaim never deletes a steal mutex another reclaimer created" +} + +test_self_orphaned_reclaim_marker_is_reclaimable_by_its_owner() { + local dir state ownerdir reclaim out rc + dir=$(make_case lock-reclaim-marker-self-orphan) + state="$dir/state" + ownerdir="$state/.owner" + reclaim="$ownerdir/reclaim" + mkdir -p "$ownerdir" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + mkdir "$2" + printf "%s\n" "${BASHPID:-$$}" > "$2/pid" + fm_lock_reclaim_marker_claim "$2" + claim=$? + if [ "$(cat "$2/pid" 2>/dev/null || true)" = "${BASHPID:-$$}" ]; then owned=self; else owned=other; fi + printf "claim=%s owned=%s\n" "$claim" "$owned" + ' _ "$LIB" "$reclaim" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "self-orphaned marker fixture shell failed (rc=$rc): $out" + [ "$out" = "claim=0 owned=self" ] \ + || fail "a process could not reclaim the marker it orphaned itself, so it would spin forever: $out" + pass "a reclaim marker orphaned by the caller itself stays reclaimable by that caller" +} + +test_self_abandoned_steal_mutex_is_reclaimed_only_by_its_own_frame() { + local dir state lockdir dead out rc + dir=$(make_case lock-self-abandoned-steal) + state="$dir/state" + lockdir="$state/.contend.lock" + dead=$(dead_pid) + mkdir "$lockdir" + printf '%s\n' "$dead" > "$lockdir/pid" + + rc=0 + out=$(FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_try_acquire "$2.steal" || exit 7 + ( fm_lock_try_acquire "$2" >/dev/null 2>&1; printf "sub=%s " "$?" ) + fm_lock_try_acquire "$2" + printf "self=%s\n" "$?" + ' _ "$LIB" "$lockdir" 2>&1) || rc=$? + [ "$rc" -eq 0 ] || fail "self-abandoned steal-mutex fixture shell failed (rc=$rc): $out" + case "$out" in + "sub=1 self="*) : ;; + *) fail "a child frame reclaimed the steal mutex its live parent still holds: $out" ;; + esac + [ "$out" = "sub=1 self=0" ] \ + || fail "a process could not reclaim the steal mutex its own interrupted frame abandoned, so the exit path would spin forever: $out" + pass "a self-abandoned steal mutex is reclaimed by its own frame and never by a child frame" +} + test_lock_steals_dead_pid_lock() { local dir state lockdir dead rc newpid dir=$(make_case lock-dead-steal) @@ -319,7 +933,8 @@ test_lock_does_not_steal_live_lock() { printf '%s\n' "$live" > "$lockdir/pid" out=$(FM_STATE_OVERRIDE="$state" bash -c ' . "$1" - if fm_lock_try_acquire "$2"; then rc=0; else rc=1; fi + rc=0 + fm_lock_try_acquire "$2" || rc=$? printf "rc=%s held=%s\n" "$rc" "${FM_LOCK_HELD_PID:-}" ' _ "$LIB" "$lockdir") kill "$live" 2>/dev/null || true @@ -1111,6 +1726,21 @@ test_stale_watch_reclaim_publishes_before_clear test_live_stale_watch_lock_is_actionable test_guard_warnings test_lock_single_winner_under_concurrency +test_lock_missing_parent_returns_typed_failure_with_bounded_launches +test_lock_owner_record_failure_returns_typed_failure +test_stale_or_malformed_steal_mutex_never_claims_nested_mutex +test_abandoned_reclaim_marker_does_not_wedge_stale_steal_recovery +test_legacy_nested_steal_residue_is_retired_only_when_stale +test_reclaim_marker_is_not_stolen_from_a_live_owner +test_dangling_steal_owner_reclaim_yields_one_winner +test_reclaim_marker_takeover_is_bound_to_the_marker_it_inspected +test_reclaim_marker_takeover_never_vacates_the_marker_slot +test_legacy_directory_steal_mutex_is_reclaimed_without_recursion +test_legacy_directory_steal_mutex_survives_known_recovery_debris +test_legacy_directory_steal_mutex_without_owner_record_is_reclaimable_when_aged +test_legacy_directory_reclaim_never_deletes_a_racer_mutex +test_self_orphaned_reclaim_marker_is_reclaimable_by_its_owner +test_self_abandoned_steal_mutex_is_reclaimed_only_by_its_own_frame test_lock_steals_dead_pid_lock test_lock_stale_steal_single_winner_under_concurrency test_lock_live_steal_mutex_is_not_reclaimed