From e2b0fc60e047dbe3218fc66d075a1b18be8c897c Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Wed, 23 Sep 2026 14:22:56 -0700 Subject: [PATCH 1/3] feat(bin): guard the partition, harness pin, and bounded exec for a non-Pi supervision host Lease liveness is now the pure record test in every calling context, so an unmarked main honors a live branch lease held by a separate process, and a lease file engages the guard's claim serialization for any caller; a home with no lease files still takes no lock. bin/fm-harness.sh honors FM_SUPERVISION_PRIMARY_HARNESS while FM_SUPERVISION_ACTOR=branch, so a supervision branch running under another harness resolves own, crew, and secondmate to the primary's harness. fm_tasks_axi's watchdog moves into bin/fm-timeout-lib.sh as fm_exec_timed with a separate grace: the perl watchdog is preferred, runs the command in its own process group against wall-clock deadlines, forwards TERM/INT/HUP, and reaps the group, so a descendant holding captured output can no longer keep the caller waiting past the bound on a host without timeout. The Claude Stop auto-arm header records that Claude drops the exit 2 of a hook it terminated at the configured timeout, re-measured on Claude Code 2.1.281. --- bin/fm-backlog-transition-lib.sh | 75 ++------ bin/fm-claude-stop-autoarm.sh | 11 +- bin/fm-harness.sh | 34 +++- bin/fm-lease-lib.sh | 88 +++++----- bin/fm-test-run.sh | 2 + bin/fm-timeout-lib.sh | 108 +++++++++++- docs/turnend-guard.md | 1 + docs/verification/supervision.md | 18 ++ tests/fm-backlog-atomicity.test.sh | 27 +-- tests/fm-branch-supervision.test.sh | 145 +++++++++++++++- tests/fm-harness-precedence.test.sh | 86 +++++++++- tests/fm-timeout-lib.test.sh | 256 ++++++++++++++++++++++++++++ 12 files changed, 711 insertions(+), 140 deletions(-) create mode 100755 tests/fm-timeout-lib.test.sh diff --git a/bin/fm-backlog-transition-lib.sh b/bin/fm-backlog-transition-lib.sh index 1d14f4ef80b..d7dc67bee53 100644 --- a/bin/fm-backlog-transition-lib.sh +++ b/bin/fm-backlog-transition-lib.sh @@ -319,28 +319,17 @@ fm_backlog_transition_applies() { # # Run `tasks-axi` with an optional FM_TASKS_AXI_TIMEOUT bound. A caller that # holds a lock across the call - the spawn commit and its preservation # read-back run under the per-task meta lock - sets the bound, so an -# unresponsive tasks-axi cannot hold that lock open indefinitely; a timed-out -# call exits 124, or 137 when the kill-after had to fire (GNU timeout's own -# status for a KILL-forced expiry), and the callers treat either as the bound -# expiring and report the timeout as the reason through their existing error -# plumbing. GNU timeout is used where it exists, -# gtimeout where coreutils ships under that name, and a small perl watchdog -# elsewhere (a stock macOS host has perl but no timeout variant; perl is -# already a hard dependency of this library's byte validators, so the -# fallback adds no new tool). Every bounded path forces termination: a -# tasks-axi that ignores SIGTERM must not outlive the bound, since an -# unbounded call under the lock is exactly the hang the bound exists to -# prevent - so the GNU variants carry a kill-after of one further bound -# (TERM at the bound, KILL after that grace) and the watchdog kills the -# same way. When a bound was requested but no bounding mechanism exists at -# all, the call fails closed instead of running unbounded. Must be the last -# command of a subshell: the exec keeps the tasks-axi process exactly where -# the plain call sat, and the bound kills the child, not the caller. +# unresponsive tasks-axi cannot hold that lock open indefinitely. The bound is +# fm_exec_timed's (bin/fm-timeout-lib.sh), with one further bound of grace +# before KILL so a tasks-axi that ignores SIGTERM cannot outlive it either; the +# callers treat fm_timed_out statuses as the bound expiring and report the +# timeout as the reason through their existing error plumbing. A bound that +# cannot be enforced on this host fails closed instead of running unbounded. +# Must be the last command of a subshell: the exec keeps the tasks-axi process +# exactly where the plain call sat, and the bound kills the child, not the +# caller. fm_tasks_axi_timeout_expired() { # - case $1 in - 124 | 137) return 0 ;; - esac - return 1 + fm_timed_out "$1" } fm_tasks_axi() { @@ -348,49 +337,7 @@ fm_tasks_axi() { if [ -z "$bound" ]; then exec tasks-axi "$@" fi - if command -v timeout >/dev/null 2>&1; then - exec timeout -k "$bound" "$bound" tasks-axi "$@" - elif command -v gtimeout >/dev/null 2>&1; then - exec gtimeout -k "$bound" "$bound" tasks-axi "$@" - elif command -v perl >/dev/null 2>&1; then - # Fork, run tasks-axi in the child, and poll waitpid(WNOHANG) until the - # child exits or the bound expires: the same contract as - # `timeout $bound tasks-axi ...`. Expiry kills the child with TERM, waits - # one further bound of grace, then KILL, and exits 124 so the callers' - # timeout plumbing reports it. Polling rather than alarm+die keeps the - # bound off perl's platform-dependent syscall-restart signal semantics. - exec perl -MPOSIX=WNOHANG -e ' - my $bound = shift; - exit 127 unless defined $bound && $bound =~ /\A[0-9]+\z/; - my $pid = fork; - exit 127 unless defined $pid; - if ($pid == 0) { exec @ARGV; exit 127 } - my $step = 0.05; - my $elapsed = 0; - while (1) { - my $done = waitpid $pid, WNOHANG; - exit(($? & 127) ? 128 + ($? & 127) : $? >> 8) if $done == $pid; - exit 127 if $done == -1; - if ($elapsed >= $bound) { - kill "TERM", $pid; - my $grace = 0; - my $gone = waitpid $pid, WNOHANG; - while ($gone == 0 && $grace < $bound) { - select undef, undef, undef, $step; - $grace += $step; - $gone = waitpid $pid, WNOHANG; - } - kill "KILL", $pid if $gone == 0; - waitpid $pid, 0; - exit 124; - } - select undef, undef, undef, $step; - $elapsed += $step; - } - ' -- "$bound" tasks-axi "$@" - fi - printf 'fm_tasks_axi: cannot bound tasks-axi within %ss: none of timeout, gtimeout, or perl is available\n' "$bound" >&2 - exit 127 + fm_exec_timed "$bound" "$bound" tasks-axi "$@" } # Print one row's `tasks-axi show` output (plus stderr) from the addressing diff --git a/bin/fm-claude-stop-autoarm.sh b/bin/fm-claude-stop-autoarm.sh index bf09b78431a..92063d4ee99 100755 --- a/bin/fm-claude-stop-autoarm.sh +++ b/bin/fm-claude-stop-autoarm.sh @@ -40,7 +40,13 @@ # this hook-owned process tree (never shell &); Claude owns the process # group, so its timeout/session teardown kills arm and watcher together. # HUP, TERM, and INT are translated through the ordinary durable failure -# handoff instead of leaving the generation frozen at arming. +# handoff instead of leaving the generation frozen at arming. Claude does +# not deliver the exit 2 of a hook it terminated at the configured timeout +# as a rewake (measured on Claude Code 2.1.278 and 2.1.281, +# docs/verification/supervision.md), so a park that outlives that timeout +# records the failure durably without waking an idle primary; nothing here +# shortens a quiet park, because no-change heartbeats are absorbed without +# closing the arm. # - Translation: while supervision is still needed and AFK remains inactive, # an actionable arm close (signal:/stale:/check:/heartbeat) prints one # rewake banner to stderr and exits 2, which wakes Claude even while idle @@ -218,7 +224,8 @@ autoarm_record() { # # watcher until its next wake, so that wait cannot be shortened without adding # artificial turns. Translate a host interruption through the ordinary durable # failure protocol instead: the winning generation records a terminal outcome, -# creates the episode marker, and exits 2 so Claude delivers a recovery turn. +# creates the episode marker, and exits 2 so Claude delivers a recovery turn - +# except after Claude's own timeout kill, whose exit 2 is dropped (header). # A superseded generation remains silent, and an episode whose attended # fail-open was already consumed must not restart automatic continuation. # shellcheck disable=SC2329 # Invoked indirectly by the signal traps below. diff --git a/bin/fm-harness.sh b/bin/fm-harness.sh index da9154bc2c4..24048f17638 100755 --- a/bin/fm-harness.sh +++ b/bin/fm-harness.sh @@ -55,6 +55,16 @@ # detect_own is the single owner of how the two combine; harness_marker and # harness_ancestry only report evidence. Record each newly verified env marker # in harness_marker, and each newly verified command name in harness_ancestry. +# Supervision-branch primary pin: a supervision branch running as its own +# process under another harness (a Pi engine under a Claude primary detects as +# pi) would otherwise resolve "own" - and with it an absent or "default" +# config/crew-harness or config/secondmate-harness - to its own harness and +# dispatch crew there. While FM_SUPERVISION_ACTOR=branch, a non-empty +# FM_SUPERVISION_PRIMARY_HARNESS names the primary's harness and replaces +# detection for the own, crew, and secondmate resolutions; a value that names +# no known harness refuses (exit 2, nothing on stdout) instead of resolving. +# Outside the branch actor the pin is ignored, and the evidence-only ancestry +# verbs never consult it. set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -381,6 +391,22 @@ harness_family() { esac } +# Print the supervision-branch primary pin when it applies (header), or +# nothing. Returns 2, with the reason on stderr, for a pin naming no harness. +supervision_primary_pin() { + local pin=${FM_SUPERVISION_PRIMARY_HARNESS:-} + [ "${FM_SUPERVISION_ACTOR:-}" = branch ] && [ -n "$pin" ] || return 0 + case "$pin" in + claude|codex|opencode|pi|pi-signed|grok|kimi|cursor|gemini|muse|rovo|omp|agy|devin) + printf '%s\n' "$pin" + ;; + *) + echo "error: FM_SUPERVISION_PRIMARY_HARNESS='$pin' names no known harness; refusing to resolve the supervision branch's harness" >&2 + return 2 + ;; + esac +} + # Combine the two evidence layers. The precedence boundary, in one rule: a # marker names its harness, but only ancestry proves which harness owns this # process tree, so a structural (comm) ancestor of a DIFFERENT harness wins. @@ -395,8 +421,12 @@ harness_family() { # - Different harness, interpreter-args ancestor only: the marker wins, because # a harness-shaped path in some node process's arguments is weaker evidence # than a harness publishing its own identity. +# The supervision-branch primary pin, when it applies, answers before either +# evidence layer is read. detect_own() { - local marker ancestry strength harness + local marker ancestry strength harness pin + pin=$(supervision_primary_pin) || exit 2 + [ -z "$pin" ] || { echo "$pin"; return; } marker=$(harness_marker) ancestry=$(harness_ancestry) if [ -z "$ancestry" ]; then @@ -463,7 +493,7 @@ secondmate_field() { resolve_secondmate() { local sm sm=$(secondmate_field 1) - if [ -z "$sm" ] || [ "$sm" = "default" ]; then sm=$(resolve_crew); fi + if [ -z "$sm" ] || [ "$sm" = "default" ]; then sm=$(resolve_crew) || exit; fi echo "$sm" } diff --git a/bin/fm-lease-lib.sh b/bin/fm-lease-lib.sh index 00e311f18e5..9b3b6da042e 100755 --- a/bin/fm-lease-lib.sh +++ b/bin/fm-lease-lib.sh @@ -1,15 +1,16 @@ #!/usr/bin/env bash # fm-lease-lib.sh - the per-task supervision lease contract (one owner). # -# WHY. On the Pi supervision branch (docs/pi-supervision-branch.md), two LLM -# actors share one firstmate home inside one pi process: MAIN (the captain's -# chat) and BRANCH (the persistent supervision conversation). Most records have -# exactly one natural owner, but the overlap set - steering or stopping a -# worker, post-landing cleanup, backlog status for a task, stuck-worker -# recovery - could otherwise be mutated by both actors at once. The lease is -# the merge-conflict analog: a small per-task file saying which actor is -# changing that task right now, and the mutating entrypoints refuse the other -# actor while it exists. +# WHY. A supervision branch (docs/pi-supervision-branch.md) is a second LLM +# actor beside MAIN (the captain's chat) in one firstmate home - on Pi, a +# persistent conversation inside the same pi process - and nothing in this +# contract assumes the two actors share a process. Most records have exactly +# one natural owner, but the overlap set - steering or stopping a worker, +# post-landing cleanup, backlog status for a task, stuck-worker recovery - +# could otherwise be mutated by both actors at once. The lease is the +# merge-conflict analog: a small per-task file saying which actor is changing +# that task right now, and the mutating entrypoints refuse the other actor +# while it exists. # # CONTRACT. # - Lease file: $STATE/.lease-, one line "\t\t". @@ -18,19 +19,23 @@ # lease-command lock; leases never coordinate across firstmate homes. # - Actors: exactly "main" and "branch". The current actor is # $FM_SUPERVISION_ACTOR when set, else "main". The branch's shell gets -# FM_SUPERVISION_ACTOR=branch injected deterministically by the Pi branch -# extension's bash tool, not by agent memory. Any other value is refused -# loudly - an unknown actor is a wiring bug, not a third role. +# FM_SUPERVISION_ACTOR=branch injected deterministically by the process +# hosting it (on Pi, the branch extension's bash tool), not by agent +# memory. Any other value is refused loudly - an unknown actor is a wiring +# bug, not a third role. # - Staleness: the recorded pid is the long-lived supervising process (the -# session-lock holder, or FM_LEASE_HOLDER_PID - see bin/fm-lease.sh), and -# both actors live inside that one pi process, so a dead recorded pid -# means the process died; the lease is cleared at the next claim, guard, -# or sweep. Liveness requires a Pi calling context plus state/.lock, and -# the recorded pid must BE its current holder, so a lease left by an exited -# Pi session goes stale even if its pid was recycled by an unrelated -# process, and a non-Pi home never honors a leftover Pi lease. A lease held by the -# live current session but an abandoned branch conversation is recovered -# by the branch extension's generation-activation cleanup. +# session-lock holder, or FM_LEASE_HOLDER_PID - see bin/fm-lease.sh), so a +# dead recorded pid means the supervising session died; the lease is +# cleared at the next claim, guard, or sweep. Liveness is the pure record +# test, identical in every calling context: the recorded pid is alive and +# IS the current state/.lock holder. So a lease left by an exited session +# goes stale for every reader, whichever harness now owns the home, and an +# unmarked main honors a live branch lease exactly as a Pi main does. The +# one residual is a recorded pid recycled onto the next session-lock holder +# itself; the host that owns a branch conversation releases that actor's +# leases when it activates a new one (the Pi branch extension's +# generation-activation cleanup), which also recovers a lease held by the +# live session but an abandoned branch conversation. # # THREAT MODEL (deliberate, captain-decided): these guards are # CONFUSED-AGENT-GRADE, the same grade bin/fm-gate-refuse-lib.sh documents @@ -45,11 +50,14 @@ # ACCIDENTAL override fails loudly inside the branch's own shell as well. # - Guard semantics (fm_lease_guard): no lease, a same-actor lease, or a # provably stale lease passes; a live lease held by the OTHER actor -# refuses with exit FM_LEASE_REFUSE_EXIT. In a Pi supervision context the -# guard retains the lease-command lock until fm_lease_guard_release, so the -# other actor cannot claim between the check and the guarded mutation. A -# home without the current Pi session lock cannot have a live lease, so -# the guard is a no-op there - non-Pi behavior is unchanged by construction. +# refuses with exit FM_LEASE_REFUSE_EXIT. Whenever the guard engages - a +# supervision context (Pi, or an explicit actor) or any lease file for the +# task - it retains the lease-command lock until fm_lease_guard_release, +# so the other actor cannot claim between the check and the guarded +# mutation. An unmarked caller with no lease file for the task returns +# before taking any lock, so a home that never ran a branch is unchanged +# byte for byte; that caller does not exclude a claim that starts during +# its mutation. # - Role partition (fm_lease_forbid_branch): actions MAIN alone owns - # merging a PR, landing local-only work, spawning workers, answering a # decision - refuse the branch actor outright, lease or no lease, while @@ -151,15 +159,11 @@ fm_lease_read() { return 0 } -# fm_lease_live : 0 iff a well-formed lease exists in a Pi context, its -# recorded pid is alive, and that pid IS the current session-lock holder (see -# the staleness contract above). +# fm_lease_live : 0 iff a well-formed lease exists, its recorded pid is +# alive, and that pid IS the current session-lock holder (the staleness +# contract above). The calling context never enters the verdict. fm_lease_live() { local lock_pid - case "${PI_CODING_AGENT:-}:${FM_SUPERVISION_ACTOR:-}" in - true:*|*:main|*:branch) ;; - *) return 1 ;; - esac fm_lease_read "$1" || return 1 [ -n "$FM_LEASE_ACTOR" ] || return 1 [ -n "$FM_LEASE_PID" ] || return 1 @@ -180,19 +184,18 @@ fm_lease_clear_stale() { } # fm_lease_guard : refuse (exit FM_LEASE_REFUSE_EXIT) when -# a live lease held by the OTHER actor exists for . In a Pi supervision -# context, a successful guard retains the command lock across the caller's -# mutation; the caller must invoke fm_lease_guard_release from its EXIT cleanup. -# This closes the check/use race with a concurrent claim. Outside Pi, stale -# records are still cleaned but the lock is released before returning. +# a live lease held by the OTHER actor exists for . Once engaged (the +# guard semantics above), a successful guard retains the command lock across +# the caller's mutation; the caller must invoke fm_lease_guard_release from its +# EXIT cleanup. This closes the check/use race with a concurrent claim. fm_lease_guard() { - local task=$1 action=$2 actor lock lease_actor active=0 + local task=$1 action=$2 actor lock lease_actor fm_lease_valid_id "$task" || return 0 actor=$(fm_lease_actor) || exit "$FM_LEASE_REFUSE_EXIT" case "${PI_CODING_AGENT:-}:${FM_SUPERVISION_ACTOR:-}" in - true:*|*:main|*:branch) active=1 ;; + true:*|*:main|*:branch) ;; + *) [ -e "$(fm_lease_path "$task")" ] || return 0 ;; esac - [ "$active" = 1 ] || [ -e "$(fm_lease_path "$task")" ] || return 0 fm_lease_lock_helpers lock="$STATE/.fm-lease-command.lock" # A caller with more than one guarded phase already excludes claims until @@ -203,9 +206,6 @@ fm_lease_guard() { fi if ! fm_lease_live "$task"; then fm_lease_clear_stale "$task" || { fm_lease_guard_release; return 1; } - if [ "$active" != 1 ]; then - fm_lease_guard_release - fi return 0 fi lease_actor=$FM_LEASE_ACTOR diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 5fbe3dc51e5..3e3241ff294 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -291,6 +291,7 @@ family_for_basename() { fm-send-popup-settle.test.sh|fm-send-settle.test.sh|\ fm-subagent-pretool-check.test.sh|\ fm-supervision-instructions.test.sh|fm-task-delivery.test.sh|\ + fm-timeout-lib.test.sh|\ fm-tmux-submit-busy.test.sh|fm-trace-context-lib.test.sh|\ fm-transition-lib.test.sh|\ fm-test-run.test.sh|fm-test-isolation-proof.test.sh) @@ -824,6 +825,7 @@ tests/fm-teardown.test.sh 145174 tests/fm-test-fixture-cleanup.test.sh 937 tests/fm-test-fixtures.test.sh 1562 tests/fm-test-isolation-proof.test.sh 2692 +tests/fm-timeout-lib.test.sh 8541 tests/fm-tmux-agent-liveness.test.sh 1953 tests/fm-tool-update-check.test.sh 13832 tests/fm-trace-context-lib.test.sh 227 diff --git a/bin/fm-timeout-lib.sh b/bin/fm-timeout-lib.sh index 7b572ac3d48..6bf7b873211 100644 --- a/bin/fm-timeout-lib.sh +++ b/bin/fm-timeout-lib.sh @@ -15,16 +15,38 @@ # except 124, which means the bound was hit (GNU timeout's convention, # reproduced by the perl and bash fallbacks). # +# fm_exec_timed [args...] +# Replaces the calling shell with the bounded command, so it must be the +# last command of a subshell: the bound kills the command, not the +# caller. The command runs in its own process group; TERM goes to that +# group at the bound, and KILL once more have passed, +# for a command that ignores TERM or is mid-way through work it will not +# abandon. A TERM, INT, or HUP delivered to the bounding process is +# forwarded to the group and starts the same grace. Exit status is the +# command's own, except 124 (the bound was hit) or 137 (GNU timeout's +# status when its KILL had to fire); fm_timed_out accepts both. Both +# values must be positive integers (125 otherwise). The perl watchdog is +# preferred: once termination has begun it also KILLs whatever the group +# left behind, so a descendant that outlives the command and holds its +# output cannot keep a capturing caller waiting, and GNU timeout, the +# fallback, cannot be followed by that reap from a replaced shell. With +# no perl, timeout, or gtimeout on the host it refuses with 127 rather +# than run unbounded: there is no bash fallback, because a monitor-mode +# watchdog cannot replace the caller. +# +# fm_timed_out +# 0 iff is how fm_run_timed or fm_exec_timed reports the bound. +# # A non-positive bound is not a bound: `timeout 0` and the perl fallback's # `alarm 0` both disable the deadline, so callers must reject 0 before calling. # -# All four mechanisms terminate the whole process GROUP, not just the direct -# child, so a hung grandchild (a vendor CLI spawned by a wrapper script, a git -# fetch spawned by a sweep) cannot outlive the bound. GNU/BSD `timeout` does -# this by default because it does not run the command in the foreground process -# group; the perl fallback does it explicitly with setpgrp plus a negative pid, -# and the bash fallback uses monitor mode to give the bounded child its own -# process group before signaling its negative pid. +# All four fm_run_timed mechanisms terminate the whole process GROUP, not just +# the direct child, so a hung grandchild (a vendor CLI spawned by a wrapper +# script, a git fetch spawned by a sweep) cannot outlive the bound. GNU/BSD +# `timeout` does this by default because it does not run the command in the +# foreground process group; the perl fallback does it explicitly with setpgrp +# plus a negative pid, and the bash fallback uses monitor mode to give the +# bounded child its own process group before signaling its negative pid. set -u fm_timeout_mechanism() { @@ -139,3 +161,75 @@ fm_run_timed() { # *) return 124 ;; esac } + +fm_timed_out() { # + case ${1:-} in + 124 | 137) return 0 ;; + esac + return 1 +} + +# The perl watchdog forks the command into its own process group (both sides +# call setpgid, so the group exists before either can signal it) and polls +# waitpid(WNOHANG) against wall-clock deadlines rather than using alarm+die, +# which keeps the bound off perl's platform-dependent syscall-restart signal +# semantics and off the drift of counting sleep intervals. +fm_exec_timed() { # + local seconds=${1:-} grace=${2:-} value + for value in "$seconds" "$grace"; do + case "$value" in + '' | 0* | *[!0-9]*) + echo "fm_exec_timed: usage: fm_exec_timed [args...]" >&2 + exit 125 + ;; + esac + done + shift 2 + if [ "$#" -eq 0 ]; then + echo "fm_exec_timed: usage: fm_exec_timed [args...]" >&2 + exit 125 + fi + if command -v perl >/dev/null 2>&1; then + exec perl -MPOSIX=WNOHANG,setpgid -MTime::HiRes=time -e ' + my ($bound, $grace) = (shift, shift); + my $pid = fork; + exit 127 unless defined $pid; + if ($pid == 0) { setpgid(0, 0); exec @ARGV; exit 127 } + setpgid($pid, $pid); + my $deadline = time + $bound; + my ($kill_at, $timed_out) = (0, 0); + for my $sig (qw(TERM INT HUP)) { + $SIG{$sig} = sub { kill $sig, -$pid; $kill_at ||= time + $grace }; + } + sub finish { + my $status = shift; + kill "KILL", -$pid if $kill_at; + exit 124 if $timed_out; + exit(($status & 127) ? 128 + ($status & 127) : $status >> 8); + } + while (1) { + my $done = waitpid $pid, WNOHANG; + finish($?) if $done == $pid; + exit 127 if $done == -1; + if ($kill_at) { + if (time >= $kill_at) { + kill "KILL", -$pid; + waitpid $pid, 0; + finish($?); + } + } elsif (time >= $deadline) { + $timed_out = 1; + $kill_at = time + $grace; + kill "TERM", -$pid; + } + select undef, undef, undef, 0.05; + } + ' -- "$seconds" "$grace" "$@" + elif command -v timeout >/dev/null 2>&1; then + exec timeout -k "$grace" "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + exec gtimeout -k "$grace" "$seconds" "$@" + fi + printf 'fm_exec_timed: cannot bound %s within %ss: none of perl, timeout, or gtimeout is available\n' "${1##*/}" "$seconds" >&2 + exit 127 +} diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index f4715f1db0d..bd293490d58 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -114,6 +114,7 @@ A legacy build's lock-holding claim (recognizable by its `autoarm` role file) st Fresh `failed` and `failed-suppressed` outcomes enter or advance the failure progression instead of acting as unconditional recovery proof. The auto-arm itself rechecks the healthy watcher predicate and retries a bounded number of times before reporting a genuine failure. The foreground arm legitimately follows a healthy watcher until its next wake, so the hook catches HUP, TERM, and INT from host timeout or teardown and commits the ordinary durable failed outcome and failure-notice marker before exiting 2 for a recovery turn. +Claude drops that exit 2 when it terminated the hook at the configured timeout itself, so a park that outlives the timeout ends without a rewake (`bin/fm-claude-stop-autoarm.sh` header). The first fresh exhausted-failure epoch preserves its handoff without consuming a blocked-stop count, while later fresh failed epochs advance the same monotonic progression instead of resetting it. When none of those proofs appears, it re-blocks up to `FM_CLAUDE_TURNEND_BLOCK_BUDGET` times (default 3, below Claude's 8-block override). In Claude mode, positive watcher recovery clears the block budget, failure notice, and attended alarm together under the existing budget lock before either hook reports ordinary recovery. diff --git a/docs/verification/supervision.md b/docs/verification/supervision.md index eebdc622b81..ad6f769bd63 100644 --- a/docs/verification/supervision.md +++ b/docs/verification/supervision.md @@ -471,6 +471,24 @@ Observed output: fm-claude-stop-autoarm: ok ``` +### Claude drops the exit 2 of a hook it timed out, 2026-09-23 + +This supports the `bin/fm-claude-stop-autoarm.sh` header statement that a park outliving the hook timeout ends without a rewake. +It was first measured on Claude Code 2.1.278 and re-measured on 2.1.281 on macOS arm64, in a scratch git project on a private tmux socket with no Firstmate hooks loaded. +Each arm registered one one-shot async `Stop` hook through `--settings`, with `asyncRewake: true` and `timeout: 30`, in an interactive `claude --model haiku --tools ''` session given one short prompt. + +```json +{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"/hook-timeout.sh","asyncRewake":true,"timeout":30}]}]}} +``` + +The control hook slept 10 seconds, printed a reply request to stderr, and exited 2 on its own. +The timeout hook trapped `TERM`, backgrounded `sleep 300`, waited, and on `TERM` printed a reply request to stderr and exited 2. + +| Arm | Hook log (seconds after the prompt) | Pane afterwards | +| --- | --- | --- | +| Control, exit 2 before the timeout | started +2, exited 2 at +12 | `Stop hook feedback` followed by the requested reply | +| Timeout, exit 2 from the `TERM` handler | started +2, `TERM` and exit 2 at +32 | no `Stop hook feedback` and no reply, still idle at +111 | + ## Watcher continuity The cross-harness evidence combines the 2026-07-17 live pass with Claude's replacement Stop-owned path revalidated on 2026-09-21, all against isolated project and home state. diff --git a/tests/fm-backlog-atomicity.test.sh b/tests/fm-backlog-atomicity.test.sh index 7cf8aa93ee8..1c98935a85a 100755 --- a/tests/fm-backlog-atomicity.test.sh +++ b/tests/fm-backlog-atomicity.test.sh @@ -25,6 +25,8 @@ set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=bin/fm-timeout-lib.sh +. "$ROOT/bin/fm-timeout-lib.sh" # An exported TASKS_AXI_BACKEND would outrank each case's .tasks.toml fixture # in fm_tasks_axi_backend, so the backend cases must start from a clean slate. @@ -329,17 +331,15 @@ make_fallback_bin() { # } run_bounded_fm_tasks_axi() { # [args...] - local fb=$1 bound=$2 out rc=0 saved_path=$PATH + local fb=$1 bound=$2 out rc=0 shift 2 - # The fallback shape itself: a PATH with no timeout variant on it. Set and - # restored here, never in a subshell, so the change cannot leak into other - # tests. - PATH="$fb" + # The fallback shape itself: a PATH with no timeout variant on it, in force + # for the bounded call only. The library is sourced first under the ordinary + # PATH, as every real caller does. out=$( . "$ROOT/bin/fm-backlog-transition-lib.sh" - FM_TASKS_AXI_TIMEOUT="$bound" fm_tasks_axi "$@" 2>&1 + PATH="$fb" FM_TASKS_AXI_TIMEOUT="$bound" fm_tasks_axi "$@" 2>&1 ) || rc=$? - PATH=$saved_path printf '%s' "$out" return "$rc" } @@ -1475,14 +1475,15 @@ test_deferred_signal_verification_outlives_an_unresponsive_tasks_axi() { # The read-back's own `start` never answers, so the spawn must bound it # (FM_TASKS_AXI_TIMEOUT=3), print the attempted wording naming the timeout, - # and exit - the outer `timeout -k 5 30` only turns a regression back into - # the lock-held-forever hang it exists to catch. + # and exit - the outer 30s bound (fm_run_timed, portable to a host with no + # timeout binary) only turns a regression back into the lock-held-forever + # hang it exists to catch. mkdir -p "$case_dir/user-home" - out=$(FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$(home_of "$case_dir")" \ + out=$(fm_run_timed 30 env FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$(home_of "$case_dir")" \ HOME="$case_dir/user-home" FM_SPAWN_NO_GUARD=1 \ FM_FAKE_PANE_PATH="$case_dir/wt" TMUX="fake,1,0" CLAUDE_CONFIG_DIR='' \ FM_TASKS_AXI_TIMEOUT=3 PATH="$case_dir/fakebin:$PATH" \ - timeout -k 5 30 "$SPAWN" "$id" "$case_dir/project" \ + "$SPAWN" "$id" "$case_dir/project" \ --mode no-mistakes --yolo off 2>&1) || rc=$? [ "$rc" -ne 0 ] || fail "an interrupted spawn reported success" case "$rc" in @@ -2775,11 +2776,11 @@ test_spawn_refuses_a_special_file_tasks_config() { rm -f "$home/.tasks.toml" mkfifo "$home/.tasks.toml" - out=$(FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" \ + out=$(fm_run_timed 60 env FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" \ FM_SPAWN_NO_GUARD=1 FM_FAKE_PANE_PATH="$case_dir/wt" TMUX="fake,1,0" \ CLAUDE_CONFIG_DIR='' \ PATH="$case_dir/fakebin:$PATH" \ - timeout 60 "$SPAWN" "$id" "$case_dir/project" --mode no-mistakes --yolo off 2>&1) || rc=$? + "$SPAWN" "$id" "$case_dir/project" --mode no-mistakes --yolo off 2>&1) || rc=$? [ "$rc" -ne 124 ] || fail "spawn hung reading a special-file tasks-axi config" [ "$rc" -ne 0 ] || fail "spawn accepted a special-file tasks-axi config" assert_contains "$out" "tasks-axi config is not a regular file" \ diff --git a/tests/fm-branch-supervision.test.sh b/tests/fm-branch-supervision.test.sh index 7a4cedd370c..2ee72337a3e 100644 --- a/tests/fm-branch-supervision.test.sh +++ b/tests/fm-branch-supervision.test.sh @@ -597,8 +597,19 @@ test_home_without_branch_is_untouched() { [ -z "$(find "$home/state" -name '.lease-*' -o -name 'branch-outcomes*' -o -name '.branch-*' 2>/dev/null)" ] \ || fail "guard layer created branch state in a home that never ran the branch" - # A stale Pi marker and recycled-but-live lease pid cannot activate leases in - # a no-lock Claude home; the guard removes the leftover and passes silently. + # An unmarked caller with no lease file for the task takes no lock at all, so + # the guard leaves a home that never ran a branch byte-for-byte unchanged. + # The positional parameter belongs to the nested shell. + # shellcheck disable=SC2016 + out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR STATE="$home/state" bash -c ' + . "$1" + fm_lease_guard task-none "probe" + if [ -e "$STATE/.fm-lease-command.lock" ]; then echo lock-taken; else echo no-lock; fi + ' _ "$ROOT/bin/fm-lease-lib.sh" 2>&1) + [ "$out" = no-lock ] || fail "an unmarked guard with no lease file engaged the lease-command lock: $out" + + # A stale Pi marker and a leftover lease cannot bind a no-lock Claude home; + # the guard removes the leftover and passes silently. printf 'harness=claude\n' > "$home/state/fake.meta" printf '%s\n' "$PPID" > "$home/state/.pi-branch-extension-loaded" printf 'branch\t%s\t123\n' "$PPID" > "$home/state/.lease-task-reused" @@ -606,14 +617,134 @@ test_home_without_branch_is_untouched() { [ "$out" = "silent-pass" ] || fail "guard helpers honored a leftover Pi lease in a no-lock Claude home: $out" [ ! -e "$home/state/.lease-task-reused" ] || fail "guard kept a leftover Pi lease without a session lock" - printf '%s\n' "$PPID" > "$home/state/.lock" + # A leftover lease whose pid is alive but is not the current lock holder - a + # session that exited while its pid lives on - is stale for a Claude main. + printf '%s\n' "$$" > "$home/state/.lock" printf 'branch\t%s\t123\n' "$PPID" > "$home/state/.lease-task-reused" # The positional parameter belongs to the nested shell. # shellcheck disable=SC2016 out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 STATE="$home/state" bash -c '. "$1"; fm_lease_guard task-reused "probe"; echo silent-pass' _ "$ROOT/bin/fm-lease-lib.sh" 2>&1) - [ "$out" = "silent-pass" ] || fail "guard helpers honored a reused-pid Pi lease in a Claude context: $out" - [ ! -e "$home/state/.lease-task-reused" ] || fail "Claude context kept a Pi lease whose old pid matched its current lock" - pass "a non-Pi home ignores stale Pi leases even when the recycled pid owns its lock" + [ "$out" = "silent-pass" ] || fail "guard helpers honored a lease whose pid no longer holds the lock: $out" + [ ! -e "$home/state/.lease-task-reused" ] || fail "Claude context kept a lease whose pid is not the current lock holder" + pass "a home without a live branch lease takes no lock and clears leftover leases in any calling context" +} + +# --- the partition across two processes, off Pi ------------------------------- + +# A branch that runs as its own process beside an unmarked main (no Pi marker, +# no actor variable - how every non-Pi primary's own shell looks) must bind that +# main exactly as it binds a Pi main: liveness is the lease record alone. +test_unmarked_main_honors_a_live_branch_lease() { + local home fakebin out status lease_before + home="$TMP_ROOT/unmarked-main-home" + fakebin="$TMP_ROOT/unmarked-main-bin" + mkdir -p "$home/state" "$fakebin" + printf '%s\n' "$$" > "$home/state/.lock" + fm_write_meta "$home/state/task-held.meta" "window=fm-task-held" "backend=tmux" "harness=claude" + # A delivery that got past the guard would reach tmux; record it instead. + printf '#!/usr/bin/env bash\nprintf "%%s\\n" "$*" >> "%s"\nexit 1\n' "$home/tmux-calls" > "$fakebin/tmux" + chmod +x "$fakebin/tmux" + + env -u PI_CODING_AGENT FM_HOME="$home" FM_SUPERVISION_ACTOR=branch FM_LEASE_HOLDER_PID=$$ \ + "$ROOT/bin/fm-lease.sh" claim task-held --actor branch || fail "the branch process could not claim its lease" + lease_before=$(cat "$home/state/.lease-task-held") + + out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" \ + "$ROOT/bin/fm-lease.sh" check task-held) || fail "an unmarked main could not see the branch lease" + case "$out" in + "branch $$ "*" live") ;; + *) fail "an unmarked main read the live branch lease as: $out" ;; + esac + + out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" FM_LEASE_HOLDER_PID=$$ \ + "$ROOT/bin/fm-lease.sh" claim task-held 2>&1) + status=$? + [ "$status" -eq 6 ] || fail "an unmarked main claim over the live branch lease exited $status, not 6: $out" + env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" \ + "$ROOT/bin/fm-lease.sh" sweep || fail "sweep from an unmarked main failed" + [ "$(cat "$home/state/.lease-task-held")" = "$lease_before" ] \ + || fail "an unmarked main overwrote or swept the live branch lease" + + out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" PATH="$fakebin:$PATH" \ + "$ROOT/bin/fm-send.sh" fm-task-held "steer while leased" 2>&1) + status=$? + [ "$status" -eq 6 ] || fail "an unmarked main steer through the live branch lease exited $status, not 6: $out" + assert_contains "$out" "steer (fm-send) refused" "the fm-send refusal lost its action label" + [ ! -e "$home/tmux-calls" ] || fail "the refused steer still reached the endpoint: $(cat "$home/tmux-calls")" + [ -z "$(find "$home/state" -path '*.inbox*' -name '*.msg' 2>/dev/null)" ] \ + || fail "the refused steer still wrote an inbox record" + + out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" PATH="$fakebin:$PATH" \ + "$ROOT/bin/fm-control.sh" task-held interrupt 2>&1) + status=$? + [ "$status" -eq 6 ] || fail "an unmarked main fm-control exited $status, not 6: $out" + assert_contains "$out" "leased to the branch supervision actor" "the fm-control refusal lost the holder" + out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" PATH="$fakebin:$PATH" \ + "$ROOT/bin/fm-teardown.sh" task-held 2>&1) + status=$? + [ "$status" -eq 6 ] || fail "an unmarked main fm-teardown exited $status, not 6: $out" + [ -e "$home/state/task-held.meta" ] || fail "the refused teardown still removed the task record" + + # Once the branch releases, the same unmarked main proceeds. + env -u PI_CODING_AGENT FM_HOME="$home" FM_SUPERVISION_ACTOR=branch \ + "$ROOT/bin/fm-lease.sh" release task-held --actor branch || fail "branch release failed" + env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" FM_LEASE_HOLDER_PID=$$ \ + "$ROOT/bin/fm-lease.sh" claim task-held || fail "an unmarked main could not claim after the branch released" + + # A new session owning the lock makes the old session's lease stale for the + # unmarked main too, and its guard clears it. + env -u PI_CODING_AGENT FM_HOME="$home" FM_SUPERVISION_ACTOR=branch FM_LEASE_HOLDER_PID=$$ \ + "$ROOT/bin/fm-lease.sh" claim task-old --actor branch || fail "branch claim for the old session failed" + printf '%s\n' "$PPID" > "$home/state/.lock" + out=$(env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 FM_HOME="$home" \ + "$ROOT/bin/fm-lease.sh" check task-old) || fail "check missed the old session's lease" + case "$out" in + *" stale") ;; + *) fail "a lease from a session that no longer holds the lock read as: $out" ;; + esac + pass "an unmarked main honors a live branch lease across processes and ignores a previous session's" +} + +# A lease file engages the guard's claim serialization for an unmarked caller +# too, so the branch cannot claim between that caller's check and its mutation. +test_unmarked_guard_with_a_lease_file_holds_exclusivity_through_mutation() { + local home operation_pid claim_pid claim_status + home="$TMP_ROOT/unmarked-guard-mutation-home" + mkdir -p "$home/state" + printf '%s\n' "$$" > "$home/state/.lock" + printf 'branch\t999999\t123\n' > "$home/state/.lease-task-race" + + # The positional parameter belongs to the nested shell. + # shellcheck disable=SC2016 + # The mutation stand-in waits for release under a bound, so a failed + # assertion below cannot leave it holding the suite open. + env -u PI_CODING_AGENT -u FM_SUPERVISION_ACTOR CLAUDECODE=1 STATE="$home/state" \ + FM_TEST_READY="$home/operation-ready" FM_TEST_RELEASE="$home/operation-release" bash -c ' + . "$1" + fm_lease_guard task-race "probe" + trap "fm_lease_guard_release" EXIT + : > "$FM_TEST_READY" + i=0 + while [ ! -e "$FM_TEST_RELEASE" ] && [ "$i" -lt 1500 ]; do sleep 0.01; i=$((i + 1)); done + ' _ "$ROOT/bin/fm-lease-lib.sh" >/dev/null 2>&1 & + operation_pid=$! + while [ ! -e "$home/operation-ready" ]; do sleep 0.01; done + [ ! -e "$home/state/.lease-task-race" ] || fail "the unmarked guard kept the dead session's lease" + + env -u PI_CODING_AGENT FM_HOME="$home" FM_SUPERVISION_ACTOR=branch FM_LEASE_HOLDER_PID=$$ \ + "$ROOT/bin/fm-lease.sh" claim task-race --actor branch >/dev/null 2>&1 & + claim_pid=$! + sleep 0.2 + kill -0 "$claim_pid" 2>/dev/null \ + || fail "the branch claimed while the unmarked guarded mutation was still running" + [ ! -e "$home/state/.lease-task-race" ] \ + || fail "the concurrent claim published a lease before the unmarked guarded mutation ended" + + : > "$home/operation-release" + wait "$operation_pid" || fail "unmarked guarded mutation fixture failed" + wait "$claim_pid"; claim_status=$? + [ "$claim_status" -eq 0 ] || fail "claim did not proceed after the unmarked guarded mutation ended: $claim_status" + pass "a lease file makes an unmarked guard exclude a concurrent claim for the complete mutation" } # --- session-bound staleness and the loud accidental-override guard --------- @@ -1108,6 +1239,8 @@ test_lease_exclusivity_release_stale_and_sweep test_mutating_scripts_refuse_the_other_actors_lease test_main_owned_actions_refuse_the_branch_actor test_home_without_branch_is_untouched +test_unmarked_main_honors_a_live_branch_lease +test_unmarked_guard_with_a_lease_file_holds_exclusivity_through_mutation test_lease_liveness_binds_to_the_session_lock test_concurrent_stale_lease_claims_have_one_winner test_guard_stale_clear_cannot_delete_a_new_claim diff --git a/tests/fm-harness-precedence.test.sh b/tests/fm-harness-precedence.test.sh index 0d4999984a3..926fc6b2acd 100755 --- a/tests/fm-harness-precedence.test.sh +++ b/tests/fm-harness-precedence.test.sh @@ -29,7 +29,8 @@ set -u # This suite states the markers it means to test in every case. Drop the ambient # ones so a verdict never depends on which harness launched the suite. -unset CLAUDECODE PI_CODING_AGENT FM_PI_HARNESS GROK_AGENT CURSOR_AGENT CURSOR_INVOKED_AS +unset CLAUDECODE PI_CODING_AGENT FM_PI_HARNESS GROK_AGENT CURSOR_AGENT CURSOR_INVOKED_AS \ + FM_SUPERVISION_ACTOR FM_SUPERVISION_PRIMARY_HARNESS HARNESS="$ROOT/bin/fm-harness.sh" RENDER="$ROOT/bin/fm-supervision-instructions.sh" @@ -715,7 +716,86 @@ SH pass "equal-depth descent ties prefer the comm-strength leaf regardless of spawn order" } -# --- 7. Session start's supervision protocol follows the corrected verdict --- +# --- 7. A supervision branch resolves the primary's harness, not its own ----- + +# A supervision branch running as its own process under another harness sees +# its own harness in both evidence layers: a Pi engine under a Claude primary +# carries PI_CODING_AGENT and a pi ancestor. Left alone, an absent or "default" +# crew or secondmate config would then dispatch workers on Pi. The primary's +# pin must win while the branch actor is set, and only then. +pin_probe() { # [VAR=VAL ...] + local bin=$1 home=$2 verb=$3 + shift 3 + env -u CLAUDECODE -u PI_CODING_AGENT -u FM_PI_HARNESS -u GROK_AGENT \ + -u CURSOR_AGENT -u CURSOR_INVOKED_AS -u FM_SUPERVISION_ACTOR \ + -u FM_SUPERVISION_PRIMARY_HARNESS FM_HOME="$home" "$@" \ + "$bin" -c "r=\$(\"$HARNESS\" $verb 2>\"$home/stderr\"); rc=\$?; printf '%s|%s' \"\$r\" \"\$rc\"" +} + +test_supervision_branch_resolves_the_primary_pin() { + local dir home bin verb got + dir="$TMP_ROOT/primary-pin" + home="$dir/home" + mkdir -p "$home/config" + bin=$(named_bin "$dir/pi-tree" pi) + + # Without the pin the branch reads as its own engine, which is the hazard. + for verb in '' crew secondmate; do + got=$(pin_probe "$bin" "$home" "$verb" PI_CODING_AGENT=true FM_SUPERVISION_ACTOR=branch) + [ "$got" = 'pi|0' ] \ + || fail "an unpinned branch under a Pi engine resolved '${verb:-own}' as '$got', expected pi (the hazard is not live)" + done + + for verb in '' crew secondmate; do + got=$(pin_probe "$bin" "$home" "$verb" PI_CODING_AGENT=true \ + FM_SUPERVISION_ACTOR=branch FM_SUPERVISION_PRIMARY_HARNESS=claude) + [ "$got" = 'claude|0' ] \ + || fail "a pinned branch resolved '${verb:-own}' as '$got', expected the primary's claude" + done + printf 'default\n' > "$home/config/crew-harness" + got=$(pin_probe "$bin" "$home" crew PI_CODING_AGENT=true \ + FM_SUPERVISION_ACTOR=branch FM_SUPERVISION_PRIMARY_HARNESS=claude) + [ "$got" = 'claude|0' ] || fail "a pinned branch resolved a default crew config as '$got', expected claude" + + # An explicit crew config is still the captain's choice, pin or no pin. + printf 'codex\n' > "$home/config/crew-harness" + got=$(pin_probe "$bin" "$home" crew PI_CODING_AGENT=true \ + FM_SUPERVISION_ACTOR=branch FM_SUPERVISION_PRIMARY_HARNESS=claude) + [ "$got" = 'codex|0' ] || fail "the pin overrode an explicit crew config: '$got'" + rm -f "$home/config/crew-harness" + + # Main, or no actor at all, ignores the pin. + got=$(pin_probe "$bin" "$home" '' PI_CODING_AGENT=true FM_SUPERVISION_PRIMARY_HARNESS=claude) + [ "$got" = 'pi|0' ] || fail "an unmarked process honored the branch-only pin: '$got'" + got=$(pin_probe "$bin" "$home" '' PI_CODING_AGENT=true \ + FM_SUPERVISION_ACTOR=main FM_SUPERVISION_PRIMARY_HARNESS=claude) + [ "$got" = 'pi|0' ] || fail "the main actor honored the branch-only pin: '$got'" + + # The ancestry evidence verb reports evidence only and never consults it. + got=$(pin_probe "$bin" "$home" ancestry PI_CODING_AGENT=true \ + FM_SUPERVISION_ACTOR=branch FM_SUPERVISION_PRIMARY_HARNESS=claude) + [ "$got" = 'comm pi|0' ] || fail "the ancestry verb consulted the pin: '$got'" + pass "a supervision branch resolves own, crew, and secondmate to the primary's pinned harness" +} + +test_supervision_branch_refuses_an_unknown_primary_pin() { + local dir home bin verb got + dir="$TMP_ROOT/primary-pin-bad" + home="$dir/home" + mkdir -p "$home/config" + bin=$(named_bin "$dir/pi-tree" pi) + for verb in '' crew secondmate; do + got=$(pin_probe "$bin" "$home" "$verb" PI_CODING_AGENT=true \ + FM_SUPERVISION_ACTOR=branch FM_SUPERVISION_PRIMARY_HARNESS=unknown) + [ "$got" = '|2' ] \ + || fail "an unknown pin resolved '${verb:-own}' as '$got', expected a refusal with nothing on stdout" + assert_contains "$(cat "$home/stderr")" "FM_SUPERVISION_PRIMARY_HARNESS='unknown' names no known harness" \ + "the refusal did not name the bad pin" + done + pass "a supervision branch refuses to resolve a harness from a pin that names none" +} + +# --- 8. Session start's supervision protocol follows the corrected verdict --- # The consequence the captain actually hit: the wrong verdict emitted Claude's # Stop-owned protocol to a Codex primary, so every turn end was blocked for @@ -758,4 +838,6 @@ test_descent_probe_reaches_a_strength_the_top_of_session_cannot test_descent_probe_ignores_a_sibling_branch_the_walk_cannot_reach test_descent_probe_tolerates_an_args_only_foreign_verdict_at_the_deepest_vantage test_descent_probe_prefers_comm_strength_when_deepest_leaves_tie +test_supervision_branch_resolves_the_primary_pin +test_supervision_branch_refuses_an_unknown_primary_pin test_supervision_protocol_follows_corrected_verdict diff --git a/tests/fm-timeout-lib.test.sh b/tests/fm-timeout-lib.test.sh new file mode 100755 index 00000000000..a035fb45297 --- /dev/null +++ b/tests/fm-timeout-lib.test.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# Behavior tests for bin/fm-timeout-lib.sh's exec-style bound, fm_exec_timed: +# TERM to the command's process group at the bound, KILL once the grace has +# passed, a forwarded signal, the caller replaced rather than wrapped, and a +# refusal instead of an unbounded run when nothing on the host can enforce the +# bound. Most cases pin the perl watchdog, the preferred mechanism and the only +# one a stock macOS host has, under a PATH that holds no timeout variant; the +# GNU fallback case runs only where a real timeout exists. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-timeout-lib) + +# A PATH with perl and the shell tools the bounded commands use, and no +# timeout variant: fm_exec_timed must take its perl watchdog here. +PERL_ONLY="$TMP_ROOT/perl-only-bin" +mkdir -p "$PERL_ONLY" +for tool in perl bash sleep; do + ln -s "$(command -v "$tool")" "$PERL_ONLY/$tool" +done + +# exec_timed : source the library under +# the ordinary PATH, then run the bounded call under as the last command +# of a subshell, exactly as a real caller does. +exec_timed() { + local path=$1 + shift + ( + . "$ROOT/bin/fm-timeout-lib.sh" + PATH=$path + fm_exec_timed "$@" + ) +} + +wait_for_file() { # + local i=0 + while [ ! -s "$1" ]; do + i=$((i + 1)) + [ "$i" -lt 500 ] || fail "timed out waiting for $1" + sleep 0.02 + done +} + +test_passes_the_command_status_and_output_through() { + local out rc=0 + out=$(exec_timed "$PERL_ONLY" 5 1 bash -c 'echo to-stdout; echo to-stderr >&2; exit 7' 2>&1) || rc=$? + [ "$rc" -eq 7 ] || fail "the watchdog did not pass the command's own status through (rc=$rc)" + assert_contains "$out" "to-stdout" "the watchdog lost the command's stdout" + assert_contains "$out" "to-stderr" "the watchdog lost the command's stderr" + pass "fm_exec_timed passes a command's status and output through unchanged" +} + +# A command that honors TERM ends at the bound, long before the grace would +# have forced it, and is gone afterwards. +test_term_ends_a_cooperative_command_at_the_bound() { + local dir rc=0 started elapsed pid + dir="$TMP_ROOT/term" + mkdir -p "$dir" + started=$SECONDS + exec_timed "$PERL_ONLY" 1 30 bash -c 'echo $$ > "$1"; exec sleep 300' _ "$dir/pid" || rc=$? + elapsed=$((SECONDS - started)) + [ "$rc" -eq 124 ] || fail "an expired bound did not report 124 (rc=$rc)" + [ "$elapsed" -ge 1 ] || fail "the bound fired before it elapsed (${elapsed}s)" + [ "$elapsed" -lt 15 ] || fail "a TERM-honoring command waited out the grace (${elapsed}s): TERM was not sent at the bound" + pid=$(cat "$dir/pid") + ! kill -0 "$pid" 2>/dev/null || fail "the bounded command outlived its bound" + pass "fm_exec_timed sends TERM at the bound and a cooperative command ends there" +} + +# A command that ignores TERM survives the bound and is killed only once the +# grace has passed, so the grace is what separates the two. +test_kill_ends_a_term_ignoring_command_after_the_grace() { + local dir rc=0 started elapsed pid + dir="$TMP_ROOT/kill" + mkdir -p "$dir" + started=$SECONDS + exec_timed "$PERL_ONLY" 1 2 bash -c 'trap "" TERM; echo $$ > "$1"; exec sleep 300' _ "$dir/pid" || rc=$? + elapsed=$((SECONDS - started)) + [ "$rc" -eq 124 ] || fail "a KILL-forced expiry did not report 124 (rc=$rc)" + [ "$elapsed" -ge 3 ] || fail "a TERM-ignoring command ended before bound plus grace (${elapsed}s): the grace was skipped" + [ "$elapsed" -lt 20 ] || fail "a TERM-ignoring command was not killed after the grace (${elapsed}s)" + pid=$(cat "$dir/pid") + ! kill -0 "$pid" 2>/dev/null || fail "the TERM-ignoring command survived the KILL" + pass "fm_exec_timed kills a TERM-ignoring command once the grace has passed" +} + +# The bounded command sits where the plain call sat: the calling subshell is +# replaced by the bounding process, whose child the command is. This holds for +# whichever mechanism the host selects, and for the perl watchdog explicitly. +test_the_bound_replaces_the_calling_shell() { + local dir path caller parent + dir="$TMP_ROOT/replace" + mkdir -p "$dir" + for path in "$PATH" "$PERL_ONLY"; do + rm -f "$dir/caller" "$dir/parent" + ( + . "$ROOT/bin/fm-timeout-lib.sh" + printf '%s\n' "$BASHPID" > "$dir/caller" + PATH=$path + fm_exec_timed 5 1 bash -c 'echo "$PPID" > "$1"' _ "$dir/parent" + ) || fail "the bounded probe failed under PATH=$path" + caller=$(cat "$dir/caller") + parent=$(cat "$dir/parent") + [ "$caller" = "$parent" ] \ + || fail "the command's parent $parent is not the replaced caller $caller under PATH=$path" + done + pass "fm_exec_timed replaces the calling shell instead of wrapping it" +} + +# The regression a direct-child watchdog had: the command dies at the bound +# but a descendant that ignores TERM keeps the captured output open, so the +# caller waits for the descendant instead of the bound. +test_a_descendant_holding_the_output_cannot_outlast_the_bound() { + local dir out rc=0 started elapsed pid + dir="$TMP_ROOT/descendant" + mkdir -p "$dir" + started=$SECONDS + # The positional parameter belongs to the bounded shell. + # shellcheck disable=SC2016 + out=$(exec_timed "$PERL_ONLY" 1 30 bash -c ' + ( trap "" TERM; exec sleep 300 ) & + echo $! > "$1" + wait + ' _ "$dir/pid") || rc=$? + elapsed=$((SECONDS - started)) + [ "$rc" -eq 124 ] || fail "an expired bound did not report 124 (rc=$rc)" + [ "$elapsed" -lt 15 ] \ + || fail "a TERM-ignoring descendant held the captured output for ${elapsed}s past a 1s bound" + pid=$(cat "$dir/pid") + ! kill -0 "$pid" 2>/dev/null || fail "the TERM-ignoring descendant survived the bound" + pass "fm_exec_timed reaps a descendant that would otherwise hold the output past the bound" +} + +# A TERM delivered to the bounding process itself - a harness tearing down a +# hook, an operator stopping the caller - reaches the command, and a command +# that then exits on its own reports its own status, not the bound's. +test_a_signal_to_the_bounding_process_reaches_the_command() { + local dir watchdog rc=0 + dir="$TMP_ROOT/forward" + mkdir -p "$dir" + # Backgrounded directly, the subshell's pid is the watchdog it becomes. + # The positional parameters belong to the bounded shell. + # shellcheck disable=SC2016 + ( + . "$ROOT/bin/fm-timeout-lib.sh" + PATH=$PERL_ONLY + fm_exec_timed 60 30 bash -c ' + trap "echo forwarded > \"\$2\"; exit 3" TERM + echo $$ > "$1" + while :; do sleep 0.1; done + ' _ "$dir/pid" "$dir/term" + ) 2>/dev/null & + watchdog=$! + wait_for_file "$dir/pid" + kill -TERM "$watchdog" || fail "could not signal the bounding process" + wait "$watchdog" || rc=$? + [ "$(cat "$dir/term" 2>/dev/null)" = forwarded ] || fail "the TERM never reached the bounded command" + [ "$rc" -eq 3 ] || fail "a forwarded TERM did not report the command's own status (rc=$rc)" + pass "fm_exec_timed forwards a TERM it receives to the bounded command" +} + +# perl is preferred whenever it exists, because only its watchdog can reap a +# leftover descendant after replacing the caller. +test_perl_is_preferred_over_timeout() { + local dir out + dir="$TMP_ROOT/prefer" + mkdir -p "$dir/bin" + for tool in perl bash; do + ln -s "$(command -v "$tool")" "$dir/bin/$tool" + done + printf '#!/bin/sh\necho timeout-used > "%s"\nexit 99\n' "$dir/timeout-used" > "$dir/bin/timeout" + chmod +x "$dir/bin/timeout" + out=$(exec_timed "$dir/bin" 5 1 bash -c 'echo ran') || fail "the bounded call failed: $out" + [ "$out" = ran ] || fail "the bounded call printed '$out'" + [ ! -e "$dir/timeout-used" ] || fail "fm_exec_timed used timeout although perl was available" + pass "fm_exec_timed prefers its perl watchdog over timeout" +} + +test_refuses_rather_than_running_unbounded() { + local dir out rc=0 + dir="$TMP_ROOT/unboundable" + mkdir -p "$dir/bin" + ln -s "$(command -v bash)" "$dir/bin/bash" + out=$(exec_timed "$dir/bin" 5 1 bash -c ': > "$1"' _ "$dir/ran" 2>&1) || rc=$? + [ "$rc" -eq 127 ] || fail "fm_exec_timed ran with nothing to bound it (rc=$rc)" + assert_contains "$out" "cannot bound bash within 5s" "the refusal did not say what it could not bound" + [ ! -e "$dir/ran" ] || fail "the command ran although nothing could bound it" + pass "fm_exec_timed refuses instead of running unbounded when no mechanism exists" +} + +test_rejects_malformed_bounds_before_running_anything() { + local dir out rc + dir="$TMP_ROOT/malformed" + mkdir -p "$dir" + for args in '0 1' '5 0' '05 1' '5 x' '' '5'; do + rc=0 + # shellcheck disable=SC2086 # deliberate splitting of the bound pair + out=$(exec_timed "$PERL_ONLY" $args bash -c ': > "$1"' _ "$dir/ran" 2>&1) || rc=$? + [ "$rc" -eq 125 ] || fail "bounds '$args' were not rejected (rc=$rc: $out)" + [ ! -e "$dir/ran" ] || fail "bounds '$args' still ran the command" + done + rc=0 + out=$(exec_timed "$PERL_ONLY" 5 1 2>&1) || rc=$? + [ "$rc" -eq 125 ] || fail "a call with no command was not rejected (rc=$rc: $out)" + assert_contains "$out" "usage: fm_exec_timed" "the rejection did not print the usage" + pass "fm_exec_timed rejects a zero, padded, non-numeric, or missing bound and a missing command" +} + +test_gnu_timeout_kills_a_term_ignoring_command_after_the_grace() { + local dir fb rc=0 started elapsed verdict + if ! command -v timeout >/dev/null 2>&1; then + pass "fm_exec_timed's GNU timeout fallback (skipped: no timeout binary on this host)" + return 0 + fi + dir="$TMP_ROOT/gnu" + fb="$dir/bin" + mkdir -p "$fb" + # No perl here, so the call falls back to GNU timeout. + for tool in timeout bash sleep; do + ln -s "$(command -v "$tool")" "$fb/$tool" + done + started=$SECONDS + exec_timed "$fb" 1 2 bash -c 'trap "" TERM; exec sleep 300' || rc=$? + elapsed=$((SECONDS - started)) + verdict=$( . "$ROOT/bin/fm-timeout-lib.sh"; fm_timed_out "$rc" && echo expired) + [ "$verdict" = expired ] || fail "the GNU path's expiry status $rc is not a timed-out status" + [ "$elapsed" -ge 3 ] || fail "the GNU path ended a TERM-ignoring command before bound plus grace (${elapsed}s)" + [ "$elapsed" -lt 20 ] || fail "the GNU path did not kill a TERM-ignoring command after the grace (${elapsed}s)" + pass "fm_exec_timed's GNU timeout fallback kills a TERM-ignoring command once the grace has passed" +} + +test_timed_out_names_exactly_the_bound_statuses() { + local status verdict + for status in 124 137 0 1 125 127 143 ''; do + verdict=$( . "$ROOT/bin/fm-timeout-lib.sh"; if fm_timed_out "$status"; then echo yes; else echo no; fi) + case "$status" in + 124|137) [ "$verdict" = yes ] || fail "status '$status' was not read as the bound" ;; + *) [ "$verdict" = no ] || fail "status '$status' was misread as the bound" ;; + esac + done + pass "fm_timed_out accepts 124 and 137 and nothing else" +} + +test_passes_the_command_status_and_output_through +test_term_ends_a_cooperative_command_at_the_bound +test_kill_ends_a_term_ignoring_command_after_the_grace +test_the_bound_replaces_the_calling_shell +test_a_descendant_holding_the_output_cannot_outlast_the_bound +test_a_signal_to_the_bounding_process_reaches_the_command +test_perl_is_preferred_over_timeout +test_refuses_rather_than_running_unbounded +test_rejects_malformed_bounds_before_running_anything +test_gnu_timeout_kills_a_term_ignoring_command_after_the_grace +test_timed_out_names_exactly_the_bound_statuses From 7651fbacd379d82c15086f0b5ee15928652aacc9 Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Wed, 23 Sep 2026 14:52:03 -0700 Subject: [PATCH 2/3] fix(bin): state that fm_exec_timed cannot reach a descendant in its own process group Live runs of real Claude and Pi engine turns under the bound showed both CLIs start every tool command in a process group of its own, so those processes end through the engine's own TERM handling rather than the group signal or reap. Also clears the new timeout test's ShellCheck findings. --- bin/fm-timeout-lib.sh | 7 ++++++- tests/fm-timeout-lib.test.sh | 7 +++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bin/fm-timeout-lib.sh b/bin/fm-timeout-lib.sh index 6bf7b873211..db62342ac67 100644 --- a/bin/fm-timeout-lib.sh +++ b/bin/fm-timeout-lib.sh @@ -29,7 +29,12 @@ # preferred: once termination has begun it also KILLs whatever the group # left behind, so a descendant that outlives the command and holds its # output cannot keep a capturing caller waiting, and GNU timeout, the -# fallback, cannot be followed by that reap from a replaced shell. With +# fallback, cannot be followed by that reap from a replaced shell. A +# descendant that moves into a process group of its own is outside both +# signals and the reap (the Claude and Pi CLIs do this for every tool +# command they run), so it ends only through the command's own TERM +# handling; that is what the grace is for, and a command KILLed after +# the grace can leave such a descendant running. With # no perl, timeout, or gtimeout on the host it refuses with 127 rather # than run unbounded: there is no bash fallback, because a monitor-mode # watchdog cannot replace the caller. diff --git a/tests/fm-timeout-lib.test.sh b/tests/fm-timeout-lib.test.sh index a035fb45297..0d82bcc7922 100755 --- a/tests/fm-timeout-lib.test.sh +++ b/tests/fm-timeout-lib.test.sh @@ -6,6 +6,7 @@ # bound. Most cases pin the perl watchdog, the preferred mechanism and the only # one a stock macOS host has, under a PATH that holds no timeout variant; the # GNU fallback case runs only where a real timeout exists. +# shellcheck disable=SC2016 # each bounded bash -c script expands its own arguments set -u # shellcheck source=tests/lib.sh @@ -29,8 +30,7 @@ exec_timed() { shift ( . "$ROOT/bin/fm-timeout-lib.sh" - PATH=$path - fm_exec_timed "$@" + PATH=$path fm_exec_timed "$@" ) } @@ -98,8 +98,7 @@ test_the_bound_replaces_the_calling_shell() { ( . "$ROOT/bin/fm-timeout-lib.sh" printf '%s\n' "$BASHPID" > "$dir/caller" - PATH=$path - fm_exec_timed 5 1 bash -c 'echo "$PPID" > "$1"' _ "$dir/parent" + PATH=$path fm_exec_timed 5 1 bash -c 'echo "$PPID" > "$1"' _ "$dir/parent" ) || fail "the bounded probe failed under PATH=$path" caller=$(cat "$dir/caller") parent=$(cat "$dir/parent") From 1d7661222695130626b31cc44ca927e293e9ea2f Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Wed, 23 Sep 2026 15:24:43 -0700 Subject: [PATCH 3/3] no-mistakes(document): Clarify cross-harness lease documentation --- docs/configuration.md | 2 +- docs/pi-supervision-branch.md | 4 ++-- docs/watcher-continuity.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c310293ad59..c26164143f8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -46,7 +46,7 @@ A genuinely no-op heartbeat is absorbed in bash and never reaches Pi, and every A broken branch still falls back to today's wake-to-main path in both postures, and the legacy `state/.afk` daemon flag means nothing on Pi. While the away-posture record `state/.afk-contract` exists the branch takes every actionable row, no processing turn opens on the parked main, and main's standing authority relocates to the branch through the guarded scripts, each keeping its own gate; [docs/pi-supervision-branch.md](pi-supervision-branch.md#postures) owns that posture. While attended the branch's role stays bounded exactly as the captain-approved architecture set it: it cannot merge a PR, land local work, freshly spawn, or answer a decision, and every existing captain gate remains unchanged in either posture. -Homes on any other primary harness never load this feature and are entirely unaffected. +Homes on other primary harnesses do not load the Pi branch extension; shared per-task lease behavior is owned by `bin/fm-lease-lib.sh`. `AGENTS.md`'s `state/` inventory routes the branch's runtime files to their format and lifecycle owners. While attended, a captain-facing (verdict `captain`) branch outcome persists as one exact, sequence-keyed visible transcript entry and then opens one sequence-keyed processing turn on main, which stays open until main acknowledges that sequence through its `fm_branch_processed` tool; while away, the entry persists but processing waits until the record is archived. The branch prompt's "Verdict: routine or captain" section owns the distinction between captain-facing, unsolicited routine, and unchanged-review outcomes. diff --git a/docs/pi-supervision-branch.md b/docs/pi-supervision-branch.md index a0d3caffd2b..26836b8a9e1 100644 --- a/docs/pi-supervision-branch.md +++ b/docs/pi-supervision-branch.md @@ -13,10 +13,10 @@ All of that describes the attended posture; the away posture, recorded by `state While attended, captain-relevant branch outcomes persist as exact, sequence-keyed visible transcript entries and then open one sequence-keyed processing turn on main, which stays open until main acknowledges that sequence; while away, the entries persist but processing waits until the record is archived. The design source is the captain-approved forked-supervision architecture board, a captain-private fleet record (a self-contained HTML explainer with the measured cache and judgment evidence); this document records the shape it landed as, and the delivering PR cites the board artifact itself. -The supervision branch itself is Pi-only by construction: +This in-process supervision branch is Pi-only by construction: - The branch lives in `.pi/extensions/fm-branch-supervision.ts`, which only a Pi primary ever loads; no other harness gains branch supervision behavior. -- The bash-side additions (leases, the outcome store, session-start recovery) are inert in a home with no branch state: no lease files exist, no actor variable is set, every guard passes silently, and no new state appears (`tests/fm-branch-supervision.test.sh` holds this). +- In a home with no branch state, the bash-side additions remain inert (`tests/fm-branch-supervision.test.sh`); `bin/fm-lease-lib.sh` owns how a pre-existing lease is honored on any harness. A home on any harness that already has an outcome store still receives the shared drain compatibility recovery described in [Lost-wake outcome backstop](#lost-wake-outcome-backstop). - It does not change which harness is primary and never moves a home to Pi. diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 1caf220fe1b..ca829fb43b9 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -68,7 +68,7 @@ An acknowledged episode does not freeze the generation, because the next downtim ## Per-actor acknowledgement -`bin/fm-wake-drain.sh` consumes the queue per actor, not per whole-queue cutoff, using `bin/fm-lease-lib.sh`'s existing `fm_lease_actor` identity (`FM_SUPERVISION_ACTOR`, unset or `main` for every non-Pi harness and Pi's own main session; `branch` only inside the Pi supervision branch's own bash tool calls, injected deterministically by the extension - never agent memory). +`bin/fm-wake-drain.sh` consumes the queue per actor, not per whole-queue cutoff, using the `fm_lease_actor` identity owned by `bin/fm-lease-lib.sh`; the Pi branch extension injects its branch actor into its own bash tool calls. Every presented row is claimed to exactly one actor under the durable queue lock. An ordinary presentation drain bounds both its initial queue-lock acquire and its later status-presentation-lock acquire at the deadline owned by the script header. A live initial queue-lock holder produces one PID-naming advisory and skips the whole drain before any claim or mutation, while a live status-presentation-lock holder produces one such advisory after raw wake presentation and leaves status annotations, sections, and cursors retriable on the next drain.