Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 35 additions & 5 deletions bin/fm-turnend-guard.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,12 @@
# with the repair banner, bounded to FM_CLAUDE_TURNEND_BLOCK_BUDGET
# (default 3) consecutive blocks per session - safely below Claude Code's
# hard 8-consecutive-block override - then allow one loud attended
# fail-open only for an already verified failure episode.
# fail-open only for an already verified failure episode. The budget
# charges each event epoch once, and it also charges every re-block
# against an epoch the auto-arm never advanced past the previous
# re-block (budget_account_current_epoch owns that rule), so an inert
# hook that leaves the ledger frozen cannot hold the guard in an
# unbounded re-block loop below that override.
set -u

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
Expand Down Expand Up @@ -249,12 +254,31 @@ fi
# The Stop-owned auto-arm fires on the same Stop event. Give it a brief bounded
# window to prove it owns recovery for this event epoch before consuming one of
# Claude's bounded continuations.
budget_account_current_epoch() {
local current_epoch outcome old_session old_count old_epoch tmp initialized
#
# Budget accounting, under the budget lock. Sets COUNT (the session's
# consumed continuations, including this one) and BUDGET_INITIALIZED_FAILURE.
# The ledger's epoch identity is what is charged: a new epoch charges once,
# and an epoch this same invocation already charged is never charged again,
# because the wait loop above can observe one fresh terminal epoch many times
# before the block decision. Across Stops the two callers differ:
# - observe (the allow paths in autoarm_owns_recovery): seeing an
# already-charged epoch again is free - it is the same claim, seen again.
# - block (the re-block path): a re-block against the epoch the previous
# re-block already charged is a new consumed continuation, because the
# auto-arm advanced nothing between the two Stops - it did not participate
# at all, which is exactly the absence this budget bounds. Charging only
# epoch changes let an inert hook (identity-gated, never fired, or failing
# before its generation claim) freeze the ledger and the count together,
# so the guard re-blocked without limit and the attended fail-open below
# never became reachable.
BUDGET_CHARGED_EPOCH=
budget_account_current_epoch() { # [observe|block]
local mode=${1:-observe} current_epoch outcome old_session old_count old_epoch tmp initialized charged
fm_lock_try_acquire "$BUDGET_LOCK" || return 1
current_epoch=$(sed -n '1s/^epoch=\([0-9][0-9]*\) .*/\1/p' "$STATE/.claude-autoarm-epoch" 2>/dev/null || true)
outcome=$(sed -n '1s/^.*outcome=\([a-z][a-z-]*\) .*$/\1/p' "$STATE/.claude-autoarm-epoch" 2>/dev/null || true)
initialized=0
charged=0
COUNT=0
if [ -f "$BUDGET_FILE" ]; then
old_session=$(sed -n '1s/^session=//p' "$BUDGET_FILE" 2>/dev/null || true)
Expand All @@ -266,13 +290,18 @@ budget_account_current_epoch() {
if [ "$old_session" = "$SESSION_ID" ]; then
COUNT=$old_count
if [ -n "$current_epoch" ] && [ "$old_epoch" = "$current_epoch" ]; then
:
if [ "$mode" = block ] && [ "$BUDGET_CHARGED_EPOCH" != "$current_epoch" ]; then
COUNT=$((COUNT + 1))
charged=1
fi
else
COUNT=$((COUNT + 1))
charged=1
fi
fi
fi
if [ ! -f "$BUDGET_FILE" ] || [ "${old_session:-}" != "$SESSION_ID" ]; then
charged=1
case "$outcome" in
failed|failed-suppressed)
if [ -e "$FAILURE_NOTICE" ]; then
Expand All @@ -293,6 +322,7 @@ budget_account_current_epoch() {
return 1
fi
rm -f "$tmp" 2>/dev/null || true
[ "$charged" -eq 0 ] || BUDGET_CHARGED_EPOCH=$current_epoch
BUDGET_INITIALIZED_FAILURE=$initialized
fm_lock_release "$BUDGET_LOCK"
return 0
Expand Down Expand Up @@ -456,7 +486,7 @@ fi

# The auto-arm genuinely failed to establish: consume the bounded re-block
# budget before considering the verified one-time attended fail-open.
budget_account_current_epoch || block_stop
budget_account_current_epoch block || block_stop
terminal_fail_open
terminal_status=$?
if [ "$terminal_status" -eq 0 ]; then
Expand Down
6 changes: 4 additions & 2 deletions docs/turnend-guard.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,9 @@ The first fresh exhausted-failure epoch preserves its handoff without consuming
When none of those proofs appears, it re-blocks up to `FM_CLAUDE_TURNEND_BLOCK_BUDGET` times (default 3, below Claude's 8-block override).
In Claude mode, positive watcher recovery clears the block budget, failure notice, and attended alarm together under the existing budget lock before either hook reports ordinary recovery.
The one loud attended fail-open is available only when the auto-arm has recorded an exhausted failure, its one notice is already consumed, the block budget is exhausted, and a final check finds neither a healthy watcher nor an automatic continuation.
Each epoch identity is accounted at most once under the budget lock.
Each epoch identity is charged at most once per Stop under the budget lock, and a re-block against an epoch the auto-arm did not advance past the previous re-block is charged as well.
That second rule is what bounds an inert auto-arm: a hook kept silent by a session lock held by a live harness outside its ancestry, a hook that never fires, or a hook failing before its generation claim leaves the ledger frozen at its last outcome.
Charging only epoch changes let the count freeze with that ledger, so the guard re-blocked without limit and the attended fail-open was never reachable; `budget_account_current_epoch` in `bin/fm-turnend-guard.sh` owns the rule.
Whenever both coordination locks are needed, positive auto-arm recovery and the terminal check acquire the auto-arm owner lock before the budget lock.
After that alarm, the Stop auto-arm suppresses further exit-2 continuations until positive watcher recovery, so the final fail-open remains reachable.
The alarm cannot repeat during that failure episode, and a later unhealthy stop blocks again.
Expand Down Expand Up @@ -183,7 +185,7 @@ That warning uses `bin/fm-supervision-instructions.sh --repair-line`, so it alwa

## Regression coverage

`tests/fm-turnend-guard.test.sh` covers the predicate, main and secondmate primary scope, child-worktree exclusion, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, the live-lock and fresh-beacon guard predicate, the cooperative `--claude` open-generation claim wait, monotonic failed-epoch progression, bounded attended fail-open, post-alarm continuation suppression, positive recovery reset, generation and legacy claim cases that must block or clear instead of allowing a blind stop, away-mode daemon ownership between watcher cycles and over a watcher lock left behind by an exited watcher, plus its dead, pid-reused, absent, stale-beacon, and away-mode-off negatives, the away-mode beacon's poll-derived grace widening for a live daemon still mid-cycle and its bound against a dead daemon, a beacon older than that wider grace, and FM_POLL's inapplicability with away mode off, Pi logical-run latching, missing-`jq` behavior, all five primary registrations, Grok native and legacy selection, typed field precedence, malformed input, and exactly-one-path safety.
`tests/fm-turnend-guard.test.sh` covers the predicate, main and secondmate primary scope, child-worktree exclusion, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, the live-lock and fresh-beacon guard predicate, the cooperative `--claude` open-generation claim wait, monotonic failed-epoch progression, bounded attended fail-open, the same bound against a ledger frozen by an inert auto-arm with and without a verified failure episode, post-alarm continuation suppression, positive recovery reset, generation and legacy claim cases that must block or clear instead of allowing a blind stop, away-mode daemon ownership between watcher cycles and over a watcher lock left behind by an exited watcher, plus its dead, pid-reused, absent, stale-beacon, and away-mode-off negatives, the away-mode beacon's poll-derived grace widening for a live daemon still mid-cycle and its bound against a dead daemon, a beacon older than that wider grace, and FM_POLL's inapplicability with away mode off, Pi logical-run latching, missing-`jq` behavior, all five primary registrations, Grok native and legacy selection, typed field precedence, malformed input, and exactly-one-path safety.
`tests/fm-guard-stale-banner.test.sh` covers the pull-guard predicate, including the persistent-model fresh-leftover-beacon negative control; the auto-arm model's healthy fresh-beacon-without-a-watcher case, session-and-recovery-bound long-turn rewake tolerance, independently broken tolerance signals, open-claim negative control, stale-beacon alarm, and isolation from other models; and the extension model's live-watcher path, ownership-qualified fresh hand-off, held-lock failures, independently broken ownership signals, stale-beacon alarm, queued-wake warning, and Pi and pi-signed harness routing.
It also covers true-reason banner wording and reason-keyed episode dedup surviving a beacon mtime change.
`tests/fm-cursor-primary.test.sh` covers the Cursor park end to end over real processes with no harness installed: each tracked Claude-shaped entrypoint standing down on a Cursor payload, both follow-up sources, the bounded repair nag and its reset, the nested loop bounds, supersession, away-mode and lock-ownership inertness, Pi-host stand-down without Cursor identity and continued parking when `PI_CODING_AGENT` leaks alongside `CURSOR_AGENT` or `CURSOR_INVOKED_AS`, child-worktree exclusion, and that the adapter never exits 2.
Expand Down
124 changes: 124 additions & 0 deletions tests/fm-turnend-guard.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1220,6 +1220,17 @@ run_integrated_autoarm() {
' 2>&1
}

# The same real hook, fired from a harness-named process that does NOT write
# state/.lock: whoever already holds that lock decides whether this firing is
# the owning session's or a competing one.
run_integrated_autoarm_unowned() {
local dir=$1 home
home=$(cd "$dir" && pwd)
# shellcheck disable=SC2016 # the fake harness expands FM_HOME inside its child shell.
printf '{"session_id":"sess-claude-mode","stop_hook_active":false}\n' \
| FM_HOME="$home" "$dir/fake-claude" -c '"$FM_HOME/bin/fm-claude-stop-autoarm.sh"' 2>&1
}

write_integrated_failed_arm() {
local dir=$1
cat > "$dir/bin/fm-watch-arm.sh" <<'SH'
Expand Down Expand Up @@ -1614,6 +1625,115 @@ test_hook_claude_mode_integrated_monotonic_fail_open() {
pass "fm-turnend-guard --claude: integrated fresh failures reach one bounded fail-open, stop continuation, and reset on recovery"
}

# The auto-arm's ledger epoch advances only when the hook reaches its
# generation claim. A live harness-named process outside the hook's ancestry
# holding state/.lock keeps the hook inert by its identity contract, so the
# ledger stays at the exhausted-failure epoch the hook wrote before it went
# quiet. The block budget used to advance only on an epoch change, so this
# shape re-blocked without limit and the attended fail-open never fired: the
# budget must count consecutive re-blocks against an unchanged epoch instead.
hold_session_lock_from_foreign_harness() { # sets FOREIGN_LOCK_HOLDER
local dir=$1
# `bash -c` execs a single command in place, which would rename the process
# to sleep; the trailing no-op keeps the harness-named shell as the holder.
# Started in this shell, not a command substitution, so the caller can reap
# it and no inherited pipe keeps a substitution waiting on the sleeper.
"$dir/fake-claude" -c 'sleep 60; true' >/dev/null 2>&1 &
FOREIGN_LOCK_HOLDER=$!
printf '%s\n' "$FOREIGN_LOCK_HOLDER" > "$dir/state/.lock"
}

test_hook_claude_mode_frozen_epoch_reaches_bounded_fail_open() {
local dir out status guard_out guard_status holder i pid identity count epoch_line
dir=$(make_primary_dir "$TMP_ROOT/hook-claude-frozen-epoch")
: > "$dir/state/task1.meta"
install_integrated_autoarm "$dir"
write_integrated_failed_arm "$dir"

out=$(run_integrated_autoarm "$dir"); status=$?
expect_code 2 "$status" "the exhausted auto-arm cycle must emit its one failure notice before going quiet"
guard_out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); guard_status=$?
expect_code 0 "$guard_status" "the first failed epoch must own its Stop handoff"
epoch_line=$(sed -n '1p' "$dir/state/.claude-autoarm-epoch")

hold_session_lock_from_foreign_harness "$dir"
holder=$FOREIGN_LOCK_HOLDER
for i in 1 2 3 4; do
out=$(run_integrated_autoarm_unowned "$dir"); status=$?
expect_code 0 "$status" "an auto-arm outside the lock owner's ancestry must stay inert at stop $i"
[ -z "$out" ] || fail "inert auto-arm produced output at stop $i: $out"
[ "$(sed -n '1p' "$dir/state/.claude-autoarm-epoch")" = "$epoch_line" ] \
|| fail "the ledger epoch advanced at stop $i, so this case no longer drives a frozen epoch"
guard_out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); guard_status=$?
if [ "$i" -lt 4 ]; then
expect_code 2 "$guard_status" "frozen-epoch stop $i must still re-block within the budget"
assert_contains "$guard_out" "TURN WOULD END BLIND" "frozen-epoch re-block $i lost the blind-turn banner"
assert_not_contains "$guard_out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' "fail-open fired before the frozen-epoch budget was spent"
assert_absent "$dir/state/.claude-autoarm-failure-alarmed" "frozen-epoch re-block $i consumed the attended alarm early"
else
expect_code 0 "$guard_status" "the frozen-epoch progression must reach the attended fail-open"
assert_contains "$guard_out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' "the frozen-epoch fail-open alarm is missing"
assert_present "$dir/state/.claude-autoarm-failure-alarmed" "the frozen-epoch fail-open did not consume its episode alarm"
fi
done

guard_out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); guard_status=$?
expect_code 2 "$guard_status" "a later unhealthy stop after the frozen-epoch alarm must remain attended"
assert_not_contains "$guard_out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' "the attended alarm repeated against the frozen epoch"

# The other direction: the bound must not outlive the failure. A verified
# healthy watcher still lets the stop through and clears the whole episode.
sleep 60 &
pid=$!
identity=$(watcher_identity "$dir" "$pid") || {
kill "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
kill "$holder" 2>/dev/null || true
wait "$holder" 2>/dev/null || true
fail "could not identify the frozen-epoch recovery watcher"
}
record_watcher_lock "$dir" "$pid" "$identity"
touch "$dir/state/.last-watcher-beat"
guard_out=$(run_hook_claude "$dir" true); guard_status=$?
kill "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
kill "$holder" 2>/dev/null || true
wait "$holder" 2>/dev/null || true
rm -rf "$dir/state/.watch.lock"
expect_code 0 "$guard_status" "a healthy watcher must still allow the stop after a frozen-epoch alarm"
[ -z "$guard_out" ] || fail "healthy allow after the frozen-epoch alarm produced output: $guard_out"
assert_absent "$dir/state/.turnend-claude-blocks" "positive recovery left the frozen-epoch block budget"
assert_absent "$dir/state/.claude-autoarm-failure-notified" "positive recovery left the failure notice"
assert_absent "$dir/state/.claude-autoarm-failure-alarmed" "positive recovery left the attended alarm"
guard_out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); guard_status=$?
expect_code 2 "$guard_status" "a later unhealthy stop must re-block from a fresh budget"
count=$(sed -n '2s/^count=//p' "$dir/state/.turnend-claude-blocks")
[ "$count" = 1 ] || fail "the post-recovery episode must restart its budget at 1, got $count"
pass "fm-turnend-guard --claude: an inert auto-arm's frozen epoch reaches one bounded fail-open and resets on recovery"
}

# The same frozen ledger without a verified failure episode: the budget must
# still provably run out, and the verified-failure gate - not a stuck counter -
# is what keeps the stop blocking after that.
test_hook_claude_mode_frozen_epoch_without_verified_failure_spends_budget_and_keeps_blocking() {
local dir out status i count
dir=$(make_primary_dir "$TMP_ROOT/hook-claude-frozen-unverified")
: > "$dir/state/task1.meta"
printf 'epoch=7 owner_pid=999 outcome=clean updated_at=1\n' > "$dir/state/.claude-autoarm-epoch"
touch -t 202001010000 "$dir/state/.claude-autoarm-epoch"
for i in 1 2 3 4 5; do
out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" false); status=$?
expect_code 2 "$status" "frozen unverified stop $i must keep blocking"
assert_not_contains "$out" 'systemMessage' "an unverified frozen epoch must never fail open"
[ "$(sed -n '1p' "$dir/state/.claude-autoarm-epoch")" = 'epoch=7 owner_pid=999 outcome=clean updated_at=1' ] \
|| fail "the guard rewrote the frozen ledger at stop $i"
done
count=$(sed -n '2s/^count=//p' "$dir/state/.turnend-claude-blocks")
[ "$count" -gt 3 ] 2>/dev/null || fail "the block budget must run out against a frozen epoch, but the recorded count is ${count:-absent}"
assert_absent "$dir/state/.claude-autoarm-failure-alarmed" "an unverified frozen epoch recorded an attended alarm"
pass "fm-turnend-guard --claude: a frozen unverified epoch spends the budget yet still blocks"
}

test_hook_claude_mode_recovery_contention_is_not_ordinary_allow() {
local dir pid identity holder out status
dir=$(make_primary_dir "$TMP_ROOT/hook-claude-recovery-contention")
Expand Down Expand Up @@ -1703,6 +1823,8 @@ test_hook_claude_mode_budget_without_verified_failure_keeps_blocking() {
out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" false); status=$?
expect_code 2 "$status" "--claude block $i must exit 2 within the budget"
done
count=$(sed -n '2s/^count=//p' "$dir/state/.turnend-claude-blocks")
[ "$count" -gt 3 ] 2>/dev/null || fail "four consecutive blocks must spend the budget, but the recorded count is ${count:-absent}"
assert_not_contains "$out" 'systemMessage' "budget exhaustion without verified auto-arm failure must not fail open"
assert_absent "$dir/state/.claude-autoarm-failure-alarmed" "unverified budget exhaustion recorded an attended alarm"
pass "fm-turnend-guard --claude: budget exhaustion alone cannot permit a blind stop"
Expand Down Expand Up @@ -2135,6 +2257,8 @@ test_hook_claude_mode_blocks_on_stuck_generation_claim
test_hook_claude_mode_terminal_fail_open_clears_abandoned_claim
test_hook_claude_mode_preserves_fresh_failed_progression
test_hook_claude_mode_integrated_monotonic_fail_open
test_hook_claude_mode_frozen_epoch_reaches_bounded_fail_open
test_hook_claude_mode_frozen_epoch_without_verified_failure_spends_budget_and_keeps_blocking
test_hook_claude_mode_recovery_contention_is_not_ordinary_allow
test_hook_claude_mode_concurrent_recovery_resets_are_idempotent
test_hook_claude_mode_stale_rewake_epoch_blocks
Expand Down
Loading