diff --git a/.agents/skills/stuck-crewmate-recovery/SKILL.md b/.agents/skills/stuck-crewmate-recovery/SKILL.md index ffef22777f0..67a80593185 100644 --- a/.agents/skills/stuck-crewmate-recovery/SKILL.md +++ b/.agents/skills/stuck-crewmate-recovery/SKILL.md @@ -40,10 +40,9 @@ Do not sweep another home's endpoints or infer ownership from a matching window Before relaunch, prove that no live agent still owns the recorded task and that the existing worktree remains available. Preserve its uncommitted changes and commits, keep the same task identity, and resume or relaunch the recorded harness in that existing worktree with the same brief plus a concise progress note. -A HERDR endpoint that is not merely idle but destroyed - a pane or workspace removed in Herdr churn - is recovered by that same relaunch, which creates one fresh endpoint in the existing worktree and rebinds the task's record to it; nothing special is needed, and the worktree is untouched ([`docs/agent-control.md`](../../../docs/agent-control.md) "Reclaiming a task whose endpoint is gone"). -That relaunch proves the endpoint is destroyed before it rebinds, so a Herdr server that was merely stopped is adopted back rather than duplicated. -On tmux there is no reclaim: a task record carries no socket identity for its endpoint, so a `missing` window cannot be told apart from one on a tmux server this seat cannot address, and both `exit` and `relaunch` refuse. -Do not work around either refusal by respawning - it means a live agent may still hold that worktree. +An endpoint that is not merely idle but destroyed - a Herdr pane or workspace removed in churn, a tmux window or server that no longer exists - is recovered by that same `bin/fm-control.sh relaunch`, which creates one fresh endpoint in the existing worktree and rebinds the task's record to it; nothing special is needed, the worktree is untouched, and a parked validation run is left exactly as it was. The Herdr reclaim works on every platform, while the tmux one reads `/proc` and so is Linux-only - on macOS a tmux task whose endpoint was destroyed refuses instead, and the refusal says so ([`docs/agent-control.md`](../../../docs/agent-control.md) "Reclaiming a task whose endpoint is gone"). +That relaunch proves the endpoint is gone before it rebinds, so a Herdr server that was merely stopped is adopted back and an agent still working in the worktree refuses rather than being duplicated. +Do not work around a refusal by respawning or by editing the task record - it means a live agent may still hold that worktree, and the refusal names what to do next. That reclaim is the owning home's operation only, and a secondmate is the one exception: recover it through `bin/fm-spawn.sh --secondmate` as above. Do not use a fresh generic spawn while the recorded worktree is unaccounted for, because allocating another worktree can split one task across two copies. If the worktree or ownership cannot be reconciled safely, leave all state intact and report the task failed or blocked with the conflicting evidence. diff --git a/bin/backends/tmux.sh b/bin/backends/tmux.sh index 2bc1c8aa0d7..8a463d2227f 100644 --- a/bin/backends/tmux.sh +++ b/bin/backends/tmux.sh @@ -80,10 +80,24 @@ fm_backend_tmux_container_ensure() { } # fm_backend_tmux_create_task: create the task's window in , -# refusing an existing in . Mirrors fm-spawn.sh's -# duplicate-check-then-new-window sequence, including the exact error text -# (session:window, matching how fm-spawn.sh composed its own $T). Prints the -# created window's stable window id on stdout for the caller to target. +# refusing an existing in . Prints the created window's +# stable window id on stdout for the caller to target. +# +# The refusal never becomes a close-and-replace, unlike the Herdr adapter's +# husk path: a herdr tab is addressed inside the session the record names, +# while `fm-` carries no home identity and every seat outside tmux +# resolves to the one session `firstmate`, so two homes holding equal task ids +# collide here. Closing the window this seat did not record would destroy +# another home's deliberately preserved endpoint - the very dead-end the +# reclaim exists to remove - so whose window it is stays the operator's call. +# +# Because that refusal is the only thing standing between a reclaim and a +# foreign window, absence of the name has to be READ, never inferred from an +# empty stream: fm_backend_tmux_window_inventory's three-way status separates +# "this session has no such window" from "the read failed", and only the first +# licenses the create. tmux permits duplicate window names, and the record +# persists the NAME form, so a second `fm-` would leave every later +# name-targeted read and write resolving to whichever one tmux matches first. # # Robustness (fm-spawn tmux window handling under a non-default captain config): # - Capture a STABLE window id with -P -F '#{window_id}', and let tmux append @@ -95,9 +109,15 @@ fm_backend_tmux_container_ensure() { # The returned window id lets callers target the window even if its name is ever # lost, so worktree discovery cannot fall back to the active client's window. fm_backend_tmux_create_task() { # -> prints window id - local ses=$1 wname=$2 proj_abs=$3 wid - if tmux list-windows -t "$ses" -F '#{window_name}' | grep -qx "$wname"; then - echo "error: window $ses:$wname already exists" >&2 + local ses=$1 wname=$2 proj_abs=$3 wid windows inventory_status + windows=$(fm_backend_tmux_window_inventory "$ses") + inventory_status=$? + if [ "$inventory_status" -eq 1 ]; then + echo "error: session $ses's windows could not be read, so whether $ses:$wname already exists is unknown; refusing rather than risk a second window under one name - retry once tmux answers again" >&2 + return 1 + fi + if [ "$inventory_status" -eq 0 ] && printf '%s\n' "$windows" | grep -qxF -- "$wname"; then + echo "error: window $ses:$wname already exists; read it with bin/fm-peek.sh $ses:$wname to see what holds it. No control-plane command closes a window this home's records do not name, so a leftover or another home's window has to be closed where it runs before this task can open its own" >&2 return 1 fi wid=$(tmux new-window -dP -F '#{window_id}' -t "$ses:" -n "$wname" -c "$proj_abs") || return 1 diff --git a/bin/fm-agent-process-lib.sh b/bin/fm-agent-process-lib.sh index 11c092a88b6..27ca4a5187b 100644 --- a/bin/fm-agent-process-lib.sh +++ b/bin/fm-agent-process-lib.sh @@ -111,3 +111,75 @@ fm_agent_process_classify() { # [pid] -> agent|shell|othe printf 'other' fi } + +# fm_agent_process_worktree_scan: whether a verified harness process is still +# working in anywhere on this host, read from the kernel's own +# per-process working directory rather than from any terminal endpoint. It +# answers the one question a lost endpoint leaves open - would launching a +# replacement join an agent that is still running on this task's local copy? - +# without trusting a runtime backend that could not see the endpoint. +# +# Only processes whose /proc entry this user owns are read: a worker is always +# launched as that user, and another user's working directory is not readable +# anyway. A non-dumpable process is skipped along with them, because Linux +# gives its /proc entry to root - no verified harness is non-dumpable, so none +# is skipped that way. Each process is classified by fm_agent_process_classify, +# so this answers from the same name vocabulary as every liveness probe. +# +# Prints "\t", always exactly one TAB: +# none - every readable process was read and none is an agent there +# agent - " " of an agent process working in +# unreadable - the reason the process table could not be read; the caller +# must treat this as "an agent may be there" +# The only reader is /proc, whose per-process `cwd` link is the kernel's own +# answer for one exact process. A host without /proc answers `unreadable`, so +# no tmux endpoint is proven absent there: every other way to ask - lsof's +# file-set records, a ps listing - reports a path it may have failed to +# resolve without saying so for the one process that matters, and reading that +# silence as "no agent" is what launches a duplicate onto a live worktree. +# A process that exits mid-scan, and a zombie that holds no working directory, +# are skipped. +fm_agent_process_worktree_scan() { # + local wt=${1-} wt_real dir pid cwd name argv0 args stat rest + if [ -z "$wt" ] || ! wt_real=$(cd "$wt" 2>/dev/null && pwd -P); then + printf 'unreadable\tthe worktree %s cannot be resolved' "'$wt'" + return 0 + fi + if [ ! -r /proc/self/cmdline ] || [ ! -L /proc/self/cwd ]; then + printf 'unreadable\tthe absence proof reads /proc and this host has none, so reclaiming a tmux task is unavailable here. The only route left is a seat that still addresses the recorded tmux server, where the window classifies directly and no absence proof is needed - and once that server is gone there is no such seat, and no route' + return 0 + fi + for dir in /proc/[0-9]*; do + [ -O "$dir" ] || continue + pid=${dir#/proc/} + if cwd=$(readlink "$dir/cwd" 2>/dev/null); then + case "$cwd" in + "$wt_real"|"$wt_real"/*) ;; + *) continue ;; + esac + else + # A process that exited mid-scan, or a zombie, has no working directory + # left. The refusal below covers only the race that reaches here: the + # link vanishing from a /proc directory this user still owns, mid-exec + # or mid-exit. + [ -d "$dir" ] || continue + stat=$(cat "$dir/stat" 2>/dev/null) || continue + rest=${stat##*) } + case "${rest%% *}" in + Z|X) continue ;; + esac + cwd= + fi + name=$(cat "$dir/comm" 2>/dev/null) || continue + argv0=$(tr '\0' '\n' < "$dir/cmdline" 2>/dev/null | head -n 1) || argv0= + args=$(tr '\0' ' ' < "$dir/cmdline" 2>/dev/null) || args= + [ "$(fm_agent_process_classify "$name" "$argv0" "$args" "$pid")" = agent ] || continue + if [ -z "$cwd" ]; then + printf 'unreadable\tthe working directory of agent process %s (%s) could not be read' "$pid" "${name:-$argv0}" + return 0 + fi + printf 'agent\t%s %s' "$pid" "${name:-$argv0}" + return 0 + done + printf 'none\t' +} diff --git a/bin/fm-control-lib.sh b/bin/fm-control-lib.sh index d3fcbcb043d..98b5a57a547 100644 --- a/bin/fm-control-lib.sh +++ b/bin/fm-control-lib.sh @@ -15,7 +15,7 @@ # This file owns three capability tables plus their pure artifact-path tables, # and ONE named exception to that purity - fm_control_endpoint_absence_verdict, # the single owner of the per-backend endpoint-absence proof, which does run -# backend reads. Everything else has no side effects, runs no backend command, +# backend reads and a process-table scan. Everything else has no side effects, runs no backend command, # and reads no state, so sourcing this file is still free and the tables can be # read by a test as a pure contract: # @@ -302,8 +302,9 @@ fm_control_backend_state_verified() { # } # fm_control_endpoint_absence_verdict: the ONE owner of the per-backend proof -# that an endpoint reading `missing` is actually GONE rather than merely -# unreachable from this seat. Call it only for a `missing` raw state. +# that an endpoint reading `missing` holds no agent this home could duplicate, +# so the task may be reclaimed rather than stranded. Call it only for a +# `missing` raw state. # # Prints "\t" - always exactly one TAB, so a caller splits # unambiguously with ${raw%%$'\t'*} and ${raw#*$'\t'}. The reason is empty @@ -313,7 +314,17 @@ fm_control_backend_state_verified() { # # assignment made here could never reach them. # # The verdicts: -# gone - absence is PROVEN. There is no endpoint and therefore no agent. +# gone - absence is PROVEN, in the terms the backend can prove, and +# there is nothing here to adopt. Herdr proves the ENDPOINT: it +# re-read the recorded pane through the recorded session's own +# socket and the pane is not there. tmux proves only the AGENT: +# a record's `session:window` carries no socket, so the same +# address can name a live window on another server, and what the +# scan establishes is that no agent process works in the recorded +# worktree. A tmux window may therefore still survive elsewhere - +# on a server this seat cannot address, or under a session name +# the record no longer matches, even on this seat's own server - +# and it holds no agent either way. # dead - the endpoint is there after all and holds no agent. # alive - the endpoint is there and an agent is running in it. # unproven - neither could be established; the caller must refuse. @@ -323,31 +334,41 @@ fm_control_backend_state_verified() { # # endpoint can still hold a live agent on the task's worktree, so every caller # that would act on absence - `exit` claiming the agent stopped, `relaunch` # re-creating the endpoint - must come through here rather than trusting the -# raw verdict. -# -# Whether absence is provable AT ALL is a property of the backend, not of the -# reading: -# herdr CAN prove it. Every read goes through fm_backend_herdr_cli, which -# passes `--session `, so the recheck starts and reads the session -# the RECORD names, through that session's own socket. The answer is about -# the task's endpoint and nothing else. -# tmux CANNOT. `list-windows -a` describes only the server the CURRENT -# process addresses (its TMUX_TMPDIR/socket), and a task's record does not -# carry the endpoint's socket identity - so a different but running server -# would answer "not anywhere" about a window it was never able to see. -# There is no read available here that closes that gap, so tmux always -# returns `unproven` and both verbs refuse. tmux is left exactly as -# deadlocked as it was before this change - no worse - but deliberately. +# raw verdict. How that is proven is a property of the backend: +# herdr proves the ENDPOINT absent. Every read goes through +# fm_backend_herdr_cli, which passes `--session `, so the recheck +# starts and reads the session the RECORD names, through that session's +# own socket. The answer is about the task's endpoint and nothing else. +# tmux proves the AGENT absent. `list-windows` describes only the server the +# CURRENT process addresses, and a task record carries no socket identity, +# so no tmux read can tell a destroyed window from one on a server this +# seat cannot address. What a duplicate launch would actually collide with +# is an agent process working in the recorded worktree, and the kernel +# knows every process's working directory regardless of which terminal +# server holds it (fm_agent_process_worktree_scan). No agent process there +# is the proof; one found there, or a process table that cannot be read, +# refuses. A window that outlived its agent on a server this +# seat cannot address is left alone - it holds no agent, and the reclaim +# opens its replacement on the server this seat does address. # # Both control-plane callers share this one implementation so the proof cannot # drift into two answers for the same endpoint. -fm_control_endpoint_absence_verdict() { # - local backend=${1-} target=${2-} +fm_control_endpoint_absence_verdict() { # + local backend=${1-} target=${2-} worktree=${3-} scan fm_backend_source "$backend" \ || { printf 'unproven\tbackend %s could not be loaded to prove anything about that endpoint' "'$backend'"; return 0; } case "$backend" in tmux) - printf 'unproven\ttmux absence cannot be proven from a task record: the record does not carry the endpoint'"'"'s socket identity, and a server-wide window inventory only describes the tmux server this process addresses, so a window absent from it may still be alive on another' + scan=$(fm_agent_process_worktree_scan "$worktree") + case "${scan%%$'\t'*}" in + none) printf 'gone\t' ;; + agent) + printf 'unproven\tagent process %s is still working in the recorded worktree %s, so its window may be on a tmux server this seat cannot address; stop that process where it runs, or address its tmux server from this seat, then retry' "${scan#*$'\t'}" "$worktree" + ;; + *) + printf 'unproven\tno tmux read can prove that window absent, and the process scan that would prove no agent is still working in %s failed: %s' "$worktree" "${scan#*$'\t'}" + ;; + esac ;; herdr) # Start the RECORDED session's server (only the server - nothing is @@ -366,6 +387,26 @@ fm_control_endpoint_absence_verdict() { # esac } +# fm_control_unclassified_next_step: the one sentence every lifecycle refusal +# of an endpoint that is not positively classified ends with, so `exit`, +# `relaunch`, and the launch owner never point an operator at each other for a +# state none of them acts on. `alive`, `dead`, and a proven-gone `missing` are +# all actionable, and is the one the caller was actually asked to +# perform: a refused `exit` names `exit`, because sending a stop request off to +# the verb that REPLACES the agent is the same cross-pointing this sentence +# exists to end. +fm_control_unclassified_next_step() { # + local id=${1-} state=${2-} verb=${3-} + case "$state" in + ambiguous) + printf 'no lifecycle command acts on it until it reads alive, dead, or gone: inspect it with bin/fm-peek.sh %s - a foreground process that is not a verified agent reads ambiguous - and once it is back at a shell or its agent, retry bin/fm-control.sh %s %s' "$id" "$id" "$verb" + ;; + *) + printf 'no lifecycle command acts on it until it reads alive, dead, or gone: the read itself failed, so retry bin/fm-control.sh %s %s once bin/fm-peek.sh %s can read the endpoint again' "$id" "$verb" "$id" + ;; + esac +} + # The per-task wiring artifacts a harness leaves behind, so a relaunch that # changes harness (or re-arms the same one with a fresh busy generation) can # clear the previous incarnation's wiring instead of leaving a stale hook diff --git a/bin/fm-control.sh b/bin/fm-control.sh index aa4c9af2004..40f589de806 100755 --- a/bin/fm-control.sh +++ b/bin/fm-control.sh @@ -40,30 +40,34 @@ # `missing` is put through the control plane's per-backend absence # proof (fm_control_endpoint_absence_verdict) before anything is # claimed about it, because `missing` also covers an endpoint that -# is merely unreachable from this seat. That proof exists only on -# HERDR, whose reads are scoped to the session the record names: -# proven gone reports `endpoint-gone` rather than -# `already-stopped`, because the endpoint this verb normally -# preserves did not survive; a pane that turns out to be there and -# idle is the ordinary `already-stopped`; one whose agent is back -# takes the ordinary interrupt-then-exit path. A tmux `missing` -# always REFUSES: a task record carries no socket identity for its -# endpoint, so this verb cannot tell a destroyed window from one on -# a tmux server it cannot address, and it will not claim a stop it -# cannot see. +# is merely unreachable from this seat: herdr re-reads the pane in +# the session the record names, and tmux proves no agent process +# is still working in the recorded worktree. Proven gone reports +# `endpoint-gone` rather than `already-stopped`, because there is +# no endpoint here for this verb to preserve - on tmux that rests +# on the agent-absence proof, not on proving the window +# destroyed; a herdr pane that turns out to be there and idle is +# the ordinary `already-stopped`; one whose agent is back takes +# the ordinary interrupt-then-exit path. An absence that cannot be proven - +# an agent process still in the worktree, or a process table that +# cannot be read - REFUSES rather than claim a stop it cannot see. # relaunch Transactionally replace the running agent with a new one, in the # SAME worktree - and the same endpoint whenever that endpoint # still exists - on the same or a newly chosen # harness/model/effort - so switching harness is one ordinary use # of this verb. When the recorded endpoint is instead proven gone - -# a Herdr pane or workspace destroyed in churn - the launch owner -# re-creates one in that worktree, in the herdr session the record -# names, and the task's record rebinds to it; that is how a task -# whose terminal was destroyed is reclaimed by the home that owns -# it, rather than being stranded with a parked approval nobody can -# answer. Reclaim is HERDR-ONLY for the reason `exit` gives above: -# a tmux `missing` cannot be proven absent from a task record, so -# it refuses. +# a Herdr pane or workspace destroyed in churn, a tmux window or +# server that no longer exists - the launch owner re-creates one in +# that worktree (herdr: in the session the record names; tmux: on +# the server and in the session THIS seat addresses, which need not +# be the recorded one), and the task's record rebinds to it; that +# is how a task whose terminal was destroyed is +# reclaimed by the home that owns it, rather than being stranded +# with a parked approval nobody can answer. The absence proof is +# the one `exit` uses above, so a refusal there refuses here too. +# A validation run already under way for the branch survives the +# reclaim untouched; nothing here ever aborts, answers, or starts +# a validation run. # An explicit `default` model or effort clears that # axis for the replacement. With no explicit axis, a secondmate # re-resolves its durable config/secondmate-harness pin (harness @@ -230,6 +234,9 @@ if ! fm_control_verb_allowed "$VERB"; then exit 2 fi +VERB_RETRY=$VERB +[ "$VERB" != relaunch ] || VERB_RETRY='relaunch --note ""' + NEW_HARNESS= NEW_MODEL= NEW_EFFORT= @@ -571,15 +578,16 @@ do_exit() { # "destroyed" with "unreachable from this seat". Route it through the # control plane's one absence proof - the same one the relaunch gate uses # - and report what that proof actually established, never more. - absence=$(fm_control_endpoint_absence_verdict "$BACKEND" "$T") + absence=$(fm_control_endpoint_absence_verdict "$BACKEND" "$T" "$WT") case "${absence%%$'\t'*}" in gone) # Proven gone, so the agent that lived in it went with it: exit's # postcondition already holds and there is nothing to send. Its own - # outcome rather than `already-stopped`, because the endpoint this - # verb normally preserves did not survive. The worktree and every - # uncommitted change are untouched, and `relaunch` re-creates the - # endpoint from here. + # outcome rather than `already-stopped`, because there is no + # endpoint here for this verb to preserve - on tmux that rests on + # the agent-absence proof, not on proving the window destroyed. The + # worktree and every uncommitted change are untouched, and + # `relaunch` re-creates the endpoint from here. printf 'endpoint-gone' return 0 ;; @@ -596,11 +604,11 @@ do_exit() { # alive path: interrupt if busy, then the harness's exit command. ;; *) - die "task $ID's endpoint $T reads 'missing', but ${absence#*$'\t'}; exit will not claim an agent stopped at an address it cannot trust, nor send lifecycle input to one" + die "task $ID's endpoint $T reads 'missing', but ${absence#*$'\t'}; exit will not claim an agent stopped at an address it cannot trust, nor send lifecycle input to one, and relaunch refuses for the same reason" ;; esac ;; - *) die "task $ID's endpoint reads '$state' rather than a positively classified state; refusing to send a lifecycle command into an unattributed endpoint" ;; + *) die "task $ID's endpoint reads '$state' rather than a positively classified state; refusing to send a lifecycle command into an unattributed endpoint - $(fm_control_unclassified_next_step "$ID" "$state" "$VERB_RETRY")" ;; esac # A busy agent is interrupted first before the exit command is submitted. case "$(busy_verdict)" in @@ -738,7 +746,7 @@ relaunch_rollback() { ;; dead) journal_write "failed:$RELAUNCH_PHASE" "rollback=prior-record-kept-agent-dead" || true - echo "error: $ID's agent stopped but relaunch did not reach replacement launch; no agent is running, and its work plus progress note are preserved at $WT" >&2 + echo "error: $ID's agent stopped but relaunch did not reach replacement launch; no agent is running, and its work plus progress note are preserved at $WT; rerun bin/fm-control.sh $ID relaunch --note \"\" to launch the replacement" >&2 ;; *) # The old agent was NOT proven stopped, so no replacement is coming @@ -767,10 +775,10 @@ relaunch_rollback() { # reconciles. Rewriting it back to the old harness would be a second, # worse inaccuracy. journal_write "failed:$RELAUNCH_PHASE" "rollback=none-new-record-kept" || true - echo "error: $ID was relaunched on $TARGET_HARNESS but no running agent could be confirmed; its work is preserved at $WT" >&2 + echo "error: $ID was relaunched on $TARGET_HARNESS but no running agent could be confirmed; its work is preserved at $WT; inspect the endpoint with bin/fm-peek.sh $ID, then rerun bin/fm-control.sh $ID relaunch --note \"\", which adopts it once it reads dead and replaces its agent once it reads alive" >&2 else journal_write "failed:$RELAUNCH_PHASE" "rollback=prior-record-kept" || true - echo "error: $ID's agent was stopped but the replacement did not launch; no agent is running, and its work plus the recorded progress note are preserved at $WT" >&2 + echo "error: $ID's agent was stopped but the replacement did not launch; no agent is running, and its work plus the recorded progress note are preserved at $WT; rerun bin/fm-control.sh $ID relaunch --note \"\" once the launch failure above is resolved" >&2 fi ;; esac diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 82226064e56..457826276d3 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -61,12 +61,12 @@ # ADOPTED as-is, while an endpoint PROVEN gone is RE-CREATED in the recorded # worktree and the republished record rebinds the task to it. That proof is # its own step, because a backend's `missing` also covers an endpoint that is -# merely unreachable from here - and it is only available on HERDR, which must -# still read the recorded pane as gone once that session's server is running -# again. A tmux `missing` always refuses: a task record carries no socket -# identity for its endpoint, so no read here can tell a destroyed window from -# one on a tmux server this process cannot address. An endpoint that turns out -# to have survived refuses too. The worktree is reused untouched either way; a +# merely unreachable from here (bin/fm-control-lib.sh's +# fm_control_endpoint_absence_verdict owns it): herdr must still read the +# recorded pane as gone once that session's server is running again, and tmux +# must find no agent process still working in the recorded worktree. An +# endpoint that turns out to have survived, or an absence that cannot be +# proven, refuses. The worktree is reused untouched either way; a # rebind is a recovery, never a teardown. Only a crewmate or scout rebinds: a # secondmate whose endpoint is gone is respawned by its own owner # (`--secondmate`, driven by the session-start liveness sweep). @@ -1186,6 +1186,7 @@ SPAWN_TREEHOUSE_PROJECT_LOCK= SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 SPAWN_SLOT_CLAIMED=0 RELAUNCH_REPLACEMENT_PENDING=0 +TMUX_REBIND_ABORT_TARGET= RELAUNCH_REPLACEMENT_BUSY_GEN= RELAUNCH_REPLACEMENT_HARNESS= RELAUNCH_REPLACEMENT_STATE= @@ -1249,6 +1250,13 @@ spawn_abort_cleanup() { fi fi fi + if [ -n "$TMUX_REBIND_ABORT_TARGET" ]; then + # A tmux reclaim's fresh window, refused before the record named it. It + # holds only the shell it was created with, so closing it loses nothing and + # keeps a retry from tripping over a same-named window. + fm_backend_kill tmux "$TMUX_REBIND_ABORT_TARGET" 2>/dev/null || true + TMUX_REBIND_ABORT_TARGET= + fi if [ "$HERDR_PROJECTION_ABORT_CLEANUP" = 1 ] && [ "$HERDR_PRESENTATION_ORDER_LOCK_HELD" != 1 ]; then if ! spawn_herdr_presentation_order_lock_acquire "${HERDR_PROJECTION_ABORT_SESSION:-}"; then @@ -1714,32 +1722,25 @@ if [ "$RELAUNCH" -eq 1 ]; then # endpoint was DESTROYED" with "the endpoint is UNREACHABLE from here right # now", and an unreachable endpoint can still hold the live agent this # relaunch would duplicate. So absence is PROVEN before it may rebind, never - # inferred from a failed read - and only HERDR can prove it: - # herdr - the recorded session's server is started, and the recorded pane is - # RE-READ through that session's own socket. `dead` means the pane - # survived the restart and is adopted after all; `alive` means the - # agent came back and refuses; only a second `missing` proves the - # pane itself did not survive. - # tmux - REFUSES, always. A task record carries no socket identity for its - # endpoint, and a server-wide inventory describes only the server - # this process addresses, so no read available here can tell "gone" - # from "on a server I cannot see". A tmux `missing` therefore stays - # as deadlocked as it was before this change - deliberately, and - # with the reason stated rather than guessed past. - # Every transient or self-contradicting read stays `unreadable`/`ambiguous` - # and refuses as it always did (bin/fm-backend.sh's fm_backend_agent_state - # owns that vocabulary). The proof itself lives in one place for the whole - # control plane - fm_control_endpoint_absence_verdict - so `exit` and - # `relaunch` cannot reach two different answers about one endpoint. + # inferred from a failed read: herdr re-reads the pane in the session the + # record names (a pane that survived is adopted as `dead`, one whose agent + # came back refuses as `alive`), and tmux proves no agent process is still + # working in the recorded worktree. Every transient or self-contradicting + # read stays `unreadable`/`ambiguous` and refuses as it always did + # (bin/fm-backend.sh's fm_backend_agent_state owns that vocabulary). The + # proof itself lives in one place for the whole control plane - + # fm_control_endpoint_absence_verdict - so `exit` and `relaunch` cannot reach + # two different answers about one endpoint. RELAUNCH_STATE=$(fm_backend_agent_state "$BACKEND" "$RELAUNCH_TARGET") if [ "$RELAUNCH_STATE" = missing ]; then - RELAUNCH_ABSENCE=$(fm_control_endpoint_absence_verdict "$BACKEND" "$RELAUNCH_TARGET") + RELAUNCH_ABSENCE=$(fm_control_endpoint_absence_verdict "$BACKEND" "$RELAUNCH_TARGET" \ + "$(fm_meta_get "$RELAUNCH_META" worktree)") case "${RELAUNCH_ABSENCE%%$'\t'*}" in gone) RELAUNCH_STATE=missing ;; dead) RELAUNCH_STATE=dead ;; alive) RELAUNCH_STATE=alive ;; *) - echo "error: task $ID's recorded endpoint $RELAUNCH_TARGET reads 'missing', but ${RELAUNCH_ABSENCE#*$'\t'}. An endpoint that cannot be proven absent may still hold a live agent on this task's worktree; refusing rather than launching a second agent into it" >&2 + echo "error: task $ID's recorded endpoint $RELAUNCH_TARGET reads 'missing', but ${RELAUNCH_ABSENCE#*$'\t'}. An endpoint that cannot be proven absent may still hold a live agent on this task's worktree; refusing rather than launching a second agent into it (bin/fm-control.sh $ID exit and relaunch refuse it for the same reason)" >&2 exit 1 ;; esac @@ -1747,8 +1748,12 @@ if [ "$RELAUNCH" -eq 1 ]; then case "$RELAUNCH_STATE" in dead) ;; missing) RELAUNCH_REBIND=1 ;; + alive) + echo "error: task $ID's endpoint reads 'alive': an agent still runs there, and a relaunch requires a positively agent-free endpoint, so replace it with bin/fm-control.sh $ID relaunch --note \"\", which stops that agent before launching this one" >&2 + exit 1 + ;; *) - echo "error: task $ID's endpoint reads '$RELAUNCH_STATE'; a relaunch requires a positively agent-free endpoint (stop the agent first with bin/fm-control.sh $ID exit)" >&2 + echo "error: task $ID's endpoint reads '$RELAUNCH_STATE'; a relaunch requires a positively agent-free endpoint, and $(fm_control_unclassified_next_step "$ID" "$RELAUNCH_STATE" 'relaunch --note ""')" >&2 exit 1 ;; esac @@ -3440,6 +3445,33 @@ if [ "$RELAUNCH" -eq 1 ]; then T=$RELAUNCH_TARGET WT_TARGET=$T SES=${T%%:*} + elif [ "$BACKEND" = tmux ]; then + # The recorded endpoint is authoritatively gone, so there is nothing to + # adopt: create ONE fresh endpoint for the same task, opened directly in the + # recorded worktree. The record published below writes window= from these + # values, which is the whole rebind - the task id, brief, worktree, armed + # poll and status log are untouched. + # + # The window goes on the tmux server and session THIS seat addresses, the + # same placement an ordinary spawn from here uses. Only one of the states + # that reach here forces that: the recorded session, or its whole server, + # was absent, and a record carries no socket identity to name another. It + # is not forced when the recorded session was read successfully and merely + # no longer holds the window; the placement is the same there, so a reclaim + # run from outside that session moves the task into this seat's session. + # Pinning the recorded session is the herdr rebind's invariant, not this + # one. + # The absence proof already established that no agent works in the + # worktree, so a window that outlived its agent on some other server holds + # nothing this could duplicate. create_task refuses a same-named window + # already in the session, so a reclaim never shares an endpoint either. + # Until the record is republished the new window holds only a shell, and + # the abort trap closes it if anything refuses before then. + SES=$(fm_backend_tmux_container_ensure) || exit 1 + WID=$(fm_backend_tmux_create_task "$SES" "$W" "$WT") || exit 1 + T="$SES:$W" + WT_TARGET=$WID + TMUX_REBIND_ABORT_TARGET=$T else # The recorded endpoint is authoritatively gone, so there is nothing to # adopt: create ONE fresh endpoint for the same task, opened directly in the @@ -3447,11 +3479,10 @@ if [ "$RELAUNCH" -eq 1 ]; then # ids) from these values, which is the whole rebind - the task id, brief, # worktree, armed poll and status log are untouched. # - # Herdr is the ONLY backend that reaches here: the gate above rebinds only - # on a PROVEN-gone endpoint, and absence is provable only on herdr, whose - # every read is scoped to the session the record names - # (fm_control_endpoint_absence_verdict owns that argument). tmux and every - # secondmate were already refused, so there is no dispatch left to make. + # Only herdr reaches here besides tmux above: the gate rebinds only on a + # PROVEN-gone endpoint, only tmux and herdr can prove one + # (fm_control_endpoint_absence_verdict owns that argument), and every + # secondmate was already refused. # # This deliberately uses the FLAT container shape rather than Herdr's # presentation projection: projection is a presentation-only layout that is @@ -4879,6 +4910,7 @@ if [ "$RELAUNCH" -eq 1 ]; then exit 1 fi RELAUNCH_REPLACEMENT_PENDING=0 + TMUX_REBIND_ABORT_TARGET= SPAWN_META_PUBLISH_STARTED=0 SPAWN_META_TMP= fi diff --git a/docs/agent-control.md b/docs/agent-control.md index 2a6a80fb02f..7ab0c46043c 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -23,7 +23,7 @@ The failure repeated across harnesses and homes, and the workaround (remember to `bin/fm-send.sh`'s `--key` path reads the composer-clear table from this owner too, rather than keeping a second copy of it. - **Per-backend capability**: which named keys a runtime backend can deliver, and whether it has a recovery-grade agent-state classifier able to prove an agent stopped. -The [endpoint-absence proof](#reclaiming-a-task-whose-endpoint-is-gone) below is the only function here that runs backend reads; sourcing the file is still free. +The [endpoint-absence proof](#reclaiming-a-task-whose-endpoint-is-gone) below is the only function here that runs backend reads or a process-table scan; sourcing the file is still free. A recorded `harness=` is not always an exact adapter name: a task launched from a raw command records that command's basename instead. `fm_control_harness_family` is the one place that prefix rule is stated, and an unrecognized value resolves to no adapter rather than being guessed into one. @@ -33,7 +33,7 @@ A recorded `harness=` is not always an exact adapter name: a task launched from | Verb | Effect | Postcondition | | --- | --- | --- | | `interrupt` | Deliver the harness's verified interrupt sequence while leaving the agent running. | Delivery succeeds while the endpoint still exists and the agent is still alive where the backend can classify that; cancellation is confirmed only from an adapter-owned acknowledgement and otherwise reports `cancel=unconfirmed`. | -| `exit` | Stop the agent, preserving the endpoint, the worktree, and every uncommitted change. | The backend's recovery-grade classifier reports the agent gone. Already-stopped is idempotent success. An endpoint reading `missing` goes through the same [absence proof](#reclaiming-a-task-whose-endpoint-is-gone) the reclaim uses before anything is claimed about it, and only Herdr can supply one: proven gone reports `endpoint-gone` (the agent went with it, and the endpoint this verb normally preserves did not survive), a pane that turns out to be there and idle is the ordinary `already-stopped`, one whose agent is back takes the ordinary interrupt-then-exit path. A tmux `missing` always refuses rather than claim a stop it cannot see. | +| `exit` | Stop the agent, preserving the endpoint, the worktree, and every uncommitted change. | The backend's recovery-grade classifier reports the agent gone. Already-stopped is idempotent success. An endpoint reading `missing` goes through the same [absence proof](#reclaiming-a-task-whose-endpoint-is-gone) the reclaim uses before anything is claimed about it: proven gone reports `endpoint-gone` (the agent is gone and there is no endpoint here for this verb to preserve - on tmux that rests on proving no agent works in the recorded worktree, not on proving the window destroyed), a Herdr pane that turns out to be there and idle is the ordinary `already-stopped`, one whose agent is back takes the ordinary interrupt-then-exit path. An absence the proof cannot establish refuses rather than claim a stop it cannot see. | | `relaunch` | Replace the running agent with a new one in the same worktree - and the same endpoint whenever that endpoint still exists - on the exact recorded adapter or an explicitly chosen harness, model, and effort. | The new agent is alive on the endpoint the task's record now names, and that record names the harness that is actually running. | An exit that delivers lifecycle input but cannot prove the agent stopped fails with `exit=unconfirmed`, reports the observed agent state and any interrupt cancellation claim, and never claims that nothing changed. @@ -80,7 +80,7 @@ A relaunch does take one session reference when the endpoint's own runtime recor A secondmate relaunch does not require one and never rewrites its standing charter. 4. **Stop the old agent** through the `exit` verb, with its postcondition. 5. **Launch the replacement** through its single owner, `bin/fm-spawn.sh --relaunch`, which reuses the recorded worktree instead of creating one, adopts the recorded endpoint when it still exists, clears the previous harness's per-task wiring, and arms a fresh busy generation. - When the recorded endpoint is proven gone rather than merely idle or unreachable - which only Herdr can establish - the launch owner creates one fresh endpoint in that same worktree and the republished record rebinds the task to it - see [Reclaiming a task whose endpoint is gone](#reclaiming-a-task-whose-endpoint-is-gone). + When the recorded endpoint is proven gone rather than merely idle or unreachable, the launch owner creates one fresh endpoint in that same worktree and the republished record rebinds the task to it - see [Reclaiming a task whose endpoint is gone](#reclaiming-a-task-whose-endpoint-is-gone). 6. **Preserve runtime-bound status authority where supported.** The endpoint's runtime may bind pane status to one session identity; the launch owner preserves it only when that runtime records a reference the replacement adapter can consume, and otherwise launches the ordinary fresh session. This reference is a launch input, never authority to send, close, or act on the pane. @@ -90,27 +90,34 @@ Switching harness is therefore one ordinary relaunch rather than a separate mech ### Reclaiming a task whose endpoint is gone -A Herdr pane or workspace can be destroyed out from under a live task by churn or a session restart. +A Herdr pane or workspace can be destroyed out from under a live task by churn or a session restart, and a tmux window or whole server can be closed or killed. The task's worktree, branch, commits, and uncommitted changes all survive that; only its terminal does not. - -**Reclaim is Herdr-only.** On tmux, both verbs refuse a `missing` endpoint, leaving it exactly as deadlocked as it was before this mechanism existed - deliberately, and with the reason stated rather than guessed past. +`bin/fm-control.sh relaunch` reclaims such a task on both state-verified backends, so no operator ever re-creates a terminal or edits the task record by hand. Two endpoint verdicts are agent-free, and both license a relaunch: - `dead` - the endpoint exists and confidently holds no agent. It is **adopted**, so the task keeps its exact recorded address. -- gone, **proven** - there is no endpoint and therefore no agent, and it cannot be adopted, so the launch owner **creates one fresh endpoint in the recorded worktree** and the republished record rebinds the task to it. +- gone, **proven** - this seat has nothing agent-free left to adopt, which each backend's own proof below establishes in its own terms: the launch owner **creates one fresh endpoint in the recorded worktree** and the republished record rebinds the task to it. That proof is its own step, because the classifier's `missing` is not one state: it conflates *the endpoint was destroyed* with *the endpoint is unreachable from here right now*. An unreachable endpoint can still hold the live agent a rebind would duplicate, so absence is proven and never inferred from a failed read - and whether it is provable at all is a property of the backend: -- **Herdr can prove it.** Every read goes through the adapter's `--session ` CLI, so the recheck starts and reads the session the *record* names, through that session's own socket. +- **Herdr proves the endpoint absent.** Every read goes through the adapter's `--session ` CLI, so the recheck starts and reads the session the *record* names, through that session's own socket. It starts that server (only the server: no workspace and no tab are created) and **re-reads the recorded pane**. `dead` means the pane survived the restart and is adopted after all, with no second tab; `alive` means the agent came back and refuses; only a second `missing` proves the pane itself did not survive ([`docs/herdr-backend.md`](herdr-backend.md) "Restart and liveness behavior"). That server start is a real side effect, and the parenthetical above does not cover it: when the recorded session's server no longer exists at all, the probe stands a fresh empty one up in order to ask, and nothing afterwards uses it. So in that state `exit` - which otherwise reads as a read-only inspection - leaves an idle herdr server behind. -- **tmux cannot.** `list-windows -a` describes only the tmux server the *current process* addresses (its `TMUX_TMPDIR`/socket), and a task record carries no socket identity for its endpoint. - A different but running server would answer "not anywhere" about a window it was never able to see, so a server-wide read cannot tell a destroyed window from one on a server this process cannot address. - There is no read available that closes that gap, so tmux always refuses - for a renamed session, a moved window, a foreign socket, and a dead server alike. +- **tmux proves the agent absent.** `list-windows` describes only the tmux server the *current process* addresses (its `TMUX_TMPDIR`/socket), and a task record carries no socket identity for its endpoint, so no tmux read can tell a destroyed window from one on a server this process cannot address. + What a replacement would actually collide with is an agent still working in the task's worktree, and the kernel records every process's working directory whichever terminal server holds it. + So the proof reads the process table instead (`fm_agent_process_worktree_scan` in `bin/fm-agent-process-lib.sh`, through `/proc`): no process of this user that classifies as a verified harness is working anywhere under the recorded worktree. + An agent found there refuses and names its process id, and so does any agent process whose working directory could not be read - a process that classifies as something other than an agent is skipped either way, so its unreadable directory refuses nothing. + `/proc` is the only reader, because it answers for one exact process and says so when it cannot: a host without it refuses every tmux absence proof, so **the tmux reclaim is Linux-only**. + On macOS, or any other host without `/proc`, a tmux task whose endpoint was destroyed cannot be reclaimed at all; the Herdr reclaim works on every platform, because its proof is a re-read of the recorded pane rather than a process scan. + The refusal says that plainly and names the only route left - a seat that still addresses the recorded tmux server, where the window classifies directly and no absence proof is needed - and says that once that server is gone there is no such seat and no route. + The replacement window opens on the tmux server and session this seat addresses, which is where an ordinary spawn from here would place it - even when the recorded session was read successfully and merely no longer holds the window. + So a tmux reclaim can move the task into the reclaiming seat's session, unlike the Herdr rebind below, which pins the session the record names. + A window already carrying the task's `fm-` name in that session refuses the create rather than being closed and replaced, unlike a Herdr husk: `fm-` carries no home identity and every seat outside tmux resolves to the one session `firstmate`, so closing it could destroy another home's deliberately preserved endpoint. The refusal names how to read that window, and says plainly that no control-plane command closes one this home's records do not name. + A window that outlived its agent is left alone whether it sits on a server this seat cannot address or under a session name the record no longer matches on this seat's own server: it holds no agent, and it is not the task's endpoint any more. Every transient or self-contradicting read stays `unreadable` or `ambiguous` and still refuses, so a momentary backend failure can never be mistaken for absence. @@ -120,7 +127,8 @@ That proof has one owner for the whole control plane (`fm_control_endpoint_absen What a reclaim is not: - It is **not a teardown**. The worktree is reused exactly as the previous agent left it; nothing unlanded is ever discarded, and the ordinary `--note` requirement still applies. -- It does **not** change the task's identity. The task id, its armed poll and registration, and its status log are untouched; only the endpoint binding in the record moves. +- It does **not** touch validation. A no-mistakes run belongs to the branch rather than to the agent that started it, so an active or parked run survives the reclaim untouched; nothing in the reclaim aborts, answers, or starts one. +- It does **not** change the task's identity. The task id, its armed poll and registration, and its status log are untouched; only the endpoint binding in the record moves - and on tmux that binding includes the session, because the replacement lands in the session this seat addresses rather than the recorded one. Its instructions are the one exception, and only in the way an ordinary relaunch already changes them: a ship or scout reclaim appends the required `--note` under a `## Progress note ()` heading in `data//brief.md`, so re-read that brief rather than assuming it is byte-identical - a reclaim that failed and was retried leaves one block per attempt. A secondmate's standing charter is never rewritten. - It is **not** a peer seat's operation. `fm-control` resolves an exact task id against **this** home's `state/`, so only the home that owns the task can reclaim it. @@ -134,11 +142,12 @@ The record is republished consistently and no work is lost, but the task's `herd The pane id necessarily changes (the pane did not survive), and the record follows it. A Herdr reclaim deliberately uses the flat container shape rather than presentation projection: projection is a presentation-only layout that is never endpoint or ownership authority, and flat is already the documented fallback for every recovery it cannot bind exactly ([`docs/herdr-backend.md`](herdr-backend.md)). -**Known limitation - a refusal before the record is republished leaves a stray husk pane** (follow-up bead `fm-herdr-rebind-leak-20260913`). -The rebind registers no abort cleanup, so a refusal in the window between the new tab being created and the record being republished leaves that pane behind while the record still names the old, gone one. +**Known limitation on Herdr - a refusal before the record is republished leaves a stray husk pane** (follow-up bead `fm-herdr-rebind-leak-20260913`). +The Herdr rebind registers no abort cleanup, so a refusal in the window between the new tab being created and the record being republished leaves that pane behind while the record still names the old, gone one. The stray pane holds a bare shell - the harness is not delivered until after publication - so the next reclaim cleans up after it: the re-created tab carries the same `fm-` label, `tab create` finds it, classifies it a husk, and closes and replaces it. That self-heals only when the retry resolves the *same* workspace, which the placement rule above does not guarantee. The worktree and the task's records are unaffected either way. +A tmux rebind does register that cleanup: a refusal between creating the fresh window and republishing the record closes that window, so a retry never meets a same-named window of its own making. ### Failure and rollback @@ -167,8 +176,10 @@ The worktree and the task's records are unaffected either way. - An ambiguous or unreadable endpoint state refuses. Only a positively classified state acts. - `exit`'s composer-empty check, above, is itself a fail-closed boundary that `relaunch` inherits by stopping the old agent through `exit`. -- `fm-spawn --relaunch` independently refuses unless the endpoint is positively agent-free - either a `dead` endpoint that survives, or a Herdr endpoint proven gone by the absence proof above - so a replacement can never join a live agent. - An `alive`, `ambiguous`, or `unreadable` verdict all refuse, and so does any endpoint whose absence is not provable, which on tmux is every `missing`; absence is claimed only from positive evidence of it. +- `fm-spawn --relaunch` independently refuses unless the endpoint is positively agent-free - either a `dead` endpoint that survives, or an endpoint proven gone by the absence proof above - so a replacement can never join a live agent. + An `alive`, `ambiguous`, or `unreadable` verdict all refuse; absence is claimed only from positive evidence of it. + Every such refusal names a command that acts on the state it read, spelled as the operator can run it: an `alive` endpoint points at `bin/fm-control.sh relaunch --note ""`, which stops that agent first, and an `ambiguous` or `unreadable` one points at `bin/fm-peek.sh ` and the same retry, because no lifecycle verb acts on it until it reads positively. + The `--note` placeholder is part of every printed `relaunch`, because a ship or scout relaunch refuses without one. It also requires the shell to be in the recorded worktree: tmux refuses immediately when it is not, while Herdr sends one `cd` to the recorded path and refuses unless a subsequent path read confirms the move. ## Capability matrix @@ -183,11 +194,15 @@ Backend capability comes from each adapter's real surface, not from a policy cho | cmux | yes | yes | yes | yes | no | | orca | no | yes | yes | no | no | +Reclaiming a destroyed endpoint is a narrower capability than recovery-grade agent state, because it needs a proof of absence rather than a reading of one endpoint. +Herdr has it on every platform: it re-reads the recorded pane through that session's own socket. +tmux has it on Linux only, because its proof is the `/proc` process scan above; on a host without `/proc` every tmux absence proof refuses, and no other backend proves absence at all. + Per-harness interrupt keys, repeat counts, composer clears, exit commands, and supported task kinds live in `bin/fm-control-lib.sh` and are exercised for every verified harness by `tests/fm-control.test.sh`, with adapters outside its lane pinning their control mechanics in their own harness suites. The empirical basis for each adapter's value is the `harness-adapters` skill's verification record for that adapter. ## Verification - `tests/fm-control.test.sh` - the adapter contract for its verified-harness lane (adapters outside the lane pin their control mechanics in their own harness suites), the backend capability matrix, exact-id scoping, the closed verb list, the busy, idle, dead, and idempotent lifecycle cases, and marker non-regression, all against a stubbed session provider. -- `tests/fm-control-relaunch.test.sh` - the relaunch transaction: identity preservation, harness switching, the progress note, checkpoint refusals, rollback after a failed launch, and the endpoint-absence proof both verbs share - the Herdr reclaim of a destroyed endpoint, and tmux refusing one it cannot prove absent. +- `tests/fm-control-relaunch.test.sh` - the relaunch transaction: identity preservation, harness switching, the progress note, checkpoint refusals, rollback after a failed launch, and the endpoint-absence proof both verbs share - the Herdr and tmux reclaim of a destroyed endpoint with its unlanded work and parked validation run intact, and the refusal while an agent still works in the worktree. - `tests/fm-control-herdr-smoke.test.sh` - the second state-verified backend against the real herdr binary, on an isolated throwaway lab session. diff --git a/docs/scripts.md b/docs/scripts.md index 70830d9a9a5..8c53281b4c1 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -65,7 +65,7 @@ The shared no-mistakes gate lifecycle boundary is summarized in [architecture.md | `fm-backend.sh` | Runtime-backend selection, meta helpers, selector resolution, and operation dispatch | | `fm-backend-hometag-lib.sh` | Shared per-installation home-tag derivation for zellij tab and cmux workspace titles | | `fm-composer-lib.sh` | Single fleet-wide owner of composer shapes, capability-aware screen classification, and verdicts | -| `fm-agent-process-lib.sh` | Backend-neutral harness-process name classifier shared by the tmux and herdr adapters | +| `fm-agent-process-lib.sh` | Backend-neutral harness-process name classifier shared by the tmux and herdr adapters, plus the worktree process scan the control plane's endpoint-absence proof reads | | `backends/tmux.sh` | Verified tmux session-provider adapter | | `backends/herdr.sh` | Herdr session-provider adapter with its own required CI lane | | `backends/zellij.sh` | Experimental zellij session-provider adapter | diff --git a/tests/fm-backend-tmux-smoke.test.sh b/tests/fm-backend-tmux-smoke.test.sh index aa1e07c3269..c7ddaa8d98c 100755 --- a/tests/fm-backend-tmux-smoke.test.sh +++ b/tests/fm-backend-tmux-smoke.test.sh @@ -43,6 +43,10 @@ cleanup_all() { SHIM_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-backend-smoke.XXXXXX") cat > "$SHIM_DIR/tmux" <&2 + exit 1 +fi exec "$REAL_TMUX" -L "$SOCKET" "\$@" SH chmod +x "$SHIM_DIR/tmux" @@ -61,17 +65,34 @@ TARGET="$SESSION:$WINDOW" tmux new-session -d -s "$SESSION" -x 200 -y 50 \ || fail "real tmux: new-session failed" -fm_backend_tmux_create_task "$SESSION" "$WINDOW" "$HOME" \ +FIRST_WID=$(fm_backend_tmux_create_task "$SESSION" "$WINDOW" "$HOME") \ || fail "fm_backend_tmux_create_task failed to create the task window" tmux list-windows -t "$SESSION" -F '#{window_name}' | grep -qx "$WINDOW" \ || fail "created window is not visible in the real session" # A second create for the SAME window name must refuse (mirrors fm-spawn.sh's -# duplicate-window guard). +# duplicate-window guard). The existing window is left exactly as it was: the +# name carries no home identity, so closing it could destroy an endpoint that +# another home's record still names and deliberately preserved. if fm_backend_tmux_create_task "$SESSION" "$WINDOW" "$HOME" 2>/dev/null; then fail "fm_backend_tmux_create_task should refuse an existing window name" fi -pass "real tmux: fm_backend_tmux_create_task creates a window and refuses a duplicate" +[ "$(tmux list-windows -t "$SESSION" -F '#{window_id}' | grep -cx "$FIRST_WID")" = 1 ] \ + || fail "a refused create must leave the existing window open" +[ "$(tmux list-windows -t "$SESSION" -F '#{window_name}' | grep -cx "$WINDOW")" = 1 ] \ + || fail "a refused create must not add a second window under the name" +pass "real tmux: fm_backend_tmux_create_task creates a window and refuses a duplicate without closing it" + +# An inventory read that FAILED is not evidence that the name is free. tmux +# permits duplicate window names, so reading a failed read as "no duplicate" +# would put a second window under one name and leave every later name-targeted +# command resolving to whichever one tmux matches first. +if FM_SMOKE_LIST_WINDOWS_FAILS=1 fm_backend_tmux_create_task "$SESSION" "$WINDOW" "$HOME" 2>/dev/null; then + fail "fm_backend_tmux_create_task should refuse when the window inventory cannot be read" +fi +[ "$(tmux list-windows -t "$SESSION" -F '#{window_name}' | grep -cx "$WINDOW")" = 1 ] \ + || fail "a create refused for an unreadable inventory must not add a second window under the name" +pass "real tmux: fm_backend_tmux_create_task refuses an unreadable window inventory rather than assuming the name is free" # --- send text + Enter ------------------------------------------------------- diff --git a/tests/fm-control-relaunch.test.sh b/tests/fm-control-relaunch.test.sh index 75cbd9756e3..d750823f66a 100755 --- a/tests/fm-control-relaunch.test.sh +++ b/tests/fm-control-relaunch.test.sh @@ -166,6 +166,9 @@ case "${1:-}" in *) shift ;; esac done + # Creating the window stands its session (and server) back up, as a real + # new-window after new-session does. + rm -f "$D/server-dead" "$D/session-missing" printf '%s\n' "$name" >> "$D/windows" printf '%s\n' "$name" >> "$D/created-windows" printf '@9\n' @@ -236,7 +239,7 @@ run_control() { # # without it this suite would write the developer's real ~/.claude.json. mkdir -p "$dir/user-home" env -u HERDR_ENV -u HERDR_PANE_ID -u HERDR_SESSION -u HERDR_SOCKET_PATH \ - -u HERDR_TAB_ID -u HERDR_WORKSPACE_ID \ + -u HERDR_TAB_ID -u HERDR_WORKSPACE_ID -u TMUX \ PATH="$dir/fakebin:$PATH" FM_HOME="$dir/home" FM_FAKE_DIR="$dir/fake" \ HOME="$dir/user-home" CLAUDE_CONFIG_DIR='' \ FM_SPAWN_NO_GUARD=1 GROK_HOME="$dir/grokhome" \ @@ -258,7 +261,7 @@ run_spawn() { # # without it this suite would write the developer's real ~/.claude.json. mkdir -p "$dir/user-home" env -u HERDR_ENV -u HERDR_PANE_ID -u HERDR_SESSION -u HERDR_SOCKET_PATH \ - -u HERDR_TAB_ID -u HERDR_WORKSPACE_ID \ + -u HERDR_TAB_ID -u HERDR_WORKSPACE_ID -u TMUX \ PATH="$dir/fakebin:$PATH" FM_HOME="$dir/home" FM_FAKE_DIR="$dir/fake" \ HOME="$dir/user-home" CLAUDE_CONFIG_DIR='' \ FM_SPAWN_NO_GUARD=1 GROK_HOME="$dir/grokhome" \ @@ -1702,8 +1705,15 @@ test_spawn_relaunch_refuses_a_live_agent() { out=$(run_spawn "$dir" rl15 --relaunch --harness claude); rc=$? expect_code 1 "$rc" "relaunching into a live endpoint should refuse" assert_contains "$out" "positively agent-free endpoint" "the refusal should demand an agent-free endpoint" - assert_contains "$out" "fm-control.sh rl15 exit" "the refusal should point at the way to stop it" - pass "fm-spawn --relaunch: refuses to launch a second agent into a live endpoint" + assert_contains "$out" 'fm-control.sh rl15 relaunch --note ""' \ + "the refusal should name the one command that replaces a live agent, spelled as it can be run" + [ ! -s "$dir/fake/literal" ] || fail "a refused relaunch must send nothing into the live endpoint" + # The named command must actually work for this state, rather than send the + # operator on to another refusal: a ship relaunch refuses without --note, so + # the printed command has to carry it. + out=$(run_control "$dir" rl15 relaunch --note "replace the live agent"); rc=$? + expect_code 0 "$rc" "the command the refusal names should replace the live agent"$'\n'"$out" + pass "fm-spawn --relaunch: refuses a live endpoint and names the command that replaces its agent" } test_spawn_relaunch_refuses_a_symlinked_task_record_before_inspection() { @@ -1831,13 +1841,12 @@ test_spawn_relaunch_refuses_a_pane_outside_the_worktree() { # --- 7. reclaiming a task whose endpoint is gone ---------------------------- # -# Before this, `missing` was a terminal state: fm-spawn --relaunch accepted only -# `dead` and told the caller to stop the agent first, while fm-control exit -# refused `missing` outright and told the caller to reconcile the task first - -# and there is no reconcile verb. Each command named the other as its -# prerequisite, so a task whose pane or workspace was destroyed could not be -# reclaimed by anything, and any no-mistakes approval it was parked on had no -# seat left to answer it. +# A task whose endpoint was destroyed rather than stopped used to be terminal: +# fm-spawn --relaunch accepted only `dead` and told the caller to stop the agent +# first, while fm-control exit refused `missing` outright and told the caller to +# reconcile the task first - and there is no reconcile verb. Each command named +# the other as its prerequisite, so the task could not be reclaimed by anything, +# and any no-mistakes approval it was parked on had no seat left to answer it. # strand_endpoint : make a tmux endpoint read `missing` the way # a destroyed window does - a successful session inventory that omits the exact @@ -1846,67 +1855,212 @@ strand_endpoint() { # : > "$1/fake/windows" } -# Every tmux `missing` refuses on BOTH verbs, whatever produced it. tmux is the -# one verified backend whose absence cannot be proven from a task record: the -# record carries no socket identity for the endpoint, and any inventory -# describes only the server this process happens to address. So a window that -# is merely on a server this seat cannot reach is indistinguishable from one -# that was destroyed, and neither verb will guess. -assert_tmux_missing_refuses() { # - local dir=$1 id=$2 what=$3 out rc brief_before - - out=$(run_spawn "$dir" "$id" --relaunch --harness claude); rc=$? - expect_code 1 "$rc" "relaunch must refuse a tmux endpoint whose absence cannot be proven ($what)"$'\n'"$out" - assert_absent "$dir/fake/created-windows" "a refused relaunch must not create a window ($what)" - assert_absent "$dir/fake/created-sessions" "a refused relaunch must not create a session ($what)" - [ ! -s "$dir/fake/literal" ] || fail "a refused relaunch must send nothing into any pane ($what)" - - brief_before=$(cat "$dir/home/data/$id/brief.md") - out=$(run_control "$dir" "$id" exit); rc=$? - expect_code 1 "$rc" "exit must refuse a tmux endpoint whose absence cannot be proven ($what)"$'\n'"$out" - assert_not_contains "$out" "endpoint-gone" \ - "exit must not report a stop it cannot see ($what)" - [ ! -s "$dir/fake/literal" ] || fail "a refused exit must send nothing into any pane ($what)" +# A no-mistakes that only records being called, so a reclaim can prove it never +# aborted, answered, or started the validation run the task is parked on; a +# read-only status query is allowed. +make_nm_recorder() { # + cat > "$1/fakebin/no-mistakes" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$FM_FAKE_DIR/nm-calls" +exit 0 +SH + chmod +x "$1/fakebin/no-mistakes" +} + +# start_worktree_agent : a real process working in the task's +# worktree whose argv[0] names a verified harness - what the tmux absence proof +# reads, whichever terminal server holds it. Sets WORKTREE_AGENT_PID. +start_worktree_agent() { # + (cd "$1/wt" && exec -a claude /bin/sleep 60) & + WORKTREE_AGENT_PID=$! + fm_wait_for_agent_argv0 "$WORKTREE_AGENT_PID" \ + || fail "the planted worktree agent never reported a harness argv[0]" +} + +stop_worktree_agent() { + [ -z "${WORKTREE_AGENT_PID:-}" ] || kill "$WORKTREE_AGENT_PID" 2>/dev/null || true + wait "$WORKTREE_AGENT_PID" 2>/dev/null || true + WORKTREE_AGENT_PID= +} + +# assert_tmux_reclaims : a tmux endpoint that +# is gone, with no agent left working in the worktree, is reclaimed through +# the ordinary control plane - `exit` reports it gone, `relaunch` opens one +# fresh window in the SAME worktree under the SAME task identity, and the +# record follows the new window without anyone editing it. Everything else +# about the task, including unlanded work and a parked validation run, is left +# exactly where it was. +assert_tmux_reclaims() { # + local dir=$1 id=$2 what=$3 out rc=0 head_before window + make_nm_recorder "$dir" + printf 'landed on the branch\n' > "$dir/wt/committed.txt" + git -C "$dir/wt" add committed.txt + git -C "$dir/wt" -c user.email=t@example.com -c user.name=t commit -qm "work in progress" + head_before=$(git -C "$dir/wt" rev-parse HEAD) + printf 'never committed\n' > "$dir/wt/dirty.txt" + printf 'working: parked at a validation gate\n' >> "$dir/home/state/$id.status" + + out=$(run_control "$dir" "$id" exit) || rc=$? + expect_code 0 "$rc" "exit must report a proven-gone tmux endpoint rather than refuse it ($what)"$'\n'"$out" + assert_contains "$out" "endpoint-gone $id" "exit must say the endpoint did not survive ($what)" + [ ! -s "$dir/fake/literal" ] || fail "exit must send nothing when there is no endpoint ($what)" + + out=$(run_control "$dir" "$id" relaunch --note "the window was destroyed; pick the work back up") || rc=$? + expect_code 0 "$rc" "relaunch must reclaim a proven-gone tmux endpoint ($what)"$'\n'"$out" + assert_contains "$(cat "$dir/fake/created-windows")" "fm-$id" \ + "the reclaim must open one fresh window for the task ($what)" + [ "$(grep -c . "$dir/fake/created-windows")" = 1 ] \ + || fail "the reclaim must open exactly one window ($what)" + window=$(meta_field "$dir" "$id" window) + case "$window" in + *":fm-$id") ;; + *) fail "the record must rebind to the new window, got '$window' ($what)" ;; + esac + [ "$(meta_field "$dir" "$id" endpoint_task_id)" = "$id" ] \ + || fail "the reclaim must keep the task identity ($what)" + [ "$(meta_field "$dir" "$id" worktree)" = "$dir/wt" ] \ + || fail "the reclaim must keep the recorded worktree ($what)" + assert_contains "$(cat "$dir/fake/literal")" "launch-brief" \ + "the replacement agent must be launched into the new window ($what)" - out=$(run_control "$dir" "$id" relaunch --note "this note must never reach a live agent"); rc=$? - expect_code 1 "$rc" "the relaunch transaction must fail closed ($what)"$'\n'"$out" - [ "$(cat "$dir/home/data/$id/brief.md")" = "$brief_before" ] \ - || fail "a refused relaunch edited instructions an agent that may still be running is reading ($what)" - assert_absent "$dir/fake/created-windows" "a refused transaction must not create a window ($what)" - assert_absent "$dir/fake/created-sessions" "a refused transaction must not create a session ($what)" - [ ! -s "$dir/fake/literal" ] || fail "a refused transaction must launch nothing ($what)" + [ "$(git -C "$dir/wt" rev-parse HEAD)" = "$head_before" ] \ + || fail "the reclaim moved the worktree's HEAD ($what)" + assert_contains "$(cat "$dir/wt/dirty.txt")" "never committed" \ + "the reclaim destroyed or rewrote an uncommitted change ($what)" + assert_contains "$(cat "$dir/home/state/$id.status")" "parked at a validation gate" \ + "the reclaim truncated the status log ($what)" + + # Reading the run is fine; acting on it is not. + if [ -f "$dir/fake/nm-calls" ] && grep -Eq '(^|[[:space:]])(run|respond|abort|sync|cancel|rerun)([[:space:]]|$)' "$dir/fake/nm-calls"; then + fail "the reclaim must never abort, answer, or start the task's validation run ($what): $(cat "$dir/fake/nm-calls")" + fi + assert_contains "$(cat "$dir/home/data/$id/brief.md")" "the window was destroyed" \ + "the replacement must inherit the progress note ($what)" + [ "$(journal_field "$dir" "$id" exit_result)" = endpoint-gone ] \ + || fail "the transaction should record that the endpoint was already gone ($what)" } -test_tmux_refuses_a_window_missing_from_its_session() { +test_tmux_reclaims_a_window_missing_from_its_session() { local dir + fm_proc_scan_available || { echo "skip - the tmux absence proof reads /proc"; return 0; } dir=$(new_case tmux-gone rl60) add_ship_task "$dir" rl60 claude strand_endpoint "$dir" rl60 - assert_tmux_missing_refuses "$dir" rl60 "window absent from a readable session inventory" - pass "tmux: a window absent from its session refuses both verbs rather than being assumed gone" + assert_tmux_reclaims "$dir" rl60 "window absent from a readable session inventory" + pass "tmux: a destroyed window is reclaimed in the same worktree under the same task" } -test_tmux_refuses_a_session_that_cannot_be_found() { +test_tmux_reclaims_a_session_that_cannot_be_found() { local dir + fm_proc_scan_available || { echo "skip - the tmux absence proof reads /proc"; return 0; } dir=$(new_case tmux-nosession rl61) add_ship_task "$dir" rl61 claude - # Real tmux's answer to a renamed session, and to a different - # TMUX_TMPDIR/socket: definitive about the SESSION, silent about whether the - # window and its agent survived elsewhere. : > "$dir/fake/session-missing" - assert_tmux_missing_refuses "$dir" rl61 "recorded session not found" - pass "tmux: an unfindable session refuses both verbs, so a live agent is never duplicated" + assert_tmux_reclaims "$dir" rl61 "recorded session not found" + pass "tmux: a destroyed session is reclaimed in the same worktree under the same task" } -test_tmux_refuses_when_the_server_is_gone() { +test_tmux_reclaims_when_the_server_is_gone() { local dir + fm_proc_scan_available || { echo "skip - the tmux absence proof reads /proc"; return 0; } dir=$(new_case tmux-noserver rl62) add_ship_task "$dir" rl62 claude - # No server on the socket this process addresses. Another server may still be - # running the task's window, and the record cannot say which socket is its. : > "$dir/fake/server-dead" - assert_tmux_missing_refuses "$dir" rl62 "no tmux server on this socket" - pass "tmux: a dead server on this socket refuses both verbs rather than proving absence" + assert_tmux_reclaims "$dir" rl62 "no tmux server on this socket" + pass "tmux: a dead server's task is reclaimed in the same worktree under the same task" +} + +test_spawn_relaunch_alone_reclaims_a_gone_tmux_endpoint() { + local dir out rc=0 + fm_proc_scan_available || { echo "skip - the tmux absence proof reads /proc"; return 0; } + dir=$(new_case tmux-spawn-gone rl64) + add_ship_task "$dir" rl64 claude + strand_endpoint "$dir" rl64 + # The launch owner reaches the same verdict on its own, so the two commands + # can never disagree about one endpoint again. + out=$(run_spawn "$dir" rl64 --relaunch --harness claude) || rc=$? + expect_code 0 "$rc" "fm-spawn --relaunch must reclaim a proven-gone tmux endpoint"$'\n'"$out" + assert_contains "$(cat "$dir/fake/created-windows")" "fm-rl64" \ + "the launch owner must open the task's fresh window" + [ "$(meta_field "$dir" rl64 worktree)" = "$dir/wt" ] \ + || fail "the launch owner must keep the recorded worktree" + pass "fm-spawn --relaunch: a proven-gone tmux endpoint is reclaimed by the launch owner itself" +} + +# An agent process still working in the worktree means the window may only be +# on a tmux server this seat cannot address; neither verb may guess past it. +test_tmux_refuses_a_gone_window_while_an_agent_works_in_the_worktree() { + local dir out rc brief_before + fm_proc_scan_available || { echo "skip - the tmux absence proof reads /proc"; return 0; } + dir=$(new_case tmux-gone-agent rl65) + add_ship_task "$dir" rl65 claude + strand_endpoint "$dir" rl65 + start_worktree_agent "$dir" + + out=$(run_spawn "$dir" rl65 --relaunch --harness claude); rc=$? + expect_code 1 "$rc" "relaunch must refuse while an agent still works in the worktree"$'\n'"$out" + assert_contains "$out" "agent process $WORKTREE_AGENT_PID" \ + "the refusal must name the agent process it found" + assert_contains "$out" "stop that process where it runs" \ + "the refusal must name what would have to change before the task can be reclaimed" + brief_before=$(cat "$dir/home/data/rl65/brief.md") + out=$(run_control "$dir" rl65 exit); rc=$? + expect_code 1 "$rc" "exit must not claim a stop while an agent still works in the worktree"$'\n'"$out" + assert_not_contains "$out" "endpoint-gone" "exit must not report a stop it cannot see" + out=$(run_control "$dir" rl65 relaunch --note "this note must never reach a live agent"); rc=$? + expect_code 1 "$rc" "the relaunch transaction must refuse"$'\n'"$out" + [ "$(cat "$dir/home/data/rl65/brief.md")" = "$brief_before" ] \ + || fail "a refused relaunch edited instructions a live agent may be reading" + assert_absent "$dir/fake/created-windows" "a refused reclaim must not create a window" + [ ! -s "$dir/fake/literal" ] || fail "a refused reclaim must launch nothing" + kill -0 "$WORKTREE_AGENT_PID" 2>/dev/null || fail "a refused reclaim must leave the working agent alone" + stop_worktree_agent + # Once that agent is gone, the command the refusal named does reclaim it. + out=$(run_control "$dir" rl65 relaunch --note "the agent has stopped; pick the work back up"); rc=$? + expect_code 0 "$rc" "the command the refusal named should reclaim the task once no agent works there"$'\n'"$out" + pass "tmux: a gone window is refused while an agent works in its worktree, then reclaimed by the command the refusal names" +} + +# The tmux absence proof reads the kernel's per-process working directories, so +# pin what it counts: only a verified harness, anywhere under the worktree, and +# never a sibling directory that merely shares its prefix. +test_worktree_agent_scan_counts_only_agents_inside_the_worktree() { + local base wt scan pid_plain pid_sub pid_sibling + fm_proc_scan_available || { echo "skip - the worktree scan reads /proc"; return 0; } + base="$TMP_ROOT/scan-$RANDOM" + wt="$base/wt" + mkdir -p "$wt/sub" "$base/wt-sibling" + # shellcheck source=/dev/null + . "$ROOT/bin/fm-agent-process-lib.sh" + + (cd "$wt" && exec /bin/sleep 60) & + pid_plain=$! + (cd "$base/wt-sibling" && exec -a claude /bin/sleep 60) & + pid_sibling=$! + fm_wait_for_agent_argv0 "$pid_sibling" \ + || fail "the sibling-directory agent never reported a harness argv[0], so excluding it would prove nothing" + scan=$(fm_agent_process_worktree_scan "$wt") + [ "$scan" = "none"$'\t' ] \ + || fail "a non-agent process, or an agent in a sibling directory, must not count, got '$scan'" + + (cd "$wt/sub" && exec -a claude /bin/sleep 60) & + pid_sub=$! + fm_wait_for_agent_argv0 "$pid_sub" \ + || fail "the subdirectory agent never reported a harness argv[0], so counting it would prove nothing" + scan=$(fm_agent_process_worktree_scan "$wt") + case "$scan" in + "agent"$'\t'"$pid_sub "*) ;; + *) fail "an agent working in a subdirectory of the worktree must count, got '$scan'" ;; + esac + + scan=$(fm_agent_process_worktree_scan "$base/never-created") + case "$scan" in + unreadable$'\t'*) ;; + *) fail "a worktree that cannot be resolved must read unreadable, got '$scan'" ;; + esac + kill "$pid_plain" "$pid_sub" "$pid_sibling" 2>/dev/null || true + wait "$pid_plain" "$pid_sub" "$pid_sibling" 2>/dev/null || true + pass "worktree agent scan: only a verified harness inside the worktree counts" } test_reclaim_refuses_an_unreadable_endpoint() { @@ -1922,9 +2076,17 @@ test_reclaim_refuses_an_unreadable_endpoint() { expect_code 1 "$rc" "an unreadable endpoint must still refuse" assert_contains "$out" "positively agent-free endpoint" \ "only a POSITIVELY proven agent-free endpoint may be relaunched into" + assert_not_contains "$out" "fm-control.sh rl63 exit" \ + "the refusal must not send the operator to a verb that refuses this state too" + assert_contains "$out" "bin/fm-peek.sh rl63" \ + "the refusal must name how to see why the endpoint cannot be read" assert_absent "$dir/fake/created-windows" \ "a refused relaunch must not create an endpoint" [ ! -s "$dir/fake/literal" ] || fail "a refused relaunch must launch nothing" + out=$(run_control "$dir" rl63 exit); rc=$? + expect_code 1 "$rc" "exit must refuse an unreadable endpoint too" + assert_contains "$out" "bin/fm-peek.sh rl63" \ + "exit's refusal must give the same next step as relaunch's" pass "reclaim: an unclassifiable endpoint is still refused, so two agents cannot share one" } @@ -2443,9 +2605,12 @@ test_spawn_relaunch_refuses_a_pending_authoritative_close test_spawn_relaunch_refuses_contradicting_flags test_spawn_relaunch_refuses_an_unrecorded_task test_spawn_relaunch_refuses_a_pane_outside_the_worktree -test_tmux_refuses_a_window_missing_from_its_session -test_tmux_refuses_a_session_that_cannot_be_found -test_tmux_refuses_when_the_server_is_gone +test_tmux_reclaims_a_window_missing_from_its_session +test_tmux_reclaims_a_session_that_cannot_be_found +test_tmux_reclaims_when_the_server_is_gone +test_spawn_relaunch_alone_reclaims_a_gone_tmux_endpoint +test_tmux_refuses_a_gone_window_while_an_agent_works_in_the_worktree +test_worktree_agent_scan_counts_only_agents_inside_the_worktree test_reclaim_refuses_an_unreadable_endpoint test_herdr_relaunch_resumes_only_the_registered_pi_session test_herdr_reclaim_adopts_a_pane_that_outlived_its_server diff --git a/tests/fm-control.test.sh b/tests/fm-control.test.sh index 832c3fd7a49..db8b8ed748d 100755 --- a/tests/fm-control.test.sh +++ b/tests/fm-control.test.sh @@ -778,23 +778,35 @@ test_already_stopped_exit_is_idempotent() { pass "fm-control exit: an already-stopped agent is idempotent success with no bytes sent" } -test_missing_tmux_endpoint_refuses_rather_than_claiming_a_stop() { - local dir out rc +test_missing_tmux_endpoint_reports_gone_only_when_no_agent_works_there() { + local dir out rc agent + fm_proc_scan_available || { echo "skip - the tmux absence proof reads /proc"; return 0; } dir=$(new_case gone) add_task "$dir" t1 claude : > "$dir/fake/windows" + # `missing` on tmux is not on its own a finding about the endpoint: a task + # record carries no socket identity, so a window merely on a server this seat + # cannot reach reads the same as one that was destroyed. The absence proof + # asks what a duplicate agent would collide with instead - an agent process + # still working in the recorded worktree (docs/agent-control.md "Reclaiming a + # task whose endpoint is gone"). + (cd "$dir/wt-t1" && exec -a claude /bin/sleep 60) & + agent=$! + fm_wait_for_agent_argv0 "$agent" \ + || fail "the planted agent never reported a harness argv[0], so the refusal below would prove nothing" out=$(run_control "$dir" t1 exit); rc=$? - # `missing` on tmux is not a finding about the endpoint. A task record carries - # no socket identity for it, and any inventory describes only the tmux server - # this process addresses, so a window that is merely on a server this seat - # cannot reach is indistinguishable from one that was destroyed. exit refuses - # rather than claim a stop it cannot see, and sends nothing to an address it - # cannot trust. Reclaim of a destroyed endpoint is Herdr-only - # (docs/agent-control.md "Reclaiming a task whose endpoint is gone"). - expect_code 1 "$rc" "a tmux endpoint whose absence cannot be proven must refuse" + kill "$agent" 2>/dev/null || true + wait "$agent" 2>/dev/null || true + expect_code 1 "$rc" "a tmux endpoint whose absence cannot be proven must refuse"$'\n'"$out" assert_not_contains "$out" "endpoint-gone" "exit must not report a stop it could not prove" + assert_contains "$out" "agent process $agent" "the refusal must name the agent it found" [ -z "$(literals "$dir")" ] || fail "nothing may be sent into an endpoint exit cannot trust" - pass "fm-control exit: an unprovable tmux endpoint refuses instead of claiming the agent stopped" + + out=$(run_control "$dir" t1 exit); rc=$? + expect_code 0 "$rc" "a gone tmux endpoint with no agent in its worktree is proven gone"$'\n'"$out" + assert_contains "$out" "endpoint-gone t1" "exit must say the endpoint did not survive" + [ -z "$(literals "$dir")" ] || fail "nothing may be sent when there is no endpoint" + pass "fm-control exit: a gone tmux endpoint reports endpoint-gone only once no agent works in its worktree" } test_interrupt_refuses_when_no_agent_runs() { @@ -817,8 +829,18 @@ test_ambiguous_endpoint_refuses() { out=$(run_control "$dir" t1 exit); rc=$? expect_code 1 "$rc" "an unattributed endpoint should refuse" assert_contains "$out" "positively classified" "the refusal should name the missing attribution" + assert_contains "$out" "retry bin/fm-control.sh t1 exit" \ + "the refusal must name the verb the operator invoked" + assert_not_contains "$out" "bin/fm-control.sh t1 relaunch" \ + "an exit refusal must not send the operator to the verb that launches a replacement" + [ -z "$(literals "$dir")" ] || fail "an unattributed endpoint must receive no bytes" + + out=$(run_control "$dir" t1 relaunch --note "the pane holds something unattributable"); rc=$? + expect_code 1 "$rc" "the same unattributed endpoint must refuse a relaunch too" + assert_contains "$out" 'retry bin/fm-control.sh t1 relaunch --note ""' \ + "a relaunch refusal must still name relaunch, with the note it requires" [ -z "$(literals "$dir")" ] || fail "an unattributed endpoint must receive no bytes" - pass "fm-control exit: an endpoint whose process cannot be attributed refuses" + pass "fm-control: an endpoint whose process cannot be attributed refuses and names the invoked verb" } test_busy_agent_is_interrupted_before_the_exit_command() { @@ -1095,7 +1117,7 @@ test_verb_allowlist_is_closed test_resume_is_refused_with_its_reason test_relaunch_only_flags_are_rejected_on_other_verbs test_already_stopped_exit_is_idempotent -test_missing_tmux_endpoint_refuses_rather_than_claiming_a_stop +test_missing_tmux_endpoint_reports_gone_only_when_no_agent_works_there test_interrupt_refuses_when_no_agent_runs test_ambiguous_endpoint_refuses test_busy_agent_is_interrupted_before_the_exit_command diff --git a/tests/lib.sh b/tests/lib.sh index d56373d1b10..aee7dd813f7 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -495,6 +495,34 @@ SH chmod +x "$fakebin/$tool" } +# --- planted processes ------------------------------------------------------- + +# fm_proc_scan_available: whether this host can run the tmux absence proof at +# all. fm_agent_process_worktree_scan reads /proc and refuses everywhere else, +# so a case that needs a positive `none` or `agent` verdict has nothing to pin +# on a host without it. +fm_proc_scan_available() { + [ -r /proc/self/cmdline ] && [ -L /proc/self/cwd ] +} + +# fm_wait_for_agent_argv0 : block until reports a verified harness +# name as its argv[0], which is the identity the process-table absence proof +# reads (bin/fm-agent-process-lib.sh). A process planted as +# `(cd && exec -a claude /bin/sleep 60) &` is still the forking shell +# until its exec lands, and a shell classifies as `shell` rather than `agent`, +# so any reader of the process table must wait for the identity itself instead +# of for a fixed interval that a loaded host can overrun. Returns nonzero if it +# never appears, so a caller can fail with that as the reason. +fm_wait_for_agent_argv0() { # + local pid=$1 _ + for _ in $(seq 1 100); do + [ "$(tr '\0' '\n' 2>/dev/null < "/proc/$pid/cmdline" | head -n 1)" = claude ] && return 0 + ps -o args= -p "$pid" 2>/dev/null | grep -q '^claude' && return 0 + /bin/sleep 0.05 + done + return 1 +} + # --- portable file timestamps ----------------------------------------------- # fm_touch_epoch [path...]: set each path's modification time to