Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .agents/skills/stuck-crewmate-recovery/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@ Do not sweep another home's endpoints or infer ownership from a matching window

Before relaunch, prove that no live agent still owns the recorded task and that the existing worktree remains available.
Preserve its uncommitted changes and commits, keep the same task identity, and resume or relaunch the recorded harness in that existing worktree with the same brief plus a concise progress note.
A HERDR endpoint that is not merely idle but destroyed - a pane or workspace removed in Herdr churn - is recovered by that same relaunch, which creates one fresh endpoint in the existing worktree and rebinds the task's record to it; nothing special is needed, and the worktree is untouched ([`docs/agent-control.md`](../../../docs/agent-control.md) "Reclaiming a task whose endpoint is gone").
That relaunch proves the endpoint is destroyed before it rebinds, so a Herdr server that was merely stopped is adopted back rather than duplicated.
For a missing HERDR endpoint, follow [Reclaiming a task whose endpoint is gone](../../../docs/agent-control.md#reclaiming-a-task-whose-endpoint-is-gone), which owns the absence proof and recovery path.
On tmux there is no reclaim: a task record carries no socket identity for its endpoint, so a `missing` window cannot be told apart from one on a tmux server this seat cannot address, and both `exit` and `relaunch` refuse.
Do not work around either refusal by respawning - it means a live agent may still hold that worktree.
That reclaim is the owning home's operation only, and a secondmate is the one exception: recover it through `bin/fm-spawn.sh <id> --secondmate` as above.
Expand Down
218 changes: 160 additions & 58 deletions bin/backends/herdr.sh

Large diffs are not rendered by default.

43 changes: 38 additions & 5 deletions bin/fm-backend.sh
Original file line number Diff line number Diff line change
Expand Up @@ -748,11 +748,33 @@ fm_backend_resolve_selector() { # <raw-target> <state-dir>
# at every call site. Each verified backend adds its own arm here, without
# changing call sites.

fm_backend_endpoint_ready() {
if [ "$1" = herdr ]; then
fm_backend_herdr_target_ready "$2" || return 1
fi
! fm_backend_endpoint_foreign "$@"
}

fm_backend_endpoint_foreign() { # <backend> <target> [expected-label]
local target=$2 session pane
[ "$1" = herdr ] || return 1
declare -F fm_backend_herdr_endpoint_foreign >/dev/null 2>&1 || return 1
session=${target%%:*}
pane=${target#*:}
[ -n "$session" ] && [ -n "$pane" ] && [ "$pane" != "$target" ] || return 1
if fm_backend_herdr_endpoint_foreign "$session" "$pane" "${3:-}"; then
return 0
else
[ "$?" -ne 1 ]
fi
}

# fm_backend_capture: bounded plain-text session capture.
fm_backend_capture() { # <backend> <target> <lines> [expected-label]
local backend=$1
shift
fm_backend_source "$backend" || return 1
fm_backend_endpoint_ready "$backend" "$1" "${3:-}" || return 1
case "$backend" in
tmux) fm_backend_tmux_capture "$@" ;;
herdr) fm_backend_herdr_capture "$@" ;;
Expand Down Expand Up @@ -790,6 +812,7 @@ fm_backend_visible_capture() { # <backend> <target> [expected-label]
return 1
}
fm_backend_source "$backend" || return 1
fm_backend_endpoint_ready "$backend" "$1" "${2:-}" || return 1
"fm_backend_${backend}_visible_capture" "$@"
}

Expand All @@ -798,6 +821,7 @@ fm_backend_send_key() { # <backend> <target> <key> [expected-label]
local backend=$1
shift
fm_backend_source "$backend" || return 1
fm_backend_endpoint_ready "$backend" "$1" "${3:-}" || return 1
case "$backend" in
tmux) fm_backend_tmux_send_key "$@" ;;
herdr) fm_backend_herdr_send_key "$@" ;;
Expand All @@ -815,6 +839,7 @@ fm_backend_send_text_submit() { # <backend> <target> <text> <retries> <enter-sl
local backend=$1
shift
fm_backend_source "$backend" || return 1
fm_backend_endpoint_ready "$backend" "$1" "${6:-}" || return 1
case "$backend" in
tmux) fm_backend_tmux_send_text_submit "$@" ;;
herdr) fm_backend_herdr_send_text_submit "$@" ;;
Expand Down Expand Up @@ -883,6 +908,7 @@ fm_backend_busy_state() { # <backend> <target>
local backend=$1
shift
fm_backend_source "$backend" || { printf 'unknown'; return 0; }
fm_backend_endpoint_ready "$backend" "$1" || { printf 'unknown'; return 0; }
case "$backend" in
herdr) fm_backend_herdr_busy_state "$@" ;;
*) printf 'unknown' ;;
Expand All @@ -905,6 +931,7 @@ fm_backend_composer_state() { # <backend> <target> [expected-label] -> empty|pe
local backend=$1
shift
fm_backend_source "$backend" || { printf 'unknown'; return 0; }
fm_backend_endpoint_ready "$backend" "$1" "${2:-}" || { printf 'unknown'; return 0; }
case "$backend" in
tmux) fm_tmux_composer_state "$@" ;;
herdr) fm_backend_herdr_composer_state "$@" ;;
Expand Down Expand Up @@ -938,6 +965,7 @@ fm_backend_target_exists() { # <backend> <target> [expected-label]
session=${target%%:*}
pane=${target#*:}
[ -n "$session" ] && [ -n "$pane" ] && [ "$pane" != "$target" ] || return 1
! fm_backend_endpoint_foreign herdr "$target" "$expected_label" || return 1
# fm_backend_herdr_cli (not a raw HERDR_SESSION-only call): verified
# empirically (docs/herdr-backend.md "Session targeting") that the bare
# env var alone is NOT reliably honored once another herdr server is
Expand Down Expand Up @@ -979,28 +1007,33 @@ fm_backend_target_exists() { # <backend> <target> [expected-label]
# process level through the shared classifier in bin/fm-agent-process-lib.sh,
# never from a registration or a rendered title alone. The tmux adapter
# requires a successful session inventory and returns `missing` only when it
# omits the exact window; the Herdr adapter reuses its strict husk classifier -
# omits the exact window; Herdr first applies recorded-endpoint ownership
# (docs/herdr-backend.md "Endpoints from a previous session"), treating a proven
# foreign pane as `missing` and an unreadable bound identity as `unreadable`
# unless pane/server absence is independently proven. A caller's `fm-<id>`
# selects its task record in the owning state directory. If ownership permits
# classification, the Herdr adapter reuses its strict husk classifier -
# which verifies a registered agent against `pane process-info` and the real
# process table, so a registration Herdr kept over a shell-only pane reads
# `dead` here (issue #4115) - then maps a positively stopped session server to
# `missing` only in this recovery-grade view. Zellij remains unverified because
# its secondmate ghost-tab and agent-process recovery path has not been
# empirically validated. Orca and cmux do not support secondmate spawns.
fm_backend_agent_state() { # <backend> <target>
fm_backend_agent_state() { # <backend> <target> [expected-label]
local backend=$1 target=$2
fm_backend_source "$backend" || { printf 'unverified'; return 0; }
case "$backend" in
tmux) fm_backend_tmux_agent_state "$target" ;;
herdr) fm_backend_herdr_agent_state "$target" ;;
herdr) fm_backend_herdr_agent_state "$target" "${3:-}" ;;
*) printf 'unverified' ;;
esac
}

# Backward-compatible three-state view for existing callers. An
# authoritatively missing endpoint is confidently not a live agent, while every
# ambiguous, unreadable, or unverified result stays unknown.
fm_backend_agent_alive() { # <backend> <target>
case "$(fm_backend_agent_state "$1" "$2")" in
fm_backend_agent_alive() { # <backend> <target> [expected-label]
case "$(fm_backend_agent_state "$1" "$2" "${3:-}")" in
alive) printf 'alive' ;;
dead|missing) printf 'dead' ;;
*) printf 'unknown' ;;
Expand Down
17 changes: 8 additions & 9 deletions bin/fm-control-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -313,13 +313,13 @@ fm_control_backend_state_verified() { # <backend>
# assignment made here could never reach them.
#
# The verdicts:
# gone - absence is PROVEN. There is no endpoint and therefore no agent.
# gone - the task's endpoint is PROVEN absent at its recorded address.
# dead - the endpoint is there after all and holds no agent.
# alive - the endpoint is there and an agent is running in it.
# unproven - neither could be established; the caller must refuse.
#
# fm_backend_agent_state's `missing` conflates "the endpoint was DESTROYED"
# with "the endpoint is UNREACHABLE from here right now". An unreachable
# fm_backend_agent_state's `missing` covers both task-endpoint absence and
# "the endpoint is UNREACHABLE from here right now". An unreachable
# endpoint can still hold a live agent on the task's worktree, so every caller
# that would act on absence - `exit` claiming the agent stopped, `relaunch`
# re-creating the endpoint - must come through here rather than trusting the
Expand All @@ -341,19 +341,18 @@ fm_control_backend_state_verified() { # <backend>
#
# Both control-plane callers share this one implementation so the proof cannot
# drift into two answers for the same endpoint.
fm_control_endpoint_absence_verdict() { # <backend> <target>
local backend=${1-} target=${2-}
fm_control_endpoint_absence_verdict() { # <backend> <target> [expected-label]
local backend=${1-} target=${2-} label=${3-}
fm_backend_source "$backend" \
|| { printf 'unproven\tbackend %s could not be loaded to prove anything about that endpoint' "'$backend'"; return 0; }
case "$backend" in
tmux)
printf 'unproven\ttmux absence cannot be proven from a task record: the record does not carry the endpoint'"'"'s socket identity, and a server-wide window inventory only describes the tmux server this process addresses, so a window absent from it may still be alive on another'
;;
herdr)
# Start the RECORDED session's server (only the server - nothing is
# created) and re-read the recorded pane. A pane that comes back with the
# server was never destroyed.
case "$(fm_backend_herdr_endpoint_absence_recheck "$target")" in
# Start only the RECORDED session's server, then re-read the endpoint
# with the task's label so another claimant cannot legitimize its address.
case "$(fm_backend_herdr_endpoint_absence_recheck "$target" "$label")" in
dead) printf 'dead\t' ;;
alive) printf 'alive\t' ;;
missing) printf 'gone\t' ;;
Expand Down
7 changes: 4 additions & 3 deletions bin/fm-control.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@
# still exists - on the same or a newly chosen
# harness/model/effort - so switching harness is one ordinary use
# of this verb. When the recorded endpoint is instead proven gone -
# a Herdr pane or workspace destroyed in churn - the launch owner
# under Herdr's recorded-endpoint ownership and absence policy
# (docs/herdr-backend.md) - the launch owner
# re-creates one in that worktree, in the herdr session the record
# names, and the task's record rebinds to it; that is how a task
# whose terminal was destroyed is reclaimed by the home that owns
Expand Down Expand Up @@ -357,7 +358,7 @@ fm_backend_validate "$BACKEND" || exit 1
# --- shared helpers ---------------------------------------------------------

agent_state() {
fm_backend_agent_state "$BACKEND" "$T"
fm_backend_agent_state "$BACKEND" "$T" "$LABEL"
}

busy_verdict() {
Expand Down Expand Up @@ -571,7 +572,7 @@ do_exit() {
# "destroyed" with "unreachable from this seat". Route it through the
# control plane's one absence proof - the same one the relaunch gate uses
# - and report what that proof actually established, never more.
absence=$(fm_control_endpoint_absence_verdict "$BACKEND" "$T")
absence=$(fm_control_endpoint_absence_verdict "$BACKEND" "$T" "$LABEL")
case "${absence%%$'\t'*}" in
gone)
# Proven gone, so the agent that lived in it went with it: exit's
Expand Down
2 changes: 1 addition & 1 deletion bin/fm-crew-state.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1298,7 +1298,7 @@ if ! pane_readable "$BACKEND_TARGET"; then
# Backends with no classifier (orca, zellij, and cmux all report unverified)
# keep their historical capture-failure-means-gone reading.
case "$TASK_BACKEND" in
tmux|herdr) AGENT_STATE=$(fm_backend_agent_state "$TASK_BACKEND" "$BACKEND_TARGET") ;;
tmux|herdr) AGENT_STATE=$(fm_backend_agent_state "$TASK_BACKEND" "$BACKEND_TARGET" "$EXPECTED_LABEL") ;;
*) AGENT_STATE=none ;;
esac
case "$TASK_BACKEND:$AGENT_STATE" in
Expand Down
2 changes: 1 addition & 1 deletion bin/fm-fleet-snapshot.sh
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,7 @@ prefetch_task_observations() { # <meta> <id>
endpoint_exists=false
fi
if [ "$kind" = secondmate ]; then
agent_alive=$(fm_backend_agent_alive "$backend" "$target" 2>/dev/null || printf unknown)
agent_alive=$(fm_backend_agent_alive "$backend" "$target" "fm-$id" 2>/dev/null || printf unknown)
fi
fi
else
Expand Down
2 changes: 1 addition & 1 deletion bin/fm-herdr-session-cleanup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ fm_herdr_cleanup_one() { # <session> <workspace> <title> <home-real>
# This unconditional retirement is the authorized containment documented
# with the presentation floor ownership in bin/backends/herdr.sh.
fm_backend_herdr_projection_close_pane_focus_preserving \
"$session" "$pane" no-agent || close_status=$?
"$session" "$pane" no-agent "fm-$id" || close_status=$?
state=$(fm_backend_herdr_pane_agent_state "$session" "$pane")
if [ "$state" = dead ]; then
if [ -f "$journal" ] && [ ! -L "$journal" ] \
Expand Down
9 changes: 5 additions & 4 deletions bin/fm-remote-secondmate-control.sh
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}"
TARGET_HOME=${FM_HOME:?FM_HOME is required}
CONTROL_STATE="$TARGET_HOME/state/parent-route"
export FM_STATE_OVERRIDE="$CONTROL_STATE"
CONTROL_DATA="$TARGET_HOME/data/.parent-route"
REMOTE_HERDR_SESSION=fm-remote

Expand Down Expand Up @@ -128,7 +129,7 @@ state_value() { # <id>; prints recovery-grade state
printf 'unverified\n'
return 0
fi
fm_backend_agent_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" 2>/dev/null || printf 'unreadable\n'
fm_backend_agent_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" "fm-$id" 2>/dev/null || printf 'unreadable\n'
}

print_route() { # <id>
Expand Down Expand Up @@ -181,7 +182,7 @@ cmd_launch() {
meta=$(meta_path "$id")
if [ -f "$meta" ]; then
remote_endpoint_require "$id"
current=$(fm_backend_agent_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" 2>/dev/null || printf 'unreadable\n')
current=$(fm_backend_agent_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" "fm-$id" 2>/dev/null || printf 'unreadable\n')
case "$current" in
alive)
print_route "$id"
Expand Down Expand Up @@ -316,7 +317,7 @@ cmd_key() {
validate_id "$id"
validate_home "$id"
remote_endpoint_require "$id"
FM_HOME="$TARGET_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" FM_STATE_OVERRIDE="$TARGET_HOME/state" \
FM_HOME="$TARGET_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" \
"$SCRIPT_DIR/fm-send.sh" "$REMOTE_ENDPOINT_TARGET" --key "$key"
}

Expand Down Expand Up @@ -393,7 +394,7 @@ cmd_update() {
local id=$1 update_out root_status
validate_id "$id"
validate_home "$id"
if ! update_out=$(FM_HOME="$FM_ROOT" FM_ROOT_OVERRIDE="$FM_ROOT" \
if ! update_out=$(FM_HOME="$FM_ROOT" FM_ROOT_OVERRIDE="$FM_ROOT" FM_STATE_OVERRIDE="$FM_ROOT/state" \
"$SCRIPT_DIR/fm-update.sh" 2>&1); then
[ -z "$update_out" ] || printf '%s\n' "$update_out" >&2
die "remote code root update failed"
Expand Down
2 changes: 1 addition & 1 deletion bin/fm-secondmate-liveness-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ fm_secondmate_liveness_probe() { # <meta> <id> <full|poll>
backend=$(fm_backend_of_meta "$meta")
target=$(fm_backend_target_of_meta "$meta")
[ -n "$target" ] || target="$window"
agent_state=$(fm_backend_agent_state "$backend" "$target" 2>/dev/null) || agent_state=unreadable
agent_state=$(fm_backend_agent_state "$backend" "$target" "fm-$id" 2>/dev/null) || agent_state=unreadable
case "$harness" in
claude|codex|opencode|pi|pi-signed|grok|kimi|omp) ;;
*)
Expand Down
Loading
Loading