From 2ed23b760f1ffcd7879fa2708831eab5cae7c82d Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:57:00 -0700 Subject: [PATCH 01/23] fix: verify Treehouse slot ownership before teardown (#3837) * fix(bin): verify pool-slot ownership before returning a worktree slot Workers were killed when cleanup returned a Treehouse pool slot that a different, live task had already taken. Teardown now proves the slot is genuinely this task's before releasing it: it refuses when another task record claims the same live worktree path, or when the endpoint's working directory contradicts the recorded slot, and that refusal holds under --force. Slot allocation, metadata publication, ownership verification, and slot return are serialized across linked firstmate homes, and forced secondmate cleanup verifies descendant slot ownership before returning any child worktree. Regression coverage drives the scripts with two task records naming one slot path and asserts the live worker survives and its slot is not reset. * no-mistakes(review): Protect slots across cloned Firstmate homes * no-mistakes(test): Gate teardown locking on genuine Treehouse slots * no-mistakes(test): Clarify pooled descendant slot gating * no-mistakes(test): Synchronize watcher re-arm test on process exit * no-mistakes(test): Wait for watcher cleanup before timeout escalation * no-mistakes(document): Document pool-slot ownership safeguards * no-mistakes(ci): Fixed all reported CI issues: normalized bare local Git origins to the same Treehouse project-lock identity as absolute clone origins; resolved ShellCheck SC1091 with explicit conditional sourcing; and taught concurrent Herdr teardown coverage to retry expected Treehouse lock contention. Added behavioral regression coverage for bare/absolute origin lock identity. Verified endpoint-safety tests, watcher tests, full CI lint, and the previously failing Herdr teardown assertion * fix(bin): resolve relative origins from repository root * no-mistakes(ci): Fixed teardown so an exact recorded endpoint may change cwd without falsely vetoing cleanup. Removed cwd-based ownership refusal while preserving cross-home record exclusivity and project locking. Updated behavioral coverage for both foreign slot ownership refusal and moved-cwd teardown success. Endpoint-safety, backend, watcher, checkpoint, and targeted lint checks pass. Real Herdr presentation E2E progressed successfully but exceeded the 600s local timeout (cherry picked from commit b028e8b14a1fcf8b16bdafc99fba616cd8577050) --- .../skills/secondmate-provisioning/SKILL.md | 1 + bin/fm-spawn.sh | 31 +- bin/fm-teardown.sh | 363 +++++++++++++++++- bin/fm-wake-lib.sh | 64 +++ bin/fm-watch-checkpoint.sh | 11 +- docs/architecture.md | 4 +- .../fm-backend-herdr-presentation-e2e.test.sh | 35 +- tests/fm-secondmate-safety.test.sh | 61 ++- tests/fm-spawn-batch.test.sh | 2 + tests/fm-task-delivery.test.sh | 1 + tests/fm-teardown-endpoint-safety.test.sh | 241 +++++++++++- 11 files changed, 799 insertions(+), 15 deletions(-) diff --git a/.agents/skills/secondmate-provisioning/SKILL.md b/.agents/skills/secondmate-provisioning/SKILL.md index b4528c65194..dc1dc89b8da 100644 --- a/.agents/skills/secondmate-provisioning/SKILL.md +++ b/.agents/skills/secondmate-provisioning/SKILL.md @@ -241,5 +241,6 @@ It refuses retirement while that cleanup is uncertain or unavailable, preserving Raw deletion is unsupported because a blocking process-event child can outlive its home. With `--force`, teardown is the explicit discard path. +The worktree-slot ownership contract in `bin/fm-teardown.sh` still applies: `--force` never authorizes returning a descendant pool slot that another task may own. It kills child windows, discards child work and state inside the secondmate home, removes the route, releases the lease, and removes the retired secondmate home. Never use `--force` unless the captain explicitly said to discard the work. diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 6cab04e9d85..65ddf47d960 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -103,7 +103,12 @@ # focus-sensitive presentation mutation. # Every single-task invocation holds one task-id-scoped lock across backend # creation through metadata publication, so concurrent same-id spawns serialize -# even when they select different backends. +# even when they select different backends. A fresh Treehouse-backed spawn also +# takes the project-identity lock in the root Firstmate home's state directory +# before slot allocation and holds it through task metadata publication. +# Teardown holds that same lock while proving and returning a slot, so +# allocation cannot reuse a slot before its owner record is published; +# contention refuses rather than waits. # With no harness arg, a crewmate/scout spawn resolves the CREW harness only when # config/crew-dispatch.json is absent. When that file exists, crewmate/scout # spawns require an explicit harness so firstmate cannot silently skip dispatch @@ -940,6 +945,8 @@ SPAWN_TASK_LOCK= SPAWN_TASK_LOCK_HELD=0 SPAWN_META_LOCK= SPAWN_META_LOCK_HELD=0 +SPAWN_TREEHOUSE_PROJECT_LOCK= +SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 CONFIG_INHERIT_LOCK= CONFIG_INHERIT_LOCK_HELD=0 TREEHOUSE_READY_DIR= @@ -1137,6 +1144,10 @@ spawn_abort_cleanup() { SPAWN_META_LOCK_HELD=0 fm_lock_release "$SPAWN_META_LOCK" || true fi + if [ "$SPAWN_TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then + SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 + fm_lock_release "$SPAWN_TREEHOUSE_PROJECT_LOCK" || true + fi if [ "$CONFIG_INHERIT_LOCK_HELD" = 1 ]; then CONFIG_INHERIT_LOCK_HELD=0 fm_lock_release "$CONFIG_INHERIT_LOCK" || true @@ -2925,6 +2936,17 @@ else fi BRIEF="$DATA/$ID/brief.md" fi +if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then + SPAWN_TREEHOUSE_PROJECT_LOCK=$(fm_treehouse_project_lock_path "$PROJ_ABS") || { + echo "error: could not resolve the shared Treehouse project lock for $PROJ_ABS" >&2 + exit 1 + } + if ! fm_lock_try_acquire "$SPAWN_TREEHOUSE_PROJECT_LOCK"; then + echo "error: another Treehouse slot allocation or return is in progress for $PROJ_ABS; refusing to race it" >&2 + exit 1 + fi + SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=1 +fi [ -f "$BRIEF" ] || { echo "error: no brief at $BRIEF" >&2; exit 1; } # Orchestration opt-in is explicit task data, never a keyword scan of prose: @@ -4370,6 +4392,13 @@ SPAWN_META_LOCK_HELD=1 echo "projects=$SECONDMATE_PROJECTS" fi } > "$STATE/$ID.meta" +# The record is published, so a teardown's slot-ownership scan can now name this +# task. The Treehouse project lock is only needed across slot allocation through +# that publication. +if [ "$SPAWN_TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then + SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 + fm_lock_release "$SPAWN_TREEHOUSE_PROJECT_LOCK" +fi [ "$BACKEND" = orca ] && ORCA_ABORT_CLEANUP=0 if [ "$HARNESS" = omp ]; then OMP_ABORT_CLEANUP=1 diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 17c3c3ddb0c..a2a9e06afeb 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -34,6 +34,29 @@ # task state when that proof fails; otherwise it removes the task's check, # trust record, PR sidecar, and publication record with the rest of the # volatile state. +# Worktree-slot ownership (teardown-slot-collision): a treehouse pool slot is +# reused across tasks, so a stale, duplicated, or drifted worktree= record can +# name a slot a DIFFERENT live task now holds. Cleanup kills every process under +# that path and hard-resets it before returning it, so releasing a slot that is +# not genuinely this task's destroys another worker's live work. Before the first +# cleanup step, teardown verifies record exclusivity: no OTHER task record in +# this home or any locally registered Firstmate home may name the same live path +# in its worktree= or home=. One live path with two task records is the reuse +# collision itself, whichever record is stale. The recorded endpoint's exact +# task identity and the record's spawn incarnation are validated separately +# before cleanup. Its current working directory is only incidental process +# state: the same worker remains the owner after changing directory, so cwd can +# never veto teardown of that exact recorded endpoint. +# The scan and destructive return hold a project-identity lock in the root +# Firstmate home's state directory. Fresh Treehouse spawns for that project in +# every local Firstmate home hold the same lock from before slot allocation +# through metadata publication, closing the publication +# gap; forced secondmate teardown takes it and runs the same checks for every +# descendant Treehouse slot before touching any child. +# This refusal is not relaxed by --force: --force authorizes discarding THIS +# task's unlanded work, never another task's live work. Reconcile whichever +# record is wrong and re-run. Orca is not a pool slot and proves its path through +# require_orca_worktree_path_match instead. # Orca tasks use the same safety checks, then close the recorded terminal and # remove the recorded worktree through `orca worktree rm`; teardown never guesses # an Orca target from ambient CLI state. @@ -204,23 +227,82 @@ if [ "$FORCE" = --force ] && [ "$TEARDOWN_ACTOR" = branch ]; then exit "$FM_LEASE_REFUSE_EXIT" fi teardown_exit_cleanup() { + local status=$? i if declare -F teardown_release_herdr_locks >/dev/null 2>&1; then teardown_release_herdr_locks || true fi + for ((i=${#DESCENDANT_LOCK_PATHS[@]} - 1; i >= 0; i--)); do + fm_lock_release "${DESCENDANT_LOCK_PATHS[$i]}" || true + done + DESCENDANT_LOCK_PATHS=() if [ "${META_LOCK_HELD:-0}" = 1 ]; then fm_lock_release "$META_LOCK" || true META_LOCK_HELD=0 fi + if [ "$TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then + fm_lock_release "$TREEHOUSE_PROJECT_LOCK" || true + TREEHOUSE_PROJECT_LOCK_HELD=0 + fi fm_lease_guard_release || true + return "$status" } trap teardown_exit_cleanup EXIT fm_lease_guard "$ID" "teardown" + +# A Treehouse slot has the managed pool's fixed // layout. +# Require both its pool state and the same Git common directory as the recorded +# project; an ordinary linked worktree is not evidence that Treehouse owns it. +is_treehouse_pool_slot() { # + local project=$1 worktree=$2 slot pool state project_common slot_common + [ -d "$project" ] && [ -d "$worktree" ] || return 1 + slot=$(CDPATH='' cd -- "$worktree" 2>/dev/null && pwd -P) || return 1 + pool=$(dirname "$(dirname "$slot")") + state="$pool/treehouse-state.json" + [ -f "$state" ] && [ ! -L "$state" ] || return 1 + project_common=$(git -C "$project" rev-parse --path-format=absolute --git-common-dir 2>/dev/null) || return 1 + slot_common=$(git -C "$slot" rev-parse --path-format=absolute --git-common-dir 2>/dev/null) || return 1 + project_common=$(CDPATH='' cd -- "$project_common" 2>/dev/null && pwd -P) || return 1 + slot_common=$(CDPATH='' cd -- "$slot_common" 2>/dev/null && pwd -P) || return 1 + [ "$project_common" = "$slot_common" ] +} + +META="$STATE/$ID.meta" +TREEHOUSE_PROJECT_LOCK= +TREEHOUSE_PROJECT_LOCK_HELD=0 +TREEHOUSE_SLOT_LOCK_REQUIRED=0 +if [ -f "$META" ] && [ ! -L "$META" ]; then + TEARDOWN_LOCK_KIND=$(fm_meta_get "$META" kind) + [ -n "$TEARDOWN_LOCK_KIND" ] || TEARDOWN_LOCK_KIND=ship + TEARDOWN_LOCK_BACKEND=$(fm_meta_get "$META" backend) + [ -n "$TEARDOWN_LOCK_BACKEND" ] || TEARDOWN_LOCK_BACKEND=tmux + TEARDOWN_LOCK_WT=$(fm_meta_get "$META" worktree) + TEARDOWN_LOCK_PROJECT=$(fm_meta_get "$META" project) + if [ "$TEARDOWN_LOCK_KIND" != secondmate ] \ + && [ "$TEARDOWN_LOCK_BACKEND" != orca ] \ + && is_treehouse_pool_slot "$TEARDOWN_LOCK_PROJECT" "$TEARDOWN_LOCK_WT"; then + TREEHOUSE_SLOT_LOCK_REQUIRED=1 + TREEHOUSE_PROJECT_LOCK=$(fm_treehouse_project_lock_path "$TEARDOWN_LOCK_PROJECT") || { + echo "REFUSED: cannot resolve the shared Treehouse project lock for ${TEARDOWN_LOCK_PROJECT:-}; nothing was changed" >&2 + exit 1 + } + fm_lock_try_acquire "$TREEHOUSE_PROJECT_LOCK" || { + echo "REFUSED: another Treehouse slot allocation or return is in progress for $TEARDOWN_LOCK_PROJECT; nothing was changed" >&2 + exit 1 + } + TREEHOUSE_PROJECT_LOCK_HELD=1 + fi +fi +DESCENDANT_LOCK_PATHS=() +DESCENDANT_TASK_STATES=() +DESCENDANT_TASK_IDS=() +DESCENDANT_TASK_KINDS=() +DESCENDANT_TASK_HOMES=() +DESCENDANT_TREEHOUSE_LOCK_PATHS=() # Fail closed before any fleet mutation: a no-mistakes gate agent must never tear # down a worktree (see bin/fm-gate-refuse-lib.sh). fm_refuse_if_gate_agent FM_LOCK_LOG_PREFIX=teardown -META="$STATE/$ID.meta" META_LOCK=$(fm_meta_lock_path "$META") || exit 1 META_LOCK_HELD=0 fm_lock_acquire_wait "$META_LOCK" @@ -466,6 +548,22 @@ ORCA_PATH_MATCH_VERIFIED=0 KIND=$(grep '^kind=' "$META" | cut -d= -f2- || true) [ -n "$KIND" ] || KIND=ship +EXPECTED_TREEHOUSE_PROJECT_LOCK= +if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] \ + && is_treehouse_pool_slot "$PROJ" "$WT"; then + EXPECTED_TREEHOUSE_PROJECT_LOCK=$(fm_treehouse_project_lock_path "$PROJ") || { + echo "REFUSED: cannot resolve the shared Treehouse project lock for ${PROJ:-}; nothing was changed" >&2 + exit 1 + } + if [ "$TREEHOUSE_PROJECT_LOCK_HELD" != 1 ] \ + || [ "$TREEHOUSE_PROJECT_LOCK" != "$EXPECTED_TREEHOUSE_PROJECT_LOCK" ]; then + echo "REFUSED: task $ID's Treehouse project identity changed while teardown acquired its locks; nothing was changed" >&2 + exit 1 + fi +elif [ "$TREEHOUSE_SLOT_LOCK_REQUIRED" = 1 ]; then + echo "REFUSED: task $ID stopped naming a live Treehouse slot while teardown acquired its locks; nothing was changed" >&2 + exit 1 +fi MODE=$(grep '^mode=' "$META" | cut -d= -f2- || true) [ -n "$MODE" ] || MODE=no-mistakes PUBLIC_FOLLOWUP_HOME=$FM_HOME @@ -1822,6 +1920,92 @@ require_orca_worktree_path_match_if_present() { require_orca_worktree_path_match "$worktree_id" "$inspected" } +# The task's own live slot, canonicalized, or empty when this record has no slot +# to release (a secondmate home, a record with no worktree=, or a path that is +# already gone). Every slot-ownership check below is scoped to that value, so a +# record with nothing live to return skips them rather than refusing. +teardown_live_slot_path() { + [ "$KIND" != secondmate ] || return 1 + is_treehouse_pool_slot "$PROJ" "$WT" || return 1 + canonical_existing_dir "$WT" +} + +collect_local_firstmate_states() { + local record_state=$1 root home reg line child known existing i=0 + local -a homes + TREEHOUSE_OWNER_STATES=("$record_state") + root=$(fm_firstmate_root_home "$FM_HOME") || { + echo "REFUSED: cannot resolve the root Firstmate home; nothing was changed" >&2 + return 1 + } + homes=("$root") + while [ "$i" -lt "${#homes[@]}" ]; do + home=${homes[$i]} + i=$((i + 1)) + known=0 + for existing in "${TREEHOUSE_OWNER_STATES[@]}"; do + [ "$existing" != "$home/state" ] || known=1 + done + [ "$known" = 1 ] || TREEHOUSE_OWNER_STATES+=("$home/state") + reg="$home/data/secondmates.md" + [ ! -e "$reg" ] && [ ! -L "$reg" ] && continue + [ -f "$reg" ] && [ ! -L "$reg" ] || { + echo "REFUSED: local Firstmate registry is unsafe at $reg; nothing was changed" >&2 + return 1 + } + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + "- "*) + secondmate_registry_parse_line "$line" || { + echo "REFUSED: malformed local Firstmate registry entry in $reg; nothing was changed" >&2 + return 1 + } + [ "$SECONDMATE_REGISTRY_REMOTE" -eq 0 ] || continue + child=$(canonical_existing_dir "$SECONDMATE_REGISTRY_HOME") || { + echo "REFUSED: registered local Firstmate home is unavailable: $SECONDMATE_REGISTRY_HOME; nothing was changed" >&2 + return 1 + } + known=0 + for existing in "${homes[@]}"; do + [ "$existing" != "$child" ] || known=1 + done + [ "$known" = 1 ] || homes+=("$child") + ;; + esac + done < "$reg" + done +} + +require_exclusive_worktree_slot_record() { + local record_meta=$1 record_id=$2 record_state=$3 worktree=$4 + local slot state_dir other other_id field other_path other_slot + slot=$(canonical_existing_dir "$worktree") || return 0 + collect_local_firstmate_states "$record_state" || return 1 + for state_dir in "${TREEHOUSE_OWNER_STATES[@]}"; do + for other in "$state_dir"/*.meta; do + [ -f "$other" ] && [ ! -L "$other" ] || continue + [ "$other" != "$record_meta" ] || continue + other_id=$(basename "$other" .meta) + for field in worktree home; do + other_path=$(fm_meta_get "$other" "$field") + [ -n "$other_path" ] || continue + other_slot=$(canonical_existing_dir "$other_path") || continue + [ "$other_slot" = "$slot" ] || continue + echo "REFUSED: task $record_id's recorded worktree $slot is also task $other_id's recorded $field." >&2 + echo "Returning that pool slot would kill $other_id's processes and reset its copy, so nothing was changed - not even with --force." >&2 + echo "Reconcile whichever record is wrong (bin/fm-crew-state.sh $record_id; bin/fm-crew-state.sh $other_id), then re-run teardown." >&2 + return 1 + done + done + done +} + +require_exclusive_task_worktree_slot() { + local slot + slot=$(teardown_live_slot_path) || return 0 + require_exclusive_worktree_slot_record "$META" "$ID" "$STATE" "$slot" +} + firstmate_home_has_treehouse_slot() { local home=$1 worktree_registered_for_project "$FM_ROOT" "$home" @@ -2171,6 +2355,178 @@ preflight_firstmate_home_process_event_tree() { preflight_firstmate_home_process_events "$home" "$label" } +collect_descendant_task_locks() { + local home=$1 sub_state child_meta child_id child_kind child_wt child_home task_set_lock + local -a child_ids + sub_state="$home/state" + if [ -L "$sub_state" ]; then + echo "REFUSED: secondmate home $home has a symbolic-link state path at $sub_state; forced teardown changed nothing" >&2 + return 1 + fi + if [ -e "$sub_state" ] && [ ! -d "$sub_state" ]; then + echo "REFUSED: secondmate home $home has a non-directory state path at $sub_state; forced teardown changed nothing" >&2 + return 1 + fi + if ! mkdir -p -- "$sub_state"; then + echo "REFUSED: secondmate home $home state directory could not be established at $sub_state; forced teardown changed nothing" >&2 + return 1 + fi + if [ -L "$sub_state" ] || [ ! -d "$sub_state" ]; then + echo "REFUSED: secondmate home $home state path is not a safe directory at $sub_state; forced teardown changed nothing" >&2 + return 1 + fi + # Freeze this home's task SET before reading it. Everything below locks the + # tasks that exist right now, but the later cleanup re-enumerates, so without + # this a fresh spawn could publish a record into the gap and be mutated + # without ever having been lifecycle-locked (bin/fm-wake-lib.sh's + # fm_task_set_lock_path owns why). Taken per home, parent before child, and + # held until this teardown exits. + task_set_lock=$(fm_task_set_lock_path "$sub_state") || { + echo "REFUSED: secondmate home $home has an invalid task-set lock path; forced teardown changed nothing" >&2 + return 1 + } + if ! fm_lock_try_acquire "$task_set_lock"; then + echo "REFUSED: secondmate home $home is publishing a task right now (task-set lock is held); forced teardown changed nothing" >&2 + return 1 + fi + DESCENDANT_LOCK_PATHS+=("$task_set_lock") + child_ids=() + for child_meta in "$sub_state"/*.meta; do + [ -e "$child_meta" ] || continue + child_ids+=("$(basename "$child_meta" .meta)") + done + [ "${#child_ids[@]}" -gt 0 ] || return 0 + while IFS= read -r child_id; do + child_meta="$sub_state/$child_id.meta" + child_kind=$(meta_value "$child_meta" kind) + [ -n "$child_kind" ] || child_kind=ship + child_home= + if [ "$child_kind" = secondmate ]; then + child_wt=$(meta_value "$child_meta" worktree) + child_home=$(meta_value "$child_meta" home) + [ -n "$child_home" ] || child_home=$child_wt + fi + DESCENDANT_TASK_STATES+=("$sub_state") + DESCENDANT_TASK_IDS+=("$child_id") + DESCENDANT_TASK_KINDS+=("$child_kind") + DESCENDANT_TASK_HOMES+=("$child_home") + [ "$child_kind" != secondmate ] \ + || collect_descendant_task_locks "$child_home" \ + || return 1 + done < <(printf '%s\n' "${child_ids[@]}" | LC_ALL=C sort) +} + +preflight_descendant_task_locks() { + local home=$1 i state task_id meta control_lock meta_lock kind child_wt child_home + DESCENDANT_TASK_STATES=() + DESCENDANT_TASK_IDS=() + DESCENDANT_TASK_KINDS=() + DESCENDANT_TASK_HOMES=() + DESCENDANT_TREEHOUSE_LOCK_PATHS=() + collect_descendant_task_locks "$home" || return 1 + # Acquisition order, which every other holder of these locks must match so + # they cannot cycle: each home's task-set lock first (parent home before child + # home, during collection above), then per-task locks in that same + # parent-before-child preorder, sorted by id within each home, each control + # lock before its matching metadata lock. No child lock holder ever reaches + # back for a parent lock. bin/fm-spawn.sh takes the same task-set lock before + # its own per-task locks when it publishes a fresh record. + for ((i=0; i < ${#DESCENDANT_TASK_IDS[@]}; i++)); do + state=${DESCENDANT_TASK_STATES[$i]} + task_id=${DESCENDANT_TASK_IDS[$i]} + meta="$state/$task_id.meta" + control_lock="$state/.control-$task_id.lock" + meta_lock=$(fm_meta_lock_path "$meta") || { + echo "REFUSED: descendant task $task_id has an invalid metadata lock path; forced teardown changed nothing" >&2 + return 1 + } + if ! fm_lock_try_acquire "$control_lock"; then + echo "REFUSED: descendant task $task_id has a lifecycle action in flight (control lock is held); forced teardown changed nothing" >&2 + return 1 + fi + DESCENDANT_LOCK_PATHS+=("$control_lock") + if ! fm_lock_try_acquire "$meta_lock"; then + echo "REFUSED: descendant task $task_id has a metadata update in flight (metadata lock is held); forced teardown changed nothing" >&2 + return 1 + fi + DESCENDANT_LOCK_PATHS+=("$meta_lock") + [ -f "$meta" ] || { + echo "REFUSED: descendant task $task_id changed while forced teardown acquired its locks; forced teardown changed nothing" >&2 + return 1 + } + kind=$(meta_value "$meta" kind) + [ -n "$kind" ] || kind=ship + [ "$kind" = "${DESCENDANT_TASK_KINDS[$i]}" ] || { + echo "REFUSED: descendant task $task_id changed kind while forced teardown acquired its locks; forced teardown changed nothing" >&2 + return 1 + } + if [ "$kind" = secondmate ]; then + child_wt=$(meta_value "$meta" worktree) + child_home=$(meta_value "$meta" home) + [ -n "$child_home" ] || child_home=$child_wt + [ "$child_home" = "${DESCENDANT_TASK_HOMES[$i]}" ] || { + echo "REFUSED: descendant task $task_id changed home while forced teardown acquired its locks; forced teardown changed nothing" >&2 + return 1 + } + fi + done +} + +preflight_descendant_treehouse_slots() { + local i state task_id meta kind backend target worktree project lock_path held + for ((i=0; i < ${#DESCENDANT_TASK_IDS[@]}; i++)); do + state=${DESCENDANT_TASK_STATES[$i]} + task_id=${DESCENDANT_TASK_IDS[$i]} + meta="$state/$task_id.meta" + kind=$(meta_value "$meta" kind) + [ -n "$kind" ] || kind=ship + backend=$(fm_backend_of_meta "$meta") + worktree=$(meta_value "$meta" worktree) + project=$(meta_value "$meta" project) + if [ "$kind" = secondmate ] || [ "$backend" = orca ]; then + continue + fi + if ! is_treehouse_pool_slot "$project" "$worktree"; then + continue + fi + lock_path=$(fm_treehouse_project_lock_path "$project") || { + echo "REFUSED: cannot resolve the shared Treehouse project lock for child $task_id; forced teardown changed nothing" >&2 + return 1 + } + held=0 + [ "$TREEHOUSE_PROJECT_LOCK_HELD" != 1 ] || [ "$TREEHOUSE_PROJECT_LOCK" != "$lock_path" ] || held=1 + for target in "${DESCENDANT_TREEHOUSE_LOCK_PATHS[@]}"; do + [ "$target" != "$lock_path" ] || held=1 + done + if [ "$held" = 0 ]; then + fm_lock_try_acquire "$lock_path" || { + echo "REFUSED: another Treehouse slot allocation or return is in progress for child $task_id; forced teardown changed nothing" >&2 + return 1 + } + DESCENDANT_TREEHOUSE_LOCK_PATHS+=("$lock_path") + DESCENDANT_LOCK_PATHS+=("$lock_path") + fi + done + for ((i=0; i < ${#DESCENDANT_TASK_IDS[@]}; i++)); do + state=${DESCENDANT_TASK_STATES[$i]} + task_id=${DESCENDANT_TASK_IDS[$i]} + meta="$state/$task_id.meta" + kind=$(meta_value "$meta" kind) + [ -n "$kind" ] || kind=ship + backend=$(fm_backend_of_meta "$meta") + worktree=$(meta_value "$meta" worktree) + project=$(meta_value "$meta" project) + if [ "$kind" = secondmate ] || [ "$backend" = orca ]; then + continue + fi + if ! is_treehouse_pool_slot "$project" "$worktree"; then + continue + fi + fm_backend_validate_task_endpoint "$meta" "$task_id" || return 1 + require_exclusive_worktree_slot_record "$meta" "$task_id" "$state" "$worktree" || return 1 + done +} + validate_firstmate_home_children_removal() { local home=$1 sub_state child_meta child_id child_wt child_proj child_kind child_home child_backend child_orca_worktree_id sub_state="$home/state" @@ -2456,6 +2812,8 @@ remove_secondmate_registry_entry() { return "$rc" } +require_exclusive_task_worktree_slot || exit 1 + validate_pr_poll_cleanup "$STATE" "$ID" || exit 1 if [ "$KIND" = secondmate ]; then @@ -2463,6 +2821,9 @@ if [ "$KIND" = secondmate ]; then validate_firstmate_home_for_removal "$HOME_PATH" "secondmate home" "$ID" >/dev/null || exit 1 if [ "$FORCE" = "--force" ]; then validate_firstmate_home_children_removal "$HOME_PATH" || exit 1 + preflight_descendant_task_locks "$HOME_PATH" || exit 1 + validate_firstmate_home_children_removal "$HOME_PATH" || exit 1 + preflight_descendant_treehouse_slots || exit 1 if [ "$BACKEND" = herdr ]; then teardown_herdr_preflight_target "$T" "$ID" || exit 1 fi diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 12c77ac2c13..e4bbcf6f6e5 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -892,6 +892,70 @@ fm_meta_lock_path() { # /.meta printf '%s/.meta-%s.lock\n' "$dir" "$id" } +# fm_task_set_lock_path: the per-home lock guarding WHICH tasks exist in a home, +# as opposed to fm_meta_lock_path, which guards one task's record. +# +# A per-task lock cannot protect a task that does not exist yet. Forced +# secondmate teardown enumerates a home's task set, locks what it found, and +# then re-enumerates while removing; a fresh spawn publishing a record inside +# that window is invisible to the first enumeration and visible to the second, +# so it gets destructively processed while never lifecycle-locked (reproduced +# with real agents: a record published 0.249s after teardown began was removed +# and its worktree returned to the pool, with both commands reporting success). +# Holding this lock from enumeration through cleanup makes the two operations +# serialize: either the spawn publishes first and the teardown's preflight +# covers it, or the teardown owns the set and the spawn refuses. Both directions +# fail closed. +fm_task_set_lock_path() { # + local state=$1 + [ -n "$state" ] || return 1 + case "$state" in *[$'\n\r\t']*) return 1 ;; esac + printf '%s/.task-set.lock\n' "$state" +} + +fm_firstmate_root_home() { + local home=${1:-$FM_HOME} marker parent seen="|" depth=0 + home=$(CDPATH='' cd -- "$home" 2>/dev/null && pwd -P) || return 1 + while [ -e "$home/.fm-secondmate-parent" ] || [ -L "$home/.fm-secondmate-parent" ]; do + marker="$home/.fm-secondmate-parent" + if ! command -v fm_secondmate_parent_record_parse >/dev/null 2>&1; then + # shellcheck source=bin/fm-secondmate-parent-lib.sh + . "$FM_WAKE_LIB_DIR/fm-secondmate-parent-lib.sh" + fi + fm_secondmate_parent_record_parse "$marker" || return 1 + [ "$FM_SECONDMATE_PARENT_ROUTE" = local ] || return 1 + parent=$(CDPATH='' cd -- "$FM_SECONDMATE_PARENT_HOME" 2>/dev/null && pwd -P) || return 1 + case "$seen" in *"|$parent|"*) return 1 ;; esac + seen="$seen$home|" + home=$parent + depth=$((depth + 1)) + [ "$depth" -le 64 ] || return 1 + done + printf '%s\n' "$home" +} + +fm_treehouse_project_lock_path() { # + local project=$1 root origin identity hash top + [ -d "$project" ] || return 1 + root=$(fm_firstmate_root_home "$FM_HOME") || return 1 + origin=$(git -C "$project" remote get-url origin 2>/dev/null || true) + if [ -n "$origin" ]; then + case "$origin" in + /*) [ ! -d "$origin" ] || origin=$(CDPATH='' cd -- "$origin" 2>/dev/null && pwd -P) || return 1 ;; + *://*|*:* ) ;; + *) [ ! -d "$project/$origin" ] || origin=$(CDPATH='' cd -- "$project/$origin" 2>/dev/null && pwd -P) || return 1 ;; + esac + identity=$origin + else + top=$(git -C "$project" rev-parse --show-toplevel 2>/dev/null) || return 1 + top=$(CDPATH='' cd -- "$top" 2>/dev/null && pwd -P) || return 1 + identity=$top + fi + hash=$(printf '%s' "$identity" | git hash-object --stdin 2>/dev/null) || return 1 + [ -d "$root/state" ] || return 1 + printf '%s/.treehouse-project-%s.lock\n' "$root/state" "$hash" +} + fm_failure_episode_reset() { local state=$1 mode=${2:-acquire} lock current pid acquired=0 path lock="$state/.turnend-claude-blocks.lock" diff --git a/bin/fm-watch-checkpoint.sh b/bin/fm-watch-checkpoint.sh index 1fb2b118b2a..35280f1f6f4 100755 --- a/bin/fm-watch-checkpoint.sh +++ b/bin/fm-watch-checkpoint.sh @@ -63,12 +63,19 @@ run_with_perl_timeout() { } local $SIG{ALRM} = sub { kill "TERM", -$pid; - select undef, undef, undef, 0.2; - kill "KILL", -$pid; + my $grace = $ENV{FM_SIGNAL_GRACE} || 5; + local $SIG{ALRM} = sub { + kill "KILL", -$pid; + waitpid $pid, 0; + exit 124; + }; + alarm $grace; + waitpid $pid, 0; exit 124; }; alarm $seconds; waitpid $pid, 0; + alarm 0; exit($? >> 8); ' "$SECONDS_ARG" "$SCRIPT_DIR/fm-watch.sh" } diff --git a/docs/architecture.md b/docs/architecture.md index 7fc12c53df9..08bc4e365d7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -291,7 +291,9 @@ After the forge command returns, the script reads the pull request's live state A verified merge leaves a durable role-routed outcome instead of living only in the merging agent's memory, and [`bin/fm-merge-outcome-lib.sh`](../bin/fm-merge-outcome-lib.sh)'s header owns its destination, shape, identity, normal-case deduplication, and at-least-once recovery. The same emitter handles a merge firstmate performed and one its poll detected, while the watcher immediately delivers the emitter's local actionable poll row. Teardown is fail-closed for ship worktrees: dirty worktrees refuse, and committed work must be landed before the worktree is returned. -[`bin/fm-teardown.sh`](../bin/fm-teardown.sh)'s header owns the landed-work proofs, PR-discovery fallback, and stale-lock recovery procedure. +A pool worktree is only returned after teardown passes the slot-ownership proof: a contradictory task record or supported live endpoint refuses without touching either task, and no discard authority relaxes that. +Before the worktree is returned, teardown concludes the task's own no-mistakes run when it is parked at a gate, including a run whose head the task copy cannot resolve - the shared runs-ledger continuation proof is the only recognition for that case, so cleanup never orphans a parked run the pipeline advanced past the submitted head. +[`bin/fm-teardown.sh`](../bin/fm-teardown.sh)'s header owns the landed-work proofs, slot-ownership proof, PR-discovery fallback, pre-teardown run conclusion, and stale-lock recovery procedure; [`tests/fm-teardown-endpoint-safety.test.sh`](../tests/fm-teardown-endpoint-safety.test.sh) and [`tests/fm-secondmate-safety.test.sh`](../tests/fm-secondmate-safety.test.sh) pin the slot-collision boundary. ## Optional X mode diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 8b172dceded..9302467b795 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -427,6 +427,17 @@ teardown_task() { # "$ROOT/bin/fm-teardown.sh" "$id" --force } +finish_concurrent_teardown() { # + local id=$1 status=$2 out=$3 err=$4 + [ "$status" -ne 0 ] || return 0 + if ! grep -F "session presentation lock is contended" "$err" >/dev/null 2>&1 \ + && ! grep -F "another Treehouse slot allocation or return is in progress" "$err" >/dev/null 2>&1; then + fail "projected teardown $id failed unexpectedly: $(cat "$err")" + fi + teardown_task "$id" "$HOME_DIR" > "$out" 2> "$err" \ + || fail "projected teardown $id retry failed after presentation cleanup completed: $(cat "$err")" +} + normalize_meta() { # sed -E \ -e 's|^worktree=.*$|worktree=|' \ @@ -876,8 +887,10 @@ teardown_task order-a "$HOME_DIR" > "$TMP_ROOT/order-a-teardown.out" 2> "$TMP_RO ORDER_A_TEARDOWN_PID=$! teardown_task order-b "$HOME_DIR" > "$TMP_ROOT/order-b-teardown.out" 2> "$TMP_ROOT/order-b-teardown.err" & ORDER_B_TEARDOWN_PID=$! -wait "$ORDER_A_TEARDOWN_PID" || fail "projected ordering fixture A teardown failed" -wait "$ORDER_B_TEARDOWN_PID" || fail "projected ordering fixture B teardown failed" +if wait "$ORDER_A_TEARDOWN_PID"; then ORDER_A_TEARDOWN_STATUS=0; else ORDER_A_TEARDOWN_STATUS=$?; fi +if wait "$ORDER_B_TEARDOWN_PID"; then ORDER_B_TEARDOWN_STATUS=0; else ORDER_B_TEARDOWN_STATUS=$?; fi +finish_concurrent_teardown order-a "$ORDER_A_TEARDOWN_STATUS" "$TMP_ROOT/order-a-teardown.out" "$TMP_ROOT/order-a-teardown.err" +finish_concurrent_teardown order-b "$ORDER_B_TEARDOWN_STATUS" "$TMP_ROOT/order-b-teardown.out" "$TMP_ROOT/order-b-teardown.err" assert_focus_is "$CAPTAIN_FOCUS" "concurrent projected teardowns" teardown_task order-fail "$HOME_DIR" > "$TMP_ROOT/order-fail-teardown.out" 2> "$TMP_ROOT/order-fail-teardown.err" \ || fail "projected ordering failure fixture teardown failed" @@ -916,8 +929,10 @@ for ROUND in 1 2 3; do WAVE_A_TEARDOWN_PID=$! teardown_task "focus-$ROUND-b" "$HOME_DIR" > "$TMP_ROOT/focus-$ROUND-b-teardown.out" 2> "$TMP_ROOT/focus-$ROUND-b-teardown.err" & WAVE_B_TEARDOWN_PID=$! - wait "$WAVE_A_TEARDOWN_PID" || fail "focus wave $ROUND teardown A failed" - wait "$WAVE_B_TEARDOWN_PID" || fail "focus wave $ROUND teardown B failed" + if wait "$WAVE_A_TEARDOWN_PID"; then WAVE_A_TEARDOWN_STATUS=0; else WAVE_A_TEARDOWN_STATUS=$?; fi + if wait "$WAVE_B_TEARDOWN_PID"; then WAVE_B_TEARDOWN_STATUS=0; else WAVE_B_TEARDOWN_STATUS=$?; fi + finish_concurrent_teardown "focus-$ROUND-a" "$WAVE_A_TEARDOWN_STATUS" "$TMP_ROOT/focus-$ROUND-a-teardown.out" "$TMP_ROOT/focus-$ROUND-a-teardown.err" + finish_concurrent_teardown "focus-$ROUND-b" "$WAVE_B_TEARDOWN_STATUS" "$TMP_ROOT/focus-$ROUND-b-teardown.out" "$TMP_ROOT/focus-$ROUND-b-teardown.err" assert_focus_is "$CAPTAIN_FOCUS" "focus wave $ROUND concurrent teardowns" WAVE_REMAINING=$(lab workspace list | jq -r '.result.workspaces[].label') [ "$WAVE_REMAINING" = $'firstmate\n2ndmate-alpha\n2ndmate-bravo' ] \ @@ -1170,6 +1185,10 @@ for RESTART_ID in fm-hibit-resume-r1 wheelhouse-healing-r1; do PATH="$HERDR_ORIGINAL_PATH" \ "$HERDR_LAB_HELPER" provision "$HERDR_LAB_SESSION" \ || fail "could not reprovision the isolated session for $RESTART_ID validation" + # Stopping the whole Herdr session also ends the anchor's agent. Its restored + # shell remains useful as the durable layout anchor, but its task record no + # longer represents a live slot owner and must not poison later slot reuse. + rm -f "$ANCHOR_META" lab pane get "$OLD_RESTART_PANE" >/dev/null 2>&1 \ || fail "$RESTART_ID restart did not preserve the projected pane structurally" if lab agent get "$OLD_RESTART_PANE" >/dev/null 2>&1; then @@ -1210,7 +1229,9 @@ for RESTART_ID in fm-hibit-resume-r1 wheelhouse-healing-r1; do || fail "$RESTART_ID repeated reclaim changed workspace identity" [ "$NEW_RESTART_PANE" != "$PRIOR_RESTART_PANE" ] \ || fail "$RESTART_ID repeated reclaim reused the prior husk pane" - "$REAL_TREEHOUSE" return --force "$PRIOR_RESTART_WT" >/dev/null 2>&1 || true + if [ "$PRIOR_RESTART_WT" != "$NEW_RESTART_WT" ]; then + "$REAL_TREEHOUSE" return --force "$PRIOR_RESTART_WT" >/dev/null 2>&1 || true + fi fi teardown_task "$RESTART_ID" "$HOME_DIR" > "$TMP_ROOT/$RESTART_ID-teardown.out" 2> "$TMP_ROOT/$RESTART_ID-teardown.err" \ @@ -1303,9 +1324,9 @@ if lab pane get "$PRIMARY_WAVE_OLD_PANE" >/dev/null 2>&1 \ fi assert_focus_is "$CONCURRENT_RECOVERY_FOCUS" "concurrent cross-home recovery" teardown_task "$PRIMARY_WAVE_ID" "$HOME_DIR" > "$TMP_ROOT/primary-wave-teardown.out" 2> "$TMP_ROOT/primary-wave-teardown.err" \ - || fail "concurrent primary recovery teardown failed" + || fail "concurrent primary recovery teardown failed: $(cat "$TMP_ROOT/primary-wave-teardown.err")" teardown_task "$BRAVO_WAVE_ID" "$SECOND_HOME_B" > "$TMP_ROOT/bravo-wave-teardown.out" 2> "$TMP_ROOT/bravo-wave-teardown.err" \ - || fail "concurrent secondmate recovery teardown failed" + || fail "concurrent secondmate recovery teardown failed: $(cat "$TMP_ROOT/bravo-wave-teardown.err")" "$REAL_TREEHOUSE" return --force "$PRIMARY_WAVE_OLD_WT" >/dev/null 2>&1 || true "$REAL_TREEHOUSE" return --force "$BRAVO_WAVE_OLD_WT" >/dev/null 2>&1 || true "$REAL_TREEHOUSE" return --force "$PRIMARY_WAVE_NEW_WT" >/dev/null 2>&1 || true diff --git a/tests/fm-secondmate-safety.test.sh b/tests/fm-secondmate-safety.test.sh index 3a369d04d44..0401c965422 100755 --- a/tests/fm-secondmate-safety.test.sh +++ b/tests/fm-secondmate-safety.test.sh @@ -2008,15 +2008,71 @@ EOF pass "secondmate force teardown discards child work" } +test_secondmate_force_teardown_refuses_duplicated_child_slot() { + local home subhome childproj childwt fakebin log err rc + home="$TMP_ROOT/force-duplicate-slot-home" + subhome="$TMP_ROOT/force-duplicate-slot-subhome" + childproj="$subhome/projects/alpha" + childwt="$TMP_ROOT/force-duplicate-slot-pool/1/alpha" + err="$TMP_ROOT/force-duplicate-slot.err" + mkdir -p "$home/state" "$home/data" "$subhome/state" "$(dirname "$childwt")" + fm_git_worktree "$childproj" "$childwt" duplicate-child + printf '{"worktrees":[{"name":"1","path":"%s"}]}\n' "$childwt" \ + > "$TMP_ROOT/force-duplicate-slot-pool/treehouse-state.json" + printf 'domain\n' > "$subhome/.fm-secondmate-home" + cat > "$home/state/domain.meta" < "$home/data/secondmates.md" + for child in stale-child live-child; do + cat > "$subhome/state/$child.meta" </dev/null 2>"$err" + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "forced secondmate teardown returned a duplicated child slot" + [ -d "$childwt" ] || fail "forced secondmate teardown removed the duplicated child slot" + [ -e "$subhome/state/stale-child.meta" ] || fail "forced secondmate teardown removed the stale child record" + [ -e "$subhome/state/live-child.meta" ] || fail "forced secondmate teardown removed the live child record" + grep -F 'kill-window' "$log" >/dev/null && fail "forced secondmate teardown killed a child before detecting its slot collision" + grep -F 'live-child' "$err" >/dev/null || grep -F 'stale-child' "$err" >/dev/null \ + || fail "forced secondmate teardown did not identify the duplicated child slot" + pass "forced secondmate teardown refuses duplicated descendant pool slots" +} + test_secondmate_force_teardown_preserves_child_on_unproven_lock() { local home subhome childproj childwt fakebin log err rc lock home="$TMP_ROOT/force-lock-home" subhome="$TMP_ROOT/force-lock-subhome" childproj="$subhome/projects/alpha" - childwt="$TMP_ROOT/force-lock-child-worktree" + childwt="$TMP_ROOT/force-lock-child-pool/1/alpha" err="$TMP_ROOT/force-lock-child.err" - mkdir -p "$home/state" "$home/data" "$subhome/state" + mkdir -p "$home/state" "$home/data" "$subhome/state" "$(dirname "$childwt")" fm_git_worktree "$childproj" "$childwt" force-child-lock + printf '{"worktrees":[{"name":"1","path":"%s"}]}\n' "$childwt" \ + > "$TMP_ROOT/force-lock-child-pool/treehouse-state.json" printf 'domain\n' > "$subhome/.fm-secondmate-home" cat > "$home/state/domain.meta" <&1) else mkdir -p "$home/projects/alpha" + git -C "$home/projects/alpha" init -q || fail "$label: could not initialize home project fixture" out=$(FM_ROOT_OVERRIDE='' FM_STATE_OVERRIDE='' FM_DATA_OVERRIDE='' FM_PROJECTS_OVERRIDE='' FM_CONFIG_OVERRIDE='' \ FM_HOME="$home" FM_SPAWN_NO_GUARD=1 \ "$SPAWN" "$id" projects/alpha codex --mode no-mistakes --yolo off 2>&1) diff --git a/tests/fm-task-delivery.test.sh b/tests/fm-task-delivery.test.sh index 694efc7bead..b60ef5930c4 100755 --- a/tests/fm-task-delivery.test.sh +++ b/tests/fm-task-delivery.test.sh @@ -33,6 +33,7 @@ make_home() { # [...] projects="$TMP_ROOT/$name/projects" fakebin="$TMP_ROOT/$name/bin" mkdir -p "$home/data" "$home/state" "$home/config" "$projects/proj" "$fakebin" + git -C "$projects/proj" init -q || fail "could not initialize project fixture" printf '#!/bin/sh\nexit 1\n' > "$fakebin/tmux" chmod +x "$fakebin/tmux" if [ "$#" -gt 0 ]; then diff --git a/tests/fm-teardown-endpoint-safety.test.sh b/tests/fm-teardown-endpoint-safety.test.sh index 5786102cd3e..23cd6815d47 100755 --- a/tests/fm-teardown-endpoint-safety.test.sh +++ b/tests/fm-teardown-endpoint-safety.test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Regression tests for cleanup endpoint identity validation. +# Regression tests for cleanup endpoint and worktree-slot identity validation. set -u # shellcheck source=tests/lib.sh @@ -14,6 +14,7 @@ make_case() { # mkdir -p "$TMP_ROOT/$dir/home/state" "$TMP_ROOT/$dir/home/data" \ "$TMP_ROOT/$dir/home/config" "$TMP_ROOT/$dir/fakebin" \ "$TMP_ROOT/$dir/worktree" "$TMP_ROOT/$dir/project" + git init -q "$TMP_ROOT/$dir/project" : > "$TMP_ROOT/$dir/worktree/sentinel" : > "$TMP_ROOT/$dir/runtime.log" cat > "$TMP_ROOT/$dir/fakebin/tmux" <<'SH' @@ -34,6 +35,19 @@ SH printf '%s\n' "$TMP_ROOT/$dir" } +mark_case_as_treehouse_pool() { # + local dir=$1 + rm -rf "$dir/worktree" + mkdir -p "$dir/pool/1" + git -C "$dir/project" -c user.name=test -c user.email=test@example.invalid \ + commit --allow-empty -qm pool-fixture + git -C "$dir/project" worktree add -q --detach "$dir/pool/1/project" + ln -s "pool/1/project" "$dir/worktree" + printf '{"worktrees":[{"name":"1","path":"%s"}]}\n' \ + "$dir/pool/1/project" > "$dir/pool/treehouse-state.json" + : > "$dir/worktree/sentinel" +} + run_case() { # local dir=$1 id=$2 FM_HOME="$dir/home" FM_ROOT_OVERRIDE="$ROOT" \ @@ -93,6 +107,42 @@ test_invalid_endpoint_records_refuse_before_mutation() { pass "fm-teardown: missing, empty, malformed, ambiguous, and task-mismatched endpoints refuse before every mutation or runtime call" } +test_non_pool_teardown_ignores_task_set_lock() { + local dir id=non-pool-task lock ready holder i=0 + dir=$(make_case non-pool-task-set-lock) + fm_write_meta "$dir/home/state/$id.meta" \ + "window=isolated:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/missing-worktree" "project=$dir/project" "kind=scout" + lock="$dir/home/state/.task-set.lock" + ready="$dir/task-set-lock-ready" + ( + # shellcheck source=/dev/null + . "$ROOT/bin/fm-wake-lib.sh" + fm_lock_try_acquire "$lock" || exit 1 + trap 'fm_lock_release "$lock"' EXIT + : > "$ready" + sleep 30 + ) & + holder=$! + while [ ! -e "$ready" ] && [ "$i" -lt 100 ]; do + sleep 0.1 + i=$((i + 1)) + done + [ -e "$ready" ] || { + kill "$holder" 2>/dev/null || true + wait "$holder" 2>/dev/null || true + fail "could not stage an in-progress task publication" + } + + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" \ + || fail "non-pool teardown was blocked by an unrelated task publication: $(cat "$dir/stderr")" + assert_absent "$dir/home/state/$id.meta" "non-pool teardown left task metadata" + assert_present "$lock" "non-pool teardown removed the publisher's lock" + kill "$holder" 2>/dev/null || true + wait "$holder" 2>/dev/null || true + pass "fm-teardown: non-pool cleanup ignores unrelated task publication locks" +} + test_supported_backend_endpoint_records_validate() { local dir id backend target dir=$(make_case valid-backends) @@ -268,8 +318,197 @@ SH pass "fm-teardown: exact tmux cleanup preserves invalid and prefix-matched neighbors while removing only the recorded target" } +test_bare_relative_origin_shares_project_lock_with_clone() { + local dir second_project primary_lock clone_lock + dir=$(make_case bare-relative-origin-lock) + git -C "$dir/project" -c user.name=test -c user.email=test@example.invalid \ + commit --allow-empty -qm lock-fixture + mkdir -p "$dir/project/remotes" + git clone -q --bare "$dir/project" "$dir/project/remotes/origin.git" + git -C "$dir/project" remote add origin remotes/origin.git + second_project="$dir/second-project" + git clone -q "$dir/project/remotes/origin.git" "$second_project" + + primary_lock=$(FM_HOME="$dir/home" bash -c \ + '. "$1"; fm_treehouse_project_lock_path "$2"' _ \ + "$ROOT/bin/fm-wake-lib.sh" "$dir/project") \ + || fail "could not resolve the primary project's bare-origin lock" + clone_lock=$(FM_HOME="$dir/home" bash -c \ + '. "$1"; fm_treehouse_project_lock_path "$2"' _ \ + "$ROOT/bin/fm-wake-lib.sh" "$second_project") \ + || fail "could not resolve the clone project's absolute-origin lock" + [ "$primary_lock" = "$clone_lock" ] \ + || fail "bare and absolute forms of the same local origin resolved different project locks" + + pass "Treehouse locking resolves a bare local origin against its source project, matching the provisioned clone" +} + +test_reused_pool_slot_refuses_before_touching_the_other_task() { + local dir id=stale-task other=live-task worker rc + + dir=$(make_case slot-reuse) + mark_case_as_treehouse_pool "$dir" + # The reuse collision: the pool slot recorded for a finished task has already + # been handed to another task, whose worker is live in it right now. + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + fm_write_meta "$dir/home/state/$other.meta" \ + "window=firstmate:fm-$other" "endpoint_task_id=$other" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + # Staged in this shell, not a command substitution: a background child of a + # $(...) subshell does not outlive it, and the point of this worker is to be + # alive in the slot while teardown runs. + ( cd "$dir/worktree" && exec sleep 30 ) & + worker=$! + + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + + [ "$rc" -ne 0 ] || fail "teardown returned a pool slot a second task record still holds" + kill -0 "$worker" 2>/dev/null || fail "teardown killed the worker holding the reused pool slot" + assert_present "$dir/worktree/sentinel" "teardown reset a pool slot a second task record still holds" + assert_present "$dir/home/state/$other.meta" "teardown removed the live task's record" + assert_present "$dir/home/state/$id.meta" "teardown removed the stale task's record before refusing" + [ ! -s "$dir/runtime.log" ] \ + || fail "teardown reached the runtime on a contested pool slot: $(cat "$dir/runtime.log")" + assert_contains "$(cat "$dir/stderr")" "$other" \ + "refusal should name the other task holding the slot" + kill "$worker" 2>/dev/null || true + wait "$worker" 2>/dev/null || true + + # The same collision recorded on a secondmate home field rather than a task + # worktree is the same slot, and refuses the same way. + dir=$(make_case slot-reuse-home) + mark_case_as_treehouse_pool "$dir" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + fm_write_meta "$dir/home/state/$other.meta" \ + "window=firstmate:fm-$other" "endpoint_task_id=$other" \ + "worktree=$dir/worktree" "home=$dir/worktree" \ + "project=$dir/project" "kind=secondmate" + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "teardown returned a pool slot a secondmate home record still holds" + assert_present "$dir/worktree/sentinel" "teardown reset a pool slot a secondmate home record still holds" + assert_present "$dir/home/state/$other.meta" "teardown removed the secondmate record" + [ ! -s "$dir/runtime.log" ] \ + || fail "teardown reached the runtime on a slot held by a secondmate home: $(cat "$dir/runtime.log")" + + pass "fm-teardown: a pool slot named by a second task record is never returned, killed, or reset" +} + +test_cross_home_pool_slot_collision_refuses() { + local dir id=stale-task other=secondmate-task second_home second_project rc + dir=$(make_case slot-reuse-cross-home) + mark_case_as_treehouse_pool "$dir" + printf 'fixture\n' > "$dir/project/tracked" + git -C "$dir/project" add tracked + git -C "$dir/project" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + second_home="$dir/secondmate-home" + second_project="$second_home/projects/project" + mkdir -p "$second_home/projects" "$second_home/state" "$second_home/data" + git clone -q "$dir/project" "$second_project" + printf '%s\n' "- mate - fixture (home: $second_home; scope: test; projects: project; added 2026-01-01)" \ + > "$dir/home/data/secondmates.md" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + fm_write_meta "$second_home/state/$other.meta" \ + "window=firstmate:fm-$other" "endpoint_task_id=$other" \ + "worktree=$dir/worktree" "project=$second_project" "kind=scout" + + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "teardown returned a pool slot held by another firstmate home" + assert_present "$dir/home/state/$id.meta" "cross-home collision removed stale metadata" + assert_present "$second_home/state/$other.meta" "cross-home collision removed live metadata" + assert_present "$dir/worktree/sentinel" "cross-home collision reset the shared slot" + [ ! -s "$dir/runtime.log" ] \ + || fail "cross-home collision reached the runtime: $(cat "$dir/runtime.log")" + assert_contains "$(cat "$dir/stderr")" "$other" \ + "cross-home refusal should name the task holding the slot" + pass "fm-teardown: a pool slot held by another firstmate home is never returned" +} + +test_sole_slot_record_still_tears_down() { + local dir id=sole-task worker + + dir=$(make_case slot-sole) + mark_case_as_treehouse_pool "$dir" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + # A neighbouring task on its OWN slot must not look like a collision. + mkdir -p "$dir/other-worktree" + fm_write_meta "$dir/home/state/neighbour.meta" \ + "window=firstmate:fm-neighbour" "endpoint_task_id=neighbour" \ + "worktree=$dir/other-worktree" "project=$dir/project" "kind=scout" + ( cd "$dir/other-worktree" && exec sleep 30 ) & + worker=$! + + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" \ + || fail "teardown of a task that solely holds its slot failed: $(cat "$dir/stderr")" + assert_absent "$dir/home/state/$id.meta" "uncontested teardown left the task record" + assert_present "$dir/home/state/neighbour.meta" "uncontested teardown removed the neighbour's record" + kill -0 "$worker" 2>/dev/null || fail "uncontested teardown killed a worker in a different slot" + grep -Fq "treehouse " "$dir/runtime.log" \ + || fail "uncontested teardown did not return its own pool slot: $(cat "$dir/runtime.log")" + kill "$worker" 2>/dev/null || true + wait "$worker" 2>/dev/null || true + pass "fm-teardown: a task that solely holds its slot still returns it" +} + +test_recorded_endpoint_that_changed_directory_still_tears_down() { + local dir id=moved-task + + dir=$(make_case slot-endpoint-moved) + mark_case_as_treehouse_pool "$dir" + mkdir -p "$dir/other-directory" + # The exact recorded worker may legitimately cd outside its worktree. Its + # endpoint identity still owns the lifecycle; cwd alone must not brick it. + cat > "$dir/fakebin/tmux" <> "\${FM_RUNTIME_LOG:?}" +printf ' <%s>' "\$@" >> "\${FM_RUNTIME_LOG:?}" +printf '\n' >> "\${FM_RUNTIME_LOG:?}" +exit 0 +SH + chmod +x "$dir/fakebin/tmux" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" \ + || fail "teardown refused its recorded endpoint after it changed directory: $(cat "$dir/stderr")" + assert_absent "$dir/home/state/$id.meta" "moved-endpoint teardown left the task record" + grep -Fq "tmux <-t> <=firstmate:=fm-$id>" "$dir/runtime.log" \ + || fail "moved-endpoint teardown did not stop the exact recorded worker: $(cat "$dir/runtime.log")" + grep -Fq "treehouse " "$dir/runtime.log" \ + || fail "moved-endpoint teardown did not return its uncontested pool slot: $(cat "$dir/runtime.log")" + + pass "fm-teardown: an exact recorded endpoint still tears down after changing cwd outside its worktree" +} + test_invalid_endpoint_records_refuse_before_mutation +test_non_pool_teardown_ignores_task_set_lock test_supported_backend_endpoint_records_validate test_tmux_empty_target_refuses_without_invocation test_recorded_process_identity_cleanup_is_exact test_isolated_tmux_invalid_and_valid_cleanup +test_bare_relative_origin_shares_project_lock_with_clone +test_reused_pool_slot_refuses_before_touching_the_other_task +test_cross_home_pool_slot_collision_refuses +test_sole_slot_record_still_tears_down +test_recorded_endpoint_that_changed_directory_still_tears_down From 5148e98a3cfdf1173c174aaf6f28e1e9089d6ae4 Mon Sep 17 00:00:00 2001 From: Christoph Meise Date: Sat, 12 Sep 2026 00:57:18 +0200 Subject: [PATCH 02/23] fix(teardown): leave a Treehouse pool slot reassigned to another task untouched (#4243) * fix(teardown): refuse to return a Treehouse pool slot reassigned to another task A pool slot is reused across tasks, so a finished task's worktree= line can name a slot a different, live task now holds. Teardown already refused when a second task record named the same live path, but that scan cannot prove the record it is tearing down is the current owner: the task that took the slot next may leave no record the scan can reach - its own worker may have exited and its record been cleaned up, or it may live in a home this machine does not register. Teardown then killed every process under the path, hard-reset it and returned it, and its unlanded-work refusal never fired because it was inspecting a directory that no longer belonged to the task being torn down (observed 2026-09-07). Treehouse's own state file cannot answer the ownership question. It records a slot's owner as a live process lease (owner_pid plus owner_started_at, with `treehouse status` reporting in-use from the processes actually running under the path), which names no task and is released by the very event that makes a record stale - the worker exiting. An unleased slot therefore reads identical whether it is still this task's or has since been handed on, and a slot whose new holder has also exited but left uncommitted work reads as free. So the identity source is Firstmate's own claim, not Treehouse's lease. fm-spawn writes that claim - the task id - into the slot at the moment it takes it, under the same project lock that allocates the slot, and fm-teardown drops it only after the slot is genuinely returned. It lives at //.fm-slot-owner, a sibling of the repo checkout rather than a file inside it, so claiming a slot can never dirty the copy the landed-work checks inspect. A claim naming another task, or one that cannot be read, refuses; --force does not lift either refusal, because --force authorizes discarding this task's unlanded work, never another task's live work. A slot that cannot be claimed refuses the spawn instead. An absent claim proceeds on exactly the record-scan protection it had before: slots taken before claims existed, and slots already returned, carry none, and refusing those would strand every task in flight across this change on no evidence at all. The refusal is deliberately all-or-nothing rather than partially completing the task's own cleanup. state/.meta is the only durable record naming the worktree and endpoint, so removing it would destroy the evidence needed to reconcile which record is wrong, and its removal is one step with the backlog transition. Nothing is stranded: clearing the stale worktree= line leaves a record with no slot to release, which then tears down normally, and the refusal names that remedy. Repairing the previous claimant's stale worktree= line at spawn time is left for separate work. It would have the new owner write another task's record - the same class of cross-task mutation this bug is - and would need that record's own meta lock; with the claim in place teardown refuses on evidence rather than depending on the stale pointer having been scrubbed. For the same reason the relaunch path writes no claim: it holds no allocation lock, and a record whose worktree= is already stale would stamp the wrong task's claim onto a live sibling's slot. The regression reproduces the reuse sequence with only one discoverable record, including a clean, fully landed ship copy torn down without --force - the shape of the real incident, which the previous code returned to the pool - and fails against the previous code; the existing two-record, cross-home, own-slot and no-claim cases still pass unchanged. This builds ON upstream b028e8b1 (#3837), which is already in this branch's base (origin/main 40c50ea8) and owns the record-exclusivity scan. Nothing here replaces that scan; the claim is the positive proof it cannot supply. Claude-Session: https://claude.ai/code/session_01JTBmuqKugaPUj7k9TXQwFS * no-mistakes(review): teardown leaves reassigned slot; spawn abort drops claim * no-mistakes(review): narrow Treehouse lease evidence; gate abort claim release on lock * no-mistakes(review): pin spawn-side slot claim; narrow abort-release header * no-mistakes(document): docs: point slot-claim rationale at fm-wake-lib owner (cherry picked from commit 7d14fc126aa8965611f1958a00542d3adc4abdb5) --- bin/fm-spawn.sh | 53 ++- bin/fm-teardown.sh | 206 +++++++++--- bin/fm-wake-lib.sh | 115 +++++++ docs/architecture.md | 4 +- tests/fm-spawn-pool-base-freshen.test.sh | 56 ++++ tests/fm-teardown-endpoint-safety.test.sh | 380 ++++++++++++++++++++++ 6 files changed, 765 insertions(+), 49 deletions(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 65ddf47d960..ce7b5616984 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -107,7 +107,15 @@ # takes the project-identity lock in the root Firstmate home's state directory # before slot allocation and holds it through task metadata publication. # Teardown holds that same lock while proving and returning a slot, so -# allocation cannot reuse a slot before its owner record is published; +# allocation cannot reuse a slot before its owner record is published. Under +# that same lock it writes the slot's owner claim, which is what lets teardown +# leave a slot reassigned since untouched; bin/fm-wake-lib.sh owns the claim +# and bin/fm-teardown.sh owns what it protects. A slot that cannot be claimed +# refuses the spawn rather than launching a worker whose slot could later be +# released out from under its successor. A spawn that aborts while it still +# holds the allocation lock drops its own claim; an abort after metadata +# publication has released that lock leaves the claim in place, and the next +# spawn's claim replaces it; # contention refuses rather than waits. # With no harness arg, a crewmate/scout spawn resolves the CREW harness only when # config/crew-dispatch.json is absent. When that file exists, crewmate/scout @@ -947,6 +955,12 @@ SPAWN_META_LOCK= SPAWN_META_LOCK_HELD=0 SPAWN_TREEHOUSE_PROJECT_LOCK= SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 +SPAWN_SLOT_CLAIMED=0 +RELAUNCH_REPLACEMENT_PENDING=0 +RELAUNCH_REPLACEMENT_BUSY_GEN= +RELAUNCH_REPLACEMENT_HARNESS= +RELAUNCH_REPLACEMENT_STATE= +RELAUNCH_REPLACEMENT_WT= CONFIG_INHERIT_LOCK= CONFIG_INHERIT_LOCK_HELD=0 TREEHOUSE_READY_DIR= @@ -1144,6 +1158,23 @@ spawn_abort_cleanup() { SPAWN_META_LOCK_HELD=0 fm_lock_release "$SPAWN_META_LOCK" || true fi + # A spawn that aborts after claiming its slot but before its record survives + # must not leave a claim naming a task no record describes. The release is a + # read-then-remove, so it runs only while the project lock that wrote the + # claim is still held (aborts before metadata publication); a later abort has + # already released that lock and leaves the claim for the next spawn's + # atomic replacement rather than racing it. The release itself never removes + # another task's claim. + if [ "$SPAWN_SLOT_CLAIMED" = 1 ] && [ -n "${WT:-}" ] \ + && [ ! -e "$STATE/$ID.meta" ] && [ ! -L "$STATE/$ID.meta" ] \ + && fm_treehouse_pool_slot "$PROJ_ABS" "$WT"; then + SPAWN_SLOT_CLAIMED=0 + if [ "$SPAWN_TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then + fm_treehouse_slot_owner_release "$WT" "$ID" || true + else + echo "warning: leaving task $ID's slot claim on $WT in place; the Treehouse project lock is no longer held, so the next spawn's claim replaces it" >&2 + fi + fi if [ "$SPAWN_TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 fm_lock_release "$SPAWN_TREEHOUSE_PROJECT_LOCK" || true @@ -3854,6 +3885,26 @@ if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] if [ "$HARNESS" = omp ]; then fm_omp_clear_stale_runtime_markers "$WT" || exit 1 fi + + # Claim the pool slot for this task. The interactive `treehouse get` sent to + # the pane above records only a process lease (Treehouse's durable + # `get --lease --lease-holder`, which bin/fm-home-seed.sh uses for secondmate + # homes, is not this path), so Treehouse cannot say which task a slot belongs + # to once that task's worker exits - and that is exactly when the slot is + # handed on and this task's worktree= line goes stale. The claim is what lets + # bin/fm-teardown.sh leave a slot that has since been reassigned untouched, so + # a slot that cannot be claimed is refused here, at the cheapest point, rather + # than launching a worker whose slot teardown could later release out from + # under its successor. + # Written under the Treehouse project lock held from before slot allocation + # through metadata publication, so no other spawn or return sees a half-claim. + if fm_treehouse_pool_slot "$PROJ_ABS" "$WT"; then + if ! fm_treehouse_slot_owner_claim "$WT" "$ID" "$FM_HOME"; then + echo "error: could not claim Treehouse pool slot $WT for task $ID; refusing to launch a worker whose slot cannot later be proved to be its own; inspect window $T" >&2 + exit 1 + fi + SPAWN_SLOT_CLAIMED=1 + fi fi if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] \ && [ "$PREWALK_WORKTREE_READY" != 1 ] && [ "$RAW_LAUNCH_WORKTREE_READY" != 1 ]; then diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index a2a9e06afeb..33971d6e8e1 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -42,8 +42,36 @@ # cleanup step, teardown verifies record exclusivity: no OTHER task record in # this home or any locally registered Firstmate home may name the same live path # in its worktree= or home=. One live path with two task records is the reuse -# collision itself, whichever record is stale. The recorded endpoint's exact -# task identity and the record's spawn incarnation are validated separately +# collision itself, whichever record is stale. +# That scan alone cannot prove THIS record is the current owner, because the task +# that took the slot next may leave no record it can reach - its own worker may +# have exited and its record been cleaned up, or it may live in a home this +# machine does not register - which is how a released-then-reassigned slot was +# returned out from under a live worker (observed 2026-09-07). So teardown also +# reads the slot's own owner claim, written by bin/fm-spawn.sh at the moment the +# slot is taken and dropped here once it is genuinely returned; bin/fm-wake-lib.sh +# owns the claim, its location, and its states. A claim naming another task is +# proof of reassignment: the slot is no longer this task's, so teardown warns, +# names the claimant, and then finishes only this task's own cleanup - endpoint, +# status, records, checks, backlog - while every step that would read or touch +# that slot is skipped: no process kill under it, no dirty or landed-work +# inspection of it, no branch or hook removal in it, no Treehouse return, and +# never the other task's claim. Skipping the inspection discards nothing of this +# task's: whatever unlanded work it had in that slot was already destroyed when +# the pool handed the slot on. Refusing instead would strand the record, because +# bin/fm-backend.sh's endpoint validation refuses an empty or missing worktree= +# unconditionally, so there is no line an operator could clear to get past it. +# A claim that cannot be read proves nothing either way and refuses; inspect or +# repair the claim file at the printed path and re-run - never remove it, since +# an absent claim proceeds and would return a slot that may be another task's. An +# absent claim - a slot taken before claims existed, or already returned - keeps +# exactly the record-scan protection it had before, because refusing it would +# strand every task in flight across that change on no evidence at all. +# Why Treehouse's own state cannot answer this for crewmate slots, and why the +# claim file sits on top of it, is owned by bin/fm-wake-lib.sh's slot-owner +# claim comment. +# The recorded endpoint's exact task identity and the record's spawn incarnation +# are validated separately # before cleanup. Its current working directory is only incidental process # state: the same worker remains the owner after changing directory, so cwd can # never veto teardown of that exact recorded endpoint. @@ -53,9 +81,10 @@ # through metadata publication, closing the publication # gap; forced secondmate teardown takes it and runs the same checks for every # descendant Treehouse slot before touching any child. -# This refusal is not relaxed by --force: --force authorizes discarding THIS -# task's unlanded work, never another task's live work. Reconcile whichever -# record is wrong and re-run. Orca is not a pool slot and proves its path through +# These refusals are not relaxed by --force: --force authorizes discarding THIS +# task's unlanded work, never another task's live work. Nothing of this task's +# own is removed by a refusal; reconcile whichever record is wrong and re-run. +# Orca is not a pool slot and proves its path through # require_orca_worktree_path_match instead. # Orca tasks use the same safety checks, then close the recorded terminal and # remove the recorded worktree through `orca worktree rm`; teardown never guesses @@ -249,23 +278,6 @@ teardown_exit_cleanup() { trap teardown_exit_cleanup EXIT fm_lease_guard "$ID" "teardown" -# A Treehouse slot has the managed pool's fixed // layout. -# Require both its pool state and the same Git common directory as the recorded -# project; an ordinary linked worktree is not evidence that Treehouse owns it. -is_treehouse_pool_slot() { # - local project=$1 worktree=$2 slot pool state project_common slot_common - [ -d "$project" ] && [ -d "$worktree" ] || return 1 - slot=$(CDPATH='' cd -- "$worktree" 2>/dev/null && pwd -P) || return 1 - pool=$(dirname "$(dirname "$slot")") - state="$pool/treehouse-state.json" - [ -f "$state" ] && [ ! -L "$state" ] || return 1 - project_common=$(git -C "$project" rev-parse --path-format=absolute --git-common-dir 2>/dev/null) || return 1 - slot_common=$(git -C "$slot" rev-parse --path-format=absolute --git-common-dir 2>/dev/null) || return 1 - project_common=$(CDPATH='' cd -- "$project_common" 2>/dev/null && pwd -P) || return 1 - slot_common=$(CDPATH='' cd -- "$slot_common" 2>/dev/null && pwd -P) || return 1 - [ "$project_common" = "$slot_common" ] -} - META="$STATE/$ID.meta" TREEHOUSE_PROJECT_LOCK= TREEHOUSE_PROJECT_LOCK_HELD=0 @@ -279,7 +291,7 @@ if [ -f "$META" ] && [ ! -L "$META" ]; then TEARDOWN_LOCK_PROJECT=$(fm_meta_get "$META" project) if [ "$TEARDOWN_LOCK_KIND" != secondmate ] \ && [ "$TEARDOWN_LOCK_BACKEND" != orca ] \ - && is_treehouse_pool_slot "$TEARDOWN_LOCK_PROJECT" "$TEARDOWN_LOCK_WT"; then + && fm_treehouse_pool_slot "$TEARDOWN_LOCK_PROJECT" "$TEARDOWN_LOCK_WT"; then TREEHOUSE_SLOT_LOCK_REQUIRED=1 TREEHOUSE_PROJECT_LOCK=$(fm_treehouse_project_lock_path "$TEARDOWN_LOCK_PROJECT") || { echo "REFUSED: cannot resolve the shared Treehouse project lock for ${TEARDOWN_LOCK_PROJECT:-}; nothing was changed" >&2 @@ -550,7 +562,7 @@ KIND=$(grep '^kind=' "$META" | cut -d= -f2- || true) [ -n "$KIND" ] || KIND=ship EXPECTED_TREEHOUSE_PROJECT_LOCK= if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] \ - && is_treehouse_pool_slot "$PROJ" "$WT"; then + && fm_treehouse_pool_slot "$PROJ" "$WT"; then EXPECTED_TREEHOUSE_PROJECT_LOCK=$(fm_treehouse_project_lock_path "$PROJ") || { echo "REFUSED: cannot resolve the shared Treehouse project lock for ${PROJ:-}; nothing was changed" >&2 exit 1 @@ -1926,7 +1938,7 @@ require_orca_worktree_path_match_if_present() { # record with nothing live to return skips them rather than refusing. teardown_live_slot_path() { [ "$KIND" != secondmate ] || return 1 - is_treehouse_pool_slot "$PROJ" "$WT" || return 1 + fm_treehouse_pool_slot "$PROJ" "$WT" || return 1 canonical_existing_dir "$WT" } @@ -2006,6 +2018,72 @@ require_exclusive_task_worktree_slot() { require_exclusive_worktree_slot_record "$META" "$ID" "$STATE" "$slot" } +# Positive slot ownership, read from the claim the task that took the slot wrote +# into the slot itself (bin/fm-wake-lib.sh owns the claim and its states). +# +# The record scan above proves that no OTHER task record names this slot. It +# cannot prove that THIS record is not the stale one, because the task that took +# the slot next may leave no record this scan can reach: its own worker may have +# exited and its record been cleaned up, or it may belong to a home this machine +# does not register. The claim closes that gap from the other side - it names the +# task that actually took the slot, and it is written under the same project lock +# that allocates it - so a claim naming another task is proof the slot was +# reassigned after this record was written. +# +# A claim naming another task does not refuse: it means the slot is no longer +# this task's, so the record's own cleanup proceeds and every slot step is +# skipped (see the script header for why refusing would strand the record and +# why skipping discards nothing). Returns TEARDOWN_SLOT_REASSIGNED_RC for that +# state so each caller gates its slot steps on one determination; the claimant +# stays in FM_TREEHOUSE_SLOT_OWNER_ID and FM_TREEHOUSE_SLOT_OWNER_HOME. +# +# An absent claim proceeds as the slot's owner: a slot taken before claims +# existed, or already returned to the pool, carries none, and refusing those +# would strand every task in flight across the change for no evidence at all. +# Those keep exactly the record-scan protection they had before. +TEARDOWN_SLOT_REASSIGNED_RC=3 +require_owned_worktree_slot_record() { # + local record_id=$1 worktree=$2 marker + fm_treehouse_slot_owner_state "$worktree" "$record_id" + case "$FM_TREEHOUSE_SLOT_OWNER" in + mine|absent) return 0 ;; + other) + echo "warning: task $record_id's recorded worktree $worktree was reassigned to task $FM_TREEHOUSE_SLOT_OWNER_ID${FM_TREEHOUSE_SLOT_OWNER_HOME:+ (home $FM_TREEHOUSE_SLOT_OWNER_HOME)}, which claimed that pool slot after this record was written; that slot is no longer $record_id's, so its processes, copy, and claim are left untouched and only $record_id's own cleanup runs." >&2 + return "$TEARDOWN_SLOT_REASSIGNED_RC" + ;; + esac + marker=$(fm_treehouse_slot_owner_marker "$worktree" 2>/dev/null) || marker="beside $worktree" + echo "REFUSED: task $record_id's recorded worktree $worktree carries a slot-owner claim that cannot be read, so the slot cannot be proved to still be this task's; nothing was changed - not even with --force." >&2 + echo "Inspect or repair the claim file at $marker (task= and home= lines), then re-run teardown." >&2 + return 1 +} + +# The one ownership determination for this task's recorded slot. Every later +# step that would read or touch $WT consults teardown_owns_worktree, so a +# reassigned slot is skipped consistently rather than by each step's own guess. +TEARDOWN_SLOT_REASSIGNED=0 +TEARDOWN_SLOT_REASSIGNED_TO= +TEARDOWN_SLOT_REASSIGNED_HOME= +require_owned_task_worktree_slot() { + local slot rc=0 + slot=$(teardown_live_slot_path) || return 0 + require_owned_worktree_slot_record "$ID" "$slot" || rc=$? + case "$rc" in + 0) return 0 ;; + "$TEARDOWN_SLOT_REASSIGNED_RC") + TEARDOWN_SLOT_REASSIGNED=1 + TEARDOWN_SLOT_REASSIGNED_TO=$FM_TREEHOUSE_SLOT_OWNER_ID + TEARDOWN_SLOT_REASSIGNED_HOME=$FM_TREEHOUSE_SLOT_OWNER_HOME + return 0 + ;; + esac + return 1 +} + +teardown_owns_worktree() { + [ "$TEARDOWN_SLOT_REASSIGNED" != 1 ] +} + firstmate_home_has_treehouse_slot() { local home=$1 worktree_registered_for_project "$FM_ROOT" "$home" @@ -2473,7 +2551,7 @@ preflight_descendant_task_locks() { } preflight_descendant_treehouse_slots() { - local i state task_id meta kind backend target worktree project lock_path held + local i state task_id meta kind backend target worktree project lock_path held owner_rc for ((i=0; i < ${#DESCENDANT_TASK_IDS[@]}; i++)); do state=${DESCENDANT_TASK_STATES[$i]} task_id=${DESCENDANT_TASK_IDS[$i]} @@ -2486,7 +2564,7 @@ preflight_descendant_treehouse_slots() { if [ "$kind" = secondmate ] || [ "$backend" = orca ]; then continue fi - if ! is_treehouse_pool_slot "$project" "$worktree"; then + if ! fm_treehouse_pool_slot "$project" "$worktree"; then continue fi lock_path=$(fm_treehouse_project_lock_path "$project") || { @@ -2495,7 +2573,7 @@ preflight_descendant_treehouse_slots() { } held=0 [ "$TREEHOUSE_PROJECT_LOCK_HELD" != 1 ] || [ "$TREEHOUSE_PROJECT_LOCK" != "$lock_path" ] || held=1 - for target in "${DESCENDANT_TREEHOUSE_LOCK_PATHS[@]}"; do + for target in "${DESCENDANT_TREEHOUSE_LOCK_PATHS[@]+"${DESCENDANT_TREEHOUSE_LOCK_PATHS[@]}"}"; do [ "$target" != "$lock_path" ] || held=1 done if [ "$held" = 0 ]; then @@ -2519,11 +2597,17 @@ preflight_descendant_treehouse_slots() { if [ "$kind" = secondmate ] || [ "$backend" = orca ]; then continue fi - if ! is_treehouse_pool_slot "$project" "$worktree"; then + if ! fm_treehouse_pool_slot "$project" "$worktree"; then continue fi fm_backend_validate_task_endpoint "$meta" "$task_id" || return 1 require_exclusive_worktree_slot_record "$meta" "$task_id" "$state" "$worktree" || return 1 + owner_rc=0 + require_owned_worktree_slot_record "$task_id" "$worktree" || owner_rc=$? + case "$owner_rc" in + 0|"$TEARDOWN_SLOT_REASSIGNED_RC") ;; + *) return 1 ;; + esac done } @@ -2695,7 +2779,7 @@ preflight_firstmate_home_herdr_children() { # } cleanup_firstmate_home_children() { - local home=$1 sub_state child_meta child_id child_t child_wt child_proj child_kind child_home child_backend child_orca_worktree_id child_return_rc child_busy_gen + local home=$1 sub_state child_meta child_id child_t child_wt child_proj child_kind child_home child_backend child_orca_worktree_id child_return_rc child_busy_gen child_owner_rc sub_state="$home/state" [ -d "$sub_state" ] || return 0 for child_meta in "$sub_state"/*.meta; do @@ -2754,24 +2838,38 @@ cleanup_firstmate_home_children() { fi fm_backend_remove_worktree "$child_backend" "$child_orca_worktree_id" || return 1 elif [ -n "$child_wt" ] && [ -d "$child_wt" ]; then - validate_child_worktree_for_removal "$child_wt" "$child_proj" >/dev/null || return 1 - remove_devin_project_hook_config "$child_wt" "$child_meta" || return 1 - rm -f "$child_wt/.claude/settings.local.json" "$child_wt/.opencode/plugins/fm-turn-end.js" \ - "$child_wt/.opencode/plugins/fm-busy-state.js" \ + # The same ownership determination as the parent's own slot: a child + # slot reassigned to another task is not this child's to kill, reset, + # or return, so only its records are cleaned up. The preflight above + # already named the reassignment on stderr under the same lock. + child_owner_rc=0 + if fm_treehouse_pool_slot "$child_proj" "$child_wt"; then + require_owned_worktree_slot_record "$child_id" "$child_wt" 2>/dev/null || child_owner_rc=$? + fi + if [ "$child_owner_rc" -eq "$TEARDOWN_SLOT_REASSIGNED_RC" ]; then + : + elif [ "$child_owner_rc" -ne 0 ]; then + require_owned_worktree_slot_record "$child_id" "$child_wt" || return 1 + else + validate_child_worktree_for_removal "$child_wt" "$child_proj" >/dev/null || return 1 + remove_devin_project_hook_config "$child_wt" "$child_meta" || return 1 + rm -f "$child_wt/.claude/settings.local.json" "$child_wt/.opencode/plugins/fm-turn-end.js" \ + "$child_wt/.opencode/plugins/fm-busy-state.js" \ "$child_wt/.fm-grok-turnend" "$child_wt/.fm-kimi-turnend" "$child_wt/.fm-devin-turnend" \ - "$child_wt/.fm-hermes-turnend" - if [ -n "$child_proj" ] && [ -d "$child_proj" ] && command -v treehouse >/dev/null 2>&1; then - if teardown_treehouse_return "$child_wt" "$child_proj" "child worktree"; then - : - else - child_return_rc=$? - if [ "$child_return_rc" -eq "$TEARDOWN_TREEHOUSE_LOCK_REFUSED" ]; then - return "$child_return_rc" + "$child_wt/.fm-hermes-turnend" + if [ -n "$child_proj" ] && [ -d "$child_proj" ] && command -v treehouse >/dev/null 2>&1; then + if teardown_treehouse_return "$child_wt" "$child_proj" "child worktree"; then + fm_treehouse_slot_owner_release "$child_wt" "$child_id" + else + child_return_rc=$? + if [ "$child_return_rc" -eq "$TEARDOWN_TREEHOUSE_LOCK_REFUSED" ]; then + return "$child_return_rc" + fi + safe_rm_rf_child_worktree "$child_wt" "$child_proj" fi + else safe_rm_rf_child_worktree "$child_wt" "$child_proj" fi - else - safe_rm_rf_child_worktree "$child_wt" "$child_proj" fi fi remove_grok_turnend_auth "$sub_state" "$child_id" "$child_meta" @@ -2813,6 +2911,7 @@ remove_secondmate_registry_entry() { } require_exclusive_task_worktree_slot || exit 1 +require_owned_task_worktree_slot || exit 1 validate_pr_poll_cleanup "$STATE" "$ID" || exit 1 @@ -2898,7 +2997,7 @@ if [ "$BACKEND" = orca ] && [ "$KIND" != scout ] && [ "$KIND" != secondmate ] && ORCA_PATH_MATCH_VERIFIED=1 fi -if [ -d "$WT" ] && [ "$FORCE" != "--force" ]; then +if teardown_owns_worktree && [ -d "$WT" ] && [ "$FORCE" != "--force" ]; then if validate_worktree_teardown_safety; then : else @@ -2942,9 +3041,11 @@ fi # kind=secondmate: a secondmate home's own runtime lifecycle is owned by the # dedicated process-event and firstmate-home removal machinery further below, # not by task-worktree cleanup. -if [ "$KIND" != secondmate ]; then +if [ "$KIND" != secondmate ] && teardown_owns_worktree; then conclude_task_no_mistakes_run "$WT" reap_task_worktree_processes worktree "$WT" "$TASK_TMP" +elif [ "$KIND" != secondmate ]; then + reap_task_worktree_processes tasktmp "$TASK_TMP" fi # A Herdr close may reposition shared workspace order, so the whole @@ -2983,6 +3084,8 @@ if [ "$BACKEND" = orca ] && [ "$KIND" != secondmate ]; then fi [ -z "$T_ORCA" ] || fm_backend_kill "$BACKEND" "$T" "$(meta_value "$META" zellij_tab_id)" "fm-$ID" 2>/dev/null || true fm_backend_remove_worktree "$BACKEND" "$ORCA_WORKTREE_ID" +elif [ "$KIND" != secondmate ] && ! teardown_owns_worktree; then + : elif [ -d "$WT" ] && [ "$KIND" != secondmate ]; then branch=$(git -C "$WT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo HEAD) if [ "$branch" != "HEAD" ]; then @@ -3006,6 +3109,11 @@ elif [ -d "$WT" ] && [ "$KIND" != secondmate ]; then echo "error: treehouse return failed for worktree $WT; teardown aborted" >&2 exit 1 } + # The slot is back in the pool, so this task's claim on it is spent. Dropping + # it here - and only after a return that succeeded - keeps a returned slot + # unclaimed until its next holder claims it, and leaves the claim in place + # whenever the return did not actually happen. + fm_treehouse_slot_owner_release "$WT" "$ID" fi HERDR_PRESENTATION_JOURNAL="$STATE/$ID.herdr-presentation" @@ -3111,5 +3219,9 @@ META_LOCK_HELD=0 if [ "$KIND" != scout ] && [ "$KIND" != secondmate ] && [ "$MODE" != local-only ]; then "$FM_ROOT/bin/fm-fleet-sync.sh" "$PROJ" || true fi -echo "teardown $ID complete (window $T, worktree $WT)" +if teardown_owns_worktree; then + echo "teardown $ID complete (window $T, worktree $WT)" +else + echo "teardown $ID complete (window $T; pool slot $WT left to task $TEARDOWN_SLOT_REASSIGNED_TO${TEARDOWN_SLOT_REASSIGNED_HOME:+ (home $TEARDOWN_SLOT_REASSIGNED_HOME)}, which it was reassigned to)" +fi backlog_refresh_reminder diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index e4bbcf6f6e5..062d6185e17 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -956,6 +956,121 @@ fm_treehouse_project_lock_path() { # printf '%s/.treehouse-project-%s.lock\n' "$root/state" "$hash" } +# A Treehouse slot has the managed pool's fixed // layout. +# Require both its pool state and the same Git common directory as the recorded +# project; an ordinary linked worktree is not evidence that Treehouse owns it. +fm_treehouse_pool_slot() { # + local project=$1 worktree=$2 slot pool state project_common slot_common + [ -d "$project" ] && [ -d "$worktree" ] || return 1 + slot=$(CDPATH='' cd -- "$worktree" 2>/dev/null && pwd -P) || return 1 + pool=$(dirname "$(dirname "$slot")") + state="$pool/treehouse-state.json" + [ -f "$state" ] && [ ! -L "$state" ] || return 1 + project_common=$(git -C "$project" rev-parse --path-format=absolute --git-common-dir 2>/dev/null) || return 1 + slot_common=$(git -C "$slot" rev-parse --path-format=absolute --git-common-dir 2>/dev/null) || return 1 + project_common=$(CDPATH='' cd -- "$project_common" 2>/dev/null && pwd -P) || return 1 + slot_common=$(CDPATH='' cd -- "$slot_common" 2>/dev/null && pwd -P) || return 1 + [ "$project_common" = "$slot_common" ] +} + +# Slot-owner claim: which task a Treehouse pool slot currently belongs to. +# +# Treehouse can record ownership durably: `treehouse get --lease --lease-holder` +# reserves a slot under a label until `treehouse return --if-lease-holder` +# releases it, and Firstmate uses exactly that for secondmate homes +# (bin/fm-home-seed.sh). Crewmate spawns do not take that path: they acquire +# their slot through the interactive pane-driven `treehouse get`, whose state +# entry is a live process lease (owner_pid plus owner_started_at, and `treehouse +# status` reports in-use from the processes actually running under the path). +# That answers "is anything running here", never "which task owns this", and it +# is released by the very event that makes a task record stale - the worker +# exiting - so a slot whose lease has lapsed reads identical whether it is still +# this task's or has since been handed to another one. Firstmate therefore keeps +# its own claim on top: one file naming the task that took the slot, written by +# bin/fm-spawn.sh under the same project lock that allocates the slot and +# released by bin/fm-teardown.sh when the slot goes back to the pool. Moving +# crewmate spawns onto the durable lease is separate follow-up work. +# +# The claim lives at //.fm-slot-owner - a sibling of the repo +# checkout rather than a file inside it - so claiming a slot can never dirty the +# copy teardown's landed-work checks inspect, and a returned slot carries no +# untracked leftover from it. +fm_treehouse_slot_owner_marker() { # + local worktree=$1 slot + slot=$(CDPATH='' cd -- "$worktree" 2>/dev/null && pwd -P) || return 1 + printf '%s/.fm-slot-owner\n' "$(dirname "$slot")" +} + +# Claim a pool slot for a task, replacing whatever the previous holder left. +# The rename is atomic, so a reader either sees the old claim or the new one. +fm_treehouse_slot_owner_claim() { # + local worktree=$1 id=$2 home=$3 marker tmp + [ -n "$id" ] || return 1 + marker=$(fm_treehouse_slot_owner_marker "$worktree") || return 1 + # Only a plain claim file may be replaced: renaming onto a directory would + # move the new claim inside it and leave the slot reading as unclaimable. + if { [ -e "$marker" ] || [ -L "$marker" ]; } \ + && { [ ! -f "$marker" ] || [ -L "$marker" ]; }; then + return 1 + fi + tmp="$marker.tmp.${BASHPID:-$$}" + rm -f "$tmp" || return 1 + { + printf 'task=%s\n' "$id" + printf 'home=%s\n' "$home" + } > "$tmp" 2>/dev/null || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$marker" 2>/dev/null || { rm -f "$tmp"; return 1; } +} + +# Read the claim on a pool slot and compare it with a task id. +# Sets FM_TREEHOUSE_SLOT_OWNER to one of: +# mine - the claim names this task +# other - the claim names a different task, so the slot was reassigned +# absent - no claim: the slot was taken before claims existed, or returned since +# unsafe - a claim file exists but cannot be read as a claim +# FM_TREEHOUSE_SLOT_OWNER_ID and FM_TREEHOUSE_SLOT_OWNER_HOME carry the recorded +# claimant as evidence. The home is reported, never matched: a home that moved +# must not turn a task's own slot into a refusal. +fm_treehouse_slot_owner_state() { # + local worktree=$1 id=$2 marker line owner_id='' owner_home='' + FM_TREEHOUSE_SLOT_OWNER=unsafe + FM_TREEHOUSE_SLOT_OWNER_ID= + FM_TREEHOUSE_SLOT_OWNER_HOME= + marker=$(fm_treehouse_slot_owner_marker "$worktree") || return 0 + if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then + FM_TREEHOUSE_SLOT_OWNER=absent + return 0 + fi + [ -f "$marker" ] && [ ! -L "$marker" ] || return 0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + task=*) owner_id=${line#task=} ;; + home=*) owner_home=${line#home=} ;; + esac + done < "$marker" || return 0 + [ -n "$owner_id" ] || return 0 + # shellcheck disable=SC2034 # Output globals, read by the sourcing caller. + FM_TREEHOUSE_SLOT_OWNER_ID=$owner_id + # shellcheck disable=SC2034 # Output globals, read by the sourcing caller. + FM_TREEHOUSE_SLOT_OWNER_HOME=$owner_home + if [ "$owner_id" = "$id" ]; then + FM_TREEHOUSE_SLOT_OWNER=mine + else + FM_TREEHOUSE_SLOT_OWNER=other + fi +} + +# Drop a task's own claim once its slot is back in the pool. Never removes +# another task's claim, so a misdirected release cannot strip the evidence that +# protects the slot's real owner. +fm_treehouse_slot_owner_release() { # + local worktree=$1 id=$2 marker + fm_treehouse_slot_owner_state "$worktree" "$id" + [ "$FM_TREEHOUSE_SLOT_OWNER" = mine ] || return 0 + marker=$(fm_treehouse_slot_owner_marker "$worktree") || return 0 + rm -f "$marker" 2>/dev/null || true +} + fm_failure_episode_reset() { local state=$1 mode=${2:-acquire} lock current pid acquired=0 path lock="$state/.turnend-claude-blocks.lock" diff --git a/docs/architecture.md b/docs/architecture.md index 08bc4e365d7..23d2e0eea3d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -291,7 +291,9 @@ After the forge command returns, the script reads the pull request's live state A verified merge leaves a durable role-routed outcome instead of living only in the merging agent's memory, and [`bin/fm-merge-outcome-lib.sh`](../bin/fm-merge-outcome-lib.sh)'s header owns its destination, shape, identity, normal-case deduplication, and at-least-once recovery. The same emitter handles a merge firstmate performed and one its poll detected, while the watcher immediately delivers the emitter's local actionable poll row. Teardown is fail-closed for ship worktrees: dirty worktrees refuse, and committed work must be landed before the worktree is returned. -A pool worktree is only returned after teardown passes the slot-ownership proof: a contradictory task record or supported live endpoint refuses without touching either task, and no discard authority relaxes that. +A pool worktree is only returned after teardown passes the slot-ownership proof: a contradictory task record or a supported live endpoint refuses without touching either task, and no discard authority relaxes that. +A slot's own owner claim, written by the spawn that takes it under the allocation lock and owned by [`bin/fm-wake-lib.sh`](../bin/fm-wake-lib.sh), covers a slot reassigned to a task that left no record the scan could reach: a claim naming a different task releases nothing - teardown warns, names the claimant, and finishes only the task's own cleanup - because Treehouse's own live process lease cannot answer ownership once the worker's exit releases it. +Allocation and return serialize on one project lock per machine-local Firstmate tree: every home reachable through local parent links shares that lock, and a home seeded from another machine anchors its own, because a lock taken on this filesystem is neither held nor observable across that boundary. Before the worktree is returned, teardown concludes the task's own no-mistakes run when it is parked at a gate, including a run whose head the task copy cannot resolve - the shared runs-ledger continuation proof is the only recognition for that case, so cleanup never orphans a parked run the pipeline advanced past the submitted head. [`bin/fm-teardown.sh`](../bin/fm-teardown.sh)'s header owns the landed-work proofs, slot-ownership proof, PR-discovery fallback, pre-teardown run conclusion, and stale-lock recovery procedure; [`tests/fm-teardown-endpoint-safety.test.sh`](../tests/fm-teardown-endpoint-safety.test.sh) and [`tests/fm-secondmate-safety.test.sh`](../tests/fm-secondmate-safety.test.sh) pin the slot-collision boundary. diff --git a/tests/fm-spawn-pool-base-freshen.test.sh b/tests/fm-spawn-pool-base-freshen.test.sh index 5057e4b81a0..1d7390d114a 100755 --- a/tests/fm-spawn-pool-base-freshen.test.sh +++ b/tests/fm-spawn-pool-base-freshen.test.sh @@ -570,8 +570,64 @@ test_pool_cleanliness_predicate() { pass "pool cleanliness allows only the lone root treehouse.toml" } +# The spawn side of the slot-owner claim that bin/fm-teardown.sh later reads: +# a launched task's claim names it, a slot that cannot be claimed refuses before +# anything is published, and an abort while the allocation lock is still held +# leaves no claim naming a task with no record. The make_case pool is already a +# managed // layout, so the claim lands beside the checkout. +test_pool_slot_claim_follows_the_spawn_outcome() { + local rec id out status before slot_claim + + id='pool-slot-claim-r1' + rec=$(make_case slot-claim "$id") + read_case_record "$rec" + slot_claim="$(dirname "$POOL_DIR")/.fm-slot-owner" + out=$(run_spawn "$id" --scout) + status=$? + expect_code 0 "$status" "spawn from a Treehouse slot should launch"$'\n'"$out" + assert_grep "worktree=$POOL_DIR" "$HOME_DIR/state/$id.meta" \ + "spawn did not publish the pool slot as its worktree" + [ -f "$slot_claim" ] || fail "spawn left its Treehouse slot unclaimed: $out" + grep -Fxq -- "task=$id" "$slot_claim" \ + || fail "the slot claim does not name the spawned task: $(cat "$slot_claim")" + grep -Fxq -- "home=$HOME_DIR" "$slot_claim" \ + || fail "the slot claim does not name the spawning home: $(cat "$slot_claim")" + + id='pool-slot-unclaimable-r1' + rec=$(make_case slot-unclaimable "$id") + read_case_record "$rec" + slot_claim="$(dirname "$POOL_DIR")/.fm-slot-owner" + mkdir -p "$slot_claim" + before=$(git -C "$POOL_DIR" rev-parse HEAD) + out=$(run_spawn "$id" --scout) + status=$? + [ "$status" -ne 0 ] || fail "spawn launched a worker on a slot it could not claim" + assert_contains "$out" "could not claim Treehouse pool slot" \ + "spawn did not name the unclaimable slot as the reason" + [ -d "$slot_claim" ] || fail "spawn replaced the directory blocking its slot claim" + [ ! -e "$HOME_DIR/state/$id.meta" ] || fail "spawn published a record for an unclaimable slot" + [ "$(git -C "$POOL_DIR" rev-parse HEAD)" = "$before" ] \ + || fail "spawn moved the slot's HEAD after failing to claim it" + + id='pool-slot-claim-aborted-r1' + rec=$(make_case slot-claim-aborted "$id") + read_case_record "$rec" + slot_claim="$(dirname "$POOL_DIR")/.fm-slot-owner" + git -C "$POOL_DIR" remote set-url origin "file://$CASE_DIR/missing-origin.git" + out=$(run_spawn "$id" --mode no-mistakes --yolo off) + status=$? + [ "$status" -ne 0 ] || fail "spawn succeeded despite an unusable origin on the slot" + assert_contains "$out" "could not fetch origin" \ + "the aborted spawn did not refuse on its unusable origin" + [ ! -e "$HOME_DIR/state/$id.meta" ] || fail "the aborted spawn published task metadata" + [ ! -e "$slot_claim" ] && [ ! -L "$slot_claim" ] \ + || fail "the aborted spawn left a slot claim naming a task with no record: $(cat "$slot_claim")" + pass "a Treehouse slot claim names the launched task, refuses when unclaimable, and is dropped by a locked abort" +} + test_pool_cleanliness_predicate test_acquisition_guards_before_treehouse_reset +test_pool_slot_claim_follows_the_spawn_outcome test_stale_pool_base_refreshes_before_branching test_non_main_default_branch_refreshes_before_branching test_direct_pr_and_scout_refresh_before_launch diff --git a/tests/fm-teardown-endpoint-safety.test.sh b/tests/fm-teardown-endpoint-safety.test.sh index 23cd6815d47..2aa1bae2012 100755 --- a/tests/fm-teardown-endpoint-safety.test.sh +++ b/tests/fm-teardown-endpoint-safety.test.sh @@ -48,6 +48,11 @@ mark_case_as_treehouse_pool() { # : > "$dir/worktree/sentinel" } +claim_pool_slot() { # [home] + local dir=$1 id=$2 home=${3:-$1/home} + printf 'task=%s\nhome=%s\n' "$id" "$home" > "$dir/pool/1/.fm-slot-owner" +} + run_case() { # local dir=$1 id=$2 FM_HOME="$dir/home" FM_ROOT_OVERRIDE="$ROOT" \ @@ -501,6 +506,375 @@ SH pass "fm-teardown: an exact recorded endpoint still tears down after changing cwd outside its worktree" } +# --- Treehouse project-lock anchoring across home layouts -------------------- +# +# The lock is anchored at the local root home, so every home on this machine +# that can reach the same pool must derive the identical file. A remote parent +# binding terminates that walk at the home holding it: its parent is on another +# machine and can neither hold nor observe a lock taken here. + +write_local_parent_record() { # + cat > "$1/.fm-secondmate-parent" < + cat > "$1/.fm-secondmate-parent" <<'REC' +schema=fm-secondmate-parent.v1 +route=remote +parent_host=machine-a +REC +} + +make_home() { # + mkdir -p "$1/state" "$1/data" "$1/config" "$1/projects" +} + +resolve_project_lock() { # + FM_HOME="$1" bash -c '. "$1"; fm_treehouse_project_lock_path "$2"' _ \ + "$ROOT/bin/fm-wake-lib.sh" "$2" +} + +test_project_lock_anchors_at_the_local_root_across_home_layouts() { + local dir main_home main_project local_mate remote_mate remote_child + local main_lock mate_lock remote_lock child_lock orphan_lock rc + dir=$(make_case project-lock-anchoring) + git -C "$dir/project" -c user.name=test -c user.email=test@example.invalid \ + commit --allow-empty -qm anchor-fixture + + # Main-home layout: a root home and a local secondmate beneath it. + main_home="$dir/home" + main_project="$main_home/projects/project" + make_home "$main_home" + git clone -q "$dir/project" "$main_project" + local_mate="$dir/local-mate" + make_home "$local_mate" + write_local_parent_record "$local_mate" "$main_home" + git clone -q "$dir/project" "$local_mate/projects/project" + + # Remote layout: a home seeded from another machine, plus its own local child. + remote_mate="$dir/remote-mate" + make_home "$remote_mate" + write_remote_parent_record "$remote_mate" + git clone -q "$dir/project" "$remote_mate/projects/project" + remote_child="$dir/remote-mate-child" + make_home "$remote_child" + write_local_parent_record "$remote_child" "$remote_mate" + git clone -q "$dir/project" "$remote_child/projects/project" + + main_lock=$(resolve_project_lock "$main_home" "$main_project") \ + || fail "the root home could not resolve its project lock" + mate_lock=$(resolve_project_lock "$local_mate" "$local_mate/projects/project") \ + || fail "a local secondmate home could not resolve its project lock" + remote_lock=$(resolve_project_lock "$remote_mate" "$remote_mate/projects/project") \ + || fail "a remote-seeded secondmate home could not resolve its project lock" + child_lock=$(resolve_project_lock "$remote_child" "$remote_child/projects/project") \ + || fail "a local child of a remote-seeded home could not resolve its project lock" + + [ "$main_lock" = "$mate_lock" ] \ + || fail "the root home and its local secondmate derived different project locks" + [ "$remote_lock" = "$child_lock" ] \ + || fail "a remote-seeded home and its local child derived different project locks" + case "$remote_lock" in + "$remote_mate/state/"*) ;; + *) fail "a remote-seeded home anchored its project lock outside its own state: $remote_lock" ;; + esac + + # An origin-less local-only project still resolves, keyed on its worktree top. + git init -q "$remote_mate/projects/local-only" + orphan_lock=$(resolve_project_lock "$remote_mate" "$remote_mate/projects/local-only") \ + || fail "an origin-less local-only project could not resolve its lock in a remote-seeded home" + [ "$orphan_lock" != "$remote_lock" ] \ + || fail "an origin-less project shared the lock identity of an unrelated origin" + + # Everything other than a remote route still fails closed. + printf 'schema=fm-secondmate-parent.v1\nroute=sideways\n' \ + > "$remote_child/.fm-secondmate-parent" + set +e + resolve_project_lock "$remote_child" "$remote_child/projects/project" >/dev/null 2>&1 + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "an unsupported parent route resolved a project lock instead of refusing" + + pass "Treehouse project locking anchors at the local root for main-home, local-secondmate, and remote-seeded layouts" +} + +test_remote_seeded_home_returns_its_uncontested_slot() { + local dir id=remote-task rc + dir=$(make_case remote-home-teardown) + mark_case_as_treehouse_pool "$dir" + write_remote_parent_record "$dir/home" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + [ "$rc" -eq 0 ] \ + || fail "teardown in a remote-seeded home refused its own uncontested slot: $(cat "$dir/stderr")" + assert_absent "$dir/home/state/$id.meta" "remote-seeded teardown left the task record" + grep -Fq "treehouse " "$dir/runtime.log" \ + || fail "remote-seeded teardown did not return its own pool slot: $(cat "$dir/runtime.log")" + if [ "${FM_TEST_EVIDENCE:-0}" = 1 ]; then + printf '# remote-seeded Treehouse teardown command\n' + printf '$ FM_HOME=%s bin/fm-teardown.sh %s --force\n' "$dir/home" "$id" + printf 'stdout:\n'; cat "$dir/stdout" + printf 'stderr:\n'; cat "$dir/stderr" + printf 'exit=%s\nruntime calls:\n' "$rc"; cat "$dir/runtime.log" + printf 'task metadata=%s\nslot sentinel=%s\n' \ + "$([ -e "$dir/home/state/$id.meta" ] && printf present || printf removed)" \ + "$([ -e "$dir/worktree/sentinel" ] && printf present || printf removed)" + fi + + pass "fm-teardown: a remote-seeded secondmate home returns its own uncontested pool slot" +} + +test_remote_seeded_home_still_refuses_a_slot_its_child_holds() { + local dir id=remote-stale other=child-task child_home child_project rc + dir=$(make_case remote-home-collision) + mark_case_as_treehouse_pool "$dir" + write_remote_parent_record "$dir/home" + printf 'fixture\n' > "$dir/project/tracked" + git -C "$dir/project" add tracked + git -C "$dir/project" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + child_home="$dir/child-home" + child_project="$child_home/projects/project" + make_home "$child_home" + write_local_parent_record "$child_home" "$dir/home" + git clone -q "$dir/project" "$child_project" + printf '%s\n' "- mate - fixture (home: $child_home; scope: test; projects: project; added 2026-01-01)" \ + > "$dir/home/data/secondmates.md" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + fm_write_meta "$child_home/state/$other.meta" \ + "window=firstmate:fm-$other" "endpoint_task_id=$other" \ + "worktree=$dir/worktree" "project=$child_project" "kind=scout" + + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + [ "$rc" -ne 0 ] \ + || fail "a remote-seeded home returned a pool slot its own local child still holds" + assert_present "$dir/home/state/$id.meta" "remote-layout collision removed stale metadata" + assert_present "$child_home/state/$other.meta" "remote-layout collision removed live metadata" + assert_present "$dir/worktree/sentinel" "remote-layout collision reset the shared slot" + assert_contains "$(cat "$dir/stderr")" "$other" \ + "remote-layout refusal should name the task holding the slot" + if [ "${FM_TEST_EVIDENCE:-0}" = 1 ]; then + printf '# remote-seeded cross-home collision command\n' + printf '$ FM_HOME=%s bin/fm-teardown.sh %s --force\n' "$dir/home" "$id" + printf 'stderr:\n'; cat "$dir/stderr" + printf 'exit=%s\nruntime calls=%s\n' "$rc" \ + "$([ -s "$dir/runtime.log" ] && cat "$dir/runtime.log" || printf none)" + printf 'remote metadata=%s\nchild metadata=%s\nslot sentinel=%s\n' \ + "$([ -e "$dir/home/state/$id.meta" ] && printf preserved || printf removed)" \ + "$([ -e "$child_home/state/$other.meta" ] && printf preserved || printf removed)" \ + "$([ -e "$dir/worktree/sentinel" ] && printf preserved || printf removed)" + fi + + pass "fm-teardown: slot ownership across a remote-seeded home and its local child still refuses" +} + +test_remote_layout_homes_serialize_on_one_project_lock() { + local dir id=remote-serialize child_home child_project lock holder rc waited=0 + dir=$(make_case remote-lock-exclusion) + mark_case_as_treehouse_pool "$dir" + write_remote_parent_record "$dir/home" + printf 'fixture\n' > "$dir/project/tracked" + git -C "$dir/project" add tracked + git -C "$dir/project" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + child_home="$dir/child-home" + child_project="$child_home/projects/project" + make_home "$child_home" + write_local_parent_record "$child_home" "$dir/home" + git clone -q "$dir/project" "$child_project" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + + # The local child takes the lock its own home derives and stays alive holding + # it, standing in for a slot allocation running in that home right now. + lock=$(resolve_project_lock "$child_home" "$child_project") \ + || fail "the local child could not resolve the shared project lock" + FM_HOME="$child_home" bash -c \ + '. "$1"; fm_lock_try_acquire "$2" || exit 1; : > "$3"; exec sleep 30' _ \ + "$ROOT/bin/fm-wake-lib.sh" "$lock" "$dir/lock-held" & + holder=$! + while [ ! -e "$dir/lock-held" ] && [ "$waited" -lt 100 ]; do + kill -0 "$holder" 2>/dev/null || break + sleep 0.1 + waited=$((waited + 1)) + done + [ -e "$dir/lock-held" ] || fail "the local child never took the shared project lock" + + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + kill "$holder" 2>/dev/null || true + wait "$holder" 2>/dev/null || true + + [ "$rc" -ne 0 ] \ + || fail "a remote-seeded home returned a pool slot while its local child held the shared lock" + assert_present "$dir/home/state/$id.meta" "contended remote-layout teardown removed the task record" + assert_present "$dir/worktree/sentinel" "contended remote-layout teardown reset the slot" + [ ! -s "$dir/runtime.log" ] \ + || fail "contended remote-layout teardown reached the runtime: $(cat "$dir/runtime.log")" + assert_contains "$(cat "$dir/stderr")" "another Treehouse slot allocation or return is in progress" \ + "the refusal should name the shared project lock, not some unrelated check" + + pass "Treehouse project locking still serializes two homes across the remote-seeded boundary" +} + +# The slot-reuse sequence with only ONE discoverable record: the finished task's +# worker exited, its slot was granted to another task, and that task leaves no +# record this home can enumerate. Nothing in the record scan contradicts the +# stale worktree= line, so the slot's own owner claim is the only evidence that +# it was reassigned. The slot is no longer this task's, so teardown finishes the +# task's own cleanup and leaves the slot - its worker, its copy, its claim - +# exactly as it found it. +assert_reassigned_slot_left_alone() { # + local dir=$1 id=$2 other=$3 description=$4 + assert_absent "$dir/home/state/$id.meta" "$description: the stale task's own record was not removed" + assert_present "$dir/pool/1/.fm-slot-owner" "$description: another task's slot claim was removed" + assert_contains "$(cat "$dir/pool/1/.fm-slot-owner")" "task=$other" \ + "$description: another task's slot claim was rewritten" + assert_present "$dir/pool/1/project/.git" "$description: the reassigned slot's checkout was removed" + ! grep -Fq "treehouse " "$dir/runtime.log" \ + || fail "$description: the reassigned slot was returned to the pool: $(cat "$dir/runtime.log")" + assert_contains "$(cat "$dir/stderr")" "$other" \ + "$description: the warning should name the task the slot was reassigned to" + assert_contains "$(cat "$dir/stderr")" "reassigned" \ + "$description: the warning should name the reassignment as the cause" +} + +test_reassigned_pool_slot_finishes_own_cleanup_without_touching_the_slot() { + local dir id=stale-task other=reassigned-task worker rc + + # Dirty slot, --force, and a live worker inside it: --force authorizes + # discarding this task's unlanded work, which is already gone with the slot, + # never the other task's live work. + dir=$(make_case slot-reassigned) + mark_case_as_treehouse_pool "$dir" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + claim_pool_slot "$dir" "$other" "$dir/other-home" + # Staged in this shell, not a command substitution: a background child of a + # $(...) subshell does not outlive it, and the point of this worker is to be + # alive in the slot while teardown runs. + ( cd "$dir/worktree" && exec sleep 30 ) & + worker=$! + + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + + [ "$rc" -eq 0 ] || fail "teardown of a task whose slot was reassigned failed: $(cat "$dir/stderr")" + kill -0 "$worker" 2>/dev/null || fail "teardown killed the worker holding the reassigned pool slot" + assert_present "$dir/worktree/sentinel" "teardown reset a pool slot another task had claimed" + assert_reassigned_slot_left_alone "$dir" "$id" "$other" "dirty reassigned slot with --force" + assert_contains "$(cat "$dir/stderr")" "$dir/other-home" \ + "the warning should name the claimant's home" + kill "$worker" 2>/dev/null || true + wait "$worker" 2>/dev/null || true + + # The same reassignment on a CLEAN slot: a landed ship task torn down without + # --force, which is the shape of the real incident. A clean, fully landed copy + # passes every unlanded-work check, so only the ownership determination can + # keep this slot out of the pool; a guard keyed off dirtiness would return it + # and destroy the live task's copy. + dir=$(make_case slot-reassigned-clean) + mark_case_as_treehouse_pool "$dir" + rm -f "$dir/worktree/sentinel" + [ -z "$(git -C "$dir/worktree" status --porcelain)" ] \ + || fail "clean-slot fixture is not clean: $(git -C "$dir/worktree" status --porcelain)" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=ship" + claim_pool_slot "$dir" "$other" "$dir/other-home" + ( cd "$dir/worktree" && exec sleep 30 ) & + worker=$! + + set +e + FM_HOME="$dir/home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_RUNTIME_LOG="$dir/runtime.log" PATH="$dir/fakebin:$PATH" \ + "$TEARDOWN" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + [ "$rc" -eq 0 ] || fail "teardown of a clean ship task whose slot was reassigned failed: $(cat "$dir/stderr")" + kill -0 "$worker" 2>/dev/null || fail "teardown killed the worker holding the clean reassigned pool slot" + assert_reassigned_slot_left_alone "$dir" "$id" "$other" "clean reassigned slot without --force" + kill "$worker" 2>/dev/null || true + wait "$worker" 2>/dev/null || true + + # A claim that exists but cannot be read as a claim proves nothing either way, + # so it refuses rather than guessing the slot is still this task's. + dir=$(make_case slot-claim-unreadable) + mark_case_as_treehouse_pool "$dir" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + printf 'not-a-claim\n' > "$dir/pool/1/.fm-slot-owner" + + set +e + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "teardown returned a pool slot whose claim could not be read" + assert_present "$dir/worktree/sentinel" "teardown reset a pool slot whose claim could not be read" + assert_present "$dir/pool/1/.fm-slot-owner" "teardown removed an unreadable slot claim" + assert_present "$dir/home/state/$id.meta" "teardown removed the task record on an unreadable claim" + [ ! -s "$dir/runtime.log" ] \ + || fail "teardown reached the runtime on an unreadable slot claim: $(cat "$dir/runtime.log")" + assert_contains "$(cat "$dir/stderr")" "$dir/pool/1/.fm-slot-owner" \ + "unreadable-claim refusal should name the claim file to inspect" + + pass "fm-teardown: a pool slot claimed by another task is left alone while the task's own cleanup finishes" +} + +# The two states that must never become a false refusal: the task's own claim, +# and no claim at all (a slot taken before claims existed, or already returned). +test_own_and_absent_slot_claims_still_tear_down() { + local dir id=owned-task + + dir=$(make_case slot-claim-own) + mark_case_as_treehouse_pool "$dir" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + claim_pool_slot "$dir" "$id" + + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" \ + || fail "teardown of a task holding its own slot claim failed: $(cat "$dir/stderr")" + assert_absent "$dir/home/state/$id.meta" "own-claim teardown left the task record" + assert_absent "$dir/pool/1/.fm-slot-owner" "own-claim teardown left its spent slot claim behind" + grep -Fq "treehouse " "$dir/runtime.log" \ + || fail "own-claim teardown did not return its own pool slot: $(cat "$dir/runtime.log")" + + dir=$(make_case slot-claim-absent) + mark_case_as_treehouse_pool "$dir" + fm_write_meta "$dir/home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" \ + "worktree=$dir/worktree" "project=$dir/project" "kind=scout" + + run_case "$dir" "$id" > "$dir/stdout" 2> "$dir/stderr" \ + || fail "teardown of an unclaimed slot failed: $(cat "$dir/stderr")" + assert_absent "$dir/home/state/$id.meta" "unclaimed-slot teardown left the task record" + grep -Fq "treehouse " "$dir/runtime.log" \ + || fail "unclaimed-slot teardown did not return its pool slot: $(cat "$dir/runtime.log")" + + pass "fm-teardown: a task's own slot claim, and an unclaimed slot, both still tear down" +} + test_invalid_endpoint_records_refuse_before_mutation test_non_pool_teardown_ignores_task_set_lock test_supported_backend_endpoint_records_validate @@ -511,4 +885,10 @@ test_bare_relative_origin_shares_project_lock_with_clone test_reused_pool_slot_refuses_before_touching_the_other_task test_cross_home_pool_slot_collision_refuses test_sole_slot_record_still_tears_down +test_reassigned_pool_slot_finishes_own_cleanup_without_touching_the_slot +test_own_and_absent_slot_claims_still_tear_down test_recorded_endpoint_that_changed_directory_still_tears_down +test_project_lock_anchors_at_the_local_root_across_home_layouts +test_remote_seeded_home_returns_its_uncontested_slot +test_remote_seeded_home_still_refuses_a_slot_its_child_holds +test_remote_layout_homes_serialize_on_one_project_lock From aa92aa0c904626f708ba734510b90efdc146f59c Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 17:10:50 +0800 Subject: [PATCH 03/23] fix: terminate Treehouse lock root walk at a remote secondmate parent Adapts the lock anchoring to this fork's remote secondmate homes (the route=remote parent record bin/fm-secondmate-parent-lib.sh owns): a remote binding ends the local root walk at the home holding it, matching upstream 64d3905b, so a remote-seeded home anchors and resolves its own project lock instead of failing to derive one and refusing every pool-slot teardown. --- bin/fm-wake-lib.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 062d6185e17..8fbefbbca45 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -923,7 +923,11 @@ fm_firstmate_root_home() { . "$FM_WAKE_LIB_DIR/fm-secondmate-parent-lib.sh" fi fm_secondmate_parent_record_parse "$marker" || return 1 - [ "$FM_SECONDMATE_PARENT_ROUTE" = local ] || return 1 + case "$FM_SECONDMATE_PARENT_ROUTE" in + local) ;; + remote) break ;; + *) return 1 ;; + esac parent=$(CDPATH='' cd -- "$FM_SECONDMATE_PARENT_HOME" 2>/dev/null && pwd -P) || return 1 case "$seen" in *"|$parent|"*) return 1 ;; esac seen="$seen$home|" From 4b3dcf5b26309896ab0c11ff69fba01522ce2268 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 17:16:43 +0800 Subject: [PATCH 04/23] chore: pin Treehouse v2.3.0 in the CI installer Bumps bin/fm-install-treehouse.sh from v2.1.1 to v2.3.0 with the release's published SHA-256 for all four archives (linux/darwin x amd64/arm64). v2.3.0's acquire fails closed on slots it cannot verify - skipping them and provisioning a fresh slot instead of attempting a reset - and its status reports a marker-absent slot as damaged rather than available, which removes the reset attempt that produced the misleading ancestry refusal in the aceh pool-orphan incident. Verified locally: a fresh install of the pinned archive reports v2.3.0; on a scratch pool containing a slot with a dangling .git marker and a slot with no marker at all, status reports the marker-absent slot damaged and get --lease skips both unverifiable slots without touching either, provisioning a new slot instead. Known-remaining gaps, documented rather than fixed per the captain's decision (scope: data/fm-aceh-pool-recovery-diagnosis/decision-pool-durable.md): a slot whose .git marker exists but resolves nowhere is still reported available (StatusDamaged covers only marker-absent slots), and the meta-absent lease-release path is still unimplemented. --- bin/fm-install-treehouse.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/fm-install-treehouse.sh b/bin/fm-install-treehouse.sh index 672283a5eb7..60df3202faa 100755 --- a/bin/fm-install-treehouse.sh +++ b/bin/fm-install-treehouse.sh @@ -9,12 +9,13 @@ # Usage: # fm-install-treehouse.sh # -# Pins Treehouse v2.1.1, the version exercised by the local real-Herdr suite. +# Pins Treehouse v2.3.0: the first release whose pool bookkeeping fails closed on +# slots it cannot verify instead of attempting a reset. set -eu -FM_TREEHOUSE_CI_VERSION=2.1.1 +FM_TREEHOUSE_CI_VERSION=2.3.0 FM_TREEHOUSE_CI_TAG="v${FM_TREEHOUSE_CI_VERSION}" -# Bounded download ceiling (bytes). Official 2.1.1 archives are under 8 MiB. +# Bounded download ceiling (bytes). Official 2.3.0 archives are under 8 MiB. FM_TREEHOUSE_CI_MAX_BYTES=15000000 FM_TREEHOUSE_CI_REPO=kunchenguid/treehouse @@ -30,19 +31,19 @@ arch=$(uname -m) case "${os}-${arch}" in Linux-x86_64) ARCHIVE=treehouse-v${FM_TREEHOUSE_CI_VERSION}-linux-amd64.tar.gz - SHA256=2fe3e01220ae51a967c3e5ba6ccf10ec83bdbae8e420368d194285a8d04c9ef8 + SHA256=94fd2b2c20c35aac1ddc2941317890ad82c9916f5ccecbac4a50cda783eed10f ;; Linux-aarch64|Linux-arm64) ARCHIVE=treehouse-v${FM_TREEHOUSE_CI_VERSION}-linux-arm64.tar.gz - SHA256=980367c0233274eb3181a19a2ca8ec69d09b4a588ba27367937d336f9a2c938e + SHA256=408589ba72b58d5e942071ed863a83fd96566cfd1e514945daa59defde528bbb ;; Darwin-arm64) ARCHIVE=treehouse-v${FM_TREEHOUSE_CI_VERSION}-darwin-arm64.tar.gz - SHA256=deabeb7153bad14659e98da78de5334afecaeaac7e05988b106a4888646747d3 + SHA256=1cb09bcfa830b4eec5e54beeaa71589adb9c5d828573dda0f5150e2d80cf13d5 ;; Darwin-x86_64) ARCHIVE=treehouse-v${FM_TREEHOUSE_CI_VERSION}-darwin-amd64.tar.gz - SHA256=f6f6bd71fe8279826aa35f201e79f34106c1c4056179e3e8141942027dd992a6 + SHA256=349afcc13c2beb20d846eb560a11b30e1a5cab8e2dfb22988a36aa7f213b5881 ;; *) die "unsupported platform ${os}-${arch}; official Treehouse assets are linux/darwin amd64 and arm64" @@ -68,7 +69,7 @@ fi [ "$ACTUAL_SHA256" = "$SHA256" ] || die "checksum mismatch for $ARCHIVE (expected $SHA256, got $ACTUAL_SHA256)" tar -xzf "$TMP/$ARCHIVE" -C "$TMP" -# Archive layout: a single `treehouse` binary at the archive root (verified for v2.1.1). +# Archive layout: a single `treehouse` binary at the archive root (verified for v2.3.0). if [ -f "$TMP/treehouse" ]; then BIN="$TMP/treehouse" elif [ -f "$TMP/treehouse-v${FM_TREEHOUSE_CI_VERSION}/treehouse" ]; then From 419db4183d60a06e5d01ba076a28858ce6b34ef6 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 17:55:14 +0800 Subject: [PATCH 05/23] fix: allow descendant state links that resolve inside the secondmate home Upstream b028e8b1's collect_descendant_task_locks refuses any symbolic-link state path. This fork's secondmate contract (pinned by fm-secondmate-safety.test.sh) permits links that resolve inside the home - the target is removed with the home - while refusing links that escape it. Resolve the link target and compare it against the resolved home boundary: links escaping the home, or that cannot be resolved at all, still refuse; inside-home links proceed and the later blanket -L refusal is dropped. --- bin/fm-teardown.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 33971d6e8e1..cfa57950a92 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -2438,8 +2438,24 @@ collect_descendant_task_locks() { local -a child_ids sub_state="$home/state" if [ -L "$sub_state" ]; then - echo "REFUSED: secondmate home $home has a symbolic-link state path at $sub_state; forced teardown changed nothing" >&2 - return 1 + # A state link that resolves inside its own home is an ordinary layout - + # the target is removed with the home - so only a link that escapes the + # home, or cannot be resolved at all, refuses. + local resolved_state resolved_home + resolved_state=$(removal_target_abs_path "$sub_state" 2>/dev/null || true) + resolved_home=$(removal_target_abs_path "$home" 2>/dev/null || true) + if [ -n "$resolved_state" ] && [ -n "$resolved_home" ]; then + case "$resolved_state" in + "$resolved_home"/*) ;; + *) resolved_state= ;; + esac + else + resolved_state= + fi + if [ -z "$resolved_state" ]; then + echo "REFUSED: secondmate home $home has a symbolic-link state path at $sub_state escaping the home; forced teardown changed nothing" >&2 + return 1 + fi fi if [ -e "$sub_state" ] && [ ! -d "$sub_state" ]; then echo "REFUSED: secondmate home $home has a non-directory state path at $sub_state; forced teardown changed nothing" >&2 @@ -2449,7 +2465,7 @@ collect_descendant_task_locks() { echo "REFUSED: secondmate home $home state directory could not be established at $sub_state; forced teardown changed nothing" >&2 return 1 fi - if [ -L "$sub_state" ] || [ ! -d "$sub_state" ]; then + if [ ! -d "$sub_state" ]; then echo "REFUSED: secondmate home $home state path is not a safe directory at $sub_state; forced teardown changed nothing" >&2 return 1 fi From 0f0217cac52ddf7f6b780b54f4e029aed1ebedf9 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 17:55:14 +0800 Subject: [PATCH 06/23] chore: drop relaunch-replacement declarations this fork does not use 7d14fc12 declares RELAUNCH_REPLACEMENT_* state for upstream's relaunch-replacement machinery, which this fork lacks. Shellcheck flagged the dead globals; removing them keeps the pick limited to the claim behavior being adapted. --- bin/fm-spawn.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index ce7b5616984..dc16db4a78b 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -956,11 +956,6 @@ SPAWN_META_LOCK_HELD=0 SPAWN_TREEHOUSE_PROJECT_LOCK= SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 SPAWN_SLOT_CLAIMED=0 -RELAUNCH_REPLACEMENT_PENDING=0 -RELAUNCH_REPLACEMENT_BUSY_GEN= -RELAUNCH_REPLACEMENT_HARNESS= -RELAUNCH_REPLACEMENT_STATE= -RELAUNCH_REPLACEMENT_WT= CONFIG_INHERIT_LOCK= CONFIG_INHERIT_LOCK_HELD=0 TREEHOUSE_READY_DIR= From 53741b5c5b4710043e1ed4d6173d04ab2c6aca1a Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 17:55:14 +0800 Subject: [PATCH 07/23] test: retry concurrent spawns that lose the Treehouse project lock The shared project lock now makes a concurrent same-project spawn refuse fast with "another Treehouse slot allocation or return is in progress" instead of racing allocation. Mirror upstream's task-set-lock tolerance: capture each spawn's status, accept that one refusal message, and retry the task once the winner publishes. Applied to the order pair, the post-create abort fixtures (which still must fail on their armed validation error), and the three focus waves. The cross-home wave keeps bare waits - those homes are not parent-linked, so each anchors its own lock and cannot contend. --- .../fm-backend-herdr-presentation-e2e.test.sh | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 9302467b795..9606b55bf54 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -427,6 +427,25 @@ teardown_task() { # "$ROOT/bin/fm-teardown.sh" "$id" --force } +finish_concurrent_spawn() { # + local id=$1 status=$2 out=$3 err=$4 + [ "$status" -ne 0 ] || return 0 + grep -F "another Treehouse slot allocation or return is in progress" "$err" >/dev/null 2>&1 \ + || fail "concurrent projected spawn $id failed unexpectedly: $(cat "$err")" + spawn_task "$id" "$HOME_DIR" "$PROJECT_DIR" > "$out" 2> "$err" \ + || fail "projected spawn $id retry failed after the Treehouse project lock cleared: $(cat "$err")" +} + +finish_concurrent_expected_abort() { # + local id=$1 status=$2 out=$3 err=$4 + [ "$status" -ne 0 ] || fail "post-create abort fixture $id unexpectedly succeeded" + if grep -F "another Treehouse slot allocation or return is in progress" "$err" >/dev/null 2>&1; then + if spawn_task "$id" "$HOME_DIR" "$PROJECT_DIR" > "$out" 2> "$err"; then + fail "post-create abort fixture $id unexpectedly succeeded after the Treehouse project lock cleared" + fi + fi +} + finish_concurrent_teardown() { # local id=$1 status=$2 out=$3 err=$4 [ "$status" -ne 0 ] || return 0 @@ -734,16 +753,19 @@ cmp -s "$TMP_ROOT/off.meta.normalized" "$TMP_ROOT/on.meta.normalized" \ || fail "metadata changed beyond Herdr container IDs between opted-out and projected paths" # Two real concurrent primary spawns contend for the bounded presentation-order -# lock. Every spawn that acquires it must follow Herdr's actual serialized create -# order; a slow host may legitimately send the other through the tested flat -# fallback rather than waiting past the product's bounded contention policy. +# lock and the shared Treehouse project lock. Every spawn that acquires them must +# follow Herdr's actual serialized create order; a slow host may legitimately +# send the other through the tested flat fallback, and the project-lock loser +# refuses fast and is retried once the winner has published. CONCURRENT_FOCUS_AUDIT_START=$(focus_audit_line_count) spawn_task order-a "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/order-a.out" 2> "$TMP_ROOT/order-a.err" & ORDER_A_PID=$! spawn_task order-b "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/order-b.out" 2> "$TMP_ROOT/order-b.err" & ORDER_B_PID=$! -wait "$ORDER_A_PID" || fail "concurrent projected spawn A failed: $(cat "$TMP_ROOT/order-a.err")" -wait "$ORDER_B_PID" || fail "concurrent projected spawn B failed: $(cat "$TMP_ROOT/order-b.err")" +if wait "$ORDER_A_PID"; then ORDER_A_STATUS=0; else ORDER_A_STATUS=$?; fi +if wait "$ORDER_B_PID"; then ORDER_B_STATUS=0; else ORDER_B_STATUS=$?; fi +finish_concurrent_spawn order-a "$ORDER_A_STATUS" "$TMP_ROOT/order-a.out" "$TMP_ROOT/order-a.err" +finish_concurrent_spawn order-b "$ORDER_B_STATUS" "$TMP_ROOT/order-b.out" "$TMP_ROOT/order-b.err" assert_focus_is "$CAPTAIN_FOCUS" "concurrent projected spawns" assert_raw_presentation_mutations_preserved_since "$CONCURRENT_FOCUS_AUDIT_START" "concurrent projected spawns" ORDER_A_META="$HOME_DIR/state/order-a.meta" @@ -831,8 +853,8 @@ ABORT_A_STATUS=0 ABORT_B_STATUS=0 wait "$ABORT_A_PID" || ABORT_A_STATUS=$? wait "$ABORT_B_PID" || ABORT_B_STATUS=$? -[ "$ABORT_A_STATUS" -ne 0 ] || fail "post-create abort fixture A unexpectedly succeeded" -[ "$ABORT_B_STATUS" -ne 0 ] || fail "post-create abort fixture B unexpectedly succeeded" +finish_concurrent_expected_abort abort-a "$ABORT_A_STATUS" "$TMP_ROOT/abort-a.out" "$TMP_ROOT/abort-a.err" +finish_concurrent_expected_abort abort-b "$ABORT_B_STATUS" "$TMP_ROOT/abort-b.out" "$TMP_ROOT/abort-b.err" grep -F "yielded a dirty pool worktree" "$TMP_ROOT/abort-a.err" >/dev/null 2>&1 \ || fail "post-create abort fixture A did not reach the armed validation failure: $(cat "$TMP_ROOT/abort-a.err")" grep -F "yielded a dirty pool worktree" "$TMP_ROOT/abort-b.err" >/dev/null 2>&1 \ @@ -910,8 +932,10 @@ for ROUND in 1 2 3; do WAVE_A_PID=$! spawn_task "focus-$ROUND-b" "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/focus-$ROUND-b.out" 2> "$TMP_ROOT/focus-$ROUND-b.err" & WAVE_B_PID=$! - wait "$WAVE_A_PID" || fail "focus wave $ROUND spawn A failed: $(cat "$TMP_ROOT/focus-$ROUND-a.err")" - wait "$WAVE_B_PID" || fail "focus wave $ROUND spawn B failed: $(cat "$TMP_ROOT/focus-$ROUND-b.err")" + if wait "$WAVE_A_PID"; then WAVE_A_STATUS=0; else WAVE_A_STATUS=$?; fi + if wait "$WAVE_B_PID"; then WAVE_B_STATUS=0; else WAVE_B_STATUS=$?; fi + finish_concurrent_spawn "focus-$ROUND-a" "$WAVE_A_STATUS" "$TMP_ROOT/focus-$ROUND-a.out" "$TMP_ROOT/focus-$ROUND-a.err" + finish_concurrent_spawn "focus-$ROUND-b" "$WAVE_B_STATUS" "$TMP_ROOT/focus-$ROUND-b.out" "$TMP_ROOT/focus-$ROUND-b.err" remember_meta_worktree "$HOME_DIR/state/focus-$ROUND-a.meta" >/dev/null remember_meta_worktree "$HOME_DIR/state/focus-$ROUND-b.meta" >/dev/null assert_focus_is "$CAPTAIN_FOCUS" "focus wave $ROUND concurrent spawns" From 907ec369556ed992169a760bae3b4d1d80d19e42 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 18:16:13 +0800 Subject: [PATCH 08/23] no-mistakes(review): Initialize teardown trap state before lease guard --- bin/fm-teardown.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index cfa57950a92..32e93460c37 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -255,6 +255,11 @@ if [ "$FORCE" = --force ] && [ "$TEARDOWN_ACTOR" = branch ]; then echo "error: forced teardown refused - the supervision branch cannot discard work" >&2 exit "$FM_LEASE_REFUSE_EXIT" fi +DESCENDANT_LOCK_PATHS=() +TREEHOUSE_PROJECT_LOCK= +TREEHOUSE_PROJECT_LOCK_HELD=0 +META_LOCK= +META_LOCK_HELD=0 teardown_exit_cleanup() { local status=$? i if declare -F teardown_release_herdr_locks >/dev/null 2>&1; then @@ -279,8 +284,6 @@ trap teardown_exit_cleanup EXIT fm_lease_guard "$ID" "teardown" META="$STATE/$ID.meta" -TREEHOUSE_PROJECT_LOCK= -TREEHOUSE_PROJECT_LOCK_HELD=0 TREEHOUSE_SLOT_LOCK_REQUIRED=0 if [ -f "$META" ] && [ ! -L "$META" ]; then TEARDOWN_LOCK_KIND=$(fm_meta_get "$META" kind) @@ -304,7 +307,6 @@ if [ -f "$META" ] && [ ! -L "$META" ]; then TREEHOUSE_PROJECT_LOCK_HELD=1 fi fi -DESCENDANT_LOCK_PATHS=() DESCENDANT_TASK_STATES=() DESCENDANT_TASK_IDS=() DESCENDANT_TASK_KINDS=() @@ -316,7 +318,6 @@ fm_refuse_if_gate_agent FM_LOCK_LOG_PREFIX=teardown META_LOCK=$(fm_meta_lock_path "$META") || exit 1 -META_LOCK_HELD=0 fm_lock_acquire_wait "$META_LOCK" META_LOCK_HELD=1 [ -f "$META" ] || { echo "error: no meta for task $ID at $META" >&2; exit 1; } From 0258bbe90ef3c6b11eaad216859253d9a475fc8e Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 18:25:03 +0800 Subject: [PATCH 09/23] no-mistakes(document): Updated Treehouse verification documentation --- 15000000 | 1 + docs/verification/runtime-backends.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 15000000 diff --git a/15000000 b/15000000 new file mode 100644 index 00000000000..44d6628cdc6 --- /dev/null +++ b/15000000 @@ -0,0 +1 @@ +bad \ No newline at end of file diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index d4c11090324..9fa91eaa8f4 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -579,7 +579,8 @@ Zellij has no verified recovery-grade agent process probe, while Orca and cmux d ### Guarded Treehouse entry -The Treehouse-backed ordinary acquisition integration was inspected on 2026-08-12 against the pinned Treehouse v2.1.1 contract. +The Treehouse-backed ordinary acquisition integration was inspected on 2026-08-12 against the then-pinned Treehouse v2.1.1 contract. +The installer now pins Treehouse v2.3.0, whose fail-closed pool-slot verification is owned by [`bin/fm-install-treehouse.sh`](../../bin/fm-install-treehouse.sh) and the current spawn/teardown regression suites; this dated inspection remains historical evidence rather than a v2.3.0 verification claim. Tmux, Zellij, and cmux submit the shared guarded acquisition command, then resolve the acquired worktree through their existing current-path adapter only after the wrapper enters its verified lease. Herdr now uses the acquisition-owned ready-file handoff described in [`herdr-backend.md`](../herdr-backend.md#watching-and-task-containers), with portable coverage in `tests/fm-spawn-dispatch-profile.test.sh` and real-Herdr coverage in `tests/fm-backend-herdr-presentation-e2e.test.sh`. Orca is not applicable because it owns task worktrees and never invokes Treehouse. From 57090ab0517a32b9192e624fe28be8f28e465b71 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 18:58:43 +0800 Subject: [PATCH 10/23] no-mistakes(ci): Fixed two PR-caused failures: removed duplicate forced-secondmate child preflight (which consumed Orca mock responses twice) and allowed explicit STATE overrides to anchor the shared Treehouse project lock when the synthetic FM_HOME lacks state/. Verified fm-backend, fm-backend-orca, and fm-teardown-endpoint-safety tests pass. The Herdr presentation failure was an external timing/fixture timeout, not reproduced as a deterministic code defect --- bin/fm-teardown.sh | 1 - bin/fm-wake-lib.sh | 11 ++++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 32e93460c37..1be54ba38ad 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -2938,7 +2938,6 @@ if [ "$KIND" = secondmate ]; then if [ "$FORCE" = "--force" ]; then validate_firstmate_home_children_removal "$HOME_PATH" || exit 1 preflight_descendant_task_locks "$HOME_PATH" || exit 1 - validate_firstmate_home_children_removal "$HOME_PATH" || exit 1 preflight_descendant_treehouse_slots || exit 1 if [ "$BACKEND" = herdr ]; then teardown_herdr_preflight_target "$T" "$ID" || exit 1 diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 8fbefbbca45..cbc0f1ae240 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -939,7 +939,7 @@ fm_firstmate_root_home() { } fm_treehouse_project_lock_path() { # - local project=$1 root origin identity hash top + local project=$1 root origin identity hash top lock_state [ -d "$project" ] || return 1 root=$(fm_firstmate_root_home "$FM_HOME") || return 1 origin=$(git -C "$project" remote get-url origin 2>/dev/null || true) @@ -956,8 +956,13 @@ fm_treehouse_project_lock_path() { # identity=$top fi hash=$(printf '%s' "$identity" | git hash-object --stdin 2>/dev/null) || return 1 - [ -d "$root/state" ] || return 1 - printf '%s/.treehouse-project-%s.lock\n' "$root/state" "$hash" + lock_state="$root/state" + # Test and embedding callers may override STATE without materializing a + # complete FM_HOME tree; keep the shared lock anchored to that explicit + # state directory in that case. + [ -d "$lock_state" ] || lock_state=$STATE + [ -d "$lock_state" ] || return 1 + printf '%s/.treehouse-project-%s.lock\n' "$lock_state" "$hash" } # A Treehouse slot has the managed pool's fixed // layout. From 33e6795a107ebb318ccf389c1aeb82075f6971e7 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 23:06:09 +0800 Subject: [PATCH 11/23] fix: bound the Treehouse publication wait through FM_TREEHOUSE_READY_POLLS The real-Herdr presentation suite failed twice in CI at the multi-home section's first spawn: treehouse get did not publish its acquired worktree within the fixed 60s. Diagnosis found no defect in the lock or pane path: the pane-side get is ~1s of work (~4s per spawn on CI), the pane never touches the project lock the waiting spawn holds, and treehouse's pool flock is the only unbounded wait in the path - a transient holder or a loaded runner stalls the pane's publication without bound. The ready-file wait is now read from FM_TREEHOUSE_READY_POLLS (default unchanged at 60s); the suite exports 240, ~18x over the worst observed CI get-bearing step, so a transient stall at the suite's peak-load point stays inside the budget. --- bin/fm-spawn.sh | 6 ++++-- tests/fm-backend-herdr-presentation-e2e.test.sh | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index dc16db4a78b..853dc35804f 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -3820,12 +3820,14 @@ if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] } if [ "${IS_SANDBOX:-}" = 1 ] || [ "$BACKEND" = herdr ]; then - for _ in $(seq 1 60); do + treehouse_ready_polls=${FM_TREEHOUSE_READY_POLLS:-60} + case "$treehouse_ready_polls" in ''|*[!0-9]*|0) treehouse_ready_polls=60 ;; esac + for _ in $(seq 1 "$treehouse_ready_polls"); do [ -s "$treehouse_ready_file" ] && break sleep 1 done if [ ! -s "$treehouse_ready_file" ]; then - echo "error: treehouse get did not publish its acquired worktree within 60s; inspect window $T" >&2 + echo "error: treehouse get did not publish its acquired worktree within ${treehouse_ready_polls}s; inspect window $T" >&2 exit 1 fi WT=$(sed -n '1p' "$treehouse_ready_file") diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 9606b55bf54..980035af8cb 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -38,6 +38,12 @@ mkdir -p "$FAKEBIN" REAL_MOVER="$ROOT/bin/backends/herdr-workspace-move.py" export REAL_HERDR REAL_TREEHOUSE REAL_MOVER HERDR_CALL_LOG TREEHOUSE_CALL_LOG MOVE_CALL_LOG FOCUS_AUDIT_LOG HERDR_ORIGINAL_PATH HERDR_LAB_HELPER export ACTIVE_SEEDED_CONTROL POST_CREATE_ABORT_CONTROL TMP_ROOT +# The publication wait covers real pane shell startup, the guarded get wrapper, +# and the acquisition itself. On a loaded CI runner that pipeline crossed the +# default 60s at this suite's peak-load point twice (each ordinary get costs +# ~4s there); 240s keeps the suite's publication proof bounded while leaving +# production's 60s default untouched. +export FM_TREEHOUSE_READY_POLLS=240 # Log every production-adapter call, remove its already-validated trailing # session flag, and send the operation through the lab helper so that helper From 03dc4c8ca63d748b49206fbb8fdacc6be01fb283 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 23:18:13 +0800 Subject: [PATCH 12/23] no-mistakes(review): Serialize task publication and preserve zero signal grace --- bin/fm-spawn.sh | 43 +++++++++++++++++++++++++++++++++++--- bin/fm-watch-checkpoint.sh | 3 ++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 853dc35804f..9986cfb9495 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -537,10 +537,17 @@ if [ "$ACCEPTED_LOCAL_BASE_SET" -eq 1 ]; then fi REMOTE_RUNPOD_DELIVERY_LOCK= +REMOTE_TASK_SET_LOCK= +REMOTE_TASK_SET_LOCK_HELD=0 remote_runpod_delivery_cleanup() { - [ -n "$REMOTE_RUNPOD_DELIVERY_LOCK" ] || return 0 - fm_lock_release "$REMOTE_RUNPOD_DELIVERY_LOCK" || true - REMOTE_RUNPOD_DELIVERY_LOCK= + if [ -n "$REMOTE_RUNPOD_DELIVERY_LOCK" ]; then + fm_lock_release "$REMOTE_RUNPOD_DELIVERY_LOCK" || true + REMOTE_RUNPOD_DELIVERY_LOCK= + fi + if [ "$REMOTE_TASK_SET_LOCK_HELD" = 1 ]; then + REMOTE_TASK_SET_LOCK_HELD=0 + fm_lock_release "$REMOTE_TASK_SET_LOCK" || true + fi } trap remote_runpod_delivery_cleanup EXIT @@ -554,6 +561,15 @@ spawn_remote_secondmate() { id=${POS[0]:-} fm_task_id_creation_valid "$id" || { echo "error: invalid task id" >&2; return 2; } mkdir -p "$STATE" || { echo "error: could not create parent state directory" >&2; return 1; } + REMOTE_TASK_SET_LOCK=$(fm_task_set_lock_path "$STATE") || { + echo "error: could not resolve task-set lock for $STATE" >&2 + return 1 + } + if ! fm_lock_try_acquire "$REMOTE_TASK_SET_LOCK"; then + echo "error: another task publication or forced teardown is in progress for $STATE" >&2 + return 1 + fi + REMOTE_TASK_SET_LOCK_HELD=1 SPAWN_TASK_LOCK="$STATE/.spawn-$id.lock" if ! fm_lock_try_acquire "$SPAWN_TASK_LOCK"; then echo "error: another spawn is already creating task $id" >&2 @@ -567,6 +583,8 @@ spawn_remote_secondmate() { fi remote=$(secondmate_registry_field "$DATA/secondmates.md" "$id" remote 2>/dev/null || true) if [ "$remote" != 1 ]; then + REMOTE_TASK_SET_LOCK_HELD=0 + fm_lock_release "$REMOTE_TASK_SET_LOCK" || true fm_lock_release "$registry_lock" || true fm_lock_release "$SPAWN_TASK_LOCK" || true return 3 @@ -951,6 +969,8 @@ HERDR_PRESENTATION_ORDER_LOCK= HERDR_PRESENTATION_ORDER_LOCK_HELD=0 SPAWN_TASK_LOCK= SPAWN_TASK_LOCK_HELD=0 +SPAWN_TASK_SET_LOCK= +SPAWN_TASK_SET_LOCK_HELD=0 SPAWN_META_LOCK= SPAWN_META_LOCK_HELD=0 SPAWN_TREEHOUSE_PROJECT_LOCK= @@ -1174,6 +1194,10 @@ spawn_abort_cleanup() { SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 fm_lock_release "$SPAWN_TREEHOUSE_PROJECT_LOCK" || true fi + if [ "$SPAWN_TASK_SET_LOCK_HELD" = 1 ]; then + SPAWN_TASK_SET_LOCK_HELD=0 + fm_lock_release "$SPAWN_TASK_SET_LOCK" || true + fi if [ "$CONFIG_INHERIT_LOCK_HELD" = 1 ]; then CONFIG_INHERIT_LOCK_HELD=0 fm_lock_release "$CONFIG_INHERIT_LOCK" || true @@ -1269,6 +1293,15 @@ fm_task_id_creation_valid "$ID" || { echo "error: invalid task id" >&2; exit 2; # shellcheck source=bin/fm-lease-lib.sh . "$SCRIPT_DIR/fm-lease-lib.sh" fm_lease_forbid_branch "new-task spawn (fm-spawn)" +SPAWN_TASK_SET_LOCK=$(fm_task_set_lock_path "$STATE") || { + echo "error: could not resolve task-set lock for $STATE" >&2 + exit 1 +} +if ! fm_lock_try_acquire "$SPAWN_TASK_SET_LOCK"; then + echo "error: another task publication or forced teardown is in progress for $STATE" >&2 + exit 1 +fi +SPAWN_TASK_SET_LOCK_HELD=1 SPAWN_TASK_LOCK="$STATE/.spawn-$ID.lock" if ! fm_lock_try_acquire "$SPAWN_TASK_LOCK"; then echo "error: another spawn is already creating task $ID" >&2 @@ -4447,6 +4480,10 @@ if [ "$SPAWN_TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 fm_lock_release "$SPAWN_TREEHOUSE_PROJECT_LOCK" fi +if [ "$SPAWN_TASK_SET_LOCK_HELD" = 1 ]; then + SPAWN_TASK_SET_LOCK_HELD=0 + fm_lock_release "$SPAWN_TASK_SET_LOCK" +fi [ "$BACKEND" = orca ] && ORCA_ABORT_CLEANUP=0 if [ "$HARNESS" = omp ]; then OMP_ABORT_CLEANUP=1 diff --git a/bin/fm-watch-checkpoint.sh b/bin/fm-watch-checkpoint.sh index 35280f1f6f4..28093650ef8 100755 --- a/bin/fm-watch-checkpoint.sh +++ b/bin/fm-watch-checkpoint.sh @@ -63,7 +63,8 @@ run_with_perl_timeout() { } local $SIG{ALRM} = sub { kill "TERM", -$pid; - my $grace = $ENV{FM_SIGNAL_GRACE} || 5; + my $grace = $ENV{FM_SIGNAL_GRACE}; + $grace = 5 unless defined $grace && $grace =~ /^\d+$/; local $SIG{ALRM} = sub { kill "KILL", -$pid; waitpid $pid, 0; From 304e91eb10299e70266a56933bb398a1d56436fb Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 23:22:20 +0800 Subject: [PATCH 13/23] no-mistakes(review): Fix zero-grace timeout hang and remove stray artifact --- 15000000 | 1 - bin/fm-watch-checkpoint.sh | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) delete mode 100644 15000000 diff --git a/15000000 b/15000000 deleted file mode 100644 index 44d6628cdc6..00000000000 --- a/15000000 +++ /dev/null @@ -1 +0,0 @@ -bad \ No newline at end of file diff --git a/bin/fm-watch-checkpoint.sh b/bin/fm-watch-checkpoint.sh index 28093650ef8..56b67e320a4 100755 --- a/bin/fm-watch-checkpoint.sh +++ b/bin/fm-watch-checkpoint.sh @@ -65,6 +65,11 @@ run_with_perl_timeout() { kill "TERM", -$pid; my $grace = $ENV{FM_SIGNAL_GRACE}; $grace = 5 unless defined $grace && $grace =~ /^\d+$/; + if ($grace == 0) { + kill "KILL", -$pid; + waitpid $pid, 0; + exit 124; + } local $SIG{ALRM} = sub { kill "KILL", -$pid; waitpid $pid, 0; From e6148cf2b31a34ededf4e868234ad0aea952ee91 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 23:27:07 +0800 Subject: [PATCH 14/23] no-mistakes(review): Reject ambiguous duplicate slot-owner claims --- bin/fm-wake-lib.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index cbc0f1ae240..f54acd1090a 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -1041,7 +1041,7 @@ fm_treehouse_slot_owner_claim() { # # claimant as evidence. The home is reported, never matched: a home that moved # must not turn a task's own slot into a refusal. fm_treehouse_slot_owner_state() { # - local worktree=$1 id=$2 marker line owner_id='' owner_home='' + local worktree=$1 id=$2 marker line owner_id='' owner_home='' task_seen=0 home_seen=0 FM_TREEHOUSE_SLOT_OWNER=unsafe FM_TREEHOUSE_SLOT_OWNER_ID= FM_TREEHOUSE_SLOT_OWNER_HOME= @@ -1053,11 +1053,22 @@ fm_treehouse_slot_owner_state() { # [ -f "$marker" ] && [ ! -L "$marker" ] || return 0 while IFS= read -r line || [ -n "$line" ]; do case "$line" in - task=*) owner_id=${line#task=} ;; - home=*) owner_home=${line#home=} ;; + task=*) + [ "$task_seen" -eq 0 ] || return 0 + owner_id=${line#task=} + task_seen=1 + ;; + home=*) + [ "$home_seen" -eq 0 ] || return 0 + owner_home=${line#home=} + home_seen=1 + ;; + *) return 0 ;; esac done < "$marker" || return 0 - [ -n "$owner_id" ] || return 0 + [ "$task_seen" -eq 1 ] && [ "$home_seen" -eq 1 ] || return 0 + [ -n "$owner_id" ] && [ -n "$owner_home" ] || return 0 + case "$owner_id" in *[!A-Za-z0-9._-]*) return 0 ;; esac # shellcheck disable=SC2034 # Output globals, read by the sourcing caller. FM_TREEHOUSE_SLOT_OWNER_ID=$owner_id # shellcheck disable=SC2034 # Output globals, read by the sourcing caller. From e47543f73ae5967997678703f5429be92bd8c0dc Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 23:32:09 +0800 Subject: [PATCH 15/23] no-mistakes(review): Return leased slots when ownership claims fail --- bin/fm-spawn.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 9986cfb9495..f9dcad691b6 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -976,6 +976,7 @@ SPAWN_META_LOCK_HELD=0 SPAWN_TREEHOUSE_PROJECT_LOCK= SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 SPAWN_SLOT_CLAIMED=0 +SPAWN_POOL_LEASE_ABORT=0 CONFIG_INHERIT_LOCK= CONFIG_INHERIT_LOCK_HELD=0 TREEHOUSE_READY_DIR= @@ -1083,6 +1084,13 @@ spawn_abort_cleanup() { echo "warning: raw launch preflight could not return its leased worktree $WT" >&2 fi fi + if [ "$SPAWN_POOL_LEASE_ABORT" = 1 ] && [ -n "${WT:-}" ] \ + && [ ! -e "$STATE/$ID.meta" ] && [ ! -L "$STATE/$ID.meta" ]; then + SPAWN_POOL_LEASE_ABORT=0 + if ! (cd "$PROJ_ABS" && "$SCRIPT_DIR/fm-treehouse-command.sh" return "$WT" >/dev/null 2>&1); then + echo "warning: spawn claim failure could not return its leased worktree $WT" >&2 + fi + fi if [ "$OMP_ABORT_CLEANUP" = 1 ]; then OMP_ABORT_CLEANUP=0 meta="${STATE:-}/${ID:-}.meta" @@ -3912,6 +3920,7 @@ if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] validate_spawn_worktree "treehouse get" "$T" validate_spawn_pool_lease "treehouse get" "$T" || exit 1 + SPAWN_POOL_LEASE_ABORT=1 if [ "$HARNESS" = omp ]; then fm_omp_clear_stale_runtime_markers "$WT" || exit 1 fi @@ -4473,6 +4482,7 @@ SPAWN_META_LOCK_HELD=1 echo "projects=$SECONDMATE_PROJECTS" fi } > "$STATE/$ID.meta" +SPAWN_POOL_LEASE_ABORT=0 # The record is published, so a teardown's slot-ownership scan can now name this # task. The Treehouse project lock is only needed across slot allocation through # that publication. From caaa7da780ada37b7cff0a071cf158356d31aa08 Mon Sep 17 00:00:00 2001 From: dnth Date: Sat, 12 Sep 2026 23:40:57 +0800 Subject: [PATCH 16/23] no-mistakes(document): Updated Treehouse verification documentation --- docs/verification/runtime-backends.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index 9fa91eaa8f4..d8a3a5288dc 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -580,7 +580,7 @@ Zellij has no verified recovery-grade agent process probe, while Orca and cmux d ### Guarded Treehouse entry The Treehouse-backed ordinary acquisition integration was inspected on 2026-08-12 against the then-pinned Treehouse v2.1.1 contract. -The installer now pins Treehouse v2.3.0, whose fail-closed pool-slot verification is owned by [`bin/fm-install-treehouse.sh`](../../bin/fm-install-treehouse.sh) and the current spawn/teardown regression suites; this dated inspection remains historical evidence rather than a v2.3.0 verification claim. +The installer now pins Treehouse v2.3.0, whose release provides fail-closed pool-slot verification; Firstmate's current spawn and teardown regression suites cover the integration, while this dated inspection remains historical evidence rather than a v2.3.0 verification claim. Tmux, Zellij, and cmux submit the shared guarded acquisition command, then resolve the acquired worktree through their existing current-path adapter only after the wrapper enters its verified lease. Herdr now uses the acquisition-owned ready-file handoff described in [`herdr-backend.md`](../herdr-backend.md#watching-and-task-containers), with portable coverage in `tests/fm-spawn-dispatch-profile.test.sh` and real-Herdr coverage in `tests/fm-backend-herdr-presentation-e2e.test.sh`. Orca is not applicable because it owns task worktrees and never invokes Treehouse. From ac71affbb0b5af1e1219eb33d07f1c81d64d0ab0 Mon Sep 17 00:00:00 2001 From: dnth Date: Sun, 13 Sep 2026 00:28:17 +0800 Subject: [PATCH 17/23] no-mistakes(ci): Fixed the Herdr behavior test to recognize contention from either the Treehouse project lock or the task-publication lock, retrying after the winner completes. The full real-Herdr presentation E2E suite passes --- tests/fm-backend-herdr-presentation-e2e.test.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 980035af8cb..7b14653c058 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -436,18 +436,22 @@ teardown_task() { # finish_concurrent_spawn() { # local id=$1 status=$2 out=$3 err=$4 [ "$status" -ne 0 ] || return 0 - grep -F "another Treehouse slot allocation or return is in progress" "$err" >/dev/null 2>&1 \ - || fail "concurrent projected spawn $id failed unexpectedly: $(cat "$err")" - spawn_task "$id" "$HOME_DIR" "$PROJECT_DIR" > "$out" 2> "$err" \ - || fail "projected spawn $id retry failed after the Treehouse project lock cleared: $(cat "$err")" + if grep -F "another Treehouse slot allocation or return is in progress" "$err" >/dev/null 2>&1 \ + || grep -F "another task publication or forced teardown is in progress" "$err" >/dev/null 2>&1; then + spawn_task "$id" "$HOME_DIR" "$PROJECT_DIR" > "$out" 2> "$err" \ + || fail "projected spawn $id retry failed after the contended lock cleared: $(cat "$err")" + return 0 + fi + fail "concurrent projected spawn $id failed unexpectedly: $(cat "$err")" } finish_concurrent_expected_abort() { # local id=$1 status=$2 out=$3 err=$4 [ "$status" -ne 0 ] || fail "post-create abort fixture $id unexpectedly succeeded" - if grep -F "another Treehouse slot allocation or return is in progress" "$err" >/dev/null 2>&1; then + if grep -F "another Treehouse slot allocation or return is in progress" "$err" >/dev/null 2>&1 \ + || grep -F "another task publication or forced teardown is in progress" "$err" >/dev/null 2>&1; then if spawn_task "$id" "$HOME_DIR" "$PROJECT_DIR" > "$out" 2> "$err"; then - fail "post-create abort fixture $id unexpectedly succeeded after the Treehouse project lock cleared" + fail "post-create abort fixture $id unexpectedly succeeded after the contended lock cleared" fi fi } From 911c638680ec454bd3d91ff287c5a9afedd94de2 Mon Sep 17 00:00:00 2001 From: dnth Date: Sun, 13 Sep 2026 01:06:21 +0800 Subject: [PATCH 18/23] no-mistakes(ci): Added bounded diagnostics to the Herdr presentation E2E multi-home spawn failure paths: parses inspect targets, captures pane output/process info/cwd, checks ready-file artifacts, and snapshots treehouse processes without changing test outcomes. `bash -n`, `git diff --check`, and the full presentation E2E suite pass locally --- .../fm-backend-herdr-presentation-e2e.test.sh | 58 +++++++++++++++++-- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 7b14653c058..e09d1988b62 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -411,6 +411,52 @@ spawn_task() { # "$ROOT/bin/fm-spawn.sh" "$id" "$project" "$RAW_SLEEP_AGENT 120" --mode no-mistakes --yolo off --backend herdr } +diagnose_spawn_failure() { # + local err_file=$1 target pane workspace session pane_dump pane_get ready_path + target=$(sed -nE 's/.*inspect window ([^[:space:]]+).*/\1/p' "$err_file" 2>/dev/null | tail -n 1 || true) + printf 'diagnostic: spawn stderr: %s\n' "$(cat "$err_file" 2>/dev/null || true)" >&2 + [ -n "$target" ] || { + printf 'diagnostic: no Herdr inspect target found\n' >&2 + return 0 + } + pane=${target##*:} + workspace=${target%:*} + workspace=${workspace##*:} + session=${target%:*:*} + printf 'diagnostic: herdr target session=%s workspace=%s pane=%s\n' "$session" "$workspace" "$pane" >&2 + + pane_dump=$(PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" run "$session" pane read "$pane" --source recent --lines 200 2>&1 || true) + printf 'diagnostic: pane read (recent):\n%s\n' "$pane_dump" >&2 + pane_get=$(PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" run "$session" pane get "$pane" 2>&1 || true) + printf 'diagnostic: pane get/process cwd: %s\n' "$pane_get" >&2 + PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" run "$session" pane process-info --pane "$pane" >&2 \ + || printf 'diagnostic: pane process-info unavailable\n' >&2 + + ready_path=$(printf '%s\n%s\n' "$pane_dump" "$pane_get" \ + | sed -nE 's/.*--ready-file[[:space:]]+([^[:space:]]+).*/\1/p' | tail -n 1 || true) + ready_path=${ready_path#\'} + ready_path=${ready_path%\'} + if [ -n "$ready_path" ]; then + printf 'diagnostic: ready-file path=%s dir=%s\n' "$ready_path" "$(dirname "$ready_path")" >&2 + if [ -d "$(dirname "$ready_path")" ]; then + printf 'diagnostic: ready-file directory exists; entries:\n' >&2 + ls -la "$(dirname "$ready_path")" >&2 || true + else + printf 'diagnostic: ready-file directory missing\n' >&2 + fi + if [ -e "$ready_path" ]; then + printf 'diagnostic: ready-file exists; content:\n%s\n' "$(sed -n '1,5p' "$ready_path" 2>&1 || true)" >&2 + else + printf 'diagnostic: ready-file missing\n' >&2 + fi + else + printf 'diagnostic: ready-file path not visible in pane evidence; matching temp dirs:\n' >&2 + find "${TMPDIR:-/tmp}" -maxdepth 1 -type d -name 'fm-treehouse-ready.*' -print 2>/dev/null >&2 || true + fi + printf 'diagnostic: live treehouse processes:\n' >&2 + ps -eo pid,ppid,stat,etime,args 2>/dev/null | grep '[t]reehouse' >&2 || true +} + relaunch_task() { # local id=$1 home=$2 FM_GATE_REFUSE_BYPASS=1 FM_SPAWN_NO_GUARD=1 FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ @@ -1054,17 +1100,17 @@ printf 'Secondmate B fixture 2.\n' > "$SECOND_HOME_B/data/b2/brief.md" MULTI_FOCUS_START=$(focus_audit_line_count) spawn_task p1 "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/p1.out" 2> "$TMP_ROOT/p1.err" \ - || fail "multi-home primary p1 failed: $(cat "$TMP_ROOT/p1.err")" + || { diagnose_spawn_failure "$TMP_ROOT/p1.err"; fail "multi-home primary p1 failed: $(cat "$TMP_ROOT/p1.err")"; } spawn_task p2 "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/p2.out" 2> "$TMP_ROOT/p2.err" \ - || fail "multi-home primary p2 failed: $(cat "$TMP_ROOT/p2.err")" + || { diagnose_spawn_failure "$TMP_ROOT/p2.err"; fail "multi-home primary p2 failed: $(cat "$TMP_ROOT/p2.err")"; } spawn_task a1 "$SECOND_HOME_A" "$PROJECT_DIR" > "$TMP_ROOT/a1.out" 2> "$TMP_ROOT/a1.err" \ - || fail "multi-home secondmate A a1 failed: $(cat "$TMP_ROOT/a1.err")" + || { diagnose_spawn_failure "$TMP_ROOT/a1.err"; fail "multi-home secondmate A a1 failed: $(cat "$TMP_ROOT/a1.err")"; } spawn_task a2 "$SECOND_HOME_A" "$PROJECT_DIR" > "$TMP_ROOT/a2.out" 2> "$TMP_ROOT/a2.err" \ - || fail "multi-home secondmate A a2 failed: $(cat "$TMP_ROOT/a2.err")" + || { diagnose_spawn_failure "$TMP_ROOT/a2.err"; fail "multi-home secondmate A a2 failed: $(cat "$TMP_ROOT/a2.err")"; } spawn_task b1 "$SECOND_HOME_B" "$PROJECT_DIR" > "$TMP_ROOT/b1.out" 2> "$TMP_ROOT/b1.err" \ - || fail "multi-home secondmate B b1 failed: $(cat "$TMP_ROOT/b1.err")" + || { diagnose_spawn_failure "$TMP_ROOT/b1.err"; fail "multi-home secondmate B b1 failed: $(cat "$TMP_ROOT/b1.err")"; } spawn_task b2 "$SECOND_HOME_B" "$PROJECT_DIR" > "$TMP_ROOT/b2.out" 2> "$TMP_ROOT/b2.err" \ - || fail "multi-home secondmate B b2 failed: $(cat "$TMP_ROOT/b2.err")" + || { diagnose_spawn_failure "$TMP_ROOT/b2.err"; fail "multi-home secondmate B b2 failed: $(cat "$TMP_ROOT/b2.err")"; } for META_X in p1 p2 a1 a2 b1 b2; do case "$META_X" in p*) remember_meta_worktree "$HOME_DIR/state/$META_X.meta" >/dev/null ;; From 9484751e226738a794aa40a9e0a8332d9bb7cce2 Mon Sep 17 00:00:00 2001 From: dnth Date: Sun, 13 Sep 2026 01:35:44 +0800 Subject: [PATCH 19/23] no-mistakes(ci): Fixed ShellCheck SC2009 in the Herdr diagnostic path with a scoped suppression preserving the required full `ps` snapshot. Targeted lint, syntax, and diff checks pass; the Herdr E2E suite passes locally. The CI Herdr failure appears environment-specific (Herdr 0.7.4 pane disappeared while an orphaned treehouse process remained), not reproduced locally --- tests/fm-backend-herdr-presentation-e2e.test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index e09d1988b62..3d1b0352e29 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -454,6 +454,7 @@ diagnose_spawn_failure() { # find "${TMPDIR:-/tmp}" -maxdepth 1 -type d -name 'fm-treehouse-ready.*' -print 2>/dev/null >&2 || true fi printf 'diagnostic: live treehouse processes:\n' >&2 + # shellcheck disable=SC2009 # Preserve the full ps snapshot in failure diagnostics. ps -eo pid,ppid,stat,etime,args 2>/dev/null | grep '[t]reehouse' >&2 || true } From 923e67033b55a0384ba5522bcf52069df97e9a9d Mon Sep 17 00:00:00 2001 From: dnth Date: Sun, 13 Sep 2026 02:09:52 +0800 Subject: [PATCH 20/23] no-mistakes(ci): Fixed Herdr acquisition hangs by failing fast when the target pane disappears during ready-file polling, with an accurate pane-death error. Added a single retry for pane-disappeared multi-home spawns while retaining final diagnostics. `bin/fm-lint.sh`, shell syntax/diff checks, and the full Herdr presentation E2E suite pass locally --- bin/fm-spawn.sh | 4 +++ .../fm-backend-herdr-presentation-e2e.test.sh | 25 ++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index f9dcad691b6..0491517730e 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -3865,6 +3865,10 @@ if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] case "$treehouse_ready_polls" in ''|*[!0-9]*|0) treehouse_ready_polls=60 ;; esac for _ in $(seq 1 "$treehouse_ready_polls"); do [ -s "$treehouse_ready_file" ] && break + if [ "$BACKEND" = herdr ] && ! fm_backend_target_exists "$BACKEND" "$T"; then + echo "error: herdr pane $T disappeared during Treehouse worktree acquisition; the pane death, not treehouse, prevented publication" >&2 + exit 1 + fi sleep 1 done if [ ! -s "$treehouse_ready_file" ]; then diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 3d1b0352e29..69c2c3fe5e9 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -411,6 +411,19 @@ spawn_task() { # "$ROOT/bin/fm-spawn.sh" "$id" "$project" "$RAW_SLEEP_AGENT 120" --mode no-mistakes --yolo off --backend herdr } +spawn_task_with_pane_loss_retry() { # + local id=$1 home=$2 project=$3 out=$4 err=$5 + if spawn_task "$id" "$home" "$project" > "$out" 2> "$err"; then + return 0 + fi + if grep -F "disappeared during Treehouse worktree acquisition" "$err" >/dev/null 2>&1; then + printf 'diagnostic: retrying %s once after Herdr pane loss\n' "$id" >&2 + spawn_task "$id" "$home" "$project" > "$out" 2> "$err" + return $? + fi + return 1 +} + diagnose_spawn_failure() { # local err_file=$1 target pane workspace session pane_dump pane_get ready_path target=$(sed -nE 's/.*inspect window ([^[:space:]]+).*/\1/p' "$err_file" 2>/dev/null | tail -n 1 || true) @@ -1100,17 +1113,17 @@ printf 'Secondmate B fixture 1.\n' > "$SECOND_HOME_B/data/b1/brief.md" printf 'Secondmate B fixture 2.\n' > "$SECOND_HOME_B/data/b2/brief.md" MULTI_FOCUS_START=$(focus_audit_line_count) -spawn_task p1 "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/p1.out" 2> "$TMP_ROOT/p1.err" \ +spawn_task_with_pane_loss_retry p1 "$HOME_DIR" "$PROJECT_DIR" "$TMP_ROOT/p1.out" "$TMP_ROOT/p1.err" \ || { diagnose_spawn_failure "$TMP_ROOT/p1.err"; fail "multi-home primary p1 failed: $(cat "$TMP_ROOT/p1.err")"; } -spawn_task p2 "$HOME_DIR" "$PROJECT_DIR" > "$TMP_ROOT/p2.out" 2> "$TMP_ROOT/p2.err" \ +spawn_task_with_pane_loss_retry p2 "$HOME_DIR" "$PROJECT_DIR" "$TMP_ROOT/p2.out" "$TMP_ROOT/p2.err" \ || { diagnose_spawn_failure "$TMP_ROOT/p2.err"; fail "multi-home primary p2 failed: $(cat "$TMP_ROOT/p2.err")"; } -spawn_task a1 "$SECOND_HOME_A" "$PROJECT_DIR" > "$TMP_ROOT/a1.out" 2> "$TMP_ROOT/a1.err" \ +spawn_task_with_pane_loss_retry a1 "$SECOND_HOME_A" "$PROJECT_DIR" "$TMP_ROOT/a1.out" "$TMP_ROOT/a1.err" \ || { diagnose_spawn_failure "$TMP_ROOT/a1.err"; fail "multi-home secondmate A a1 failed: $(cat "$TMP_ROOT/a1.err")"; } -spawn_task a2 "$SECOND_HOME_A" "$PROJECT_DIR" > "$TMP_ROOT/a2.out" 2> "$TMP_ROOT/a2.err" \ +spawn_task_with_pane_loss_retry a2 "$SECOND_HOME_A" "$PROJECT_DIR" "$TMP_ROOT/a2.out" "$TMP_ROOT/a2.err" \ || { diagnose_spawn_failure "$TMP_ROOT/a2.err"; fail "multi-home secondmate A a2 failed: $(cat "$TMP_ROOT/a2.err")"; } -spawn_task b1 "$SECOND_HOME_B" "$PROJECT_DIR" > "$TMP_ROOT/b1.out" 2> "$TMP_ROOT/b1.err" \ +spawn_task_with_pane_loss_retry b1 "$SECOND_HOME_B" "$PROJECT_DIR" "$TMP_ROOT/b1.out" "$TMP_ROOT/b1.err" \ || { diagnose_spawn_failure "$TMP_ROOT/b1.err"; fail "multi-home secondmate B b1 failed: $(cat "$TMP_ROOT/b1.err")"; } -spawn_task b2 "$SECOND_HOME_B" "$PROJECT_DIR" > "$TMP_ROOT/b2.out" 2> "$TMP_ROOT/b2.err" \ +spawn_task_with_pane_loss_retry b2 "$SECOND_HOME_B" "$PROJECT_DIR" "$TMP_ROOT/b2.out" "$TMP_ROOT/b2.err" \ || { diagnose_spawn_failure "$TMP_ROOT/b2.err"; fail "multi-home secondmate B b2 failed: $(cat "$TMP_ROOT/b2.err")"; } for META_X in p1 p2 a1 a2 b1 b2; do case "$META_X" in From 0b618323fd21be7e364c2469ed2f42ee2acc0ce4 Mon Sep 17 00:00:00 2001 From: dnth Date: Sun, 13 Sep 2026 02:39:48 +0800 Subject: [PATCH 21/23] no-mistakes(ci): Updated the Herdr presentation E2E diagnostics to preserve workspace-qualified pane IDs and capture qualified pane read, pane status fields, and process-info. Bash syntax, ShellCheck, diff checks, and the full Herdr presentation E2E test pass locally --- .../fm-backend-herdr-presentation-e2e.test.sh | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 69c2c3fe5e9..44d925e8a23 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -425,23 +425,34 @@ spawn_task_with_pane_loss_retry() { # } diagnose_spawn_failure() { # - local err_file=$1 target pane workspace session pane_dump pane_get ready_path + local err_file=$1 target pane workspace session pane_dump pane_get pane_fields ready_path target=$(sed -nE 's/.*inspect window ([^[:space:]]+).*/\1/p' "$err_file" 2>/dev/null | tail -n 1 || true) printf 'diagnostic: spawn stderr: %s\n' "$(cat "$err_file" 2>/dev/null || true)" >&2 [ -n "$target" ] || { printf 'diagnostic: no Herdr inspect target found\n' >&2 return 0 } - pane=${target##*:} - workspace=${target%:*} - workspace=${workspace##*:} - session=${target%:*:*} + # Herdr pane ids are workspace-qualified (for example, wJ:p2). Keep that + # qualification when querying the pane; stripping to p2 produces a false + # pane_not_found diagnostic even while the pane is alive. + session=${target%%:*} + pane=${target#*:} + workspace=${pane%:*} printf 'diagnostic: herdr target session=%s workspace=%s pane=%s\n' "$session" "$workspace" "$pane" >&2 pane_dump=$(PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" run "$session" pane read "$pane" --source recent --lines 200 2>&1 || true) - printf 'diagnostic: pane read (recent):\n%s\n' "$pane_dump" >&2 + printf 'diagnostic: pane read (recent, qualified):\n%s\n' "$pane_dump" >&2 pane_get=$(PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" run "$session" pane get "$pane" 2>&1 || true) - printf 'diagnostic: pane get/process cwd: %s\n' "$pane_get" >&2 + printf 'diagnostic: pane get (qualified): %s\n' "$pane_get" >&2 + pane_fields=$(printf '%s' "$pane_get" | jq -c ' + if (.result.pane? // null) == null and (.result.agent? // null) == null then + {agent_status: null, foreground_cwd: null} + else + {agent_status: (.result.agent.agent_status // .result.pane.agent_status // null), + foreground_cwd: (.result.pane.foreground_cwd // null)} + end + ' 2>/dev/null || printf '%s\n' '{"agent_status":null,"foreground_cwd":null}') + printf 'diagnostic: pane get fields (agent_status/foreground_cwd): %s\n' "$pane_fields" >&2 PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" run "$session" pane process-info --pane "$pane" >&2 \ || printf 'diagnostic: pane process-info unavailable\n' >&2 From 751977ab8e0a87c041ad29de0db1f772974f623a Mon Sep 17 00:00:00 2001 From: dnth Date: Sun, 13 Sep 2026 03:01:06 +0800 Subject: [PATCH 22/23] no-mistakes(ci): Added a bounded test-only Herdr snapshotter to the multi-home E2E suite. It records workspace/pane lists, qualified pane status (agent_status, foreground_cwd, cwd), and recent pane output every ~15s, includes the snapshot tail in spawn failure diagnostics, and stops cleanly after the section. Bash syntax, ShellCheck, and diff checks pass; a local run began successfully but exceeded the 45s bounded smoke window --- .../fm-backend-herdr-presentation-e2e.test.sh | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 44d925e8a23..bc203ada8e7 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -290,8 +290,17 @@ export HERDR_SESSION="$HERDR_LAB_SESSION" HERDR_LAB_SESSION LAB_READY=0 RECORDED_WORKTREES="" LOCK_CONTENTION_OWNER_PID= +HERDR_SNAPSHOT_PID= +HERDR_SNAPSHOT_CONTROL="$TMP_ROOT/herdr-snapshot.running" +HERDR_SNAPSHOT_LOG="$TMP_ROOT/herdr-pane-snapshot.log" cleanup_all() { local wt + if [ -n "$HERDR_SNAPSHOT_PID" ]; then + rm -f "$HERDR_SNAPSHOT_CONTROL" + kill "$HERDR_SNAPSHOT_PID" 2>/dev/null || true + wait "$HERDR_SNAPSHOT_PID" 2>/dev/null || true + HERDR_SNAPSHOT_PID= + fi if [ -n "$LOCK_CONTENTION_OWNER_PID" ]; then kill "$LOCK_CONTENTION_OWNER_PID" 2>/dev/null || true wait "$LOCK_CONTENTION_OWNER_PID" 2>/dev/null || true @@ -480,6 +489,58 @@ diagnose_spawn_failure() { # printf 'diagnostic: live treehouse processes:\n' >&2 # shellcheck disable=SC2009 # Preserve the full ps snapshot in failure diagnostics. ps -eo pid,ppid,stat,etime,args 2>/dev/null | grep '[t]reehouse' >&2 || true + if [ -s "$HERDR_SNAPSHOT_LOG" ]; then + printf 'diagnostic: in-flight Herdr pane snapshot tail:\n' >&2 + tail -n 240 "$HERDR_SNAPSHOT_LOG" >&2 || true + fi +} + +start_herdr_snapshotter() { + : > "$HERDR_SNAPSHOT_LOG" + : > "$HERDR_SNAPSHOT_CONTROL" + snapshot_lab() { + timeout 10s env PATH="$HERDR_ORIGINAL_PATH" \ + "$HERDR_LAB_HELPER" run "$HERDR_LAB_SESSION" "$@" + } + ( + while [ -e "$HERDR_SNAPSHOT_CONTROL" ]; do + { + printf '\n=== herdr snapshot %s ===\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + workspaces=$(snapshot_lab workspace list 2>&1 || true) + printf '%s\n' 'workspace list:' "$workspaces" + printf '%s\n' "$workspaces" | jq -r '.result.workspaces[]?.workspace_id // empty' 2>/dev/null \ + | while IFS= read -r ws; do + [ -n "$ws" ] || continue + panes=$(snapshot_lab pane list --workspace "$ws" 2>&1 || true) + printf 'pane list workspace=%s:\n%s\n' "$ws" "$panes" + printf '%s\n' "$panes" | jq -r '.result.panes[]?.pane_id // empty' 2>/dev/null \ + | while IFS= read -r pane; do + [ -n "$pane" ] || continue + pane_get=$(snapshot_lab pane get "$pane" 2>&1 || true) + pane_fields=$(printf '%s' "$pane_get" | jq -c ' + {agent_status: (.result.agent.agent_status // .result.pane.agent_status // null), + foreground_cwd: (.result.pane.foreground_cwd // null), + cwd: (.result.pane.cwd // null)} + ' 2>/dev/null || printf '%s\n' '{"agent_status":null,"foreground_cwd":null,"cwd":null}') + printf 'pane get %s fields (agent_status/foreground_cwd/cwd): %s\n%s\n' "$pane" "$pane_fields" "$pane_get" + pane_read=$(snapshot_lab pane read "$pane" --source recent --lines 40 2>&1 || true) + printf 'pane read %s (recent):\n%s\n' "$pane" "$pane_read" + done + done + } >> "$HERDR_SNAPSHOT_LOG" 2>&1 + sleep 15 + done + ) & + HERDR_SNAPSHOT_PID=$! +} + +stop_herdr_snapshotter() { + rm -f "$HERDR_SNAPSHOT_CONTROL" + if [ -n "$HERDR_SNAPSHOT_PID" ]; then + kill "$HERDR_SNAPSHOT_PID" 2>/dev/null || true + wait "$HERDR_SNAPSHOT_PID" 2>/dev/null || true + HERDR_SNAPSHOT_PID= + fi } relaunch_task() { # @@ -1113,6 +1174,11 @@ pass "real Herdr lab: the primary presentation setting inherits into real second # Keep the pre-existing 2ndmate-alpha/bravo workspaces as owning parents and captain focus. assert_focus_is "$CAPTAIN_FOCUS" "multi-home captain focus" +# Capture pane state during the high-load multi-home sequence. Herdr can queue +# pane commands behind a busy foreground process, so post-teardown evidence is +# insufficient for diagnosing a publication stall. +start_herdr_snapshotter + mkdir -p "$SECOND_HOME_A/data/a1" "$SECOND_HOME_A/data/a2" \ "$SECOND_HOME_B/data/b1" "$SECOND_HOME_B/data/b2" \ "$HOME_DIR/data/p1" "$HOME_DIR/data/p2" @@ -1471,6 +1537,7 @@ do done assert_focus_is "$CAPTAIN_FOCUS" "multi-home teardown" pass "real Herdr lab: multi-home exact-pane teardowns restore captain focus without workspace close authority" +stop_herdr_snapshotter # Missing, renamed, and duplicate tokens are read-only recovery diagnostics. # The duplicate case allows flat fallback only when every matching pane is From d78f42113123ea84c85f1ff70d097870bac40e22 Mon Sep 17 00:00:00 2001 From: dnth Date: Sun, 13 Sep 2026 03:33:41 +0800 Subject: [PATCH 23/23] no-mistakes(ci): Implemented the v2.3.0 Treehouse guard fix: guarded `read-tree --reset` with ancestry validation and SAFE_FILE publication, propagated pane-side acquisition failures via `.failed` ready markers, and surfaced them promptly from `fm-spawn.sh`. `bash -n`, targeted ShellCheck, diff checks, and `fm-treehouse-orphan-recovery` plus `fm-spawn-pool-base-freshen` tests pass --- bin/fm-spawn.sh | 5 +++++ bin/fm-treehouse-get.sh | 15 ++++++++++++++- bin/treehouse-git-guard/git | 31 +++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 0491517730e..d027a03bb22 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -3865,6 +3865,11 @@ if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] case "$treehouse_ready_polls" in ''|*[!0-9]*|0) treehouse_ready_polls=60 ;; esac for _ in $(seq 1 "$treehouse_ready_polls"); do [ -s "$treehouse_ready_file" ] && break + if [ -s "${treehouse_ready_file}.failed" ]; then + cat "${treehouse_ready_file}.failed" >&2 + echo "error: guarded Treehouse acquisition failed in pane $T" >&2 + exit 1 + fi if [ "$BACKEND" = herdr ] && ! fm_backend_target_exists "$BACKEND" "$T"; then echo "error: herdr pane $T disappeared during Treehouse worktree acquisition; the pane death, not treehouse, prevented publication" >&2 exit 1 diff --git a/bin/fm-treehouse-get.sh b/bin/fm-treehouse-get.sh index f9cde69e276..f8f8e87636e 100755 --- a/bin/fm-treehouse-get.sh +++ b/bin/fm-treehouse-get.sh @@ -17,7 +17,20 @@ REAL_GIT=$(command -v git 2>/dev/null) || { exit 1 } GUARD_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-treehouse-get.XXXXXX") || exit 1 -trap 'rm -rf "$GUARD_DIR"' EXIT +# shellcheck disable=SC2329 # Invoked indirectly by the EXIT trap. +cleanup_guard_dir() { + local status=$? + if [ "$status" -ne 0 ] && [ -n "${ready_file:-}" ] \ + && [ ! -e "$ready_file" ] && [ ! -L "$ready_file" ]; then + { + printf 'exit_status=%s\n' "$status" + [ -s "$GUARD_DIR/error" ] && sed -n '1p' "$GUARD_DIR/error" + } > "${ready_file}.failed" 2>/dev/null || true + fi + rm -rf -- "$GUARD_DIR" + return "$status" +} +trap cleanup_guard_dir EXIT # shellcheck source=bin/fm-pool-lib.sh disable=SC1091 . "$SCRIPT_DIR/fm-pool-lib.sh" diff --git a/bin/treehouse-git-guard/git b/bin/treehouse-git-guard/git index ea67fb54fac..d78fc1e6e53 100755 --- a/bin/treehouse-git-guard/git +++ b/bin/treehouse-git-guard/git @@ -59,6 +59,37 @@ checkout|switch|reset) : > "$SAFE_FILE" exit 0 ;; +read-tree) + has_reset=0 + ref= + for arg in "${@:2}"; do + [ "$arg" = --reset ] && has_reset=1 + case "$arg" in + -*) ;; + *) [ -n "$ref" ] || ref=$arg ;; + esac + done + if [ "$has_reset" -ne 1 ]; then + printf "error: refusing pooled worktree acquisition at %s: unverified git read-tree boundary\n" \ + "$PWD" >> "$ERROR_FILE" + exit 42 + fi + case "$ref" in + ''|-*) + printf "error: refusing pooled worktree acquisition at %s: Treehouse did not supply a verifiable reset target\n" \ + "$PWD" >> "$ERROR_FILE" + exit 42 + ;; + esac + if ! "$REAL_GIT" merge-base --is-ancestor HEAD "$ref" 2>/dev/null; then + printf "error: refusing pooled worktree acquisition at %s: HEAD is not an ancestor of %s; local commits were preserved\n" \ + "$PWD" "$ref" >> "$ERROR_FILE" + exit 42 + fi + "$REAL_GIT" "$@" || exit $? + : > "$SAFE_FILE" + exit 0 + ;; restore) printf "error: refusing pooled worktree acquisition at %s: unverified git restore boundary\n" \ "$PWD" >> "$ERROR_FILE"