From d4c486f9060c36eab8bdd39a9c71b804439ffd1a Mon Sep 17 00:00:00 2001 From: twilwa Date: Thu, 24 Sep 2026 10:07:00 +0200 Subject: [PATCH 1/5] fix: bound startup reconciliation and large fleet input --- bin/fm-fleet-snapshot.sh | 14 +- bin/fm-herdr-session-cleanup.sh | 132 +++++++++++++---- bin/fm-session-start.sh | 10 +- bin/fm-startup-network.sh | 38 +++-- tests/fm-contributions.test.sh | 18 ++- tests/fm-herdr-session-cleanup.test.sh | 103 ++++++++++++++ tests/fm-session-start.test.sh | 83 +++++++++-- tests/fm-startup-network.test.sh | 188 ++++++++++++++++++++++--- 8 files changed, 516 insertions(+), 70 deletions(-) diff --git a/bin/fm-fleet-snapshot.sh b/bin/fm-fleet-snapshot.sh index 666d03b8d6c..fb9486e7741 100755 --- a/bin/fm-fleet-snapshot.sh +++ b/bin/fm-fleet-snapshot.sh @@ -1991,8 +1991,18 @@ contribution_tasks_json() { if [ "$OUTPUT_MODE" = contribution-input ]; then # Reuse the canonical backlog parser, without observing workers or other homes. contribution_tasks=$(contribution_tasks_json) || { echo "fm-fleet-snapshot: contribution task read failed" >&2; exit 1; } - jq -n --argjson backlog "$BACKLOG_JSON" --argjson tasks "$contribution_tasks" '{backlog:$backlog,tasks:$tasks}' - exit 0 + JSON_TRANSPORT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-fleet-snapshot.XXXXXX") \ + || { echo "fm-fleet-snapshot: temporary transport directory creation failed" >&2; exit 1; } + trap 'rm -f -- "$JSON_TRANSPORT_DIR/backlog.json" "$JSON_TRANSPORT_DIR/contribution-tasks.json" 2>/dev/null || true; rmdir -- "$JSON_TRANSPORT_DIR" 2>/dev/null || true' EXIT + printf '%s\n' "$BACKLOG_JSON" > "$JSON_TRANSPORT_DIR/backlog.json" \ + || { echo "fm-fleet-snapshot: temporary backlog file write failed" >&2; exit 1; } + printf '%s\n' "$contribution_tasks" > "$JSON_TRANSPORT_DIR/contribution-tasks.json" \ + || { echo "fm-fleet-snapshot: temporary contribution task file write failed" >&2; exit 1; } + jq -n --slurpfile backlog "$JSON_TRANSPORT_DIR/backlog.json" \ + --slurpfile tasks "$JSON_TRANSPORT_DIR/contribution-tasks.json" \ + '{backlog:$backlog[0],tasks:$tasks[0]}' + jq_rc=$? + exit "$jq_rc" fi prefetch_task_current_states || { echo "fm-fleet-snapshot: task observation failed" >&2; exit 1; } TASKS_JSON=$(task_json_lines) || { echo "fm-fleet-snapshot: task snapshot failed" >&2; exit 1; } diff --git a/bin/fm-herdr-session-cleanup.sh b/bin/fm-herdr-session-cleanup.sh index 259969bbf22..4a84c995bf8 100755 --- a/bin/fm-herdr-session-cleanup.sh +++ b/bin/fm-herdr-session-cleanup.sh @@ -21,6 +21,10 @@ # The script never closes a workspace. It removes only the matching journal, # and only after the exact pane is confirmed gone. Every error warns and returns # success so session startup continues conservatively. +# Discovery validates each home journal once; locked mutation checks are uncached. +# FM_HERDR_SESSION_CLEANUP_TIMEOUT bounds the complete pass (default 30 seconds); +# after expiry the parent reclaims only recorded locks it can safely acquire. +# Unfinished candidates are preserved and coverage is explicitly unconfirmed. set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -35,6 +39,12 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" fm_backend_source herdr # shellcheck source=bin/fm-pr-lib.sh . "$SCRIPT_DIR/fm-pr-lib.sh" +# shellcheck source=bin/fm-timeout-lib.sh +. "$SCRIPT_DIR/fm-timeout-lib.sh" + +FM_HERDR_CLEANUP_INDEX= +FM_HERDR_CLEANUP_INDEX_READY=0 +FM_HERDR_CLEANUP_LOCK_RECORD=${FM_HERDR_CLEANUP_LOCK_RECORD:-} fm_herdr_cleanup_warn() { printf 'warning: herdr session-start projection cleanup: %s\n' "$*" >&2 @@ -63,7 +73,14 @@ fm_herdr_cleanup_home_identity() { } fm_herdr_cleanup_journal_matches() { # <session> <home-real> - local title=$1 session=$2 home_real=$3 journal id expected journal_home + local title=$1 session=$2 home_real=$3 index + index=$(fm_herdr_cleanup_index "$session" "$home_real") || return 1 + fm_herdr_cleanup_index_matches "$title" "$index" +} + +# The index is only a discovery accelerator, never mutation authority. +fm_herdr_cleanup_index() { # <session> <home-real> + local session=$1 home_real=$2 journal id journal_home expected [ -d "$STATE" ] && [ ! -L "$STATE" ] || return 1 for journal in "$STATE"/*"$FM_BACKEND_HERDR_PRESENTATION_JOURNAL_SUFFIX"; do [ -f "$journal" ] && [ ! -L "$journal" ] || continue @@ -78,11 +95,20 @@ fm_herdr_cleanup_journal_matches() { # <title> <session> <home-real> fi expected=$(fm_backend_herdr_projection_workspace_label \ "$id" "$FM_BACKEND_HERDR_JOURNAL_PROJECTION_ID") - [ "$expected" = "$title" ] || continue - printf '%s\t%s\t%s\n' "$journal" "$id" "$FM_BACKEND_HERDR_JOURNAL_PROJECTION_ID" + printf '%s\t%s\t%s\t%s\n' "$expected" "$journal" "$id" \ + "$FM_BACKEND_HERDR_JOURNAL_PROJECTION_ID" done } +fm_herdr_cleanup_index_matches() { # <title> [index] + local title=$1 index=${2-$FM_HERDR_CLEANUP_INDEX} label record + while IFS= read -r record; do + label=${record%%$'\t'*} + [ "$label" = "$title" ] || continue + printf '%s\n' "${record#*$'\t'}" + done <<< "$index" +} + fm_herdr_cleanup_unique_match() { # <title> <session> <home-real> local title=$1 session=$2 home_real=$3 matches count record FM_HERDR_CLEANUP_JOURNAL= @@ -92,7 +118,11 @@ fm_herdr_cleanup_unique_match() { # <title> <session> <home-real> FM_HERDR_CLEANUP_BOUND_WORKSPACE= FM_HERDR_CLEANUP_BOUND_TAB= FM_HERDR_CLEANUP_BOUND_PANE= - matches=$(fm_herdr_cleanup_journal_matches "$title" "$session" "$home_real") || return 1 + if [ "${4:-fresh}" = discovery ] && [ "$FM_HERDR_CLEANUP_INDEX_READY" = 1 ]; then + matches=$(fm_herdr_cleanup_index_matches "$title") || return 1 + else + matches=$(fm_herdr_cleanup_journal_matches "$title" "$session" "$home_real") || return 1 + fi count=$(printf '%s\n' "$matches" | awk 'NF { n++ } END { print n+0 }') [ "$count" -eq 1 ] || return 1 record=$(printf '%s\n' "$matches" | awk 'NF { print; exit }') @@ -199,12 +229,47 @@ fm_herdr_cleanup_revalidate() { # <session> <workspace> <tab> <pane> <title> <to [ "${focus#*$'\t'}" != "$tab" ] } -fm_herdr_cleanup_one() { # <session> <workspace> <title> <home-real> +fm_herdr_cleanup_record_lock_paths() { # <id> <task-lock> <presentation-lock> + local id=$1 task_lock=$2 presentation_lock=$3 record=$FM_HERDR_CLEANUP_LOCK_RECORD + [ -n "$record" ] || return 0 + [ -f "$record" ] && [ ! -L "$record" ] || return 1 + printf '%s\t%s\t%s\n' "$id" "$task_lock" "$presentation_lock" > "$record" +} + +fm_herdr_cleanup_recover_interrupted_candidate() { # <lock-record> + local record=$1 id task_lock presentation_lock + [ -f "$record" ] && [ ! -L "$record" ] || return 0 + IFS=$'\t' read -r id task_lock presentation_lock < "$record" || return 0 + fm_task_id_creation_valid "$id" || return 0 + [ "$task_lock" = "$STATE/.spawn-$id.lock" ] || return 0 + case "$presentation_lock" in + /tmp/firstmate-herdr-presentation/order-????????????????????????????????.lock) ;; + *) return 0 ;; + esac + if fm_lock_try_acquire "$task_lock"; then + if fm_lock_try_acquire "$presentation_lock"; then + fm_lock_release "$presentation_lock" || true + fi + fm_lock_release "$task_lock" || true + fi + return 0 +} + +fm_herdr_cleanup_one() ( # <session> <workspace> <title> <home-real> local session=$1 workspace=$2 title=$3 home_real=$4 token journal id task_lock local version bound_workspace bound_tab bound_pane presentation_lock snapshot local tab pane state close_status=0 + task_lock='' presentation_lock='' + # A deadline may interrupt lock acquisition or a backend read. + # fm_lock_release verifies this process owns each path, so unconditional cleanup + # closes the signal window before the held flags could be set. + trap '[ -z "$presentation_lock" ] || fm_lock_release "$presentation_lock" || true + [ -z "$task_lock" ] || fm_lock_release "$task_lock" || true' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + trap 'exit 129' HUP token=$(fm_herdr_cleanup_title_token "$title") || return 0 - if ! fm_herdr_cleanup_unique_match "$title" "$session" "$home_real"; then + if ! fm_herdr_cleanup_unique_match "$title" "$session" "$home_real" discovery; then return 0 fi journal=$FM_HERDR_CLEANUP_JOURNAL @@ -215,24 +280,24 @@ fm_herdr_cleanup_one() { # <session> <workspace> <title> <home-real> bound_pane=$FM_HERDR_CLEANUP_BOUND_PANE [ "$FM_HERDR_CLEANUP_TOKEN" = "$token" ] || return 0 task_lock="$STATE/.spawn-$id.lock" - if ! fm_lock_try_acquire "$task_lock"; then - fm_herdr_cleanup_warn "$id skipped because its task lock is busy" - return 0 - fi presentation_lock=$(fm_backend_herdr_presentation_session_lock_path "$session" 2>/dev/null) || { - fm_lock_release "$task_lock" || true fm_herdr_cleanup_warn "$id skipped because the shared presentation lock is unavailable" return 0 } + fm_herdr_cleanup_record_lock_paths "$id" "$task_lock" "$presentation_lock" || { + fm_herdr_cleanup_warn "$id skipped because deadline lock recovery could not be recorded" + return 0 + } + if ! fm_lock_try_acquire "$task_lock"; then + fm_herdr_cleanup_warn "$id skipped because its task lock is busy" + return 0 + fi if ! fm_lock_try_acquire "$presentation_lock"; then - fm_lock_release "$task_lock" || true fm_herdr_cleanup_warn "$id skipped because the shared presentation lock is busy" return 0 fi if [ -e "$STATE/$id.meta" ] || [ -L "$STATE/$id.meta" ]; then - fm_lock_release "$presentation_lock" || true - fm_lock_release "$task_lock" || true return 0 fi snapshot=$(fm_backend_herdr_cli "$session" api snapshot 2>/dev/null) || snapshot= @@ -241,8 +306,6 @@ fm_herdr_cleanup_one() { # <session> <workspace> <title> <home-real> "$snapshot" "$workspace" "$title" "$token" \ "$bound_workspace" "$bound_tab" "$bound_pane"; then fm_herdr_cleanup_warn "$id preserved because its locked candidate snapshot was ambiguous" - fm_lock_release "$presentation_lock" || true - fm_lock_release "$task_lock" || true return 0 fi tab=$FM_HERDR_CLEANUP_TAB @@ -250,16 +313,12 @@ fm_herdr_cleanup_one() { # <session> <workspace> <title> <home-real> if [ "$(fm_backend_herdr_pane_agent_state "$session" "$pane")" != no-agent ] \ || ! fm_backend_herdr_pane_idle_shell_pid "$session" "$pane" >/dev/null; then fm_herdr_cleanup_warn "$id preserved because its pane is not a provably idle childless shell" - fm_lock_release "$presentation_lock" || true - fm_lock_release "$task_lock" || true return 0 fi if ! fm_herdr_cleanup_revalidate \ "$session" "$workspace" "$tab" "$pane" "$title" "$token" "$home_real" \ "$journal" "$id" "$version" "$bound_workspace" "$bound_tab" "$bound_pane"; then fm_herdr_cleanup_warn "$id preserved because immediate revalidation changed or was unreadable" - fm_lock_release "$presentation_lock" || true - fm_lock_release "$task_lock" || true return 0 fi @@ -287,10 +346,8 @@ fm_herdr_cleanup_one() { # <session> <workspace> <title> <home-real> else fm_herdr_cleanup_warn "$id preserved because exact pane closure could not be confirmed" fi - fm_lock_release "$presentation_lock" || true - fm_lock_release "$task_lock" || true return 0 -} +) fm_herdr_session_cleanup() { local session home_real list candidates workspace title journal found=0 @@ -324,6 +381,11 @@ fm_herdr_session_cleanup() { fm_herdr_cleanup_warn "session '$session' workspace discovery was unreadable; preserving every candidate" return 0 } + FM_HERDR_CLEANUP_INDEX=$(fm_herdr_cleanup_index "$session" "$home_real") || { + fm_herdr_cleanup_warn 'journal discovery failed; preserving every candidate' + return 0 + } + FM_HERDR_CLEANUP_INDEX_READY=1 while IFS=$'\t' read -r workspace title; do [ -n "$workspace" ] && [ -n "$title" ] || continue fm_herdr_cleanup_one "$session" "$workspace" "$title" "$home_real" @@ -332,6 +394,28 @@ fm_herdr_session_cleanup() { } if [ "${FM_HERDR_SESSION_CLEANUP_SOURCE_ONLY:-0}" != 1 ]; then - fm_herdr_session_cleanup + if [ "${1:-}" = --_worker ]; then + fm_herdr_session_cleanup + else + [ -d "$STATE" ] && [ ! -L "$STATE" ] || exit 0 + budget=${FM_HERDR_SESSION_CLEANUP_TIMEOUT:-30} + case "$budget" in ''|*[!0-9]*|0) budget=30 ;; esac + cleanup_lock_record=$(umask 077; mktemp "$STATE/.herdr-cleanup-locks.XXXXXX") || { + fm_herdr_cleanup_warn 'deadline lock recovery could not be prepared; preserving every candidate' + exit 0 + } + cleanup_rc=0 + FM_HERDR_CLEANUP_LOCK_RECORD="$cleanup_lock_record" \ + fm_run_timed "$budget" "$SCRIPT_DIR/fm-herdr-session-cleanup.sh" --_worker || cleanup_rc=$? + if [ "$cleanup_rc" -ne 0 ]; then + fm_herdr_cleanup_recover_interrupted_candidate "$cleanup_lock_record" + fi + if [ "$cleanup_rc" -eq 124 ]; then + fm_herdr_cleanup_warn "${budget}s deadline reached; unfinished candidates preserved; cleanup coverage is unconfirmed" + elif [ "$cleanup_rc" -ne 0 ]; then + fm_herdr_cleanup_warn "cleanup exited $cleanup_rc; unfinished candidates preserved; cleanup coverage is unconfirmed" + fi + rm -f -- "$cleanup_lock_record" 2>/dev/null || true + fi exit 0 fi diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 37b4909161f..57245f8759d 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -72,7 +72,7 @@ # convergence, pending remote handoff delivery, and the fleet-sync fetch - are # started as one detached bounded worker right after the lock (step 1) and # harvested at step 7 without ever blocking on it. The bounded inactive-outcome -# startup scan joins that worker because its local current-state reads can also +# startup scan and home-summary publication join that worker because their local current-state reads can also # be slow. bin/fm-startup-network.sh owns that stage and its safety argument; # bin/fm-bootstrap.sh and bin/fm-inactive-reconcile.sh remain the owners of the # work itself and still run it. @@ -656,13 +656,7 @@ if [ "$READ_ONLY" -eq 0 ]; then rm -f "$COMPLETION_FILE" 2>/dev/null || true fi fm_trace_context_session_start "$CONFIG" "$STATE/.trace-context-effective" - # A full locked start publishes this home's current structured summary. - # Publication is side-band and best-effort, so it can never change the - # session-start result. A context re-emit is not another session start. - if [ "$REEMIT" -eq 0 ]; then - "$SCRIPT_DIR/fm-home-summary-refresh.sh" --best-effort || true - fi - # Every network call and the potentially slow inactive-outcome startup scan + # Every network call, summary refresh, and potentially slow inactive-outcome startup scan # are launched HERE, detached and bounded, so they run concurrently with the # whole digest below instead of in front of it. Step 7 harvests whatever has # finished, without ever waiting. diff --git a/bin/fm-startup-network.sh b/bin/fm-startup-network.sh index cc9e70451d6..5143231f714 100755 --- a/bin/fm-startup-network.sh +++ b/bin/fm-startup-network.sh @@ -13,8 +13,13 @@ # composed from bounded local reads while these checks run concurrently in a # detached worker, and their result is reported back inline when it finishes in # time, or as a durable wake when it does not. The locked startup's bounded -# inactive-outcome scan also runs here because its local current-state reads can -# be just as slow; that scan publishes its own findings to the durable wake queue. +# inactive-outcome scan and best-effort home-summary refresh also run here because +# their local current-state reads can be just as slow; the scan publishes its own +# findings to the durable wake queue. Summary publication runs concurrently with +# the checks under its own single-flight lock and a deadline capped at the stage +# budget. The network result is published before the summary child is reaped, and +# that child is still reaped before the deferred stage finishes. It never inherits +# the digest stdout. # # WHAT IS PRESERVED. Nothing is dropped. bin/fm-bootstrap.sh remains the single # owner of every network sweep and still runs all of them, unchanged, via its @@ -104,7 +109,10 @@ # and the wake decision. # # The whole stage is bounded by FM_STARTUP_NETWORK_TIMEOUT (default 120s), one -# aggregate deadline covering both the inactive-outcome scan and network sweeps. +# aggregate deadline covering the inactive-outcome scan and network sweeps. +# Concurrent summary publication retains its own FM_HOME_SUMMARY_TIMEOUT, capped +# at that stage budget, and is reaped before the deferred stage exits. Its cleanup +# cannot delay publication of the network result. # Hitting the bound is reported as an actionable NETWORK_CHECKS: line, never as # silence. bin/fm-timeout-lib.sh remains the single owner of bounded execution. set -u @@ -197,7 +205,7 @@ worker_alive() { phase_label() { # <phases> case "$1" in probe) printf 'GitHub authentication' ;; - probe,sweeps) printf 'GitHub authentication, dead-secondmate relaunch, secondmate convergence, pending handoff delivery, project clone refresh with its drift reporting, and inactive terminal-outcome reconciliation' ;; + probe,sweeps) printf 'GitHub authentication, dead-secondmate relaunch, secondmate convergence, pending handoff delivery, project clone refresh with its drift reporting, inactive terminal-outcome reconciliation, and home-summary publication' ;; *) printf 'the deferred network checks' ;; esac } @@ -420,7 +428,7 @@ EOF } cmd_run() { # <locked> <lock-pid> <generation> - local locked=$1 lock_pid=$2 generation=$3 phases started budget out rc sweep_locked=0 downgraded=0 internal=0 lease_held=0 timings stage_started + local locked=$1 lock_pid=$2 generation=$3 phases started budget out rc sweep_locked=0 downgraded=0 internal=0 lease_held=0 timings stage_started summary_pid='' summary_budget mkdir -p "$STATE" 2>/dev/null || return 1 started=$(now) budget=$(stage_budget) @@ -485,12 +493,21 @@ EOF downgraded=1 fi fi - # One aggregate deadline covers both deferred operations. The inactive scan - # retains its own tighter per-scan bound inside this outer bound. Findings - # need no report translation: the scan writes its ordinary durable - # inactive-outcome wakes directly. A child shell composes the two executable - # owners only so fm_run_timed can govern them as one process group. + # The inactive scan retains its own tighter per-scan bound inside this outer + # bound. Findings need no report translation: the scan writes its ordinary + # durable inactive-outcome wakes directly. The best-effort summary is + # single-flight and runs alongside the checks, not in front of them. Keep its + # bounded wrapper outside the network process group: killing that wrapper at + # the network deadline could strand the separate process group it owns. Both + # budgets start together; the network result is published before reaping the + # summary child. if [ "$sweep_locked" -eq 1 ]; then + summary_budget=${FM_HOME_SUMMARY_TIMEOUT:-60} + case "$summary_budget" in ''|*[!0-9]*|0) summary_budget=60 ;; esac + [ "$summary_budget" -le "$budget" ] || summary_budget=$budget + FM_HOME_SUMMARY_TIMEOUT="$summary_budget" FM_HOME_SUMMARY_IF_IDLE=1 \ + "$SCRIPT_DIR/fm-home-summary-refresh.sh" --best-effort </dev/null >/dev/null 2>&1 & + summary_pid=$! # shellcheck disable=SC2016 # Child-shell variables expand inside the bound. fm_run_timed "$budget" env FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" \ FM_BOOTSTRAP_NETWORK=only FM_BOOTSTRAP_NETWORK_LOCK_PID="$lock_pid" \ @@ -524,6 +541,7 @@ EOF publish "$generation" failed "$phases" "$sweep_locked" "$started" "$rc" "$out" "$timings" ;; esac + [ -z "$summary_pid" ] || wait "$summary_pid" || true rm -f "$out" 2>/dev/null || true [ -z "$timings" ] || rm -f "$timings" 2>/dev/null || true return 0 diff --git a/tests/fm-contributions.test.sh b/tests/fm-contributions.test.sh index e9bee1b06cb..8bb70cc6ecb 100755 --- a/tests/fm-contributions.test.sh +++ b/tests/fm-contributions.test.sh @@ -795,6 +795,22 @@ test_unavailable_forge_records_error_and_wakes_once_per_episode() { # genuine ou pass 'a genuinely unavailable forge records an error and wakes once per failure episode' } +test_large_contribution_input_stays_off_argv() { + local home title i out + home=$(new_home large-contribution-input) + title=$(printf '%2048s' '' | tr ' ' x) + i=1 + while [ "$i" -le 72 ]; do + printf -- '- [ ] large-%03d - %s\n' "$i" "$title" >> "$home/data/backlog.md" + i=$((i + 1)) + done + out=$(with_home "$home" "$ROOT/bin/fm-fleet-snapshot.sh" --contribution-input) \ + || fail 'large contribution input exceeded the process argument limit' + printf '%s' "$out" | jq -e '(.backlog.records | length) == 72 and (.tasks | type) == "array"' >/dev/null \ + || fail 'large contribution input returned an incomplete backlog/task pair' + pass 'large contribution input is transported through files rather than process arguments' +} + test_late_owner_keeps_failure_episode_suppressed() { local home out line='contributions: observation unavailable for https://github.com/o/r/pull/8' local error='forge observation unavailable or changed during read' task @@ -829,7 +845,7 @@ test_late_owner_keeps_failure_episode_suppressed() { } failures=0 -for test_name in test_actor_coverage test_stale_verdict test_unchecked_is_not_silence test_newest_check_has_no_verdict test_comment_wake test_review_wake test_inline_wake test_ready_issue_wake test_fresh_issue_requires_maintainer test_missing_lane_remains_missing test_partial_freshness_keeps_measured_rows test_malformed_record_cannot_prove_silence test_issue_timeline_and_exact_ack test_verdict_retains_judged_head test_observed_replacement_refreshes_verdict test_unobserved_head_leaves_verdict_unknown test_away_yolo_is_fleet_work test_away_yolo_cross_home_is_fleet_work test_retired_and_unsupported_coverage test_unsupported_forge_is_not_fleet_work test_held_unsupported_forge_is_not_captain_work test_shared_contribution_signal_wakes_once test_watcher_keeps_diagnostics_separate_from_contribution_wakes test_expired_child_unsupported_forge_stays_unmeasured test_watcher_surfaces_new_contribution_once test_home_summary_coverage test_unreadable_pending_is_not_empty test_budget_refusal_between_calls test_budget_bounded_call_timeout test_genuine_failure_near_deadline_is_unavailable test_shared_url_observed_once test_terminal_contribution_settles test_late_owner_inherits_terminal_observation test_done_task_open_pr_still_observed test_reservation_defers_later_url_when_fifteen_seconds_do_not_remain test_three_second_pr_reads_complete_fresh_in_one_cycle test_unavailable_forge_records_error_and_wakes_once_per_episode test_late_owner_keeps_failure_episode_suppressed; do +for test_name in test_actor_coverage test_stale_verdict test_unchecked_is_not_silence test_newest_check_has_no_verdict test_comment_wake test_review_wake test_inline_wake test_ready_issue_wake test_fresh_issue_requires_maintainer test_missing_lane_remains_missing test_partial_freshness_keeps_measured_rows test_malformed_record_cannot_prove_silence test_issue_timeline_and_exact_ack test_verdict_retains_judged_head test_observed_replacement_refreshes_verdict test_unobserved_head_leaves_verdict_unknown test_away_yolo_is_fleet_work test_away_yolo_cross_home_is_fleet_work test_retired_and_unsupported_coverage test_unsupported_forge_is_not_fleet_work test_held_unsupported_forge_is_not_captain_work test_shared_contribution_signal_wakes_once test_watcher_keeps_diagnostics_separate_from_contribution_wakes test_expired_child_unsupported_forge_stays_unmeasured test_watcher_surfaces_new_contribution_once test_home_summary_coverage test_unreadable_pending_is_not_empty test_budget_refusal_between_calls test_budget_bounded_call_timeout test_genuine_failure_near_deadline_is_unavailable test_shared_url_observed_once test_terminal_contribution_settles test_late_owner_inherits_terminal_observation test_done_task_open_pr_still_observed test_large_contribution_input_stays_off_argv test_reservation_defers_later_url_when_fifteen_seconds_do_not_remain test_three_second_pr_reads_complete_fresh_in_one_cycle test_unavailable_forge_records_error_and_wakes_once_per_episode test_late_owner_keeps_failure_episode_suppressed; do ( "$test_name" ) || failures=$((failures + 1)) done [ "$failures" -eq 0 ] || fail "$failures contribution regressions" diff --git a/tests/fm-herdr-session-cleanup.test.sh b/tests/fm-herdr-session-cleanup.test.sh index 99cf4b49a1c..d66ccbbfcaa 100755 --- a/tests/fm-herdr-session-cleanup.test.sh +++ b/tests/fm-herdr-session-cleanup.test.sh @@ -169,6 +169,9 @@ fm_backend_herdr_cli() { esac ;; "api snapshot") + if [ -e "$FIXTURE_DIR/journal-race" ]; then + write_v1 "$ID" ZbCdEfGhIjKlMnOpQrStUv + fi : > "$FIXTURE_DIR/snapshotted" printf '{"result":{"snapshot":{"focused_workspace_id":"w1","focused_tab_id":"%s","focused_pane_id":"w1:p1","workspaces":' "$(cat "$FIXTURE_DIR/active-tab")" fixture_workspaces @@ -326,4 +329,104 @@ FM_HOME="$INTEGRATION_ROOT/home" FM_ROOT_OVERRIDE="$INTEGRATION_ROOT" \ [ ! -s "$TRACE" ] || fail "read-only session start ran stale projection cleanup" pass "session start runs cleanup only after acquiring its home lock" +# Cached discovery cannot authorize retirement after the journal changes. +reset_fixture +: > "$FIXTURE_DIR/journal-race" +assert_preserved "journal identity changed after indexed discovery" +[ "$(fm_backend_herdr_projection_journal_field "$FM_STATE_OVERRIDE/$ID.herdr-presentation" projection_id)" = ZbCdEfGhIjKlMnOpQrStUv ] \ + || fail "journal race did not change the identity under review" +pass "fresh locked revalidation rejects journal identity changes after indexed discovery" + +# Observe journal I/O through the executable cleanup interface. Increasing +# foreign projection titles must not reread this home's journals per workspace. +( + reset_fixture + for n in 1 2 3 4 5 6 7 8 9 10; do write_v1 "owned-$n"; done + export FM_TEST_REAL_GREP + FM_TEST_REAL_GREP=$(command -v grep) + export FM_TEST_JOURNAL_READ_LOG="$TMP_ROOT/journal-reads" + cat > "$FAKEBIN/grep" <<'SH' +#!/usr/bin/env bash +for arg in "$@"; do + case "$arg" in *.herdr-presentation) printf '%s\n' "$arg" >> "$FM_TEST_JOURNAL_READ_LOG" ;; esac +done +exec "$FM_TEST_REAL_GREP" "$@" +SH + chmod +x "$FAKEBIN/grep" + fixture_workspaces() { + local n=1 + printf '[' + while [ "$n" -le "$FM_TEST_PROJECTION_COUNT" ]; do + [ "$n" -eq 1 ] || printf ',' + printf '{"workspace_id":"foreign-%s","label":"โ”” foreign-%s ยท p:%s"}' "$n" "$n" "$TOKEN" + n=$((n + 1)) + done + printf ']' + } + : > "$FM_TEST_JOURNAL_READ_LOG" + FM_TEST_PROJECTION_COUNT=1 fm_herdr_session_cleanup + single_reads=$(wc -l < "$FM_TEST_JOURNAL_READ_LOG" | tr -d '[:space:]') + : > "$FM_TEST_JOURNAL_READ_LOG" + FM_TEST_PROJECTION_COUNT=20 fm_herdr_session_cleanup + fleet_reads=$(wc -l < "$FM_TEST_JOURNAL_READ_LOG" | tr -d '[:space:]') + [ "$single_reads" -gt 0 ] || fail "journal I/O probe checked nothing" + [ "$fleet_reads" -eq "$single_reads" ] \ + || fail "foreign projections multiplied journal reads: $single_reads -> $fleet_reads" + [ ! -s "$CLOSE_LOG" ] || fail "foreign projection discovery closed a pane" + rm -f "$FAKEBIN/grep" +) || exit 1 +pass "fleet discovery reads home journals once regardless of foreign projection count" + +# Exercise the actual executable deadline, not the sourced worker alone. +( + reset_fixture + cp "$FM_STATE_OVERRIDE/$ID.herdr-presentation" "$TMP_ROOT/deadline-journal" + export FM_TEST_HERDR_PID="$TMP_ROOT/deadline-herdr.pid" + export FM_TEST_HERDR_SOCKET="$TMP_ROOT/deadline-session.sock" + export FM_TEST_HERDR_TITLE="$TITLE" + cat > "$FAKEBIN/herdr" <<'SH' +#!/usr/bin/env bash +case "${1:-} ${2:-}" in + "session list") + printf '{"sessions":[{"name":"test","running":true,"socket_path":"%s"}]}\n' "$FM_TEST_HERDR_SOCKET" + ;; + "workspace list") + printf '{"result":{"workspaces":[{"workspace_id":"w1","label":"firstmate"},{"workspace_id":"w2","label":"%s","tab_count":1,"pane_count":1}]}}\n' "$FM_TEST_HERDR_TITLE" + ;; + "api snapshot") + printf '%s\n' "$$" > "$FM_TEST_HERDR_PID" + trap 'exit 143' TERM + sleep 30 + ;; + *) exit 1 ;; +esac +SH + chmod +x "$FAKEBIN/herdr" + presentation_lock=$(PATH="$FAKEBIN:$PATH" HERDR_SESSION=test FM_HOME="$FM_HOME" \ + FM_ROOT_OVERRIDE="$ROOT" bash -c \ + '. "$1/bin/fm-backend.sh"; fm_backend_source herdr; fm_backend_herdr_presentation_session_lock_path test' \ + _ "$ROOT") || fail "could not resolve the isolated fixture presentation lock" + started=$SECONDS + HERDR_SESSION=test FM_HERDR_SESSION_CLEANUP_TIMEOUT=2 \ + "$ROOT/bin/fm-herdr-session-cleanup.sh" > "$TMP_ROOT/deadline.out" 2>&1 \ + || fail "cleanup deadline blocked startup" + [ "$((SECONDS - started))" -lt 15 ] || fail "cleanup deadline did not bound execution" + grep -q 'unfinished candidates preserved; cleanup coverage is unconfirmed' "$TMP_ROOT/deadline.out" \ + || fail "cleanup deadline did not report unconfirmed coverage" + cmp -s "$TMP_ROOT/deadline-journal" "$FM_STATE_OVERRIDE/$ID.herdr-presentation" \ + || fail "cleanup deadline changed an unfinished journal" + [ -s "$FM_TEST_HERDR_PID" ] || fail "deadline never reached the backend read after acquiring locks" + task_lock="$FM_STATE_OVERRIDE/.spawn-$ID.lock" + [ ! -e "$task_lock" ] && [ ! -L "$task_lock" ] \ + || fail "cleanup deadline left the candidate task lock held: $(ls -ld "$task_lock" 2>/dev/null) owner=$(cat "$task_lock/pid" 2>/dev/null)" + [ ! -e "$presentation_lock" ] && [ ! -L "$presentation_lock" ] \ + || fail "cleanup deadline left the shared presentation lock held: $(ls -ld "$presentation_lock" 2>/dev/null) owner=$(cat "$presentation_lock/pid" 2>/dev/null)" + backend_pid=$(cat "$FM_TEST_HERDR_PID") + if kill -0 "$backend_pid" 2>/dev/null; then + backend_state=$(ps -p "$backend_pid" -o stat= 2>/dev/null || true) + case "$backend_state" in Z*) ;; *) fail "deadline left the backend process running" ;; esac + fi +) || exit 1 +pass "executable deadline preserves journals and releases both locks after interruption" + printf 'all fm-herdr-session-cleanup tests passed\n' diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 3beaad78ad1..6d5ab793915 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -719,6 +719,60 @@ write_omp_loaded_markers() { # --- context digest: absent vs empty vs present ----------------------------- +test_summary_refresh_never_blocks_the_digest() { + local rec root home fakebin world out started release finished waited=0 f + local SESSION_START + rec=$(new_world summary-refresh-deferred) + IFS='|' read -r root home fakebin <<EOF +$rec +EOF + world=${root%/root} + started="$world/summary.started" + release="$world/summary.release" + finished="$world/summary.finished" + mkdir -p "$root/bin" + ln -s "$ROOT/docs" "$root/docs" + for f in "$ROOT"/bin/*.sh; do ln -s "$f" "$root/bin/${f##*/}"; done + rm "$root/bin/fm-home-summary-refresh.sh" + cat > "$root/bin/fm-home-summary-refresh.sh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "${FM_HOME_SUMMARY_IF_IDLE:-0}" > "${FM_TEST_SUMMARY_STARTED:?}" +ticks=0 +while [ ! -e "${FM_TEST_SUMMARY_RELEASE:?}" ] && [ "$ticks" -lt 600 ]; do + sleep 0.1 + ticks=$((ticks + 1)) +done +: > "${FM_TEST_SUMMARY_FINISHED:?}" +SH + chmod +x "$root/bin/fm-home-summary-refresh.sh" + SESSION_START="$root/bin/fm-session-start.sh" + make_fake_toolchain "$fakebin" + make_fake_ps_claude "$fakebin" + out=$(FM_FAKE_HARNESS_PID="$SESSION_START_TEST_HARNESS_PID" \ + FM_TEST_SUMMARY_STARTED="$started" FM_TEST_SUMMARY_RELEASE="$release" \ + FM_TEST_SUMMARY_FINISHED="$finished" \ + run_session_start "$home" "$root" "$fakebin:$BASE_PATH") + assert_contains "$out" 'NEXT STEP' "digest did not reach its final section" + assert_not_contains "$out" 'โ— STARTUP TRUNCATED' "summary truncated the digest" + assert_absent "$finished" "digest waited for the unreleased summary producer" + while [ ! -s "$started" ] && [ "$waited" -lt 100 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + if ! grep -q '^1$' "$started" 2>/dev/null; then + fail "summary did not run in deferred single-flight mode: marker=$(cat "$started" 2>/dev/null), stage=$(network_stage_report "$home" "$root" 2>/dev/null), digest=$out" + fi + : > "$release" + waited=0 + while [ ! -e "$finished" ] && [ "$waited" -lt 100 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + [ -e "$finished" ] || fail "summary obligation disappeared after the digest" + wait_for_network_stage "$home" "$root" || fail "deferred startup stage never settled" + pass "session start: summary publication is deferred without losing its obligation" +} + test_context_digest_absent_empty_present() { local rec root home fakebin out rec=$(new_world context-digest) @@ -732,8 +786,11 @@ EOF : > "$home/data/captain.md" # secondmates.md, captain-shared.md, and learnings.md deliberately absent - out=$(run_session_start "$home" "$root" "$fakebin:$BASE_PATH") + out=$(FM_FAKE_HARNESS_PID="$SESSION_START_TEST_HARNESS_PID" \ + run_session_start "$home" "$root" "$fakebin:$BASE_PATH") + wait_for_network_stage "$home" "$root" \ + || fail "the deferred startup work did not finish" jq -e --arg home "$home" ' .schema == "fm-secondmate-home-summary.v1" and .home == $home @@ -1470,10 +1527,18 @@ EOF # unreachable host without touching one: if any part of the blocking path still # waits on the network, the digest cannot finish before this does. install_slow_gh() { - local fakebin=$1 seconds=$2 finished_marker=${3:-} + local fakebin=$1 seconds=$2 finished_marker=${3:-} release_gate=${4:-} cat > "$fakebin/gh" <<SH #!/usr/bin/env bash if [ "\${1:-}" = auth ]; then + if [ -n '$release_gate' ]; then + ticks=0 + while [ ! -e '$release_gate' ] && [ "\$ticks" -lt 1200 ]; do + sleep 0.1 + ticks=\$((ticks + 1)) + done + [ -e '$release_gate' ] || exit 98 + fi sleep $seconds [ -z '$finished_marker' ] || : > '$finished_marker' exit 1 @@ -1576,25 +1641,25 @@ SH } # The headline guarantee: an unreachable host delays a reported CHECK, never the -# startup. The fake host hangs for 12s; the digest must be done long before that, +# startup. The fake host waits for release; the digest must be done before that, # must say so rather than implying the checks passed, and the sweeps must still # run and land afterwards. test_unreachable_network_never_blocks_the_digest() { - local rec root home fakebin mate log spawned network_finished out started elapsed + local rec root home fakebin mate log spawned network_finished out release_gate rec=$(prepare_session_start_secondmate secondmate-slow-network) IFS='|' read -r root home fakebin mate log spawned <<EOF $rec EOF network_finished="${root%/root}/network-finished" - install_slow_gh "$fakebin" 12 "$network_finished" + release_gate="${root%/root}/network-release" + install_slow_gh "$fakebin" 0 "$network_finished" "$release_gate" - started=$(date +%s) out=$(run_session_start_secondmate "$root" "$home" "$fakebin" "$mate" "$log" "$spawned" missing) - elapsed=$(( $(date +%s) - started )) [ ! -e "$network_finished" ] \ - || fail "the digest waited for the 12s unreachable-host probe instead of returning from local state (${elapsed}s)" + || fail "the digest waited for the gated unreachable-host probe instead of returning from local state" assert_contains "$out" "SESSION START" "the digest did not complete" + assert_not_contains "$out" 'โ— STARTUP TRUNCATED' "the gated probe prevented the digest from completing" assert_contains "$out" "IN PROGRESS - the deferred network checks have not finished yet." \ "the digest did not disclose that its network checks were still running" assert_contains "$out" "NOT yet confirmed: GitHub authentication, dead-secondmate relaunch" \ @@ -1603,6 +1668,7 @@ EOF "the digest reported a GitHub-auth verdict it could not yet have" # ... and the work itself still happens, off the blocking path. + : > "$release_gate" wait_for_network_stage "$home" "$root" 60 \ || fail "the deferred stage never finished: $(network_stage_report "$home" "$root")" assert_contains "$(network_stage_report "$home" "$root")" "NEEDS_GH_AUTH" \ @@ -2699,6 +2765,7 @@ EOF pass "session start rejects Pi loaded markers from previous sessions" } +test_summary_refresh_never_blocks_the_digest test_context_digest_absent_empty_present test_lock_refusal_read_only_path test_lock_write_failure_read_only_path diff --git a/tests/fm-startup-network.test.sh b/tests/fm-startup-network.test.sh index 346b71e4277..47b0c5256fd 100755 --- a/tests/fm-startup-network.test.sh +++ b/tests/fm-startup-network.test.sh @@ -28,8 +28,8 @@ FM_TEST_CLEANUP_DIRS+=("$TMP_ROOT") trap fm_test_cleanup EXIT # new_world <name>: an FM_HOME plus a fake code root whose bin/ is a real -# firstmate bin/ except for fm-bootstrap.sh, which is replaced by a scriptable -# stand-in. The stage's contract is about WHEN and WHETHER the network half runs +# firstmate bin/ except for fm-bootstrap.sh and fm-home-summary-refresh.sh, +# which are replaced by scriptable stand-ins. The contract is when each runs # and how its result is published; bin/fm-bootstrap.sh's own behavior is owned by # tests/fm-bootstrap.test.sh, so pinning it here would duplicate that owner and # make these assertions depend on unrelated tool detection. @@ -61,10 +61,35 @@ if [ -n "${FM_TIMING_LOG:-}" ]; then "$(( $(fm_timing_now_ms) - 1500 ))" "${FM_FAKE_TIMING_DETAIL:-}" fi [ -z "${FM_FAKE_BOOTSTRAP_SLEEP:-}" ] || sleep "$FM_FAKE_BOOTSTRAP_SLEEP" +if [ -n "${FM_FAKE_BOOTSTRAP_RELEASE:-}" ]; then + ticks=0 + while [ ! -e "$FM_FAKE_BOOTSTRAP_RELEASE" ] && [ "$ticks" -lt 1200 ]; do + sleep 0.1 + ticks=$((ticks + 1)) + done + [ -e "$FM_FAKE_BOOTSTRAP_RELEASE" ] || exit 98 +fi [ -z "${FM_FAKE_BOOTSTRAP_OUT:-}" ] || printf '%s\n' "$FM_FAKE_BOOTSTRAP_OUT" exit "${FM_FAKE_BOOTSTRAP_RC:-0}" SH chmod +x "$root/bin/fm-bootstrap.sh" + rm -f "$root/bin/fm-home-summary-refresh.sh" + cat > "$root/bin/fm-home-summary-refresh.sh" <<'SH' +#!/usr/bin/env bash +set -u +[ -z "${FM_FAKE_SUMMARY_LOG:-}" ] || printf '%s %s %s\n' "$$" "${FM_HOME_SUMMARY_IF_IDLE:-0}" "$*" >> "$FM_FAKE_SUMMARY_LOG" +if [ -n "${FM_FAKE_SUMMARY_RELEASE:-}" ]; then + ticks=0 + while [ ! -e "$FM_FAKE_SUMMARY_RELEASE" ] && [ "$ticks" -lt 300 ]; do + sleep 0.1 + ticks=$((ticks + 1)) + done +fi +printf 'summary stdout must remain private\n' +printf 'summary stderr must remain private\n' >&2 +exit "${FM_FAKE_SUMMARY_RC:-0}" +SH + chmod +x "$root/bin/fm-home-summary-refresh.sh" cat > "$root/bin/ps" <<'SH' #!/usr/bin/env bash pid= @@ -132,26 +157,134 @@ wait_for_startup_network_wake() { # <home> [tenths] # --- tests ------------------------------------------------------------------- +test_summary_runs_concurrently_and_is_reaped() { + local rec home root log summary release out report harvest summary_pid waited=0 + rec=$(new_world summary-concurrent) + IFS='|' read -r home root log <<EOF +$rec +EOF + summary="${root%/root}/summary.log" + release="${root%/root}/summary.release" + printf '%s\n' $$ > "$home/state/.lock" + out=$(FM_FAKE_BOOTSTRAP_LOG="$log" FM_FAKE_SUMMARY_LOG="$summary" \ + FM_FAKE_SUMMARY_RELEASE="$release" FM_FAKE_BOOTSTRAP_RC=7 FM_FAKE_SUMMARY_RC=9 \ + run_stage "$home" "$root" start --locked 1 --harvest-pid $$) + while { [ ! -s "$summary" ] || [ ! -s "$log" ]; } && [ "$waited" -lt 100 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + assert_grep ' 1 --best-effort' "$summary" "summary did not use single-flight best-effort mode" + assert_grep 'network=only detect_only=0' "$log" "summary blocked the network checks" + read -r summary_pid _ < "$summary" + kill -0 "$summary_pid" 2>/dev/null || fail "summary gate did not remain outstanding" + report='' + while [ "$waited" -lt 100 ]; do + report=$(run_stage "$home" "$root" report) + if ! grep -q 'IN PROGRESS' <<EOF +$report +EOF + then + break + fi + sleep 0.1 + waited=$((waited + 1)) + done + assert_contains "$report" 'exited 7' "summary reaping delayed publication of the network result" + assert_not_contains "$report" 'IN PROGRESS' "network result remained unpublished behind summary reaping" + kill -0 "$summary_pid" 2>/dev/null || fail "summary did not remain outstanding after network publication" + harvest=$(run_stage "$home" "$root" harvest --pid "$$") || fail "published network result could not be harvested" + assert_contains "$harvest" 'exited 7' "harvest did not receive the published network result" + : > "$release" + run_stage "$home" "$root" wait 30 >/dev/null || fail "summary stage never settled" + waited=0 + while kill -0 "$summary_pid" 2>/dev/null && [ "$waited" -lt 100 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + if kill -0 "$summary_pid" 2>/dev/null; then fail "summary child was not reaped"; fi + assert_not_contains "$out$report$harvest" 'summary stdout' "summary stdout leaked into stage output" + assert_not_contains "$out$report$harvest" 'summary stderr' "summary stderr leaked into stage output" + pass "fm-startup-network: network results publish before concurrent summary reaping" +} + +test_summary_is_authorized_and_bounded() { + local rec home root log summary release report summary_pid waited=0 + rec=$(new_world summary-bound) + IFS='|' read -r home root log <<EOF +$rec +EOF + summary="${root%/root}/summary.log" + release="${root%/root}/summary.release" + printf '222222\n' > "$home/state/.lock" + FM_FAKE_BOOTSTRAP_LOG="$log" FM_FAKE_SUMMARY_LOG="$summary" \ + run_stage "$home" "$root" run --locked 1 --lock-pid 111111 + assert_absent "$summary" "lock-refused worker launched summary publication" + # Use the real summary timeout wrapper here: a stub without its nested + # process group would miss an orphaned collector after the network deadline. + rm "$root/bin/fm-home-summary-refresh.sh" "$root/bin/fm-fleet-snapshot.sh" + ln -s "$ROOT/bin/fm-home-summary-refresh.sh" "$root/bin/fm-home-summary-refresh.sh" + cat > "$root/bin/fm-fleet-snapshot.sh" <<'SH' +#!/usr/bin/env bash +trap '' TERM +printf '%s\n' "$$" > "${FM_FAKE_SUMMARY_LOG:?}" +while :; do sleep 1; done +SH + chmod +x "$root/bin/fm-fleet-snapshot.sh" + printf '%s\n' $$ > "$home/state/.lock" + FM_FAKE_BOOTSTRAP_LOG="$log" FM_FAKE_SUMMARY_LOG="$summary" \ + FM_FAKE_SUMMARY_RELEASE="$release" FM_STARTUP_NETWORK_TIMEOUT=5 \ + FM_HOME_SUMMARY_TIMEOUT=60 FM_FAKE_BOOTSTRAP_SLEEP=20 \ + run_stage "$home" "$root" start --locked 1 --harvest-pid $$ + while [ ! -s "$summary" ] && [ "$waited" -lt 100 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + [ -s "$summary" ] || fail "bounded summary never started" + read -r summary_pid _ < "$summary" + FM_STARTUP_NETWORK_TIMEOUT=5 run_stage "$home" "$root" wait 20 >/dev/null \ + || fail "summary exceeded the deferred-stage bound" + report=$(run_stage "$home" "$root" report) + assert_contains "$report" 'hit the 5s bound' "summary timeout was not reported" + assert_contains "$report" 'home-summary publication' "timeout omitted the summary obligation" + if kill -0 "$summary_pid" 2>/dev/null; then fail "timed-out summary child survived cleanup"; fi + assert_grep '5-second deadline' "$home/state/.home-summary-refresh.log" \ + "summary did not retain and report its capped deadline" + pass "fm-startup-network: summary requires lock authority and cannot outlive the stage bound" +} + # `start` is called from inside a session-open hook whose stdout the harness # reads to EOF. A worker that inherited that pipe would hold the session open for # exactly as long as the network work it was supposed to get off the critical # path, so this asserts both halves: start returns fast, AND the pipe closes # while the worker is still running. test_start_returns_without_holding_the_callers_stdout() { - local rec home root log started elapsed pending + local rec home root log pending release returned caller output waited=0 rec=$(new_world start-nonblocking) IFS='|' read -r home root log <<EOF $rec EOF printf '%s\n' $$ > "$home/state/.lock" - started=$(date +%s) + release="${root%/root}/bootstrap.release" + returned="${root%/root}/start.returned" # Command substitution reads to EOF, exactly like a hook harvesting hook output. - FM_FAKE_BOOTSTRAP_LOG="$log" FM_FAKE_BOOTSTRAP_SLEEP=10 \ - run_stage "$home" "$root" start --locked 1 --harvest-pid $$ >/dev/null - elapsed=$(( $(date +%s) - started )) - - [ "$elapsed" -lt 4 ] || fail "start blocked for ${elapsed}s behind a 10s worker" + ( + output=$(FM_FAKE_BOOTSTRAP_LOG="$log" FM_FAKE_BOOTSTRAP_RELEASE="$release" \ + FM_STARTUP_NETWORK_TIMEOUT=120 \ + run_stage "$home" "$root" start --locked 1 --harvest-pid $$) || exit 1 + printf '%s' "$output" > "$returned" + ) & + caller=$! + while [ ! -e "$returned" ] && [ "$waited" -lt 600 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + if [ ! -e "$returned" ]; then + : > "$release" + wait "$caller" || true + fail "start or its stdout remained blocked behind the gated worker" + fi + wait "$caller" || fail "start failed before returning its output" await_worker_record "$home" pending=$(run_stage "$home" "$root" report) [ "$(printf '%s\n' "$pending" | head -1)" = "IN PROGRESS - the deferred network checks have not finished yet." ] \ @@ -160,6 +293,7 @@ EOF "the pending guidance still promised a wake for clean success" assert_contains "$pending" "$root/bin/fm-startup-network.sh report" \ "the pending guidance omitted the durable on-demand report path" + : > "$release" run_stage "$home" "$root" wait 30 >/dev/null || fail "the worker never published" assert_grep 'network=only' "$log" "the worker did not run bootstrap's network-only phase" pass "fm-startup-network: start returns immediately and never holds the caller's stdout open" @@ -577,13 +711,16 @@ EOF } test_lock_takeover_stays_read_only_while_a_sweep_holds_the_lease() { - local rec home root log next_owner new_owner out rc started elapsed waited=0 + local rec home root log next_owner new_owner out rc release attempt waited=0 rec=$(new_world sweep-lease) IFS='|' read -r home root log <<EOF $rec EOF printf '%s\n' $$ > "$home/state/.lock" - FM_FAKE_BOOTSTRAP_LOG="$log" FM_FAKE_BOOTSTRAP_SLEEP=6 \ + release="${root%/root}/bootstrap.release" + attempt="${root%/root}/lock-attempt" + FM_FAKE_BOOTSTRAP_LOG="$log" FM_FAKE_BOOTSTRAP_RELEASE="$release" \ + FM_STARTUP_NETWORK_TIMEOUT=120 \ run_stage "$home" "$root" start --locked 1 --harvest-pid $$ while [ ! -s "$log" ] && [ "$waited" -lt 50 ]; do sleep 0.1 @@ -592,18 +729,33 @@ EOF [ -s "$log" ] || fail "the mutating sweep never started" next_owner=$(/bin/ps -o ppid= -p $$ | tr -d ' ') - started=$(date +%s) - rc=0 - out=$(PATH="$root/bin:$PATH" FM_FAKE_HARNESS_PID="$next_owner" \ - FM_HOME="$home" FM_ROOT_OVERRIDE="$root" "$root/bin/fm-lock.sh" 2>&1) || rc=$? - elapsed=$(( $(date +%s) - started )) + # Keep mutation outstanding until the real lock command returns. A fixed + # sleep races ancestry discovery on loaded hosts and can permit a valid + # takeover after the sweep has already completed. + ( + rc=0 + PATH="$root/bin:$PATH" FM_FAKE_HARNESS_PID="$next_owner" \ + FM_HOME="$home" FM_ROOT_OVERRIDE="$root" "$root/bin/fm-lock.sh" > "$attempt.out" 2>&1 || rc=$? + printf '%s\n' "$rc" > "$attempt.rc" + ) & + waited=0 + while [ ! -s "$attempt.rc" ] && [ "$waited" -lt 600 ]; do + sleep 0.1 + waited=$((waited + 1)) + done + if [ ! -s "$attempt.rc" ]; then + : > "$release" + fail "lock takeover blocked behind the gated deferred sweep" + fi + rc=$(cat "$attempt.rc") + out=$(cat "$attempt.out") [ "$rc" -ne 0 ] || fail "lock takeover succeeded while the prior sweep was mutating" - [ "$elapsed" -lt 4 ] || fail "lock takeover blocked ${elapsed}s behind deferred network work" assert_contains "$out" "operate read-only" \ "a lease-blocked takeover did not fail closed to read-only: $out" [ "$(cat "$home/state/.lock")" = "$$" ] \ || fail "the lease-blocked takeover replaced the prior owner" + : > "$release" run_stage "$home" "$root" wait 30 >/dev/null || fail "the leased sweep never settled" out=$(PATH="$root/bin:$PATH" FM_FAKE_HARNESS_PID="$next_owner" \ FM_HOME="$home" FM_ROOT_OVERRIDE="$root" "$root/bin/fm-lock.sh" 2>&1) \ @@ -759,6 +911,8 @@ GITHUB_TOKEN=ghp_supersecretvalue" \ pass "fm-startup-network: the timing artifact cannot carry a command line or forge records" } +test_summary_runs_concurrently_and_is_reaped +test_summary_is_authorized_and_bounded test_wait_fails_without_a_published_stage test_start_returns_without_holding_the_callers_stdout test_harvest_acknowledgement_suppresses_the_wake_and_no_claim_produces_it From d89b3ea85760d5750cc28c75514e77d6a3b73229 Mon Sep 17 00:00:00 2001 From: twilwa <techwilliams.warren@gmail.com> Date: Thu, 24 Sep 2026 11:32:32 +0200 Subject: [PATCH 2/5] no-mistakes(review): Drop redundant contribution-input EXIT trap in fleet snapshot --- bin/fm-fleet-snapshot.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/fm-fleet-snapshot.sh b/bin/fm-fleet-snapshot.sh index fb9486e7741..825cfd97a5d 100755 --- a/bin/fm-fleet-snapshot.sh +++ b/bin/fm-fleet-snapshot.sh @@ -1993,7 +1993,6 @@ if [ "$OUTPUT_MODE" = contribution-input ]; then contribution_tasks=$(contribution_tasks_json) || { echo "fm-fleet-snapshot: contribution task read failed" >&2; exit 1; } JSON_TRANSPORT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-fleet-snapshot.XXXXXX") \ || { echo "fm-fleet-snapshot: temporary transport directory creation failed" >&2; exit 1; } - trap 'rm -f -- "$JSON_TRANSPORT_DIR/backlog.json" "$JSON_TRANSPORT_DIR/contribution-tasks.json" 2>/dev/null || true; rmdir -- "$JSON_TRANSPORT_DIR" 2>/dev/null || true' EXIT printf '%s\n' "$BACKLOG_JSON" > "$JSON_TRANSPORT_DIR/backlog.json" \ || { echo "fm-fleet-snapshot: temporary backlog file write failed" >&2; exit 1; } printf '%s\n' "$contribution_tasks" > "$JSON_TRANSPORT_DIR/contribution-tasks.json" \ From de8411643e24b3bfe507bbea2f6934ceca87e6d3 Mon Sep 17 00:00:00 2001 From: twilwa <techwilliams.warren@gmail.com> Date: Thu, 24 Sep 2026 11:58:03 +0200 Subject: [PATCH 3/5] no-mistakes(test): Widen cleanup deadline test budget to avoid load flakes --- tests/fm-herdr-session-cleanup.test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fm-herdr-session-cleanup.test.sh b/tests/fm-herdr-session-cleanup.test.sh index d66ccbbfcaa..96ad2e3f7f6 100755 --- a/tests/fm-herdr-session-cleanup.test.sh +++ b/tests/fm-herdr-session-cleanup.test.sh @@ -407,10 +407,10 @@ SH '. "$1/bin/fm-backend.sh"; fm_backend_source herdr; fm_backend_herdr_presentation_session_lock_path test' \ _ "$ROOT") || fail "could not resolve the isolated fixture presentation lock" started=$SECONDS - HERDR_SESSION=test FM_HERDR_SESSION_CLEANUP_TIMEOUT=2 \ + HERDR_SESSION=test FM_HERDR_SESSION_CLEANUP_TIMEOUT=10 \ "$ROOT/bin/fm-herdr-session-cleanup.sh" > "$TMP_ROOT/deadline.out" 2>&1 \ || fail "cleanup deadline blocked startup" - [ "$((SECONDS - started))" -lt 15 ] || fail "cleanup deadline did not bound execution" + [ "$((SECONDS - started))" -lt 25 ] || fail "cleanup deadline did not bound execution" grep -q 'unfinished candidates preserved; cleanup coverage is unconfirmed' "$TMP_ROOT/deadline.out" \ || fail "cleanup deadline did not report unconfirmed coverage" cmp -s "$TMP_ROOT/deadline-journal" "$FM_STATE_OVERRIDE/$ID.herdr-presentation" \ From dd4ccf053617498b90f6cb1cd81168d70ab874e9 Mon Sep 17 00:00:00 2001 From: twilwa <techwilliams.warren@gmail.com> Date: Thu, 24 Sep 2026 12:22:45 +0200 Subject: [PATCH 4/5] no-mistakes(document): Document startup summary deferral and herdr cleanup deadline --- AGENTS.md | 2 +- docs/configuration.md | 3 ++- docs/herdr-backend.md | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 256e5c536de..0ace1ac1fb7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -187,7 +187,7 @@ A lock-refused session must not spawn, steer, merge, drain the wake queue, repai The digest itself makes no external-network call and never waits for one. Every network check a session start owes - GitHub auth, dead-secondmate relaunch, secondmate convergence, pending handoff delivery, and project clone refresh - runs off the digest's blocking path in a bounded worker owned by `bin/fm-startup-network.sh` and is reported in the digest's own `NETWORK CHECKS` section. -The locked startup inactive-outcome scan joins that worker so a slow local current-state read cannot block the digest; its findings use the ordinary durable wake queue. +The locked startup inactive-outcome scan and best-effort home-summary publication join that worker so a slow local current-state read cannot block the digest; the scan's findings use the ordinary durable wake queue. When that section reports its checks still in progress it names exactly what is unconfirmed; treat none of those as passed until `bin/fm-startup-network.sh report` returns the finished result, while a failed or otherwise actionable result also arrives as a `check: startup-network` wake. 1. **Lock** - acquires the per-home session lock first, before anything mutates shared state, then starts the deferred startup stage above. diff --git a/docs/configuration.md b/docs/configuration.md index f72b6ba3156..ba3ed43a816 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1179,12 +1179,13 @@ FM_BACKLOG_ROW_TIMEOUT_SECS=10 # seconds bounding each backlog row read (bin/f FM_BOOTSTRAP_DETECT_ONLY=0 # internal/read-only session-start mode: skip bootstrap's mutating sweeps and print advisory TANGLE wording FM_BOOTSTRAP_NETWORK=all # internal session-start phase split: all, skip (local steps only), or only (network steps only); see bin/fm-bootstrap.sh FM_STARTUP_NETWORK_TIMEOUT=120 # seconds bounding the deferred inactive-outcome scan plus network checks; hitting it prints an actionable NETWORK_CHECKS line +FM_HERDR_SESSION_CLEANUP_TIMEOUT=30 # herdr-only: seconds bounding the complete locked session-start projection cleanup pass; hitting it preserves unfinished candidates and warns that cleanup coverage is unconfirmed; invalid or zero values use 30 FM_TASKS_AXI_COMPATIBLE= # internal one-hop handoff of an already-computed tasks-axi compatibility verdict (0 or 1); consumed when bin/fm-tasks-axi-lib.sh is sourced FM_GUARD_READ_ONLY=0 # internal/read-only guard mode: keep alarms but suppress drain, supervision repair, and checkout repair commands FM_GUARD_CONTINUE_LINE='This is a supervision warning only; the guarded operation WILL still run.' # banner continuation line; fm-send.sh overrides it to name the requested message specifically FM_POLL=15 # seconds between watcher poll cycles FM_HOME_SUMMARY_INTERVAL=300 # seconds before a live watcher refreshes this home's state/home-summary.json even without a status signal; invalid or zero values use 300 -FM_HOME_SUMMARY_TIMEOUT=60 # seconds bounding the complete best-effort home-summary refresh, including lock acquisition, validation, atomic publication, and worker-side failure logging; invalid or zero values use 60 +FM_HOME_SUMMARY_TIMEOUT=60 # seconds bounding the complete best-effort home-summary refresh, including lock acquisition, validation, atomic publication, and worker-side failure logging; invalid or zero values use 60; the session-start refresh is capped at FM_STARTUP_NETWORK_TIMEOUT FM_HOME_SUMMARY_ERROR_LOG_MAX_BYTES=65536 # approximate size cap for state/.home-summary-refresh.log before it is trimmed to the newest 200 lines; invalid or zero values use 65536 FM_HOME_SUMMARY_FAILURE_REPORT=2 # recorded publication failures since the ledger's own last publication before session start reports a HOME_SUMMARY line; invalid or zero values use 2 FM_SNAPSHOT_CREW_STATE_TIMEOUT=10 # seconds bounding each local per-task current-state read inside bin/fm-fleet-snapshot.sh; remote endpoint liveness is not probed on the snapshot path diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 64b0fa77a1c..eab91dbaf67 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -166,6 +166,8 @@ Firstmate immediately revalidates the same journal, metadata absence, workspace It closes only that pane, never a workspace. The matching journal is retired only after the exact pane is positively confirmed gone; an unconfirmed close retains the journal, while a confirmed close may retire it even when focus restoration reported an error after the close. A second run finds no matching title or journal and is a no-op. +Discovery validates this home's journals once per pass, but every locked mutation check rereads them. +`FM_HERDR_SESSION_CLEANUP_TIMEOUT` bounds the complete pass; at the deadline Firstmate reclaims only the recorded locks it can safely acquire, preserves every unfinished candidate, and warns that cleanup coverage is unconfirmed. A malformed or missing title or token, duplicate token, zero or multiple journal matches, cross-home version 2 binding, current metadata, registered or unknown agent, extra tab or pane, active target, busy lock, changed revalidation, unreadable check, or any error preserves the candidate and lets session startup continue with at most a concise warning. Operational compromises: From 1a85fc8c39aa4b304e93efc857d501b6a0644228 Mon Sep 17 00:00:00 2001 From: twilwa <techwilliams.warren@gmail.com> Date: Thu, 24 Sep 2026 12:42:42 +0200 Subject: [PATCH 5/5] no-mistakes(ci): Lint 1 failed because ShellCheck SC2329 ("function never invoked") fired at tests/fm-herdr-session-cleanup.test.sh:356. That line is a subshell copy of fixture_workspaces that replaces the file's main version. The fake herdr command calls fixture_workspaces indirectly when it answers `workspace list` and `api snapshot`, and ShellCheck can't see that call. The fix is one comment line above the replacement: `# shellcheck disable=SC2329 # invoked indirectly by the fake herdr workspace list.` The same file already does this for its other indirectly-called replacements (lines 43 and 49), as do tests/fm-daemon.test.sh and tests/fm-bootstrap.test.sh. No behavior changed. Checked locally: `bin/fm-lint.sh tests/fm-herdr-session-cleanup.test.sh` passes with pinned ShellCheck 0.11.0 and full extended analysis, and `bash tests/fm-herdr-session-cleanup.test.sh` passes every test, including the journal-read-count, deadline, lock and identity tests. The change is not committed --- tests/fm-herdr-session-cleanup.test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fm-herdr-session-cleanup.test.sh b/tests/fm-herdr-session-cleanup.test.sh index 96ad2e3f7f6..a7c70bebf8a 100755 --- a/tests/fm-herdr-session-cleanup.test.sh +++ b/tests/fm-herdr-session-cleanup.test.sh @@ -353,6 +353,7 @@ done exec "$FM_TEST_REAL_GREP" "$@" SH chmod +x "$FAKEBIN/grep" + # shellcheck disable=SC2329 # invoked indirectly by the fake herdr workspace list. fixture_workspaces() { local n=1 printf '['