From d80a1d80b9c00ffe7bbe189b1ebd8c532ce27d83 Mon Sep 17 00:00:00 2001 From: knowttl Date: Tue, 22 Sep 2026 20:59:24 -0700 Subject: [PATCH 1/7] fix(bin): wake a home when queued work becomes ready on its own Queued backlog work gated on a hold date or on blockers could become ready without any turn in the home - a date passing, or a blocker closed by a captain answer, a hand-run tasks-axi done, or work elsewhere - and nothing noticed until the next teardown or session start. bin/fm-ready-work.sh owns the backstop: the watcher runs a ready-work scan on the base heartbeat cadence and wakes once per readiness transition, teardown names the work its own close unblocked and records it as surfaced, and live-gated queued work (a future hold date, or blockers that are in flight or live-gated themselves) now counts as supervision need while undated holds never keep a watcher alive. --- AGENTS.md | 2 +- bin/fm-guard.sh | 3 + bin/fm-ready-work.sh | 262 +++++++++++++++++++++++++ bin/fm-supervision-lib.sh | 16 +- bin/fm-teardown.sh | 6 +- bin/fm-test-run.sh | 3 +- bin/fm-turnend-guard.sh | 4 + bin/fm-watch.sh | 30 +++ docs/architecture.md | 6 +- docs/scripts.md | 1 + docs/turnend-guard.md | 3 +- tests/fm-claude-stop-autoarm.test.sh | 1 + tests/fm-cursor-primary.test.sh | 2 +- tests/fm-ready-work.test.sh | 210 ++++++++++++++++++++ tests/fm-session-lock-ancestry.test.sh | 1 + tests/fm-teardown.test.sh | 20 ++ tests/fm-turnend-guard.test.sh | 27 +++ 17 files changed, 589 insertions(+), 8 deletions(-) create mode 100755 bin/fm-ready-work.sh create mode 100755 tests/fm-ready-work.test.sh diff --git a/AGENTS.md b/AGENTS.md index 38e3cddf0f0..c8ea581f176 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,7 +154,7 @@ state/ runtime records and signals; gitignored .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .claude-autoarm-failure-notified .claude-autoarm-failure-alarmed .turnend-claude-blocks .turnend-claude-blocks.lock Claude Stop auto-arm single-flight, epoch, failure-episode, attended-alarm, guard-budget, and budget-lock records; never touch .cursor-park-owner .cursor-park-owner.lock .turnend-cursor-blocks Cursor stop-hook owner record, publication and commit lock, and bounded repair-nag budget; never touch - .hash-* .count-* .stale-* .stale-since-* .churn-since-* .paused-* .wedge-escalations-* .dead-reported-* .writing-* .waiting-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch + .hash-* .count-* .stale-* .stale-since-* .churn-since-* .paused-* .wedge-escalations-* .dead-reported-* .writing-* .waiting-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak .ready-work* watcher internals; never touch .watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete .last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it .subsuper-* .supervise-daemon.* sub-supervisor internals; never touch diff --git a/bin/fm-guard.sh b/bin/fm-guard.sh index ba9ee330465..265212bc8ec 100755 --- a/bin/fm-guard.sh +++ b/bin/fm-guard.sh @@ -173,6 +173,7 @@ fm_supervision_status "$STATE" "$GRACE" in_flight=$FM_SUP_IN_FLIGHT sources=$FM_SUP_SOURCES checks=$FM_SUP_CHECKS +gated=$FM_SUP_GATED needed=$FM_SUP_NEEDED beacon_desc=$FM_SUP_BEACON_DESC fm_watcher_supervision_verdict "$STATE" "$WATCH" "$GRACE" "$FM_HOME" "$FM_ROOT" @@ -240,6 +241,8 @@ if [ "$watcher_healthy" = false ]; then printf '● %s process-event source(s) registered, but %s.\n' "$sources" "$watcher_cause" elif [ "$checks" -gt 0 ]; then printf '● %s registered custom check(s), but %s.\n' "$checks" "$watcher_cause" + elif [ "$gated" -gt 0 ]; then + printf '● %s queued backlog item(s) wait on a date or blocker, but %s.\n' "$gated" "$watcher_cause" else printf '● X-mode relay polling needs supervision, but %s.\n' "$watcher_cause" fi diff --git a/bin/fm-ready-work.sh b/bin/fm-ready-work.sh new file mode 100755 index 00000000000..e881db53947 --- /dev/null +++ b/bin/fm-ready-work.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# fm-ready-work.sh - the ready-work backstop: surface queued backlog work that +# became dispatchable without this home acting, and report whether gated queued +# work still needs a watcher to notice it. +# +# Usage: fm-ready-work.sh surface +# Print, on one line, the queued task ids that became ready since they were +# last surfaced (nothing when none did), and record them as surfaced. +# bin/fm-teardown.sh runs it right after closing its task, so the dependents +# that close unblocked are named in its own output instead of arriving later +# as a separate wake. +# Sourced (. bin/fm-ready-work.sh): fm_ready_work_scan, fm_ready_work_commit, +# fm_ready_work_release (bin/fm-watch.sh), and fm_ready_work_live_gates +# (bin/fm-supervision-lib.sh). +# +# WHY. Queued work gated on a date (`tasks-axi hold --until`, including captain +# holds deferred with bin/fm-captain-hold.sh --until) or on blockers can become +# ready without any turn in this home: a date passes, or a blocker is closed by a +# captain answer, a hand-run `tasks-axi done`, or work elsewhere. Teardown and +# session start re-evaluate the queue, but nothing else did, so such work waited +# for the next unrelated teardown or session start. +# +# READINESS is tasks-axi's own derivation, read from one `tasks-axi list` through +# bin/fm-tasks-axi.sh: its derived `blocked` and `held` fields already apply +# dependency state and compare hold dates to the local date. Ready means queued, +# not blocked, not held, and not a public-followup obligation, which is never +# dispatchable - the same set `tasks-axi ready` lists. +# +# ONCE PER TRANSITION. state/.ready-work-surfaced lists the ready ids already +# surfaced. A scan reports the ready ids missing from it; the caller commits the +# current ready set as the new record only after it has enqueued its wake +# (enqueue before suppress), so a crash in between repeats the wake rather than +# losing it. An id leaves the record when it is dispatched, closed, or re-held, +# so its next readiness is new again. An item filed already ready is surfaced +# once too, unless it is dispatched before the next scan. A home with no record +# yet seeds it silently, so the first scan never replays the whole ready queue. +# state/.ready-work.lock serializes scan-to-commit between the watcher and +# teardown, so one transition is reported by exactly one of them. +# +# LIVE GATES (supervision need). A queued item's gate is live when it can clear +# without this home acting: a hold with a future date, or a blocker that is in +# flight or itself live-gated. An undated hold - or a blocker chain that ends in +# one, or in queued ready work this home has not dispatched - waits on this +# home's own next turn, so it never keeps a watcher alive; a dated gate keeps one +# only until its date, when the scan surfaces the item and the gate stops +# counting. +# +# STEPPING ASIDE. tasks-axi missing from PATH, config/backlog-backend=manual, a +# missing data directory, a markdown home with no backlog file, or a listing that +# fails, times out (FM_READY_WORK_TIMEOUT seconds, default 10), or cannot be +# parsed all mean nothing to surface and no need: this backstop never blocks a +# turn or a teardown on its own failure. The home's data and config directories +# are the state directory's siblings unless FM_DATA_OVERRIDE / FM_CONFIG_OVERRIDE +# name them. + +FM_READY_WORK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_READY_WORK_READY= +FM_READY_WORK_LIVE=0 +FM_READY_WORK_NEW= +FM_READY_WORK_LOCK= + +# Classify one `tasks-axi list` listing. Prints `ready ` per ready item and +# a final `live `; exits 2 when the listing lacks the expected table. +fm_ready_work_classify() { + LC_ALL=C awk ' + function split_row(line, out, n, i, c, field, inq, esc) { + n = 0; field = ""; inq = 0; esc = 0 + for (i = 1; i <= length(line); i++) { + c = substr(line, i, 1) + if (esc) { field = field c; esc = 0; continue } + if (inq) { + if (c == "\\") { esc = 1; continue } + if (c == "\"") { inq = 0; continue } + field = field c + continue + } + if (c == "\"") { inq = 1; continue } + if (c == ",") { out[++n] = field; field = ""; continue } + field = field c + } + out[++n] = field + return n + } + /^tasks: / { table = 1; next } + /^tasks\[[0-9]+\]\{/ { + header = $0 + sub(/^[^{]*\{/, "", header) + sub(/\}:.*$/, "", header) + ncol = split(header, cols, ",") + for (i = 1; i <= ncol; i++) col[cols[i]] = i + table = 1 + rows = 1 + next + } + rows && /^ / { + line = substr($0, 3) + split_row(line, f) + id = f[col["id"]] + n++ + ids[n] = id + state[id] = f[col["state"]] + kind[id] = f[col["kind"]] + blocked[id] = f[col["blocked"]] + held[id] = f[col["held"]] + until[id] = f[col["hold_until"]] + deps[id] = f[col["blocked_by"]] + next + } + { rows = 0 } + END { + if (!table) exit 2 + if (n > 0 && !("id" in col && "state" in col && "kind" in col && "blocked" in col \ + && "blocked_by" in col && "held" in col && "hold_until" in col)) exit 2 + for (i = 1; i <= n; i++) { + id = ids[i] + if (state[id] != "queued" || kind[id] == "public-followup") continue + if (blocked[id] == "no" && held[id] == "no") print "ready " id + if (held[id] == "yes" && until[id] != "-" && until[id] != "") live[id] = 1 + } + # A blocked item is live when any open blocker is in flight or live itself; + # iterate to the fixpoint so a chain inherits its root gate. An undated + # hold stays not live even when blocked, since only this home releases it. + do { + changed = 0 + for (i = 1; i <= n; i++) { + id = ids[i] + if (state[id] != "queued" || live[id] || blocked[id] != "yes") continue + if (held[id] == "yes" && (until[id] == "-" || until[id] == "")) continue + m = split(deps[id], bs, ",") + for (j = 1; j <= m; j++) { + b = bs[j] + if (state[b] == "in_flight" || live[b]) { live[id] = 1; changed = 1; break } + } + } + } while (changed) + count = 0 + for (id in live) if (live[id]) count++ + print "live " count + } + ' +} + +# fm_ready_work_read +# Sets FM_READY_WORK_READY (sorted ready ids, one per line) and +# FM_READY_WORK_LIVE (live-gated queued count). Returns 0 on a good read, 1 when +# this home has no readable tasks-axi backlog (see STEPPING ASIDE). +fm_ready_work_read() { + local state=$1 home data config root backend listing classified + FM_READY_WORK_READY= + FM_READY_WORK_LIVE=0 + home=${state%/*} + data=${FM_DATA_OVERRIDE:-$home/data} + config=${FM_CONFIG_OVERRIDE:-$home/config} + [ -d "$data" ] || return 1 + command -v tasks-axi >/dev/null 2>&1 || return 1 + # shellcheck source=bin/fm-tasks-axi-lib.sh + command -v fm_tasks_axi_backend >/dev/null 2>&1 \ + || . "$FM_READY_WORK_DIR/fm-tasks-axi-lib.sh" || return 1 + fm_backlog_backend_manual "$config" && return 1 + root=$(CDPATH='' cd -- "$data/.." 2>/dev/null && pwd -P) || return 1 + backend=$(fm_tasks_axi_backend "$root" 2>/dev/null) || return 1 + if [ "$backend" = markdown ] && [ ! -e "$data/backlog.md" ]; then + return 1 + fi + # shellcheck source=bin/fm-timeout-lib.sh + command -v fm_run_timed >/dev/null 2>&1 \ + || . "$FM_READY_WORK_DIR/fm-timeout-lib.sh" || return 1 + listing=$(FM_HOME="$home" FM_DATA_OVERRIDE="$data" \ + fm_run_timed "${FM_READY_WORK_TIMEOUT:-10}" \ + "$FM_READY_WORK_DIR/fm-tasks-axi.sh" list --fields blocked,blocked_by,held,hold_until \ + 2>/dev/null : print the live-gated queued count. +fm_ready_work_live_gates() { + if fm_ready_work_read "$1"; then + printf '%s\n' "$FM_READY_WORK_LIVE" + else + printf '0\n' + fi +} + +# fm_ready_work_scan +# Takes state/.ready-work.lock, reads the backlog, and sets FM_READY_WORK_NEW to +# the space-separated ready ids not yet surfaced (empty while seeding a home +# with no record). Returns 0 with the lock held, to be finished by +# fm_ready_work_commit; returns 1 with no lock held when there is nothing to +# read or the lock stays contended. +fm_ready_work_scan() { + local state=$1 record + FM_READY_WORK_NEW= + # shellcheck source=bin/fm-wake-lib.sh + command -v fm_lock_acquire_wait_bounded >/dev/null 2>&1 \ + || . "$FM_READY_WORK_DIR/fm-wake-lib.sh" || return 1 + FM_READY_WORK_LOCK="$state/.ready-work.lock" + fm_lock_acquire_wait_bounded "$FM_READY_WORK_LOCK" "${FM_READY_WORK_TIMEOUT:-10}" || return 1 + if ! fm_ready_work_read "$state"; then + fm_ready_work_release + return 1 + fi + record="$state/.ready-work-surfaced" + [ -e "$record" ] || return 0 + FM_READY_WORK_NEW=$(printf '%s\n' "$FM_READY_WORK_READY" | LC_ALL=C awk ' + FILENAME == ARGV[1] { if ($0 != "") seen[$0] = 1; next } + $0 != "" && !($0 in seen) { printf "%s%s", sep, $0; sep = " " } + ' "$record" -) + return 0 +} + +# fm_ready_work_commit : record the scanned ready set as surfaced and +# release the scan lock. +fm_ready_work_commit() { + local state=$1 record tmp status=0 + record="$state/.ready-work-surfaced" + if tmp=$(mktemp "$state/.ready-work-surfaced.XXXXXX"); then + if [ -n "$FM_READY_WORK_READY" ]; then + printf '%s\n' "$FM_READY_WORK_READY" > "$tmp" || status=1 + fi + if [ "$status" -eq 0 ]; then + mv -f -- "$tmp" "$record" || status=1 + fi + [ "$status" -eq 0 ] || rm -f -- "$tmp" + else + status=1 + fi + fm_ready_work_release + return "$status" +} + +fm_ready_work_release() { + [ -n "$FM_READY_WORK_LOCK" ] || return 0 + fm_lock_release "$FM_READY_WORK_LOCK" || true + FM_READY_WORK_LOCK= +} + +fm_ready_work_main() { + local state + case "${1:-}" in + surface) ;; + -h|--help) + awk 'NR == 1 { next } /^#/ { sub(/^# ?/, ""); print; next } { exit }' "$0" + return 0 + ;; + *) + printf 'usage: fm-ready-work.sh surface\n' >&2 + return 2 + ;; + esac + state=${FM_STATE_OVERRIDE:-${FM_HOME:-$(cd "$FM_READY_WORK_DIR/.." && pwd)}/state} + fm_ready_work_scan "$state" || return 0 + fm_ready_work_commit "$state" || return 0 + [ -z "$FM_READY_WORK_NEW" ] || printf '%s\n' "$FM_READY_WORK_NEW" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + fm_ready_work_main "$@" +fi diff --git a/bin/fm-supervision-lib.sh b/bin/fm-supervision-lib.sh index 1bbc5708834..2add4ceb05a 100644 --- a/bin/fm-supervision-lib.sh +++ b/bin/fm-supervision-lib.sh @@ -12,6 +12,9 @@ # live watcher process means per supervision model. The status fields here retain # the beacon-age details used in their messages. +# shellcheck source=bin/fm-ready-work.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/fm-ready-work.sh" + # Portable mtime; Linux stat lacks -f, macOS stat lacks -c. fm_sup_stat_mtime() { if [ "$(uname)" = Darwin ]; then @@ -35,10 +38,16 @@ fm_sup_stat_mtime() { # sweep's call at execution time, and a home whose check # no longer validates needs the watcher precisely so the # sweep can report the rejection instead of going quiet. +# FM_SUP_GATED count of live-gated queued backlog items: a future hold +# date, or blockers that can close without this home +# acting (bin/fm-ready-work.sh owns that definition and +# the watcher wake it waits for). Read only when nothing +# above already needs supervision, which keeps the +# backlog read off a busy home's turn boundary; 0 then. # FM_SUP_NEEDED true/false - in-flight work, an X-mode relay poll, a # registered event source (a source is a wait on an # external process, not a task, so it has no metadata), -# or a registered custom check +# a registered custom check, or live-gated queued work # FM_SUP_WATCHER_FRESH true/false - a watcher beacon within the grace window # FM_SUP_BEACON_DESC human-readable beacon age, for banners ("never" if absent) # FM_SUP_QUEUE_PENDING true/false - state/.wake-queue has unread records @@ -78,6 +87,11 @@ fm_supervision_status() { || [ "$FM_SUP_CHECKS" -gt 0 ]; then FM_SUP_NEEDED=true fi + FM_SUP_GATED=0 + if [ "$FM_SUP_NEEDED" = false ]; then + FM_SUP_GATED=$(fm_ready_work_live_gates "$state") + [ "$FM_SUP_GATED" -gt 0 ] && FM_SUP_NEEDED=true + fi beat="$state/.last-watcher-beat" if [ -e "$beat" ]; then diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 602b88cae77..7e92f5bf9a2 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -1541,7 +1541,7 @@ backlog_done_args() { # invariant). This prints what already happened, so the follow-up wording stays # only where a human still owes the edit. backlog_refresh_reminder() { - local backlog_display root backend=markdown + local backlog_display root backend=markdown newly_ready [ "$KIND" = secondmate ] && return 0 [ "$CLEANUP_RECOVERY" = orca ] && return 0 if root=$(fm_backlog_root "$DATA"); then @@ -1558,6 +1558,10 @@ backlog_refresh_reminder() { printf '%s\n' "Backlog: $ID stays open in $backlog_display, still held for the captain with its deliverable recorded. Relay the question and close it only with bin/fm-captain-hold.sh answer." elif [ "$BACKLOG_CLOSED" = 1 ]; then printf '%s\n' "Backlog: $ID is closed in $backlog_display. Run bin/fm-tasks-axi.sh ready for dependency-cleared candidates, check date gates, and dispatch only work whose blockers are gone and date is due." + # Recorded as surfaced here, so the watcher does not wake again for them. + newly_ready=$(FM_STATE_OVERRIDE="$STATE" FM_DATA_OVERRIDE="$DATA" FM_CONFIG_OVERRIDE="$CONFIG" \ + "$SCRIPT_DIR/fm-ready-work.sh" surface 2>/dev/null || true) + [ -z "$newly_ready" ] || printf '%s\n' "Backlog: newly ready queued work: $newly_ready" else printf '%s\n' "Backlog: $ID just finished ($BACKLOG_SKIP_REASON). Update $backlog_display - move $ID to Done, keep Done to the 10 most recent, then re-scan Queued and dispatch only work whose blockers are gone and date is due." fi diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 73e47d095a0..2616a673880 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -304,7 +304,7 @@ family_for_basename() { fm-mail.test.sh|fm-mail-check.test.sh|\ fm-turnend-foreign-owner-arm-fix.test.sh|\ fm-wake-queue.test.sh|fm-watch-arm.test.sh|fm-watch-checkpoint.test.sh|fm-watch-recovery-loop.test.sh|\ - fm-watch-triage.test.sh|fm-task-inbox.test.sh|\ + fm-watch-triage.test.sh|fm-task-inbox.test.sh|fm-ready-work.test.sh|\ fm-watcher-lock.test.sh|fm-inactive-reconcile.test.sh) printf '%s\n' watcher-wake-lock ;; @@ -771,6 +771,7 @@ tests/fm-project-origin.test.sh 136 tests/fm-public-followup.test.sh 153508 tests/fm-quota-array-dispatch-live-e2e.test.sh 71 tests/fm-quota-choose.test.sh 1484 +tests/fm-ready-work.test.sh 14839 tests/fm-remote-backlog-handoff.test.sh 73123 tests/fm-remote-doctor.test.sh 13889 tests/fm-remote-entrypoint.test.sh 108 diff --git a/bin/fm-turnend-guard.sh b/bin/fm-turnend-guard.sh index 7854f93b6dd..14c1cad1bf2 100755 --- a/bin/fm-turnend-guard.sh +++ b/bin/fm-turnend-guard.sh @@ -242,6 +242,8 @@ block_stop() { printf '● %s process-event source(s) registered, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_SOURCES" "$FM_SUP_BEACON_DESC" elif [ "$FM_SUP_CHECKS" -gt 0 ]; then printf '● %s registered custom check(s), but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_CHECKS" "$FM_SUP_BEACON_DESC" + elif [ "$FM_SUP_GATED" -gt 0 ]; then + printf '● %s queued backlog item(s) wait on a date or blocker, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_GATED" "$FM_SUP_BEACON_DESC" else printf '● X-mode relay polling needs supervision, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_BEACON_DESC" fi @@ -516,6 +518,8 @@ if [ "$terminal_status" -eq 0 ]; then NEED_DESC="$FM_SUP_SOURCES process-event source(s) registered" elif [ "$FM_SUP_CHECKS" -gt 0 ]; then NEED_DESC="$FM_SUP_CHECKS registered custom check(s)" + elif [ "$FM_SUP_GATED" -gt 0 ]; then + NEED_DESC="$FM_SUP_GATED queued backlog item(s) waiting on a date or blocker" else NEED_DESC="X-mode relay polling active" fi diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 443c32303f7..d84b24cd15d 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -112,6 +112,11 @@ # running a check or removing poll artifacts # heartbeat fleet-scan backstop found an unsurfaced captain-relevant # status, unless afk is active +# check: ready-work: +# queued backlog work became ready (a date gate passed +# or its blockers closed) and has not been surfaced; +# once per readiness transition, in every posture +# (bin/fm-ready-work.sh owns readiness and dedup) # check: inactive-outcome bounded poll-loop reconciliation found a suspicious # inactive terminal outcome that still lacks its durable # upstream receipt @@ -188,6 +193,8 @@ mkdir -p "$STATE" # watcher reads only its presence (afk_record_present below). # shellcheck source=bin/fm-afk-contract.sh . "$SCRIPT_DIR/fm-afk-contract.sh" +# shellcheck source=bin/fm-ready-work.sh +. "$SCRIPT_DIR/fm-ready-work.sh" WATCH_LOCK="$STATE/.watch.lock" WATCH_PATH="$SCRIPT_DIR/fm-watch.sh" @@ -229,6 +236,7 @@ POLL=${FM_POLL:-15} # seconds between cycles WATCHER_STALE_GRACE=${FM_WATCHER_STALE_GRACE:-${FM_GUARD_GRACE:-$(fm_poll_derived_grace "$POLL")}} HEARTBEAT=${FM_HEARTBEAT:-600} # base seconds between heartbeat scans HEARTBEAT_MAX=${FM_HEARTBEAT_MAX:-7200} # heartbeat backoff cap +READY_SCAN=${FM_READY_SCAN:-$HEARTBEAT} # seconds between ready-work scans, never backed off CHECK_INTERVAL=${FM_CHECK_INTERVAL:-300} # seconds between *.check.sh sweeps CHECK_TIMEOUT=${FM_CHECK_TIMEOUT:-30} # seconds allowed per *.check.sh HOME_SUMMARY_INTERVAL=${FM_HOME_SUMMARY_INTERVAL:-300} @@ -2489,6 +2497,28 @@ EOF fi fi + # Queued backlog work that became ready without this home acting: a date gate + # passed or its blockers closed. bin/fm-ready-work.sh owns readiness and the + # once-per-transition record; this block only enqueues before it commits. + # Its own unbacked-off cadence, ahead of the signal scan for the same + # starvation reason as the checks above, keeps a due date prompt even while + # the heartbeat has backed off on an idle home. + if [ "$(age_of "$STATE/.last-ready-scan")" -ge "$READY_SCAN" ]; then + touch "$STATE/.last-ready-scan" + if fm_ready_work_scan "$STATE"; then + if [ -n "$FM_READY_WORK_NEW" ]; then + reason="check: ready-work: $FM_READY_WORK_NEW" + if ! fm_wake_append check ready-work "$reason"; then + fm_ready_work_release + exit 1 + fi + fm_ready_work_commit "$STATE" || triage_log "ready-work record not updated; the next scan repeats this wake" + wake "$reason" + fi + fm_ready_work_commit "$STATE" || triage_log "ready-work record not updated" + fi + fi + # On the first changed signal, linger one grace period and re-scan before # classifying: a crewmate's final status write and the same turn's turn-end # hook land seconds apart, and reporting them as separate actionable wakes diff --git a/docs/architecture.md b/docs/architecture.md index 5494575cde1..244d9f1d947 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,6 +93,8 @@ Fresh stale panes use the same current-state read before trusting the status log No-change heartbeats are also benign. Separately from heartbeat backoff and wedge handling, the watcher poll runs `bin/fm-inactive-reconcile.sh` on its own bounded cadence, while locked session start sends the same bounded local scan through `bin/fm-startup-network.sh`'s deferred worker so current-state reads never block the digest. In each home the scan considers only that home's long-inactive direct ordinary crewmates, excludes captain-held work, and accepts only `done` or `failed` from `bin/fm-crew-state.sh`. +The poll also runs a ready-work scan on the base heartbeat cadence, never backed off, in every posture: queued backlog work that became ready without this home acting - a hold date that arrived or blockers closed by anything other than its own teardown - wakes the home once per readiness transition as `check: ready-work: `, and teardown names the work its own close unblocked instead. +Live-gated queued work - a future hold date, or blockers that are in flight or themselves live-gated - counts as supervision need until its gate clears, while undated holds never keep a watcher alive; `bin/fm-ready-work.sh` owns readiness, the once-per-transition record, and that bound. A secondmate retains a durable receipt for its idempotent report through the established parent route, and main-home captain presentation retains a separate receipt; neither path performs a forge or PR check. A secondmate home's terminal child ledger lines, PR registrations, captain holds, and merges are published on that same parent route by the scripts that record them, so no captain-facing outcome depends on the mate model appending it ([secondmate-parent-channel.md](secondmate-parent-channel.md)). Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. @@ -167,11 +169,11 @@ It suppresses failed-looking closes when the same identity-matched watcher is he Cursor's `bin/fm-turnend-guard-cursor.sh` hook is the same between-turns shape in one synchronous step: it parks the awaited `stop` hook on the arm wrapper and translates an actionable close into one `followup_message`, with a generation baton that makes an older park still running after the next `stop` claim stand down instead of leaking a stale duplicate wake. The existing turn-end guard remains the final backstop for every harness-engine protocol, with pi-signed sharing Pi's protocol, omp's blocking `session_stop` hook compelling one continuation per turn, the `--claude` mode cooperating with the auto-arm claim, and Cursor's `--cursor` mode rendering a block as one bounded follow-up because its `stop` step cannot be blocked. Its `--restart` mode signals only the watcher recorded in the current home's `state/.watch.lock`, so restarting one home cannot kill sibling secondmate watchers. -A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled or if work, process-event sources, registered custom checks, or Relay polling has an unhealthy model-aware supervision verdict; on main it also warns when queued wakes are waiting for main itself to drain. +A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled or if work, process-event sources, registered custom checks, live-gated queued work, or Relay polling has an unhealthy model-aware supervision verdict; on main it also warns when queued wakes are waiting for main itself to drain. The drain script calls that guard after presenting the queue; records remain durable until the exact generation-bound acknowledgement printed by the drain succeeds after handling, and main may keep the queued-wakes warning visible until then. The Pi supervision branch's deliberate queued-wake warning exception is owned by [`pi-supervision-branch.md`](pi-supervision-branch.md#components-and-their-owners), while [`watcher-continuity.md`](watcher-continuity.md#per-actor-acknowledgement) owns the guard's per-actor counting, the advisory main gets for rows a live branch grant holds, and main's retirement of queue rows no actor could ever present or acknowledge. It leads with a prominent bordered tangle banner, while `bin/fm-guard.sh` owns the watcher-down banner and reminder policy so repeated guarded commands stay noisy without reprinting the full banner in the same episode. -On every verified primary harness, tracked hook integration gives the primary session a push-based backstop: when work, a process-event source, a registered custom check, or Relay polling needs supervision and no supervision owner provably holds this home with a fresh beacon, blocking-capable Stop hooks block and nonblocking turn-end integrations force one bounded follow-up. +On every verified primary harness, tracked hook integration gives the primary session a push-based backstop: when work, a process-event source, a registered custom check, live-gated queued work, or Relay polling needs supervision and no supervision owner provably holds this home with a fresh beacon, blocking-capable Stop hooks block and nonblocking turn-end integrations force one bounded follow-up. The guard covers the main primary and genuinely marked secondmate homes, exempts child crewmate/scout worktrees, is loop-safe per harness, and is documented in [turnend-guard.md](turnend-guard.md). Away mode is a posture of the one supervision session, recorded in `state/.afk-contract` by `bin/fm-afk-contract.sh` in the same turn as `/afk` with no wait for a further go, read back in plain sentences only after entry, and announced at entry as hold-for-return only because no phone channel exists. diff --git a/docs/scripts.md b/docs/scripts.md index 725013870a2..36d3e2da131 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -103,6 +103,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-lock-lib.sh` | Shared "is this git lock provably abandoned?" proof used by teardown and fleet-sync | | `fm-config-inherit-lib.sh` | Shared primary-to-secondmate inherited local-material propagation and config-reread delivery | | `fm-tasks-axi.sh` | Run `tasks-axi` against this home's backlog from any working directory | +| `fm-ready-work.sh` | Surface queued backlog work that became ready without this home acting, once per readiness transition, and count live-gated queued work as supervision need | | `fm-tasks-axi-lib.sh` | Shared backlog-backend selector and `tasks-axi` compatibility probe | | `fm-backlog-transition-lib.sh` | Pair task-record changes with their backlog transitions and replay interrupted closes | | `fm-quota-axi-lib.sh` | Shared `quota-axi` compatibility floor and quota snapshot schema validation | diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index f4715f1db0d..0f9c2e135ec 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -13,7 +13,7 @@ Do not infer this guard's scope, loop safety, or compatibility tradeoffs for tho `bin/fm-guard.sh` is a pull-based warning that runs only when another supervision command invokes it. The turn-end guard closes the remaining gap at the primary's own turn boundary. -When work, a process-event source, a registered custom check, or Relay polling needs supervision at that boundary and no identity-matched watcher has a fresh beacon, the harness integration must either block the turn end or force one bounded follow-up that uses the recovery instruction from the emitted session-start protocol. +When work, a process-event source, a registered custom check, live-gated queued work, or Relay polling needs supervision at that boundary and no identity-matched watcher has a fresh beacon, the harness integration must either block the turn end or force one bounded follow-up that uses the recovery instruction from the emitted session-start protocol. The mid-turn pull warning uses the model-aware supervision verdict described below, while the turn-end guard keeps the PID-strict watcher predicate. Away and quiet mode are the one place the turn-end guard accepts a different supervisor: while `state/.afk` exists, in either mode (`bin/fm-wake-lib.sh`'s `fm_afk_mode`), the daemon owns supervision, so a live identity-matched daemon with a fresh beacon satisfies that boundary in place of a watcher process holding the lock. The guard remains a backstop; [`watcher-continuity.md`](watcher-continuity.md) owns normal continuity. @@ -32,6 +32,7 @@ Registered `state/procevent/*.source` records also require supervision even thou The default cross-harness mode exits silently with no supervision need. Every mode treats `state/x-watch.check.sh` as supervision need, so Relay polling remains guarded without an in-flight task. A custom check registered with `bin/fm-check-register.sh` counts the same way, so an operator's home-level poll keeps running after the last task is torn down. +Live-gated queued backlog work counts too, so a home whose only remaining work waits on a hold date or on blockers that can close on their own keeps the watcher whose ready-work scan will surface it; `bin/fm-ready-work.sh` owns which gates are live, and an undated hold never counts. Otherwise it calls `fm_watcher_healthy [grace-seconds] [home]` from `bin/fm-wake-lib.sh`, the same PID-strict identity-matched lock and fresh-beacon check used by `bin/fm-watch-arm.sh`: a stale beacon blocks even when a watcher pid is live, and a fresh leftover beacon blocks when the lock is missing, dead, or identity-mismatched. The turn-end guard needs that strict check because it fires at the turn boundary, where the auto-arm is bringing a fresh watcher up for the upcoming idle period, and it cooperates with that arm rather than trusting a beacon left by the cycle that just ended. When an active home instead has a live session lock held by a verified harness that the current session does not own, the Claude guard emits a read-only ownership diagnostic and allows the turn to end safely. diff --git a/tests/fm-claude-stop-autoarm.test.sh b/tests/fm-claude-stop-autoarm.test.sh index 2775994b794..3195581fd0e 100755 --- a/tests/fm-claude-stop-autoarm.test.sh +++ b/tests/fm-claude-stop-autoarm.test.sh @@ -29,6 +29,7 @@ install_autoarm_scripts() { cp "$ROOT/bin/fm-claude-stop-autoarm.sh" "$dir/bin/fm-claude-stop-autoarm.sh" cp "$ROOT/bin/fm-primary-scope-lib.sh" "$dir/bin/fm-primary-scope-lib.sh" cp "$ROOT/bin/fm-supervision-lib.sh" "$dir/bin/fm-supervision-lib.sh" + cp "$ROOT/bin/fm-ready-work.sh" "$dir/bin/fm-ready-work.sh" cp "$ROOT/bin/fm-wake-lib.sh" "$dir/bin/fm-wake-lib.sh" cp "$ROOT/bin/fm-session-lock-lib.sh" "$dir/bin/fm-session-lock-lib.sh" cp "$ROOT/bin/fm-cursor-lib.sh" "$dir/bin/fm-cursor-lib.sh" diff --git a/tests/fm-cursor-primary.test.sh b/tests/fm-cursor-primary.test.sh index fb872c520fd..f067d3e4c89 100755 --- a/tests/fm-cursor-primary.test.sh +++ b/tests/fm-cursor-primary.test.sh @@ -72,7 +72,7 @@ install_scripts() { for f in fm-turnend-guard-cursor.sh fm-turnend-guard.sh fm-sessionstart-cursor.sh \ fm-sessionstart-run.sh fm-sessionstart-nudge.sh fm-arm-pretool-check.sh \ fm-cd-pretool-check.sh fm-claude-stop-autoarm.sh fm-hook-host-lib.sh \ - fm-primary-scope-lib.sh fm-supervision-lib.sh fm-wake-lib.sh \ + fm-primary-scope-lib.sh fm-supervision-lib.sh fm-ready-work.sh fm-wake-lib.sh \ fm-session-lock-lib.sh fm-cursor-lib.sh fm-operational-input.sh \ fm-supervision-instructions.sh fm-harness.sh fm-lock.sh \ fm-gate-refuse-lib.sh; do diff --git a/tests/fm-ready-work.test.sh b/tests/fm-ready-work.test.sh new file mode 100755 index 00000000000..432e755a3a8 --- /dev/null +++ b/tests/fm-ready-work.test.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# tests/fm-ready-work.test.sh - the ready-work backstop (bin/fm-ready-work.sh): +# queued backlog work that becomes ready without this home acting is surfaced +# once per readiness transition, live-gated queued work counts as supervision +# need only while its gate can clear on its own, and a real fm-watch.sh +# subprocess wakes exactly once for a blocker cleared outside teardown. +# Every home is a scratch directory with its own real tasks-axi backlog. +set -u + +# shellcheck source=tests/wake-helpers.sh +. "$(dirname "${BASH_SOURCE[0]}")/wake-helpers.sh" + +command -v tasks-axi >/dev/null 2>&1 || { printf 'skip: tasks-axi not found\n'; exit 0; } + +WATCH="$ROOT/bin/fm-watch.sh" +READY="$ROOT/bin/fm-ready-work.sh" +TMP_ROOT=$(fm_test_tmproot fm-ready-work-tests) + +# A far-east date that is always later than today in a far-west zone, so one +# hold date reads as future under WEST and due under EAST without waiting. +EAST=Etc/GMT-14 +WEST=Etc/GMT+12 +EAST_TODAY=$(TZ=$EAST date +%F) + +make_home() { # + local home="$TMP_ROOT/$1" + mkdir -p "$home/state" "$home/data" "$home/config" + cp "$ROOT/.tasks.toml" "$home/.tasks.toml" + printf '%s\n' '# Backlog' '' '## In flight' '' '## Queued' '' '## Done' > "$home/data/backlog.md" + printf '%s\n' "$home" +} + +axi() { # + local home=$1 + shift + FM_HOME="$home" "$ROOT/bin/fm-tasks-axi.sh" "$@" >/dev/null || fail "tasks-axi $* failed in $home" +} + +surface() { # [env assignments...] + local home=$1 + shift + env FM_HOME="$home" "$@" "$READY" surface +} + +live_gates() { # [env assignments...] + local home=$1 + shift + # shellcheck disable=SC2016 # Positional parameters expand in the child shell. + env "$@" bash -c '. "$1"; fm_ready_work_live_gates "$2"' _ "$READY" "$home/state" +} + +test_surfaces_once_per_readiness_transition() { + local home out + home=$(make_home transition) + axi "$home" add blocker "the blocker" + axi "$home" add dependent "the dependent" + axi "$home" block dependent --by blocker + axi "$home" start blocker + out=$(surface "$home") + [ -z "$out" ] || fail "seeding a home with no record surfaced work: $out" + [ -e "$home/state/.ready-work-surfaced" ] || fail "the seeding scan left no record" + + # Closed by hand, not by this home's teardown. + axi "$home" "done" blocker + out=$(surface "$home") + [ "$out" = dependent ] || fail "a blocker closed outside teardown did not surface its dependent: '$out'" + out=$(surface "$home") + [ -z "$out" ] || fail "an unchanged ready item surfaced twice: $out" + + axi "$home" hold dependent --reason "wait" + out=$(surface "$home") + [ -z "$out" ] || fail "re-holding surfaced work: $out" + axi "$home" unhold dependent + out=$(surface "$home") + [ "$out" = dependent ] || fail "released work did not surface as a new transition: '$out'" + + axi "$home" start dependent + out=$(surface "$home") + [ -z "$out" ] || fail "dispatched work surfaced: $out" + grep -qx dependent "$home/state/.ready-work-surfaced" \ + && fail "dispatched work kept its surfaced marker" + pass "ready work is surfaced once per readiness transition and its marker retires on dispatch" +} + +test_date_gate_surfaces_when_due() { + local home out + home=$(make_home date-gate) + axi "$home" add dated "deferred by the captain" + axi "$home" hold dated --reason "revisit later" --kind captain --until "$EAST_TODAY" + out=$(surface "$home" TZ=$WEST) + [ -z "$out" ] || fail "seeding surfaced work: $out" + [ "$(live_gates "$home" TZ=$WEST)" = 1 ] || fail "a future-dated captain hold is not a live gate" + out=$(surface "$home" TZ=$WEST) + [ -z "$out" ] || fail "a hold not yet due surfaced: $out" + out=$(surface "$home" TZ=$EAST) + [ "$out" = dated ] || fail "a captain hold whose date passed did not surface: '$out'" + [ "$(live_gates "$home" TZ=$EAST)" = 0 ] || fail "a due hold still counts as a live gate" + pass "a dated captain hold surfaces once its date arrives and stops needing a watcher" +} + +test_live_gates_are_bounded() { + local home + home=$(make_home undated) + axi "$home" add question "a captain call" + axi "$home" hold question --reason "captain decides" --kind captain + axi "$home" add after-question "gated on the call" + axi "$home" block after-question --by question + axi "$home" add queued-ready "waiting on this home to dispatch it" + axi "$home" add after-ready "gated on undispatched work" + axi "$home" block after-ready --by queued-ready + [ "$(live_gates "$home")" = 0 ] \ + || fail "undated holds or undispatched blockers counted as live gates: $(live_gates "$home")" + ( + # shellcheck source=/dev/null + . "$ROOT/bin/fm-supervision-lib.sh" + if fm_supervision_needed "$home/state" 300; then + fail "a home with only undated holds gained a watcher need" + fi + ) || exit 1 + + home=$(make_home live) + axi "$home" add running "in flight" + axi "$home" start running + axi "$home" add after-running "gated on in-flight work" + axi "$home" block after-running --by running + axi "$home" add dated "future date" + axi "$home" hold dated --reason later --until "$EAST_TODAY" + axi "$home" add after-dated "gated on a dated item" + axi "$home" block after-dated --by dated + [ "$(live_gates "$home" TZ=$WEST)" = 3 ] \ + || fail "in-flight, dated, and inherited gates were not all live: $(live_gates "$home" TZ=$WEST)" + ( + # shellcheck source=/dev/null + . "$ROOT/bin/fm-supervision-lib.sh" + TZ=$WEST fm_supervision_needed "$home/state" 300 \ + || fail "live-gated queued work did not need supervision" + [ "$FM_SUP_GATED" = 3 ] || fail "FM_SUP_GATED reported $FM_SUP_GATED" + ) || exit 1 + + printf 'manual\n' > "$home/config/backlog-backend" + [ "$(live_gates "$home" TZ=$WEST)" = 0 ] || fail "a manual-backend home read its backlog" + pass "only gates that can clear on their own count as supervision need" +} + +test_watcher_wakes_once_for_a_cleared_blocker() { + local dir state fakebin out pid + dir=$(make_case watcher-ready) + state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + mkdir -p "$dir/data" + cp "$ROOT/.tasks.toml" "$dir/.tasks.toml" + printf '%s\n' '# Backlog' '' '## In flight' '' '## Queued' '' '## Done' > "$dir/data/backlog.md" + axi "$dir" add blocker "the blocker" + axi "$dir" add dependent "the dependent" + axi "$dir" block dependent --by blocker + axi "$dir" start blocker + [ -z "$(surface "$dir")" ] || fail "seeding surfaced work" + axi "$dir" "done" blocker + + PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_READY_SCAN=1 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || { reap "$pid"; fail "the watcher did not wake for newly ready work"; } + grep -Fx 'check: ready-work: dependent' "$out" >/dev/null \ + || fail "the watcher wake did not name the ready work: $(cat "$out")" + grep -F "$(printf '\tcheck\tready-work\tcheck: ready-work: dependent')" "$state/.wake-queue" >/dev/null \ + || fail "the ready-work wake was not queued durably" + + ack_handled_wakes "$state" || fail "the ready-work wake could not be drained and acknowledged" + : > "$out" + PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_READY_SCAN=1 "$WATCH" > "$out" & + pid=$! + rm -f "$state/.last-ready-scan" + wait_live "$pid" 40 || fail "the watcher woke again for already surfaced work: $(cat "$out")" + [ -e "$state/.last-ready-scan" ] || { reap "$pid"; fail "the second watcher never ran a ready-work scan"; } + reap "$pid" + [ ! -s "$out" ] || fail "the second watcher printed a wake: $(cat "$out")" + pass "a real watcher wakes once for work a blocker closed outside teardown made ready" +} + +reap() { kill "$1" 2>/dev/null || true; wait "$1" 2>/dev/null || true; } + +# Drain the queue and run the generation-bound acknowledgement the drain +# prints, as a supervisor does after handling its wakes. +ack_handled_wakes() { # + local state=$1 err sequence generation + err="$state/.test-drain.err" + FM_STATE_OVERRIDE="$state" "$ROOT/bin/fm-wake-drain.sh" >/dev/null 2> "$err" || return 1 + sequence=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through \([0-9][0-9]*\) --recovery-generation [A-Za-z0-9._-][A-Za-z0-9._-]*$/\1/p' "$err") + generation=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through [0-9][0-9]* --recovery-generation \([A-Za-z0-9._-][A-Za-z0-9._-]*\)$/\1/p' "$err") + rm -f "$err" + [ -n "$sequence" ] && [ -n "$generation" ] || return 1 + FM_STATE_OVERRIDE="$state" "$ROOT/bin/fm-wake-drain.sh" --ack-through "$sequence" \ + --recovery-generation "$generation" >/dev/null 2>&1 +} + +wait_live() { # [ticks] + local pid=$1 limit=${2:-30} i=0 + while [ "$i" -lt "$limit" ]; do + kill -0 "$pid" 2>/dev/null || return 1 + sleep 0.1 + i=$((i + 1)) + done + return 0 +} + +test_surfaces_once_per_readiness_transition +test_date_gate_surfaces_when_due +test_live_gates_are_bounded +test_watcher_wakes_once_for_a_cleared_blocker diff --git a/tests/fm-session-lock-ancestry.test.sh b/tests/fm-session-lock-ancestry.test.sh index 381acd6ae85..8c0c8fcf024 100755 --- a/tests/fm-session-lock-ancestry.test.sh +++ b/tests/fm-session-lock-ancestry.test.sh @@ -435,6 +435,7 @@ install_autoarm_scripts() { cp "$ROOT/bin/fm-claude-stop-autoarm.sh" "$dir/bin/fm-claude-stop-autoarm.sh" cp "$ROOT/bin/fm-primary-scope-lib.sh" "$dir/bin/fm-primary-scope-lib.sh" cp "$ROOT/bin/fm-supervision-lib.sh" "$dir/bin/fm-supervision-lib.sh" + cp "$ROOT/bin/fm-ready-work.sh" "$dir/bin/fm-ready-work.sh" cp "$ROOT/bin/fm-wake-lib.sh" "$dir/bin/fm-wake-lib.sh" cp "$ROOT/bin/fm-session-lock-lib.sh" "$dir/bin/fm-session-lock-lib.sh" cp "$ROOT/bin/fm-cursor-lib.sh" "$dir/bin/fm-cursor-lib.sh" diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index 08969300ac6..601fa23cbe5 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -727,6 +727,25 @@ test_teardown_closes_the_backlog_item_itself() { pass "teardown closes its own backlog item before reporting success" } +test_teardown_names_the_work_its_close_unblocked() { + local case_dir out again + case_dir=$(make_case tasks-axi-unblocked) + write_meta "$case_dir" no-mistakes ship + seed_backlog_in_flight "$case_dir" + tasks-axi add dep-y1 "phase two" --file "$case_dir/data/backlog.md" >/dev/null + tasks-axi block dep-y1 --by task-x1 --file "$case_dir/data/backlog.md" >/dev/null + FM_STATE_OVERRIDE="$case_dir/state" FM_DATA_OVERRIDE="$case_dir/data" \ + FM_CONFIG_OVERRIDE="$case_dir/config" "$ROOT/bin/fm-ready-work.sh" surface >/dev/null + + out=$(run_teardown "$case_dir") || fail "teardown failed with a gated dependent" + printf '%s\n' "$out" | grep -Fx 'Backlog: newly ready queued work: dep-y1' >/dev/null \ + || fail "teardown did not name the dependent its close unblocked: $out" + again=$(FM_STATE_OVERRIDE="$case_dir/state" FM_DATA_OVERRIDE="$case_dir/data" \ + FM_CONFIG_OVERRIDE="$case_dir/config" "$ROOT/bin/fm-ready-work.sh" surface) + [ -z "$again" ] || fail "work teardown already named would wake the watcher again: $again" + pass "teardown names the queued work its close unblocked and records it as surfaced" +} + test_teardown_manual_backend_leaves_the_backlog_to_the_operator() { local case_dir out backlog_path case_dir=$(make_case tasks-axi-manual-optout) @@ -3861,6 +3880,7 @@ EOF test_local_only_fork_remote_allows test_teardown_closes_the_backlog_item_itself +test_teardown_names_the_work_its_close_unblocked test_teardown_manual_backend_leaves_the_backlog_to_the_operator test_local_only_truly_unpushed_refuses test_local_only_merged_to_local_main_allows diff --git a/tests/fm-turnend-guard.test.sh b/tests/fm-turnend-guard.test.sh index a2338e2a2e5..4f47ac09553 100755 --- a/tests/fm-turnend-guard.test.sh +++ b/tests/fm-turnend-guard.test.sh @@ -190,6 +190,11 @@ install_guard_scripts() { cp "$ROOT/bin/fm-harness.sh" "$dir/bin/fm-harness.sh" cp "$ROOT/bin/fm-primary-scope-lib.sh" "$dir/bin/fm-primary-scope-lib.sh" cp "$ROOT/bin/fm-supervision-lib.sh" "$dir/bin/fm-supervision-lib.sh" + cp "$ROOT/bin/fm-ready-work.sh" "$dir/bin/fm-ready-work.sh" + cp "$ROOT/bin/fm-tasks-axi.sh" "$dir/bin/fm-tasks-axi.sh" + cp "$ROOT/bin/fm-tasks-axi-lib.sh" "$dir/bin/fm-tasks-axi-lib.sh" + cp "$ROOT/bin/fm-backlog-transition-lib.sh" "$dir/bin/fm-backlog-transition-lib.sh" + cp "$ROOT/bin/fm-timeout-lib.sh" "$dir/bin/fm-timeout-lib.sh" cp "$ROOT/bin/fm-wake-lib.sh" "$dir/bin/fm-wake-lib.sh" cp "$ROOT/bin/fm-hook-host-lib.sh" "$dir/bin/fm-hook-host-lib.sh" cp "$ROOT/bin/fm-session-lock-lib.sh" "$dir/bin/fm-session-lock-lib.sh" @@ -488,6 +493,26 @@ test_hook_registered_check_only_blocks_with_check_banner() { pass "fm-turnend-guard: registered-check-only supervision is named in the block banner" } +test_hook_gated_backlog_blocks_only_while_its_gate_can_clear() { + local dir out status + command -v tasks-axi >/dev/null 2>&1 || { printf 'skip: tasks-axi not found\n'; return 0; } + dir=$(make_primary_dir "$TMP_ROOT/hook-gated-backlog") + mkdir -p "$dir/data" + cp "$ROOT/.tasks.toml" "$dir/.tasks.toml" + printf '%s\n' '# Backlog' '' '## In flight' '' '## Queued' '' '## Done' > "$dir/data/backlog.md" + tasks-axi add question "a captain call" --file "$dir/data/backlog.md" >/dev/null + tasks-axi hold question --reason "captain decides" --kind captain --file "$dir/data/backlog.md" >/dev/null + out=$(run_hook "$dir" false); status=$? + expect_code 0 "$status" "an undated captain hold alone must not demand a watcher" + tasks-axi add later "phase two" --file "$dir/data/backlog.md" >/dev/null + tasks-axi hold later --reason "not before" --until 2099-01-01 --file "$dir/data/backlog.md" >/dev/null + out=$(run_hook "$dir" false); status=$? + expect_code 2 "$status" "a future-dated queued item must keep the home supervised" + assert_contains "$out" "1 queued backlog item(s) wait on a date or blocker, but no live watcher" "gated-only blind stop must identify its supervision need" + assert_not_contains "$out" "X-mode relay polling needs supervision" "gated-only blind stop must not be misreported as relay polling" + pass "fm-turnend-guard: dated queued work is guarded and undated captain holds are not" +} + test_hook_ignores_repo_state_when_fm_home_set() { local dir home out status dir=$(make_primary_dir "$TMP_ROOT/hook-fm-home-ignore-root") @@ -1210,6 +1235,7 @@ install_integrated_autoarm() { cp "$ROOT/bin/fm-claude-stop-autoarm.sh" "$dir/bin/fm-claude-stop-autoarm.sh" cp "$ROOT/bin/fm-primary-scope-lib.sh" "$dir/bin/fm-primary-scope-lib.sh" cp "$ROOT/bin/fm-supervision-lib.sh" "$dir/bin/fm-supervision-lib.sh" + cp "$ROOT/bin/fm-ready-work.sh" "$dir/bin/fm-ready-work.sh" cp "$ROOT/bin/fm-wake-lib.sh" "$dir/bin/fm-wake-lib.sh" cp "$ROOT/bin/fm-hook-host-lib.sh" "$dir/bin/fm-hook-host-lib.sh" cp "$ROOT/bin/fm-session-lock-lib.sh" "$dir/bin/fm-session-lock-lib.sh" @@ -2210,6 +2236,7 @@ test_hook_blocks_from_fm_home_state test_hook_x_mode_reason_sources_cadence test_hook_x_mode_only_blocks_in_default_mode test_hook_registered_check_only_blocks_with_check_banner +test_hook_gated_backlog_blocks_only_while_its_gate_can_clear test_hook_ignores_repo_state_when_fm_home_set test_hook_uses_state_override test_hook_loop_guard_allows_retry From 68991a25101ae6e1805907222dc0a766c7751d67 Mon Sep 17 00:00:00 2001 From: knowttl Date: Wed, 23 Sep 2026 08:26:47 -0700 Subject: [PATCH 2/7] no-mistakes(review): Surface gated work after durable wake delivery --- bin/fm-captain-hold.sh | 13 ++++-- bin/fm-ready-work.sh | 92 ++++++++++++++++++++----------------- bin/fm-tasks-axi.sh | 9 +++- bin/fm-teardown.sh | 9 ++-- bin/fm-watch.sh | 3 +- docs/architecture.md | 2 +- tests/fm-ready-work.test.sh | 66 +++++++++++++++++++++----- tests/fm-teardown.test.sh | 12 ++--- 8 files changed, 134 insertions(+), 72 deletions(-) diff --git a/bin/fm-captain-hold.sh b/bin/fm-captain-hold.sh index 880926494c2..b5fa23b2e2f 100755 --- a/bin/fm-captain-hold.sh +++ b/bin/fm-captain-hold.sh @@ -338,16 +338,23 @@ load_decision() { # ; sets DECISION_TEXT and DECISION_DIGEST # the root's own tasks-axi configuration, exactly like the transition library's # mutate path. tasks_axi() { - local data file root backend + local data file root backend status=0 data=$(fm_backlog_data_absolute "$DATA") || fail "data directory cannot be resolved: $DATA" root=$(fm_backlog_root "$data") || fail "$FM_BACKLOG_TRANSITION_ERROR" backend=$(fm_tasks_axi_backend "$root") || return 2 if [ "$backend" = markdown ]; then file=$(fm_backlog_file "$data") || fail "$FM_BACKLOG_TRANSITION_ERROR" - (cd "$root" && tasks-axi "$@" --file "$file") + (cd "$root" && tasks-axi "$@" --file "$file") || status=$? else - (cd "$root" && tasks-axi "$@") + (cd "$root" && tasks-axi "$@") || status=$? fi + [ "$status" -eq 0 ] || return "$status" + case "$1" in + done|unhold|hold|block|unblock|start) + FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_DATA_OVERRIDE="$DATA" \ + "$SCRIPT_DIR/fm-ready-work.sh" wake || true + ;; + esac } require_tasks_axi() { diff --git a/bin/fm-ready-work.sh b/bin/fm-ready-work.sh index e881db53947..3fce2cdc6d9 100755 --- a/bin/fm-ready-work.sh +++ b/bin/fm-ready-work.sh @@ -3,12 +3,9 @@ # became dispatchable without this home acting, and report whether gated queued # work still needs a watcher to notice it. # -# Usage: fm-ready-work.sh surface -# Print, on one line, the queued task ids that became ready since they were -# last surfaced (nothing when none did), and record them as surfaced. -# bin/fm-teardown.sh runs it right after closing its task, so the dependents -# that close unblocked are named in its own output instead of arriving later -# as a separate wake. +# Usage: fm-ready-work.sh surface|wake +# Surface queued task ids released by a date or blocker gate since the last +# delivery; wake appends them to the durable wake queue. # Sourced (. bin/fm-ready-work.sh): fm_ready_work_scan, fm_ready_work_commit, # fm_ready_work_release (bin/fm-watch.sh), and fm_ready_work_live_gates # (bin/fm-supervision-lib.sh). @@ -16,7 +13,7 @@ # WHY. Queued work gated on a date (`tasks-axi hold --until`, including captain # holds deferred with bin/fm-captain-hold.sh --until) or on blockers can become # ready without any turn in this home: a date passes, or a blocker is closed by a -# captain answer, a hand-run `tasks-axi done`, or work elsewhere. Teardown and +# captain answer, a hand-run backlog close, or work elsewhere. Teardown and # session start re-evaluate the queue, but nothing else did, so such work waited # for the next unrelated teardown or session start. # @@ -26,16 +23,12 @@ # not blocked, not held, and not a public-followup obligation, which is never # dispatchable - the same set `tasks-axi ready` lists. # -# ONCE PER TRANSITION. state/.ready-work-surfaced lists the ready ids already -# surfaced. A scan reports the ready ids missing from it; the caller commits the -# current ready set as the new record only after it has enqueued its wake -# (enqueue before suppress), so a crash in between repeats the wake rather than -# losing it. An id leaves the record when it is dispatched, closed, or re-held, -# so its next readiness is new again. An item filed already ready is surfaced -# once too, unless it is dispatched before the next scan. A home with no record -# yet seeds it silently, so the first scan never replays the whole ready queue. -# state/.ready-work.lock serializes scan-to-commit between the watcher and -# teardown, so one transition is reported by exactly one of them. +# ONCE PER TRANSITION. state/.ready-work-surfaced lists gated ready ids already +# surfaced. A scan reports gated ready ids missing from it; the caller commits +# the current gated ready set only after delivery. An id leaves the record when +# it is dispatched, closed, or re-held, so its next readiness is new again. +# state/.ready-work.lock serializes scan-to-commit across callers, so one +# transition is reported by exactly one of them. # # LIVE GATES (supervision need). A queued item's gate is live when it can clear # without this home acting: a hold with a future date, or a blocker that is in @@ -50,16 +43,15 @@ # fails, times out (FM_READY_WORK_TIMEOUT seconds, default 10), or cannot be # parsed all mean nothing to surface and no need: this backstop never blocks a # turn or a teardown on its own failure. The home's data and config directories -# are the state directory's siblings unless FM_DATA_OVERRIDE / FM_CONFIG_OVERRIDE -# name them. +# come from FM_HOME unless FM_DATA_OVERRIDE / FM_CONFIG_OVERRIDE name them. FM_READY_WORK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -FM_READY_WORK_READY= +FM_READY_WORK_ELIGIBLE= FM_READY_WORK_LIVE=0 FM_READY_WORK_NEW= FM_READY_WORK_LOCK= -# Classify one `tasks-axi list` listing. Prints `ready ` per ready item and +# Classify one `tasks-axi list` listing. Prints `eligible ` per gated ready item and # a final `live `; exits 2 when the listing lacks the expected table. fm_ready_work_classify() { LC_ALL=C awk ' @@ -103,18 +95,22 @@ fm_ready_work_classify() { blocked[id] = f[col["blocked"]] held[id] = f[col["held"]] until[id] = f[col["hold_until"]] - deps[id] = f[col["blocked_by"]] + blockers[id] = f[col["blocked_by"]] + deps[id] = f[col["deps"]] next } { rows = 0 } END { if (!table) exit 2 if (n > 0 && !("id" in col && "state" in col && "kind" in col && "blocked" in col \ - && "blocked_by" in col && "held" in col && "hold_until" in col)) exit 2 + && "blocked_by" in col && "deps" in col && "held" in col && "hold_until" in col)) exit 2 for (i = 1; i <= n; i++) { id = ids[i] if (state[id] != "queued" || kind[id] == "public-followup") continue - if (blocked[id] == "no" && held[id] == "no") print "ready " id + if (blocked[id] == "no" && held[id] == "no") { + if (deps[id] != "none" && deps[id] != "-" && deps[id] != "" \ + || until[id] != "-" && until[id] != "") print "eligible " id + } if (held[id] == "yes" && until[id] != "-" && until[id] != "") live[id] = 1 } # A blocked item is live when any open blocker is in flight or live itself; @@ -126,7 +122,7 @@ fm_ready_work_classify() { id = ids[i] if (state[id] != "queued" || live[id] || blocked[id] != "yes") continue if (held[id] == "yes" && (until[id] == "-" || until[id] == "")) continue - m = split(deps[id], bs, ",") + m = split(blockers[id], bs, ",") for (j = 1; j <= m; j++) { b = bs[j] if (state[b] == "in_flight" || live[b]) { live[id] = 1; changed = 1; break } @@ -141,14 +137,14 @@ fm_ready_work_classify() { } # fm_ready_work_read -# Sets FM_READY_WORK_READY (sorted ready ids, one per line) and +# Sets FM_READY_WORK_ELIGIBLE (sorted gated ready ids, one per line) and # FM_READY_WORK_LIVE (live-gated queued count). Returns 0 on a good read, 1 when # this home has no readable tasks-axi backlog (see STEPPING ASIDE). fm_ready_work_read() { local state=$1 home data config root backend listing classified - FM_READY_WORK_READY= + FM_READY_WORK_ELIGIBLE= FM_READY_WORK_LIVE=0 - home=${state%/*} + home=${FM_HOME:-${FM_ROOT_OVERRIDE:-$(cd "$FM_READY_WORK_DIR/.." && pwd)}} data=${FM_DATA_OVERRIDE:-$home/data} config=${FM_CONFIG_OVERRIDE:-$home/config} [ -d "$data" ] || return 1 @@ -167,10 +163,10 @@ fm_ready_work_read() { || . "$FM_READY_WORK_DIR/fm-timeout-lib.sh" || return 1 listing=$(FM_HOME="$home" FM_DATA_OVERRIDE="$data" \ fm_run_timed "${FM_READY_WORK_TIMEOUT:-10}" \ - "$FM_READY_WORK_DIR/fm-tasks-axi.sh" list --fields blocked,blocked_by,held,hold_until \ + "$FM_READY_WORK_DIR/fm-tasks-axi.sh" list --fields blocked,blocked_by,deps,held,hold_until \ 2>/dev/null # Takes state/.ready-work.lock, reads the backlog, and sets FM_READY_WORK_NEW to -# the space-separated ready ids not yet surfaced (empty while seeding a home -# with no record). Returns 0 with the lock held, to be finished by +# the space-separated gated ready ids not yet surfaced. Returns 0 with the lock held, to be finished by # fm_ready_work_commit; returns 1 with no lock held when there is nothing to # read or the lock stays contended. fm_ready_work_scan() { @@ -204,22 +199,22 @@ fm_ready_work_scan() { return 1 fi record="$state/.ready-work-surfaced" - [ -e "$record" ] || return 0 - FM_READY_WORK_NEW=$(printf '%s\n' "$FM_READY_WORK_READY" | LC_ALL=C awk ' + [ -e "$record" ] || record=/dev/null + FM_READY_WORK_NEW=$(printf '%s\n' "$FM_READY_WORK_ELIGIBLE" | LC_ALL=C awk ' FILENAME == ARGV[1] { if ($0 != "") seen[$0] = 1; next } $0 != "" && !($0 in seen) { printf "%s%s", sep, $0; sep = " " } ' "$record" -) return 0 } -# fm_ready_work_commit : record the scanned ready set as surfaced and +# fm_ready_work_commit : record the scanned gated ready set as surfaced and # release the scan lock. fm_ready_work_commit() { local state=$1 record tmp status=0 record="$state/.ready-work-surfaced" if tmp=$(mktemp "$state/.ready-work-surfaced.XXXXXX"); then - if [ -n "$FM_READY_WORK_READY" ]; then - printf '%s\n' "$FM_READY_WORK_READY" > "$tmp" || status=1 + if [ -n "$FM_READY_WORK_ELIGIBLE" ]; then + printf '%s\n' "$FM_READY_WORK_ELIGIBLE" > "$tmp" || status=1 fi if [ "$status" -eq 0 ]; then mv -f -- "$tmp" "$record" || status=1 @@ -239,22 +234,35 @@ fm_ready_work_release() { } fm_ready_work_main() { - local state + local state mode case "${1:-}" in - surface) ;; + surface|wake) mode=$1 ;; -h|--help) awk 'NR == 1 { next } /^#/ { sub(/^# ?/, ""); print; next } { exit }' "$0" return 0 ;; *) - printf 'usage: fm-ready-work.sh surface\n' >&2 + printf 'usage: fm-ready-work.sh surface|wake\n' >&2 return 2 ;; esac state=${FM_STATE_OVERRIDE:-${FM_HOME:-$(cd "$FM_READY_WORK_DIR/.." && pwd)}/state} fm_ready_work_scan "$state" || return 0 - fm_ready_work_commit "$state" || return 0 - [ -z "$FM_READY_WORK_NEW" ] || printf '%s\n' "$FM_READY_WORK_NEW" + if [ -n "$FM_READY_WORK_NEW" ]; then + if [ "$mode" = wake ]; then + . "$FM_READY_WORK_DIR/fm-wake-lib.sh" + fm_wake_append check ready-work "check: ready-work: $FM_READY_WORK_NEW" || { + fm_ready_work_release + return 1 + } + else + printf '%s\n' "$FM_READY_WORK_NEW" || { + fm_ready_work_release + return 1 + } + fi + fi + fm_ready_work_commit "$state" } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then diff --git a/bin/fm-tasks-axi.sh b/bin/fm-tasks-axi.sh index b8e2844c0e5..8e1af7248a4 100755 --- a/bin/fm-tasks-axi.sh +++ b/bin/fm-tasks-axi.sh @@ -124,4 +124,11 @@ else fi cd "$FM_BACKLOG_AXI_ROOT" || fail "cannot enter the backlog root $FM_BACKLOG_AXI_ROOT" -exec tasks-axi ${ARGS[@]+"${ARGS[@]}"} +case "${ARGS[0]:-}" in + done|unhold|hold|block|unblock|start) + tasks-axi ${ARGS[@]+"${ARGS[@]}"} || exit $? + FM_HOME="$FM_HOME" FM_DATA_OVERRIDE="$DATA" \ + "$SCRIPT_DIR/fm-ready-work.sh" wake || true + ;; + *) exec tasks-axi ${ARGS[@]+"${ARGS[@]}"} ;; +esac diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 7e92f5bf9a2..732be63786e 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -1541,7 +1541,7 @@ backlog_done_args() { # invariant). This prints what already happened, so the follow-up wording stays # only where a human still owes the edit. backlog_refresh_reminder() { - local backlog_display root backend=markdown newly_ready + local backlog_display root backend=markdown [ "$KIND" = secondmate ] && return 0 [ "$CLEANUP_RECOVERY" = orca ] && return 0 if root=$(fm_backlog_root "$DATA"); then @@ -1558,10 +1558,9 @@ backlog_refresh_reminder() { printf '%s\n' "Backlog: $ID stays open in $backlog_display, still held for the captain with its deliverable recorded. Relay the question and close it only with bin/fm-captain-hold.sh answer." elif [ "$BACKLOG_CLOSED" = 1 ]; then printf '%s\n' "Backlog: $ID is closed in $backlog_display. Run bin/fm-tasks-axi.sh ready for dependency-cleared candidates, check date gates, and dispatch only work whose blockers are gone and date is due." - # Recorded as surfaced here, so the watcher does not wake again for them. - newly_ready=$(FM_STATE_OVERRIDE="$STATE" FM_DATA_OVERRIDE="$DATA" FM_CONFIG_OVERRIDE="$CONFIG" \ - "$SCRIPT_DIR/fm-ready-work.sh" surface 2>/dev/null || true) - [ -z "$newly_ready" ] || printf '%s\n' "Backlog: newly ready queued work: $newly_ready" + FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_DATA_OVERRIDE="$DATA" \ + FM_CONFIG_OVERRIDE="$CONFIG" "$SCRIPT_DIR/fm-ready-work.sh" wake \ + || true else printf '%s\n' "Backlog: $ID just finished ($BACKLOG_SKIP_REASON). Update $backlog_display - move $ID to Done, keep Done to the 10 most recent, then re-scan Queued and dispatch only work whose blockers are gone and date is due." fi diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index d84b24cd15d..adf1eef98db 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -236,7 +236,6 @@ POLL=${FM_POLL:-15} # seconds between cycles WATCHER_STALE_GRACE=${FM_WATCHER_STALE_GRACE:-${FM_GUARD_GRACE:-$(fm_poll_derived_grace "$POLL")}} HEARTBEAT=${FM_HEARTBEAT:-600} # base seconds between heartbeat scans HEARTBEAT_MAX=${FM_HEARTBEAT_MAX:-7200} # heartbeat backoff cap -READY_SCAN=${FM_READY_SCAN:-$HEARTBEAT} # seconds between ready-work scans, never backed off CHECK_INTERVAL=${FM_CHECK_INTERVAL:-300} # seconds between *.check.sh sweeps CHECK_TIMEOUT=${FM_CHECK_TIMEOUT:-30} # seconds allowed per *.check.sh HOME_SUMMARY_INTERVAL=${FM_HOME_SUMMARY_INTERVAL:-300} @@ -2503,7 +2502,7 @@ EOF # Its own unbacked-off cadence, ahead of the signal scan for the same # starvation reason as the checks above, keeps a due date prompt even while # the heartbeat has backed off on an idle home. - if [ "$(age_of "$STATE/.last-ready-scan")" -ge "$READY_SCAN" ]; then + if [ "$(age_of "$STATE/.last-ready-scan")" -ge "$HEARTBEAT" ]; then touch "$STATE/.last-ready-scan" if fm_ready_work_scan "$STATE"; then if [ -n "$FM_READY_WORK_NEW" ]; then diff --git a/docs/architecture.md b/docs/architecture.md index 244d9f1d947..d9aa92ee7a5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,7 +93,7 @@ Fresh stale panes use the same current-state read before trusting the status log No-change heartbeats are also benign. Separately from heartbeat backoff and wedge handling, the watcher poll runs `bin/fm-inactive-reconcile.sh` on its own bounded cadence, while locked session start sends the same bounded local scan through `bin/fm-startup-network.sh`'s deferred worker so current-state reads never block the digest. In each home the scan considers only that home's long-inactive direct ordinary crewmates, excludes captain-held work, and accepts only `done` or `failed` from `bin/fm-crew-state.sh`. -The poll also runs a ready-work scan on the base heartbeat cadence, never backed off, in every posture: queued backlog work that became ready without this home acting - a hold date that arrived or blockers closed by anything other than its own teardown - wakes the home once per readiness transition as `check: ready-work: `, and teardown names the work its own close unblocked instead. +The poll also runs a ready-work scan on the base heartbeat cadence, never backed off, in every posture: queued backlog work released by a hold date or closed blockers wakes the home once per readiness transition as `check: ready-work: `; supported close commands and teardown enqueue the same wake when they clear a blocker. Live-gated queued work - a future hold date, or blockers that are in flight or themselves live-gated - counts as supervision need until its gate clears, while undated holds never keep a watcher alive; `bin/fm-ready-work.sh` owns readiness, the once-per-transition record, and that bound. A secondmate retains a durable receipt for its idempotent report through the established parent route, and main-home captain presentation retains a separate receipt; neither path performs a forge or PR check. A secondmate home's terminal child ledger lines, PR registrations, captain holds, and merges are published on that same parent route by the scripts that record them, so no captain-facing outcome depends on the mate model appending it ([secondmate-parent-channel.md](secondmate-parent-channel.md)). diff --git a/tests/fm-ready-work.test.sh b/tests/fm-ready-work.test.sh index 432e755a3a8..e04601f8006 100755 --- a/tests/fm-ready-work.test.sh +++ b/tests/fm-ready-work.test.sh @@ -36,6 +36,13 @@ axi() { # FM_HOME="$home" "$ROOT/bin/fm-tasks-axi.sh" "$@" >/dev/null || fail "tasks-axi $* failed in $home" } +external_axi() { # + local home=$1 + shift + tasks-axi "$@" --file "$home/data/backlog.md" >/dev/null \ + || fail "external tasks-axi $* failed in $home" +} + surface() { # [env assignments...] local home=$1 shift @@ -46,7 +53,7 @@ live_gates() { # [env assignments...] local home=$1 shift # shellcheck disable=SC2016 # Positional parameters expand in the child shell. - env "$@" bash -c '. "$1"; fm_ready_work_live_gates "$2"' _ "$READY" "$home/state" + env FM_HOME="$home" "$@" bash -c '. "$1"; fm_ready_work_live_gates "$2"' _ "$READY" "$home/state" } test_surfaces_once_per_readiness_transition() { @@ -57,11 +64,10 @@ test_surfaces_once_per_readiness_transition() { axi "$home" block dependent --by blocker axi "$home" start blocker out=$(surface "$home") - [ -z "$out" ] || fail "seeding a home with no record surfaced work: $out" - [ -e "$home/state/.ready-work-surfaced" ] || fail "the seeding scan left no record" + [ -z "$out" ] || fail "blocked work surfaced: $out" # Closed by hand, not by this home's teardown. - axi "$home" "done" blocker + external_axi "$home" "done" blocker out=$(surface "$home") [ "$out" = dependent ] || fail "a blocker closed outside teardown did not surface its dependent: '$out'" out=$(surface "$home") @@ -70,7 +76,7 @@ test_surfaces_once_per_readiness_transition() { axi "$home" hold dependent --reason "wait" out=$(surface "$home") [ -z "$out" ] || fail "re-holding surfaced work: $out" - axi "$home" unhold dependent + external_axi "$home" unhold dependent out=$(surface "$home") [ "$out" = dependent ] || fail "released work did not surface as a new transition: '$out'" @@ -98,6 +104,39 @@ test_date_gate_surfaces_when_due() { pass "a dated captain hold surfaces once its date arrives and stops needing a watcher" } +test_first_scan_and_source_close() { + local home out state + home=$(make_home first-scan) + state="$home/state" + axi "$home" add ready "ready from creation" + axi "$home" add due "held until today" + axi "$home" hold due --reason later --until "$EAST_TODAY" + out=$(surface "$home" TZ=$EAST) + [ "$out" = due ] || fail "the first scan did not surface an already due gate: '$out'" + [ -z "$(surface "$home" TZ=$EAST)" ] || fail "the due gate surfaced twice" + + axi "$home" add blocker "a queued blocker" + axi "$home" add dependent "dependent work" + axi "$home" block dependent --by blocker + axi "$home" done blocker + grep -F 'check: ready-work: dependent' "$state/.wake-queue" >/dev/null \ + || fail "closing a queued blocker did not queue a dependent wake" + [ -z "$(surface "$home")" ] || fail "a source wake repeated through the scanner" + pass "a first due scan surfaces its gate and a queued close wakes its dependent" +} + +test_alternate_state_keeps_home_backlog() { + local home state out + home=$(make_home alternate-state) + state="$home/other-state" + mkdir -p "$state" + axi "$home" add due "held until today" + axi "$home" hold due --reason later --until "$EAST_TODAY" + out=$(FM_HOME="$home" FM_STATE_OVERRIDE="$state" TZ=$EAST "$READY" surface) + [ "$out" = due ] || fail "alternate state read the wrong backlog: '$out'" + pass "an alternate state directory still reads the configured home backlog" +} + test_live_gates_are_bounded() { local home home=$(make_home undated) @@ -113,7 +152,7 @@ test_live_gates_are_bounded() { ( # shellcheck source=/dev/null . "$ROOT/bin/fm-supervision-lib.sh" - if fm_supervision_needed "$home/state" 300; then + if FM_HOME="$home" fm_supervision_needed "$home/state" 300; then fail "a home with only undated holds gained a watcher need" fi ) || exit 1 @@ -132,7 +171,7 @@ test_live_gates_are_bounded() { ( # shellcheck source=/dev/null . "$ROOT/bin/fm-supervision-lib.sh" - TZ=$WEST fm_supervision_needed "$home/state" 300 \ + FM_HOME="$home" TZ=$WEST fm_supervision_needed "$home/state" 300 \ || fail "live-gated queued work did not need supervision" [ "$FM_SUP_GATED" = 3 ] || fail "FM_SUP_GATED reported $FM_SUP_GATED" ) || exit 1 @@ -154,10 +193,11 @@ test_watcher_wakes_once_for_a_cleared_blocker() { axi "$dir" block dependent --by blocker axi "$dir" start blocker [ -z "$(surface "$dir")" ] || fail "seeding surfaced work" - axi "$dir" "done" blocker + external_axi "$dir" "done" blocker + [ ! -s "$state/.wake-queue" ] || fail "setup queued a wake before the watcher: $(cat "$state/.wake-queue"); $(FM_HOME="$dir" "$ROOT/bin/fm-tasks-axi.sh" list --fields blocked,blocked_by,held,hold_until)" - PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ - FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_READY_SCAN=1 "$WATCH" > "$out" & + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 100 || { reap "$pid"; fail "the watcher did not wake for newly ready work"; } grep -Fx 'check: ready-work: dependent' "$out" >/dev/null \ @@ -167,8 +207,8 @@ test_watcher_wakes_once_for_a_cleared_blocker() { ack_handled_wakes "$state" || fail "the ready-work wake could not be drained and acknowledged" : > "$out" - PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ - FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_READY_SCAN=1 "$WATCH" > "$out" & + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! rm -f "$state/.last-ready-scan" wait_live "$pid" 40 || fail "the watcher woke again for already surfaced work: $(cat "$out")" @@ -206,5 +246,7 @@ wait_live() { # [ticks] test_surfaces_once_per_readiness_transition test_date_gate_surfaces_when_due +test_first_scan_and_source_close +test_alternate_state_keeps_home_backlog test_live_gates_are_bounded test_watcher_wakes_once_for_a_cleared_blocker diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index 601fa23cbe5..41949b4b38c 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -727,7 +727,7 @@ test_teardown_closes_the_backlog_item_itself() { pass "teardown closes its own backlog item before reporting success" } -test_teardown_names_the_work_its_close_unblocked() { +test_teardown_wakes_the_work_its_close_unblocked() { local case_dir out again case_dir=$(make_case tasks-axi-unblocked) write_meta "$case_dir" no-mistakes ship @@ -738,12 +738,12 @@ test_teardown_names_the_work_its_close_unblocked() { FM_CONFIG_OVERRIDE="$case_dir/config" "$ROOT/bin/fm-ready-work.sh" surface >/dev/null out=$(run_teardown "$case_dir") || fail "teardown failed with a gated dependent" - printf '%s\n' "$out" | grep -Fx 'Backlog: newly ready queued work: dep-y1' >/dev/null \ - || fail "teardown did not name the dependent its close unblocked: $out" + grep -F 'check: ready-work: dep-y1' "$case_dir/state/.wake-queue" >/dev/null \ + || fail "teardown did not queue a wake for its dependent" again=$(FM_STATE_OVERRIDE="$case_dir/state" FM_DATA_OVERRIDE="$case_dir/data" \ FM_CONFIG_OVERRIDE="$case_dir/config" "$ROOT/bin/fm-ready-work.sh" surface) - [ -z "$again" ] || fail "work teardown already named would wake the watcher again: $again" - pass "teardown names the queued work its close unblocked and records it as surfaced" + [ -z "$again" ] || fail "work teardown already woke would surface again: $again" + pass "teardown queues a durable wake for the work its close unblocked" } test_teardown_manual_backend_leaves_the_backlog_to_the_operator() { @@ -3880,7 +3880,7 @@ EOF test_local_only_fork_remote_allows test_teardown_closes_the_backlog_item_itself -test_teardown_names_the_work_its_close_unblocked +test_teardown_wakes_the_work_its_close_unblocked test_teardown_manual_backend_leaves_the_backlog_to_the_operator test_local_only_truly_unpushed_refuses test_local_only_merged_to_local_main_allows From de5c77592ce99f71dc6384355a476c1f17487fd2 Mon Sep 17 00:00:00 2001 From: knowttl Date: Wed, 23 Sep 2026 08:32:19 -0700 Subject: [PATCH 3/7] no-mistakes(review): Remove unused surface mode and document bare mutation limit --- bin/fm-ready-work.sh | 36 ++++++++---------- docs/architecture.md | 2 +- tests/fm-ready-work.test.sh | 76 +++++++++++++++++++++---------------- tests/fm-teardown.test.sh | 14 +++---- 4 files changed, 65 insertions(+), 63 deletions(-) diff --git a/bin/fm-ready-work.sh b/bin/fm-ready-work.sh index 3fce2cdc6d9..749a3710f2b 100755 --- a/bin/fm-ready-work.sh +++ b/bin/fm-ready-work.sh @@ -3,9 +3,9 @@ # became dispatchable without this home acting, and report whether gated queued # work still needs a watcher to notice it. # -# Usage: fm-ready-work.sh surface|wake -# Surface queued task ids released by a date or blocker gate since the last -# delivery; wake appends them to the durable wake queue. +# Usage: fm-ready-work.sh wake +# Append queued task ids released by a date or blocker gate to the durable +# wake queue before recording them as surfaced. # Sourced (. bin/fm-ready-work.sh): fm_ready_work_scan, fm_ready_work_commit, # fm_ready_work_release (bin/fm-watch.sh), and fm_ready_work_live_gates # (bin/fm-supervision-lib.sh). @@ -13,9 +13,10 @@ # WHY. Queued work gated on a date (`tasks-axi hold --until`, including captain # holds deferred with bin/fm-captain-hold.sh --until) or on blockers can become # ready without any turn in this home: a date passes, or a blocker is closed by a -# captain answer, a hand-run backlog close, or work elsewhere. Teardown and -# session start re-evaluate the queue, but nothing else did, so such work waited -# for the next unrelated teardown or session start. +# captain answer or a supported backlog close. A bare tasks-axi mutation bypasses +# bin/fm-tasks-axi.sh and is unsupported; home-local blockers missed that way are +# re-evaluated at session start. An undated captain hold or queued blocker alone +# does not keep a watcher running. # # READINESS is tasks-axi's own derivation, read from one `tasks-axi list` through # bin/fm-tasks-axi.sh: its derived `blocked` and `held` fields already apply @@ -234,33 +235,26 @@ fm_ready_work_release() { } fm_ready_work_main() { - local state mode + local state case "${1:-}" in - surface|wake) mode=$1 ;; + wake) ;; -h|--help) awk 'NR == 1 { next } /^#/ { sub(/^# ?/, ""); print; next } { exit }' "$0" return 0 ;; *) - printf 'usage: fm-ready-work.sh surface|wake\n' >&2 + printf 'usage: fm-ready-work.sh wake\n' >&2 return 2 ;; esac state=${FM_STATE_OVERRIDE:-${FM_HOME:-$(cd "$FM_READY_WORK_DIR/.." && pwd)}/state} fm_ready_work_scan "$state" || return 0 if [ -n "$FM_READY_WORK_NEW" ]; then - if [ "$mode" = wake ]; then - . "$FM_READY_WORK_DIR/fm-wake-lib.sh" - fm_wake_append check ready-work "check: ready-work: $FM_READY_WORK_NEW" || { - fm_ready_work_release - return 1 - } - else - printf '%s\n' "$FM_READY_WORK_NEW" || { - fm_ready_work_release - return 1 - } - fi + . "$FM_READY_WORK_DIR/fm-wake-lib.sh" + fm_wake_append check ready-work "check: ready-work: $FM_READY_WORK_NEW" || { + fm_ready_work_release + return 1 + } fi fm_ready_work_commit "$state" } diff --git a/docs/architecture.md b/docs/architecture.md index d9aa92ee7a5..cd46c81c283 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -94,7 +94,7 @@ No-change heartbeats are also benign. Separately from heartbeat backoff and wedge handling, the watcher poll runs `bin/fm-inactive-reconcile.sh` on its own bounded cadence, while locked session start sends the same bounded local scan through `bin/fm-startup-network.sh`'s deferred worker so current-state reads never block the digest. In each home the scan considers only that home's long-inactive direct ordinary crewmates, excludes captain-held work, and accepts only `done` or `failed` from `bin/fm-crew-state.sh`. The poll also runs a ready-work scan on the base heartbeat cadence, never backed off, in every posture: queued backlog work released by a hold date or closed blockers wakes the home once per readiness transition as `check: ready-work: `; supported close commands and teardown enqueue the same wake when they clear a blocker. -Live-gated queued work - a future hold date, or blockers that are in flight or themselves live-gated - counts as supervision need until its gate clears, while undated holds never keep a watcher alive; `bin/fm-ready-work.sh` owns readiness, the once-per-transition record, and that bound. +Live-gated queued work - a future hold date, or blockers that are in flight or themselves live-gated - counts as supervision need until its gate clears, while undated holds never keep a watcher alive; `bin/fm-ready-work.sh` owns readiness, the once-per-transition record, and the unsupported bare-mutation limit. A secondmate retains a durable receipt for its idempotent report through the established parent route, and main-home captain presentation retains a separate receipt; neither path performs a forge or PR check. A secondmate home's terminal child ledger lines, PR registrations, captain holds, and merges are published on that same parent route by the scripts that record them, so no captain-facing outcome depends on the mate model appending it ([secondmate-parent-channel.md](secondmate-parent-channel.md)). Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. diff --git a/tests/fm-ready-work.test.sh b/tests/fm-ready-work.test.sh index e04601f8006..7991c39eff7 100755 --- a/tests/fm-ready-work.test.sh +++ b/tests/fm-ready-work.test.sh @@ -43,10 +43,16 @@ external_axi() { # || fail "external tasks-axi $* failed in $home" } -surface() { # [env assignments...] +wake_ready() { # [env assignments...] local home=$1 shift - env FM_HOME="$home" "$@" "$READY" surface + env FM_HOME="$home" "$@" "$READY" wake || fail "ready-work wake failed in $home" +} + +ready_wake_count() { # + local queue="$1/state/.wake-queue" + [ -f "$queue" ] || { printf '0\n'; return; } + grep -Fc "check: ready-work: $2" "$queue" || true } live_gates() { # [env assignments...] @@ -57,63 +63,65 @@ live_gates() { # [env assignments...] } test_surfaces_once_per_readiness_transition() { - local home out + local home home=$(make_home transition) axi "$home" add blocker "the blocker" axi "$home" add dependent "the dependent" axi "$home" block dependent --by blocker axi "$home" start blocker - out=$(surface "$home") - [ -z "$out" ] || fail "blocked work surfaced: $out" + wake_ready "$home" + [ "$(ready_wake_count "$home" dependent)" = 0 ] || fail "blocked work woke" # Closed by hand, not by this home's teardown. - external_axi "$home" "done" blocker - out=$(surface "$home") - [ "$out" = dependent ] || fail "a blocker closed outside teardown did not surface its dependent: '$out'" - out=$(surface "$home") - [ -z "$out" ] || fail "an unchanged ready item surfaced twice: $out" + axi "$home" "done" blocker + wake_ready "$home" + [ "$(ready_wake_count "$home" dependent)" = 1 ] || fail "a cleared blocker did not wake its dependent" + wake_ready "$home" + [ "$(ready_wake_count "$home" dependent)" = 1 ] || fail "an unchanged ready item woke twice" axi "$home" hold dependent --reason "wait" - out=$(surface "$home") - [ -z "$out" ] || fail "re-holding surfaced work: $out" - external_axi "$home" unhold dependent - out=$(surface "$home") - [ "$out" = dependent ] || fail "released work did not surface as a new transition: '$out'" + wake_ready "$home" + [ "$(ready_wake_count "$home" dependent)" = 1 ] || fail "re-holding woke dependent work" + axi "$home" unhold dependent + wake_ready "$home" + [ "$(ready_wake_count "$home" dependent)" = 2 ] || fail "released work did not wake as a new transition" axi "$home" start dependent - out=$(surface "$home") - [ -z "$out" ] || fail "dispatched work surfaced: $out" + wake_ready "$home" + [ "$(ready_wake_count "$home" dependent)" = 2 ] || fail "dispatched work woke" grep -qx dependent "$home/state/.ready-work-surfaced" \ && fail "dispatched work kept its surfaced marker" pass "ready work is surfaced once per readiness transition and its marker retires on dispatch" } test_date_gate_surfaces_when_due() { - local home out + local home home=$(make_home date-gate) axi "$home" add dated "deferred by the captain" axi "$home" hold dated --reason "revisit later" --kind captain --until "$EAST_TODAY" - out=$(surface "$home" TZ=$WEST) - [ -z "$out" ] || fail "seeding surfaced work: $out" + wake_ready "$home" TZ=$WEST + [ "$(ready_wake_count "$home" dated)" = 0 ] || fail "a future gate woke" [ "$(live_gates "$home" TZ=$WEST)" = 1 ] || fail "a future-dated captain hold is not a live gate" - out=$(surface "$home" TZ=$WEST) - [ -z "$out" ] || fail "a hold not yet due surfaced: $out" - out=$(surface "$home" TZ=$EAST) - [ "$out" = dated ] || fail "a captain hold whose date passed did not surface: '$out'" + wake_ready "$home" TZ=$WEST + [ "$(ready_wake_count "$home" dated)" = 0 ] || fail "a hold not yet due woke" + wake_ready "$home" TZ=$EAST + [ "$(ready_wake_count "$home" dated)" = 1 ] || fail "a captain hold whose date passed did not wake" [ "$(live_gates "$home" TZ=$EAST)" = 0 ] || fail "a due hold still counts as a live gate" pass "a dated captain hold surfaces once its date arrives and stops needing a watcher" } test_first_scan_and_source_close() { - local home out state + local home state home=$(make_home first-scan) state="$home/state" axi "$home" add ready "ready from creation" axi "$home" add due "held until today" axi "$home" hold due --reason later --until "$EAST_TODAY" - out=$(surface "$home" TZ=$EAST) - [ "$out" = due ] || fail "the first scan did not surface an already due gate: '$out'" - [ -z "$(surface "$home" TZ=$EAST)" ] || fail "the due gate surfaced twice" + wake_ready "$home" TZ=$EAST + [ "$(ready_wake_count "$home" due)" = 1 ] || fail "the first scan did not wake an already due gate" + [ "$(ready_wake_count "$home" ready)" = 0 ] || fail "work ready from creation woke" + wake_ready "$home" TZ=$EAST + [ "$(ready_wake_count "$home" due)" = 1 ] || fail "the due gate woke twice" axi "$home" add blocker "a queued blocker" axi "$home" add dependent "dependent work" @@ -121,19 +129,21 @@ test_first_scan_and_source_close() { axi "$home" done blocker grep -F 'check: ready-work: dependent' "$state/.wake-queue" >/dev/null \ || fail "closing a queued blocker did not queue a dependent wake" - [ -z "$(surface "$home")" ] || fail "a source wake repeated through the scanner" + wake_ready "$home" + [ "$(ready_wake_count "$home" dependent)" = 1 ] || fail "a source wake repeated through the scanner" pass "a first due scan surfaces its gate and a queued close wakes its dependent" } test_alternate_state_keeps_home_backlog() { - local home state out + local home state home=$(make_home alternate-state) state="$home/other-state" mkdir -p "$state" axi "$home" add due "held until today" axi "$home" hold due --reason later --until "$EAST_TODAY" - out=$(FM_HOME="$home" FM_STATE_OVERRIDE="$state" TZ=$EAST "$READY" surface) - [ "$out" = due ] || fail "alternate state read the wrong backlog: '$out'" + FM_HOME="$home" FM_STATE_OVERRIDE="$state" TZ=$EAST "$READY" wake + grep -F 'check: ready-work: due' "$state/.wake-queue" >/dev/null \ + || fail "alternate state read the wrong backlog" pass "an alternate state directory still reads the configured home backlog" } @@ -192,7 +202,7 @@ test_watcher_wakes_once_for_a_cleared_blocker() { axi "$dir" add dependent "the dependent" axi "$dir" block dependent --by blocker axi "$dir" start blocker - [ -z "$(surface "$dir")" ] || fail "seeding surfaced work" + wake_ready "$dir" external_axi "$dir" "done" blocker [ ! -s "$state/.wake-queue" ] || fail "setup queued a wake before the watcher: $(cat "$state/.wake-queue"); $(FM_HOME="$dir" "$ROOT/bin/fm-tasks-axi.sh" list --fields blocked,blocked_by,held,hold_until)" diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index 41949b4b38c..dc63f277679 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -728,21 +728,19 @@ test_teardown_closes_the_backlog_item_itself() { } test_teardown_wakes_the_work_its_close_unblocked() { - local case_dir out again + local case_dir case_dir=$(make_case tasks-axi-unblocked) write_meta "$case_dir" no-mistakes ship seed_backlog_in_flight "$case_dir" tasks-axi add dep-y1 "phase two" --file "$case_dir/data/backlog.md" >/dev/null tasks-axi block dep-y1 --by task-x1 --file "$case_dir/data/backlog.md" >/dev/null - FM_STATE_OVERRIDE="$case_dir/state" FM_DATA_OVERRIDE="$case_dir/data" \ - FM_CONFIG_OVERRIDE="$case_dir/config" "$ROOT/bin/fm-ready-work.sh" surface >/dev/null - - out=$(run_teardown "$case_dir") || fail "teardown failed with a gated dependent" + run_teardown "$case_dir" >/dev/null || fail "teardown failed with a gated dependent" grep -F 'check: ready-work: dep-y1' "$case_dir/state/.wake-queue" >/dev/null \ || fail "teardown did not queue a wake for its dependent" - again=$(FM_STATE_OVERRIDE="$case_dir/state" FM_DATA_OVERRIDE="$case_dir/data" \ - FM_CONFIG_OVERRIDE="$case_dir/config" "$ROOT/bin/fm-ready-work.sh" surface) - [ -z "$again" ] || fail "work teardown already woke would surface again: $again" + FM_STATE_OVERRIDE="$case_dir/state" FM_DATA_OVERRIDE="$case_dir/data" \ + FM_CONFIG_OVERRIDE="$case_dir/config" "$ROOT/bin/fm-ready-work.sh" wake + [ "$(grep -Fc 'check: ready-work: dep-y1' "$case_dir/state/.wake-queue")" = 1 ] \ + || fail "work teardown already woke was queued again" pass "teardown queues a durable wake for the work its close unblocked" } From b23159338889e4bc8ba23c10856cfd7a0ef13920 Mon Sep 17 00:00:00 2001 From: knowttl Date: Wed, 23 Sep 2026 08:39:04 -0700 Subject: [PATCH 4/7] no-mistakes(review): Use fixed ready-work timeout --- bin/fm-ready-work.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/fm-ready-work.sh b/bin/fm-ready-work.sh index 749a3710f2b..ded70d84a03 100755 --- a/bin/fm-ready-work.sh +++ b/bin/fm-ready-work.sh @@ -41,7 +41,7 @@ # # STEPPING ASIDE. tasks-axi missing from PATH, config/backlog-backend=manual, a # missing data directory, a markdown home with no backlog file, or a listing that -# fails, times out (FM_READY_WORK_TIMEOUT seconds, default 10), or cannot be +# fails, times out after 10 seconds, or cannot be # parsed all mean nothing to surface and no need: this backstop never blocks a # turn or a teardown on its own failure. The home's data and config directories # come from FM_HOME unless FM_DATA_OVERRIDE / FM_CONFIG_OVERRIDE name them. @@ -163,7 +163,7 @@ fm_ready_work_read() { command -v fm_run_timed >/dev/null 2>&1 \ || . "$FM_READY_WORK_DIR/fm-timeout-lib.sh" || return 1 listing=$(FM_HOME="$home" FM_DATA_OVERRIDE="$data" \ - fm_run_timed "${FM_READY_WORK_TIMEOUT:-10}" \ + fm_run_timed 10 \ "$FM_READY_WORK_DIR/fm-tasks-axi.sh" list --fields blocked,blocked_by,deps,held,hold_until \ 2>/dev/null /dev/null 2>&1 \ || . "$FM_READY_WORK_DIR/fm-wake-lib.sh" || return 1 FM_READY_WORK_LOCK="$state/.ready-work.lock" - fm_lock_acquire_wait_bounded "$FM_READY_WORK_LOCK" "${FM_READY_WORK_TIMEOUT:-10}" || return 1 + fm_lock_acquire_wait_bounded "$FM_READY_WORK_LOCK" 10 || return 1 if ! fm_ready_work_read "$state"; then fm_ready_work_release return 1 From f0b7c1955b99f4b76aa67a68542ae2b7c78ca956 Mon Sep 17 00:00:00 2001 From: knowttl Date: Wed, 23 Sep 2026 08:59:20 -0700 Subject: [PATCH 5/7] no-mistakes(document): Correct ready-work documentation and secondmate supervision limit --- bin/fm-watch.sh | 3 ++- docs/architecture.md | 3 +-- docs/scripts.md | 2 +- docs/turnend-guard.md | 4 ++-- tests/fm-captain-hold-lifecycle.test.sh | 2 ++ tests/fm-ready-work.test.sh | 21 +++++++++++---------- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index adf1eef98db..03846fb33c7 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -2513,8 +2513,9 @@ EOF fi fm_ready_work_commit "$STATE" || triage_log "ready-work record not updated; the next scan repeats this wake" wake "$reason" + else + fm_ready_work_commit "$STATE" || triage_log "ready-work record not updated" fi - fm_ready_work_commit "$STATE" || triage_log "ready-work record not updated" fi fi diff --git a/docs/architecture.md b/docs/architecture.md index cd46c81c283..ae7e162ba78 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,8 +93,7 @@ Fresh stale panes use the same current-state read before trusting the status log No-change heartbeats are also benign. Separately from heartbeat backoff and wedge handling, the watcher poll runs `bin/fm-inactive-reconcile.sh` on its own bounded cadence, while locked session start sends the same bounded local scan through `bin/fm-startup-network.sh`'s deferred worker so current-state reads never block the digest. In each home the scan considers only that home's long-inactive direct ordinary crewmates, excludes captain-held work, and accepts only `done` or `failed` from `bin/fm-crew-state.sh`. -The poll also runs a ready-work scan on the base heartbeat cadence, never backed off, in every posture: queued backlog work released by a hold date or closed blockers wakes the home once per readiness transition as `check: ready-work: `; supported close commands and teardown enqueue the same wake when they clear a blocker. -Live-gated queued work - a future hold date, or blockers that are in flight or themselves live-gated - counts as supervision need until its gate clears, while undated holds never keep a watcher alive; `bin/fm-ready-work.sh` owns readiness, the once-per-transition record, and the unsupported bare-mutation limit. +The watcher scans for newly ready gated backlog work on the base heartbeat cadence; [`bin/fm-ready-work.sh`](../bin/fm-ready-work.sh) owns readiness, durable wake delivery, supervision need, and the unsupported bare-mutation limit. A secondmate retains a durable receipt for its idempotent report through the established parent route, and main-home captain presentation retains a separate receipt; neither path performs a forge or PR check. A secondmate home's terminal child ledger lines, PR registrations, captain holds, and merges are published on that same parent route by the scripts that record them, so no captain-facing outcome depends on the mate model appending it ([secondmate-parent-channel.md](secondmate-parent-channel.md)). Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. diff --git a/docs/scripts.md b/docs/scripts.md index 36d3e2da131..1d47670da18 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -103,7 +103,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-lock-lib.sh` | Shared "is this git lock provably abandoned?" proof used by teardown and fleet-sync | | `fm-config-inherit-lib.sh` | Shared primary-to-secondmate inherited local-material propagation and config-reread delivery | | `fm-tasks-axi.sh` | Run `tasks-axi` against this home's backlog from any working directory | -| `fm-ready-work.sh` | Surface queued backlog work that became ready without this home acting, once per readiness transition, and count live-gated queued work as supervision need | +| `fm-ready-work.sh` | Backstop for newly ready gated backlog work and its supervision need | | `fm-tasks-axi-lib.sh` | Shared backlog-backend selector and `tasks-axi` compatibility probe | | `fm-backlog-transition-lib.sh` | Pair task-record changes with their backlog transitions and replay interrupted closes | | `fm-quota-axi-lib.sh` | Shared `quota-axi` compatibility floor and quota snapshot schema validation | diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index 0f9c2e135ec..78e0675485c 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -32,7 +32,7 @@ Registered `state/procevent/*.source` records also require supervision even thou The default cross-harness mode exits silently with no supervision need. Every mode treats `state/x-watch.check.sh` as supervision need, so Relay polling remains guarded without an in-flight task. A custom check registered with `bin/fm-check-register.sh` counts the same way, so an operator's home-level poll keeps running after the last task is torn down. -Live-gated queued backlog work counts too, so a home whose only remaining work waits on a hold date or on blockers that can close on their own keeps the watcher whose ready-work scan will surface it; `bin/fm-ready-work.sh` owns which gates are live, and an undated hold never counts. +Live-gated queued backlog work counts too; [`bin/fm-ready-work.sh`](../bin/fm-ready-work.sh) owns which gates keep a watcher running. Otherwise it calls `fm_watcher_healthy [grace-seconds] [home]` from `bin/fm-wake-lib.sh`, the same PID-strict identity-matched lock and fresh-beacon check used by `bin/fm-watch-arm.sh`: a stale beacon blocks even when a watcher pid is live, and a fresh leftover beacon blocks when the lock is missing, dead, or identity-mismatched. The turn-end guard needs that strict check because it fires at the turn boundary, where the auto-arm is bringing a fresh watcher up for the upcoming idle period, and it cooperates with that arm rather than trusting a beacon left by the cycle that just ended. When an active home instead has a live session lock held by a verified harness that the current session does not own, the Claude guard emits a read-only ownership diagnostic and allows the turn to end safely. @@ -177,7 +177,7 @@ That warning uses `bin/fm-supervision-instructions.sh --repair-line`, so it alwa ## Compatibility limits - Child crewmate and scout worktrees are outside scope. -- A valid secondmate home is in scope; an idle secondmate endpoint with no Relay poll remains healthy because it has no supervision need. +- A valid secondmate home is in scope; an idle secondmate endpoint with no Relay poll or live-gated backlog work remains healthy when it has no other supervision need. - The blocking and bounded-follow-up mechanisms are limited to the primary integrations listed above. - OpenCode headless mode and untrusted Grok project hooks remain fail-open at the host boundary. - Cursor's `stop` step does not fire in headless `cursor-agent -p`, the same class of limit as OpenCode headless; firstmate primaries run interactive. diff --git a/tests/fm-captain-hold-lifecycle.test.sh b/tests/fm-captain-hold-lifecycle.test.sh index a87dbaf9de1..5118f94d06f 100755 --- a/tests/fm-captain-hold-lifecycle.test.sh +++ b/tests/fm-captain-hold-lifecycle.test.sh @@ -855,6 +855,8 @@ test_answer_records_and_closes() { # as resolved everywhere. show=$(tasks_in "$home" show sample-guard-work --full) assert_contains "$show" "blocked: no" "the recorded answer did not release dependent work" + [ "$(grep -Fc 'check: ready-work: sample-guard-work' "$home/state/.wake-queue")" = 1 ] \ + || fail "the recorded answer did not queue exactly one dependent wake" run_captain "$home" verify "$id" >/dev/null \ || fail "an answered captain call did not satisfy the completion gate" json=$(run_bearings "$home") || fail "Bearings failed after the answer" diff --git a/tests/fm-ready-work.test.sh b/tests/fm-ready-work.test.sh index 7991c39eff7..e6219a519c3 100755 --- a/tests/fm-ready-work.test.sh +++ b/tests/fm-ready-work.test.sh @@ -191,13 +191,14 @@ test_live_gates_are_bounded() { pass "only gates that can clear on their own count as supervision need" } -test_watcher_wakes_once_for_a_cleared_blocker() { - local dir state fakebin out pid - dir=$(make_case watcher-ready) - state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" - mkdir -p "$dir/data" - cp "$ROOT/.tasks.toml" "$dir/.tasks.toml" - printf '%s\n' '# Backlog' '' '## In flight' '' '## Queued' '' '## Done' > "$dir/data/backlog.md" +test_watcher_wakes_once_for_a_cleared_blocker() ( + local dir state socket_dir out pid + dir=$(make_home watcher-ready) + state="$dir/state"; socket_dir="$dir/tmux"; out="$dir/watch.out" + mkdir -p "$socket_dir" + TMUX_TMPDIR="$socket_dir" TMUX= tmux new-session -d -s fm-ready-work-test \ + || fail "could not start an isolated tmux server" + trap 'TMUX_TMPDIR="$socket_dir" TMUX= tmux kill-server >/dev/null 2>&1 || true' EXIT axi "$dir" add blocker "the blocker" axi "$dir" add dependent "the dependent" axi "$dir" block dependent --by blocker @@ -206,7 +207,7 @@ test_watcher_wakes_once_for_a_cleared_blocker() { external_axi "$dir" "done" blocker [ ! -s "$state/.wake-queue" ] || fail "setup queued a wake before the watcher: $(cat "$state/.wake-queue"); $(FM_HOME="$dir" "$ROOT/bin/fm-tasks-axi.sh" list --fields blocked,blocked_by,held,hold_until)" - PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + TMUX_TMPDIR="$socket_dir" TMUX= FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 100 || { reap "$pid"; fail "the watcher did not wake for newly ready work"; } @@ -217,7 +218,7 @@ test_watcher_wakes_once_for_a_cleared_blocker() { ack_handled_wakes "$state" || fail "the ready-work wake could not be drained and acknowledged" : > "$out" - PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + TMUX_TMPDIR="$socket_dir" TMUX= FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! rm -f "$state/.last-ready-scan" @@ -226,7 +227,7 @@ test_watcher_wakes_once_for_a_cleared_blocker() { reap "$pid" [ ! -s "$out" ] || fail "the second watcher printed a wake: $(cat "$out")" pass "a real watcher wakes once for work a blocker closed outside teardown made ready" -} +) reap() { kill "$1" 2>/dev/null || true; wait "$1" 2>/dev/null || true; } From 050d57dcc6825dbfb9648af9b6a274f518798980 Mon Sep 17 00:00:00 2001 From: knowttl Date: Wed, 23 Sep 2026 09:01:04 -0700 Subject: [PATCH 6/7] no-mistakes(lint): Fix ShellCheck warnings in ready-work tests --- tests/fm-ready-work.test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/fm-ready-work.test.sh b/tests/fm-ready-work.test.sh index e6219a519c3..6c8b2ad17b9 100755 --- a/tests/fm-ready-work.test.sh +++ b/tests/fm-ready-work.test.sh @@ -126,7 +126,7 @@ test_first_scan_and_source_close() { axi "$home" add blocker "a queued blocker" axi "$home" add dependent "dependent work" axi "$home" block dependent --by blocker - axi "$home" done blocker + axi "$home" "done" blocker grep -F 'check: ready-work: dependent' "$state/.wake-queue" >/dev/null \ || fail "closing a queued blocker did not queue a dependent wake" wake_ready "$home" @@ -196,9 +196,9 @@ test_watcher_wakes_once_for_a_cleared_blocker() ( dir=$(make_home watcher-ready) state="$dir/state"; socket_dir="$dir/tmux"; out="$dir/watch.out" mkdir -p "$socket_dir" - TMUX_TMPDIR="$socket_dir" TMUX= tmux new-session -d -s fm-ready-work-test \ + TMUX_TMPDIR="$socket_dir" TMUX='' tmux new-session -d -s fm-ready-work-test \ || fail "could not start an isolated tmux server" - trap 'TMUX_TMPDIR="$socket_dir" TMUX= tmux kill-server >/dev/null 2>&1 || true' EXIT + trap 'TMUX_TMPDIR="$socket_dir" TMUX="" tmux kill-server >/dev/null 2>&1 || true' EXIT axi "$dir" add blocker "the blocker" axi "$dir" add dependent "the dependent" axi "$dir" block dependent --by blocker @@ -207,7 +207,7 @@ test_watcher_wakes_once_for_a_cleared_blocker() ( external_axi "$dir" "done" blocker [ ! -s "$state/.wake-queue" ] || fail "setup queued a wake before the watcher: $(cat "$state/.wake-queue"); $(FM_HOME="$dir" "$ROOT/bin/fm-tasks-axi.sh" list --fields blocked,blocked_by,held,hold_until)" - TMUX_TMPDIR="$socket_dir" TMUX= FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + TMUX_TMPDIR="$socket_dir" TMUX='' FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 100 || { reap "$pid"; fail "the watcher did not wake for newly ready work"; } @@ -218,7 +218,7 @@ test_watcher_wakes_once_for_a_cleared_blocker() ( ack_handled_wakes "$state" || fail "the ready-work wake could not be drained and acknowledged" : > "$out" - TMUX_TMPDIR="$socket_dir" TMUX= FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + TMUX_TMPDIR="$socket_dir" TMUX='' FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! rm -f "$state/.last-ready-scan" From fd8c144cfdf892ae2f2445ee22868991b07aace0 Mon Sep 17 00:00:00 2001 From: knowttl Date: Wed, 23 Sep 2026 09:36:18 -0700 Subject: [PATCH 7/7] no-mistakes(ci): Fixed both lint failures by removing a redundant wake-library load and stopping ShellCheck from recursively analyzing the new ready-work import through watcher and supervision callers. The ready-work script remains linted as its own root. Local lint, source-aware checks, and ready-work behavior tests pass --- bin/fm-ready-work.sh | 1 - bin/fm-supervision-lib.sh | 2 +- bin/fm-watch.sh | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/fm-ready-work.sh b/bin/fm-ready-work.sh index ded70d84a03..30076720413 100755 --- a/bin/fm-ready-work.sh +++ b/bin/fm-ready-work.sh @@ -250,7 +250,6 @@ fm_ready_work_main() { state=${FM_STATE_OVERRIDE:-${FM_HOME:-$(cd "$FM_READY_WORK_DIR/.." && pwd)}/state} fm_ready_work_scan "$state" || return 0 if [ -n "$FM_READY_WORK_NEW" ]; then - . "$FM_READY_WORK_DIR/fm-wake-lib.sh" fm_wake_append check ready-work "check: ready-work: $FM_READY_WORK_NEW" || { fm_ready_work_release return 1 diff --git a/bin/fm-supervision-lib.sh b/bin/fm-supervision-lib.sh index 2add4ceb05a..c5f0edd49ed 100644 --- a/bin/fm-supervision-lib.sh +++ b/bin/fm-supervision-lib.sh @@ -12,7 +12,7 @@ # live watcher process means per supervision model. The status fields here retain # the beacon-age details used in their messages. -# shellcheck source=bin/fm-ready-work.sh +# shellcheck source=/dev/null . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/fm-ready-work.sh" # Portable mtime; Linux stat lacks -f, macOS stat lacks -c. diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 03846fb33c7..977453d96c7 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -193,7 +193,7 @@ mkdir -p "$STATE" # watcher reads only its presence (afk_record_present below). # shellcheck source=bin/fm-afk-contract.sh . "$SCRIPT_DIR/fm-afk-contract.sh" -# shellcheck source=bin/fm-ready-work.sh +# shellcheck source=/dev/null . "$SCRIPT_DIR/fm-ready-work.sh" WATCH_LOCK="$STATE/.watch.lock"