diff --git a/.gitignore b/.gitignore index 5ed2da0c32a..94fcdf88f97 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ projects/ state/ data/ +ops-inbox/ +status/ .no-mistakes/ .lavish/ .fm-secondmate-home @@ -16,3 +18,4 @@ config/backend config/x-mode.env config/cmux-socket-password config/wedge-alarm +config/ops-inbox-cmd diff --git a/AGENTS.md b/AGENTS.md index 973cdbc944a..2a55fb1f4f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,7 +81,9 @@ config/backlog-backend backlog backend override; LOCAL, gitignored; absent or " config/backend runtime session-provider backend override for new tasks; LOCAL, gitignored; absent = falls through to runtime auto-detection (the runtime firstmate itself is executing inside), then tmux; tmux is the verified reference backend (docs/tmux-backend.md), while herdr, zellij, orca, and cmux are experimental spawn backends (docs/herdr-backend.md, docs/zellij-backend.md, docs/orca-backend.md, docs/cmux-backend.md) - herdr and cmux can also be selected by runtime auto-detection, zellij and orca never are (always explicit), and codex-app is not accepted; see docs/codex-app-backend.md; not inherited into secondmate homes config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md +config/ops-inbox-cmd optional local list-only command for a machine-level operations inbox; absent leaves only this home's `ops-inbox/` directory; see docs/configuration.md config/x-mode.env generated X-mode watcher cadence; LOCAL, gitignored; source before arming watcher when present +ops-inbox/ local operational-failure event files; the bounded read-only digest and monitored layout are owned by docs/configuration.md data/ personal fleet records; LOCAL, gitignored as a whole backlog.md task queue, dependencies, history captain.md this home's domain-local captain preferences and working style; LOCAL, gitignored, canonical even if harness memory mirrors it, and updated with inspect-then-update @@ -140,7 +142,7 @@ It composes today's `fm-lock.sh`, `fm-bootstrap.sh`, and `fm-wake-drain.sh` - ca When the lock could not be acquired, the queue is left untouched because another session owns it, and the guard's tangle/watcher-liveness alarms still print in read-only advisory mode without drain, supervision repair, or checkout repair commands. 4. **Context digest** - the full contents of `data/projects.md`, `data/secondmates.md`, `data/captain.md`, and `data/learnings.md`, each clearly delimited. A file that does not exist prints an explicit `ABSENT` marker, never confused with an empty-but-present file: absence is meaningful (`captain.md` absent means use this template's defaults, `projects.md` absent means rebuild it from the clones under `projects/`, etc.). -5. **Fleet-state digest** - the full `data/backlog.md`; every `state/.meta`; a bounded tail of each task's `state/.status` (labeled as wake-EVENT history, not current state, with the full log path printed for a deeper read); the `state/.afk` flag; and one cheap alive/dead read of each task's recorded backend endpoint. +5. **Fleet-state digest** - the compact backlog listing owned by `bin/fm-session-start.sh`; every `state/.meta`; a bounded tail of each task's `state/.status` (labeled as wake-EVENT history, not current state, with the full log path printed for a deeper read); the `state/.afk` flag; bounded operations-inbox signals; and one cheap alive/dead read of each task's recorded backend endpoint. That liveness line is a fast presence check only, not a full state read - when you need a crew's actual current state (a run-step, not just "is the pane there"), read it with `bin/fm-crew-state.sh ` as before; the digest deliberately skips that deeper, slower read for every task so it stays fast and bounded. 6. **Supervision operating instructions and next step** - after the wake queue and before context, the digest emits exactly one operating block for the detected primary harness. The closing reminder points back to that emitted block and preserves only the lock, afk, X-mode, and read-once reminders. diff --git a/bin/fm-ops-inbox-lib.sh b/bin/fm-ops-inbox-lib.sh new file mode 100644 index 00000000000..406597f6c2d --- /dev/null +++ b/bin/fm-ops-inbox-lib.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# Shared read-only operations-inbox discovery for the session-start digest and +# watcher. The home directory is $FM_HOME/ops-inbox; an optional local +# config/ops-inbox-cmd supplies one prompt list-only command for a machine +# inbox. This file owns the config seam and fingerprint mechanics. + +fm_ops_inbox_stat_sig() { + if [ "$(uname)" = Darwin ]; then + stat -f '%z:%Fm' "$1" 2>/dev/null + else + stat -c '%s:%y' "$1" 2>/dev/null + fi +} + +fm_ops_inbox_stat_mtime() { + if [ "$(uname)" = Darwin ]; then + stat -f %m "$1" 2>/dev/null + else + stat -c %Y "$1" 2>/dev/null + fi +} + +fm_ops_inbox_hash() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 | awk '{print "sha256:" $1}' + elif command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print "sha256:" $1}' + else + cksum | awk '{print "cksum:" $1 ":" $2}' + fi +} + +fm_ops_inbox_home_dir() { + printf '%s/ops-inbox\n' "$1" +} + +# fm_ops_inbox_external_command +# Prints the first non-empty, non-comment config line. That line is an +# operator-owned list-only shell command, intentionally generic so tracked +# firstmate code does not know any machine-specific inbox location. +fm_ops_inbox_external_command() { + local config=$1 line path + path="$config/ops-inbox-cmd" + [ -f "$path" ] || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*) continue ;; + esac + printf '%s\n' "$line" + return 0 + done < "$path" + return 1 +} + +# fm_ops_inbox_external_output +# Prints the configured command's combined output and returns its exit status. +# A command may use a non-zero exit to signal unacknowledged criticals, so +# callers must inspect the output as well as this status. +fm_ops_inbox_external_output() { + local config=$1 command + command=$(fm_ops_inbox_external_command "$config") || return 127 + command -v perl >/dev/null 2>&1 || return 124 + fm_ops_inbox_external_run "$command" +} + +FM_OPS_INBOX_TIMEOUT=${FM_OPS_INBOX_TIMEOUT:-10} +case "$FM_OPS_INBOX_TIMEOUT" in ''|*[!0-9]*|0) FM_OPS_INBOX_TIMEOUT=10 ;; esac +FM_OPS_INBOX_OUTPUT_MAX_BYTES=${FM_OPS_INBOX_OUTPUT_MAX_BYTES:-32768} +case "$FM_OPS_INBOX_OUTPUT_MAX_BYTES" in ''|*[!0-9]*|0) FM_OPS_INBOX_OUTPUT_MAX_BYTES=32768 ;; esac +FM_OPS_INBOX_MARKER_LIMIT=${FM_OPS_INBOX_MARKER_LIMIT:-256} +case "$FM_OPS_INBOX_MARKER_LIMIT" in ''|*[!0-9]*|0) FM_OPS_INBOX_MARKER_LIMIT=256 ;; esac + +fm_ops_inbox_external_run() { + local command=$1 + perl -e ' + use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK); + use IO::Select; + use POSIX qw(WNOHANG); + use Time::HiRes qw(time); + + my ($timeout, $max, $command) = @ARGV; + pipe(my $reader, my $writer) or exit 124; + my $pid = fork; + exit 124 unless defined $pid; + if (!$pid) { + close $reader; + setpgrp(0, 0) or exit 124; + open STDOUT, ">&", $writer or exit 124; + open STDERR, ">&", $writer or exit 124; + close $writer; + exec "bash", "-c", $command; + exit 127; + } + + close $writer; + my $flags = fcntl($reader, F_GETFL, 0); + fcntl($reader, F_SETFL, $flags | O_NONBLOCK) or exit 124; + my $selector = IO::Select->new($reader); + my $deadline = time + $timeout; + my $kill_deadline; + my $capture_deadline; + my $eof = 0; + my $shell_done = 0; + my $shell_status = 124; + my $timed_out = 0; + my $capped = 0; + my $killed = 0; + my $written = 0; + + while (!$eof || !$shell_done) { + my $now = time; + if (!$timed_out && !$capped && $now >= $deadline) { + kill "TERM", -$pid; + $timed_out = 1; + $kill_deadline = $now + 0.2; + $capture_deadline = $kill_deadline + 0.1; + } + if (($timed_out || $capped) && !$killed && $now >= $kill_deadline) { + kill "KILL", -$pid; + $killed = 1; + } + if (defined $capture_deadline && $now >= $capture_deadline) { + $selector->remove($reader); + close $reader; + $eof = 1; + last; + } + + my $next = $deadline; + $next = $kill_deadline if defined $kill_deadline && $kill_deadline < $next; + $next = $capture_deadline if defined $capture_deadline && $capture_deadline < $next; + my $wait = $next - time; + $wait = 0 if $wait < 0; + $wait = 0.05 if $wait > 0.05; + for my $fh ($selector->can_read($wait)) { + my $read = sysread($fh, my $chunk, 8192); + if (!defined $read) { + next; + } + if ($read == 0) { + $selector->remove($fh); + close $fh; + $eof = 1; + next; + } + my $remaining = $max - $written; + if ($read > $remaining) { + print substr($chunk, 0, $remaining) if $remaining > 0; + $written += $remaining; + kill "TERM", -$pid; + $capped = 1; + $kill_deadline = time + 0.2; + $capture_deadline = $kill_deadline + 0.1; + next; + } + print $chunk; + $written += $read; + } + + if (!$shell_done && waitpid($pid, WNOHANG) == $pid) { + $shell_status = $?; + $shell_done = 1; + } + } + + exit 125 if $capped; + exit 124 if $timed_out; + exit(128 + ($shell_status & 127)) if $shell_status & 127; + exit($shell_status >> 8); + ' "$FM_OPS_INBOX_TIMEOUT" "$FM_OPS_INBOX_OUTPUT_MAX_BYTES" "$command" +} + +# fm_ops_inbox_home_records +# Prints newest-first mtime/path records from a bounded home-inbox scan. +# A final __FM_OPS_INBOX_OVERFLOW__ record means the scan limit was reached. +fm_ops_inbox_home_records() { + local home=$1 limit=$2 dir path mtime count=0 overflow=0 + local -a records=() + dir=$(fm_ops_inbox_home_dir "$home") + [ -d "$dir" ] || return 0 + while IFS= read -r -d '' path; do + if [ "$count" -ge "$limit" ]; then + overflow=1 + break + fi + mtime=$(fm_ops_inbox_stat_mtime "$path") || continue + records+=("$mtime"$'\t'"$path") + count=$((count + 1)) + done < <(find "$dir" -mindepth 1 -maxdepth 2 -type f -print0 2>/dev/null) + ((${#records[@]})) && printf '%s\n' "${records[@]}" | LC_ALL=C sort -rn + [ "$overflow" -eq 0 ] || printf '%s\n' '__FM_OPS_INBOX_OVERFLOW__' +} + +fm_ops_inbox_home_marker() { + local home=$1 dir path sig count=0 overflow=0 + dir=$(fm_ops_inbox_home_dir "$home") + [ -d "$dir" ] || return 0 + { + while IFS= read -r -d '' path; do + if [ "$count" -ge "$FM_OPS_INBOX_MARKER_LIMIT" ]; then + overflow=1 + break + fi + sig=$(fm_ops_inbox_stat_sig "$path") || continue + printf '%s\t%s\n' "$sig" "$path" + count=$((count + 1)) + done < <(find "$dir" -mindepth 1 -maxdepth 1 -print0 2>/dev/null) + [ "$overflow" -eq 0 ] || printf '__FM_OPS_INBOX_MARKER_OVERFLOW__:%s\n' "$FM_OPS_INBOX_MARKER_LIMIT" + } | LC_ALL=C sort +} + +fm_ops_inbox_home_has_events() { + local home=$1 dir + dir=$(fm_ops_inbox_home_dir "$home") + [ -d "$dir" ] || return 1 + [ -n "$(find "$dir" -mindepth 1 -maxdepth 2 -type f -print -quit 2>/dev/null)" ] +} + +# fm_ops_inbox_has_events +# The configured list-command contract starts with `unacked_criticals: `. +# A malformed or failed configured command is treated as an event so its one +# durable wake cannot be hidden by a bad local seam. +fm_ops_inbox_has_events() { + local home=$1 config=$2 output rc count + fm_ops_inbox_home_has_events "$home" && return 0 + fm_ops_inbox_external_command "$config" >/dev/null || return 1 + output=$(fm_ops_inbox_external_output "$config") + rc=$? + count=$(printf '%s\n' "$output" | awk '/^unacked_criticals:[[:space:]]*[0-9]+$/ { sub(/^unacked_criticals:[[:space:]]*/, ""); print; exit }') + case "$count" in + ''|*[!0-9]*) return 0 ;; + 0) [ "$rc" -eq 0 ] && return 1; return 0 ;; + *) return 0 ;; + esac +} + +# fm_ops_inbox_fingerprint +# Hashes local directory markers plus the configured external list output. The +# fingerprint is safe to persist in state/.hash-ops-inbox as the watcher's +# suppressor. +fm_ops_inbox_fingerprint() { + local home=$1 config=$2 command output rc + { + printf 'home\n' + fm_ops_inbox_home_marker "$home" + if command=$(fm_ops_inbox_external_command "$config"); then + printf 'external:configured:%s\n' "$command" + output=$(fm_ops_inbox_external_output "$config") + rc=$? + printf 'external:exit:%s\n%s\n' "$rc" "$output" + else + printf 'external:absent\n' + fi + } | fm_ops_inbox_hash +} diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index 418df43f15d..ef0c6eb2fd2 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -33,10 +33,13 @@ # 3. wake-drain - mutates the durable wake queue, so it also only runs # when locked. # 4. context digest - data/projects.md, data/secondmates.md, data/captain.md, -# data/learnings.md: read-only, always safe, always runs. -# 5. fleet digest - data/backlog.md, every state/*.meta, a bounded -# state/*.status tail, state/.afk, and a cheap -# per-task endpoint-liveness read: read-only, always runs. +# data/captain-shared.md, data/learnings.md: read-only, +# always safe, always runs. +# 5. fleet digest - a compact data/backlog.md identity/metadata listing, +# every state/*.meta, a bounded state/*.status tail, +# state/.afk, bounded operations-inbox signals, and a +# cheap per-task endpoint-liveness read: +# read-only, always runs. # 6. closing reminder - prints the context-specific watcher next step; this # script points back to the emitted harness supervision # block and deliberately never arms the watcher itself. @@ -81,9 +84,20 @@ PRIMARY_HARNESS=$("$SCRIPT_DIR/fm-harness.sh" 2>/dev/null || printf unknown) # shellcheck source=bin/fm-backend.sh . "$SCRIPT_DIR/fm-backend.sh" +# shellcheck source=bin/fm-tasks-axi-lib.sh +. "$SCRIPT_DIR/fm-tasks-axi-lib.sh" +# shellcheck source=bin/fm-ops-inbox-lib.sh +. "$SCRIPT_DIR/fm-ops-inbox-lib.sh" STATUS_TAIL=${FM_SESSION_START_STATUS_TAIL:-5} case "$STATUS_TAIL" in ''|*[!0-9]*) STATUS_TAIL=5 ;; esac +BACKLOG_LIMIT=${FM_SESSION_START_BACKLOG_LIMIT:-80} +case "$BACKLOG_LIMIT" in ''|*[!0-9]*|0) BACKLOG_LIMIT=80 ;; esac +OPS_INBOX_LIMIT=${FM_SESSION_START_OPS_INBOX_LIMIT:-5} +case "$OPS_INBOX_LIMIT" in ''|*[!0-9]*|0) OPS_INBOX_LIMIT=5 ;; esac +OPS_INBOX_SCAN_LIMIT=${FM_SESSION_START_OPS_INBOX_SCAN_LIMIT:-256} +case "$OPS_INBOX_SCAN_LIMIT" in ''|*[!0-9]*|0) OPS_INBOX_SCAN_LIMIT=256 ;; esac +[ "$OPS_INBOX_SCAN_LIMIT" -ge "$OPS_INBOX_LIMIT" ] || OPS_INBOX_SCAN_LIMIT=$OPS_INBOX_LIMIT RULE='================================================================================' SUBRULE='--------------------------------------------------------------------------------' @@ -117,6 +131,69 @@ print_status_tail() { tail -n "$STATUS_TAIL" "$status" } +print_ops_inbox() { + local dir record path output rc shown event_count overflow + local records + subsection "OPS INBOX" + dir=$(fm_ops_inbox_home_dir "$FM_HOME") + if [ ! -d "$dir" ]; then + printf 'home ops-inbox: ABSENT (%s)\n' "$dir" + else + records=$(fm_ops_inbox_home_records "$FM_HOME" "$OPS_INBOX_SCAN_LIMIT") + event_count=0 + overflow=0 + while IFS= read -r record; do + [ "$record" = '__FM_OPS_INBOX_OVERFLOW__' ] && { overflow=1; continue; } + [ -n "$record" ] && event_count=$((event_count + 1)) + done </dev/null; then + printf 'external inbox: ABSENT (config/ops-inbox-cmd)\n' + return + fi + output=$(fm_ops_inbox_external_output "$CONFIG") + rc=$? + printf 'external inbox: configured list command (exit %s); bounded to %s output line(s):\n' "$rc" "$OPS_INBOX_LIMIT" + if [ -z "$output" ]; then + printf '(no output)\n' + else + printf '%s\n' "$output" | head -n "$OPS_INBOX_LIMIT" + count=$(printf '%s\n' "$output" | awk 'END { print NR + 0 }') + [ "$count" -le "$OPS_INBOX_LIMIT" ] || printf '(truncated %s additional output line(s))\n' "$((count - OPS_INBOX_LIMIT))" + fi +} + hash_file() { local file=$1 [ -f "$file" ] || return 1 @@ -286,6 +363,8 @@ else printf 'absent\n' fi +print_ops_inbox + # --- 6. closing reminder ----------------------------------------------- section "NEXT STEP" if [ "$READ_ONLY" -eq 1 ]; then diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index bb79cda1b35..9843e99204c 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -68,6 +68,10 @@ mkdir -p "$STATE" # the herdr subscriber writes them (bin/fm-transition-lib.sh). # shellcheck source=bin/fm-transition-lib.sh . "$SCRIPT_DIR/fm-transition-lib.sh" +# shellcheck source=bin/fm-x-lib.sh +. "$SCRIPT_DIR/fm-x-lib.sh" +# shellcheck source=bin/fm-ops-inbox-lib.sh +. "$SCRIPT_DIR/fm-ops-inbox-lib.sh" WATCH_LOCK="$STATE/.watch.lock" WATCH_PATH="$SCRIPT_DIR/fm-watch.sh" @@ -491,6 +495,53 @@ heartbeat_scan_finds_actionable() { return 1 } +# Operations inboxes are external failure signals, not crew-status events. +# Poll their compact fingerprint every cycle while a regular task is in flight, +# then at the existing heartbeat cadence otherwise. A changed fingerprint is +# only marked seen after its durable wake record is appended, preventing both a +# missed event on interruption and a hot loop on an unchanged inbox. +ops_inbox_tasks_in_flight() { + local meta kind + for meta in "$STATE"/*.meta; do + [ -f "$meta" ] || continue + kind=$(grep '^kind=' "$meta" | cut -d= -f2- || true) + [ "$kind" = secondmate ] && continue + return 0 + done + return 1 +} + +ops_inbox_changed() { + local fingerprint previous + fingerprint=$(fm_ops_inbox_fingerprint "$FM_HOME" "${FM_CONFIG_OVERRIDE:-$FM_HOME/config}") + previous=$(cat "$STATE/.hash-ops-inbox" 2>/dev/null || true) + if [ -z "$previous" ]; then + # A watcher first armed against an empty inbox establishes its baseline + # silently. Existing events or a broken configured command still surface, + # while normal task tests and an empty new home do not manufacture a wake. + FM_OPS_INBOX_FINGERPRINT=$fingerprint + if ! fm_ops_inbox_has_events "$FM_HOME" "${FM_CONFIG_OVERRIDE:-$FM_HOME/config}"; then + mark_ops_inbox_seen + return 1 + fi + return 0 + fi + [ "$fingerprint" != "$previous" ] || return 1 + FM_OPS_INBOX_FINGERPRINT=$fingerprint + return 0 +} + +mark_ops_inbox_seen() { + printf '%s\n' "$FM_OPS_INBOX_FINGERPRINT" > "$STATE/.hash-ops-inbox" +} + +surface_ops_inbox_change() { + local reason='check: ops-inbox changed - inspect the OPS INBOX session-start digest' + fm_wake_append check ops-inbox "$reason" || exit 1 + mark_ops_inbox_seen + wake "$reason" +} + # event_wait_or_sleep: the terminal wait of each supervision cycle. For a home # with push-capable windows (herdr), it replaces the blind `sleep POLL` with a # bounded wait on the backend's native transition stream, so a crew going @@ -648,6 +699,10 @@ while :; do # alive. Supervision scripts warn when this goes stale with tasks in flight. touch "$STATE/.last-watcher-beat" + if ops_inbox_tasks_in_flight && ops_inbox_changed; then + surface_ops_inbox_change + fi + # Slow per-task checks (firstmate writes these, e.g. a merged-PR poll). # Time-based via .last-check mtime so the cadence survives watcher restarts. # Evaluated BEFORE the signal scan: wake() exits the cycle, so a check placed @@ -900,7 +955,10 @@ EOF # no-change case (advance the schedule and back off exactly as wake() would, # without exiting); the away-mode daemon, when present, owns triage and wants # every heartbeat. - if afk_present; then + if ! ops_inbox_tasks_in_flight && ops_inbox_changed; then + touch "$STATE/.last-heartbeat" + surface_ops_inbox_change + elif afk_present; then fm_wake_append heartbeat heartbeat heartbeat || exit 1 touch "$STATE/.last-heartbeat" wake "heartbeat" diff --git a/docs/architecture.md b/docs/architecture.md index f3fe5b7690b..482de048b95 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ firstmate's full operating manual for the orchestrator agent itself is [`AGENTS. ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. -Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, check-script output such as PR merge polling or an X-mode mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, changed operations inboxes, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. Repeated provably-working stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) before detector state advances, so a missed process exit can be recovered by draining the queue. No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step for that crew's branch or a backend busy signature. diff --git a/docs/configuration.md b/docs/configuration.md index 888f27ca30d..f550a543ffc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,6 +6,41 @@ The files and environment variables you set to operate firstmate. The shared orchestrator behavior lives in [`AGENTS.md`](../AGENTS.md) - edit it like any prompt when the fleet is empty, or dispatch shared-repo edits to a crewmate while tasks are in flight. +## Operational home layout and state + +This section is the single owner of the top-level operational-home layout; producer script headers and their help own exact child-file fields and mutation contracts. +The tracked code root contains the shared instruction, skill, documentation, workflow, and `bin/` surfaces, while each effective `FM_HOME` contains private operational directories. +`data/` holds durable private fleet records such as the project and secondmate registries, captain preferences, optional shared captain preferences, learnings, backlog, briefs, and scout reports. +`state/` holds volatile runtime records such as task metadata, append-only status events, endpoint signals, watcher and wake-queue coordination, away-mode state, and generated X-mode artifacts. +`ops-inbox/` holds local operational-failure event files delivered to this home. +`config/` holds local gitignored operating choices, and `projects/` holds the local project clones that Firstmate reads but changes only through the guarded exceptions in `AGENTS.md`. + +`bin/fm-spawn.sh` owns the base task-metadata fields it emits, while the runtime-backend section below owns backend-specific fields and selector interpretation. +The producing PR and X helpers own the fields they append, `bin/fm-classify-lib.sh` owns status-event vocabulary, and `bin/fm-crew-state.sh` owns current-state reconciliation. +Wake, watcher, away-mode, and X-specific state mechanics remain with their named scripts and reference sections rather than being duplicated into one exhaustive state tree here. + +`bin/fm-session-start.sh`'s header is the single owner of session-start ordering, composed commands, digest contents, and startup mechanism. +`AGENTS.md` retains the run-once and read-once operator rules, lock-refusal safety, installation consent, and direct-report recovery boundaries because those facts apply at every session start. +Ordinary dead-direct-report recovery is owned by `stuck-crewmate-recovery`, while persistent-secondmate recovery is owned by `secondmate-provisioning`. + +## Operations inbox (ops-inbox/ / config/ops-inbox-cmd) + +Each home may receive operational-failure event files directly in its local `ops-inbox/` directory or one source directory below it (`ops-inbox//`). +Write each event once or atomically replace it so the watcher can detect the event or source marker without rescanning retained event files. +Deeper paths are outside the monitored layout. +`bin/fm-session-start.sh` reports a bounded count and newest full paths from that directory without changing any event or acknowledgement state. +Set the local, gitignored `config/ops-inbox-cmd` to one list-only shell command when this machine also has a durable machine-level inbox. +The first non-empty, non-comment line is the command, and firstmate runs it through `bash -c` with combined stdout and stderr. +The command must be trusted local code and print only its current unhandled critical listing, starting with `unacked_criticals: `; its exit status is displayed but does not suppress its output because some list commands use a non-zero status when criticals exist. +The command is intentionally operator-owned and generic, so firstmate does not encode a machine-specific inbox path or acknowledgement implementation. +The watcher fingerprints both sources, wakes immediately for a changed inbox while a regular task is in flight, and checks the same fingerprint on its existing heartbeat cadence otherwise. +`FM_SESSION_START_OPS_INBOX_LIMIT` bounds both the home-event paths and configured-command output lines in the digest, defaulting to 5. +`FM_SESSION_START_OPS_INBOX_SCAN_LIMIT` bounds retained home-event records inspected at startup, defaulting to 256, and reports an explicit sampled overflow when reached. +`FM_OPS_INBOX_TIMEOUT` bounds each configured command invocation to 10 seconds by default. +`FM_OPS_INBOX_OUTPUT_MAX_BYTES` bounds each configured command capture to 32768 bytes by default. +`FM_OPS_INBOX_MARKER_LIMIT` bounds the top-level entries tracked by each watcher fingerprint, defaulting to 256. +When that limit is exceeded, the fingerprint records an overflow sentinel and the inbox must be retained below the limit before individual changes can be surfaced again. + ## Backlog backend (.tasks.toml / config/backlog-backend) The tracked `.tasks.toml` pins the default `tasks-axi` markdown backend to `data/backlog.md`, with `done_keep = 10` and an archive at `data/done-archive.md`. @@ -341,6 +376,8 @@ FM_BACKEND_CMUX_COMPOSER_LINES=20 # cmux-only: tail lines scanned to locate the FM_BACKEND_CMUX_IDLE_RE='^Type a message\.\.\.$' # cmux-only: empty-composer placeholder regex after border/prompt stripping CMUX_SOCKET_PASSWORD= # cmux-only: socket password fallback when config/cmux-socket-password is absent (docs/cmux-backend.md) FM_SESSION_START_STATUS_TAIL=5 # state/*.status lines printed per task in the session-start digest +FM_SESSION_START_OPS_INBOX_LIMIT=5 # home-event paths and external-command output lines printed in the operations-inbox digest +FM_SESSION_START_OPS_INBOX_SCAN_LIMIT=256 # home-event records inspected for the bounded operations-inbox startup digest FM_BOOTSTRAP_DETECT_ONLY=0 # internal/read-only session-start mode: skip bootstrap's mutating sweeps and print advisory TANGLE wording FM_GUARD_READ_ONLY=0 # internal/read-only guard mode: keep alarms but suppress drain, supervision repair, and checkout repair commands FM_GUARD_CONTINUE_LINE='This is a supervision warning only; the guarded operation WILL still run.' # banner continuation line; fm-send.sh overrides it to name the requested message specifically @@ -349,6 +386,9 @@ FM_HEARTBEAT=600 # base seconds between heartbeat scans; no-change heartb FM_HEARTBEAT_MAX=7200 # heartbeat backoff cap FM_CHECK_INTERVAL=300 # seconds between slow checks (merge polls or the X-mode poll shim) FM_CHECK_TIMEOUT=30 # seconds allowed per slow check script +FM_OPS_INBOX_TIMEOUT=10 # seconds allowed for each configured operations-inbox command +FM_OPS_INBOX_OUTPUT_MAX_BYTES=32768 # byte cap for each configured operations-inbox command capture +FM_OPS_INBOX_MARKER_LIMIT=256 # top-level operations-inbox entries included in each watcher fingerprint FM_CODEX_WATCH_CHECKPOINT=180 # seconds per foreground watcher checkpoint in Codex primary supervision FM_CREW_STATE_NM_TIMEOUT=10 # seconds allowed per no-mistakes query inside fm-crew-state.sh FM_CREW_STATE_RUNS_LIMIT=200 # recent no-mistakes runs rows scanned when cross-branch attribution falls back from axi status diff --git a/docs/scripts.md b/docs/scripts.md index 3533442e438..909a73314dc 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -68,6 +68,7 @@ The shared no-mistakes gate refusal used by `fm-spawn.sh`, `fm-send.sh`, and `fm | `fm-teardown.sh` | Fail-closed teardown: return landed ship worktrees, require scout reports, retire secondmate homes | | `fm-harness.sh` | Detect the running harness and resolve crew or secondmate harness, model, and effort | | `fm-lock.sh` | Per-home firstmate session lock | +| `fm-ops-inbox-lib.sh` | Shared home and configured external operations-inbox digest and fingerprint helpers | | `fm-x-lib.sh` | Shared X-mode config, relay, and reply-threading helpers | | `fm-x-poll.sh` | One bounded X relay poll: stash pending mentions, print `x-mention ` | | `fm-x-reply.sh` | Post or dry-run preview a composed X-mode reply or follow-up | diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 6d3b0d97046..98780fcad1b 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -264,11 +264,11 @@ EOF assert_contains "$out" "data/secondmates.md" "digest did not label the secondmates.md section" assert_contains "$out" "data/learnings.md" "digest did not label the learnings.md section" - # Exactly two ABSENT markers (secondmates.md, learnings.md; backlog.md is - # covered by its own test) - and the present-but-empty captain.md must NOT - # print ABSENT. + # Exactly three standalone ABSENT markers (secondmates.md, learnings.md, + # backlog.md) - the absent home ops inbox is inline, and the present-but-empty + # captain.md must NOT print ABSENT. absent_count=$(printf '%s\n' "$out" | grep -c '^ABSENT$') - [ "$absent_count" -eq 3 ] || fail "expected 3 ABSENT markers (secondmates.md, learnings.md, backlog.md), got $absent_count: $out" + [ "$absent_count" -eq 3 ] || fail "expected 3 standalone ABSENT markers (secondmates.md, learnings.md, backlog.md), got $absent_count: $out" cap_section=$(printf '%s\n' "$out" | awk '/^data\/captain\.md$/{flag=1;next}/^data\//{flag=0}flag') assert_contains "$cap_section" "(present, empty)" "empty-but-present captain.md was not distinguished from ABSENT" @@ -276,6 +276,68 @@ EOF pass "context digest distinguishes ABSENT, empty-but-present, and populated files" } +# --- operations inbox digest: absent, bounded home paths, configured command -- + +test_ops_inbox_digest_absent_bounded_and_configured() { + local rec root home fakebin out external stat_log real_stat + rec=$(new_world ops-inbox-digest) + IFS='|' read -r root home fakebin < "$home/ops-inbox/hermes-runtime/old.event" + printf 'middle\n' > "$home/ops-inbox/hermes-runtime/middle.event" + printf 'new\n' > "$home/ops-inbox/hermes-runtime/new.event" + touch -t 202607171100 "$home/ops-inbox/hermes-runtime/old.event" + touch -t 202607171200 "$home/ops-inbox/hermes-runtime/middle.event" + touch -t 202607171300 "$home/ops-inbox/hermes-runtime/new.event" + external="$fakebin/external-ops-inbox" + cat > "$external" <<'SH' +#!/usr/bin/env bash +printf '%s\n' 'unacked_criticals: 3' 'critical-old' 'critical-new' +exit 1 +SH + chmod +x "$external" + printf '%s list\n' "$external" > "$home/config/ops-inbox-cmd" + + out=$(FM_SESSION_START_OPS_INBOX_LIMIT=2 run_session_start "$home" "$root" "$fakebin:$BASE_PATH") + assert_contains "$out" "home ops-inbox: 3 event file(s); newest 2 with full paths:" "home inbox count or bound missing" + assert_contains "$out" "$home/ops-inbox/hermes-runtime/new.event" "newest home inbox event path missing" + assert_contains "$out" "$home/ops-inbox/hermes-runtime/middle.event" "second-newest home inbox event path missing" + assert_not_contains "$out" "$home/ops-inbox/hermes-runtime/old.event" "bounded home inbox digest leaked oldest event path" + assert_contains "$out" "(truncated 1 older event file(s))" "home inbox truncation was not disclosed" + assert_contains "$out" "external inbox: configured list command (exit 1); bounded to 2 output line(s):" "external command status or bound missing" + assert_contains "$out" "unacked_criticals: 3" "configured external inbox output missing its critical count" + assert_contains "$out" "critical-old" "configured external inbox did not print bounded output" + assert_not_contains "$out" "critical-new" "configured external inbox output was not bounded" + assert_contains "$out" "(truncated 1 additional output line(s))" "external inbox truncation was not disclosed" + + stat_log="$home/stat.log" + real_stat=$(command -v stat) +cat > "$fakebin/stat" <> "$stat_log" ;; +esac +exec "$real_stat" "\$@" +SH + chmod +x "$fakebin/stat" + out=$(FM_SESSION_START_OPS_INBOX_LIMIT=2 FM_SESSION_START_OPS_INBOX_SCAN_LIMIT=2 run_session_start "$home" "$root" "$fakebin:$BASE_PATH") + assert_contains "$out" "home ops-inbox: at least 3 event file(s); bounded scan reached 2; newest 2 sampled paths:" "bounded inbox scan did not disclose overflow" + assert_contains "$out" "(scan stopped after 2 event file(s); retained inbox exceeds the bounded startup scan)" "bounded inbox scan did not disclose retained overflow" + [ "$(wc -l < "$stat_log" | tr -d '[:space:]')" -eq 2 ] || fail "bounded inbox scan statted more than its configured record limit" + + pass "OPS INBOX digest distinguishes absent sources and bounds home and configured-inbox scans" +} + # --- lock refusal: read-only path -------------------------------------------- test_lock_refusal_read_only_path() { @@ -741,6 +803,7 @@ EOF } test_context_digest_absent_empty_present +test_ops_inbox_digest_absent_bounded_and_configured test_lock_refusal_read_only_path test_output_ordering_diagnostics_lead test_herdr_backend_diagnostics_follow_real_session_start diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 22ff02bffe1..59a37011ba1 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -21,6 +21,7 @@ set -u . "$(dirname "${BASH_SOURCE[0]}")/wake-helpers.sh" # shellcheck source=bin/fm-classify-lib.sh . "$ROOT/bin/fm-classify-lib.sh" +. "$ROOT/bin/fm-ops-inbox-lib.sh" WATCH="$ROOT/bin/fm-watch.sh" DRAIN="$ROOT/bin/fm-wake-drain.sh" @@ -1176,6 +1177,195 @@ test_heartbeat_backstop_surfaces_unsurfaced_status() { pass "heartbeat backstop fail-safe surfaces a captain-relevant status the per-wake path missed" } +# --- operations inbox: task-poll and heartbeat wake classification ----------- + +seed_ops_inbox_fingerprint() { # + local home=$1 state=$2 fingerprint + fingerprint=$(bash -c '. "$1"; fm_ops_inbox_fingerprint "$2" "$3"' _ \ + "$ROOT/bin/fm-ops-inbox-lib.sh" "$home" "$home/config") \ + || fail "could not seed operations-inbox fingerprint" + printf '%s\n' "$fingerprint" > "$state/.hash-ops-inbox" +} + +test_ops_inbox_new_event_wakes_with_task_in_flight() { + local dir state fakebin out home pid + dir=$(make_case ops-inbox-task); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + home="$dir/home" + mkdir -p "$home/ops-inbox" "$home/config" + printf 'project=firstmate\nkind=ship\n' > "$state/ops.meta" + seed_ops_inbox_fingerprint "$home" "$state" + + PATH="$fakebin:$PATH" FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_STATE_OVERRIDE="$state" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_live "$pid" 15 || { reap "$pid"; fail "watcher exited before an operations-inbox event landed: $(cat "$out")"; } + printf 'P1 failure\n' > "$home/ops-inbox/new.event" + wait_for_exit "$pid" 40 || fail "watcher did not wake for a new operations-inbox event while a task was in flight" + grep -Fx 'check: ops-inbox changed - inspect the OPS INBOX session-start digest' "$out" >/dev/null \ + || fail "operations-inbox task wake did not use the actionable check classification: $(cat "$out")" + grep "$(printf '\tcheck\tops-inbox\t')" "$state/.wake-queue" >/dev/null \ + || fail "operations-inbox task wake was not durably queued as a check" + + pass "a new operations-inbox event wakes immediately with a task in flight" +} + +test_ops_inbox_new_event_wakes_on_heartbeat_without_tasks() { + local dir state fakebin out home pid + dir=$(make_case ops-inbox-heartbeat); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + home="$dir/home" + mkdir -p "$home/ops-inbox" "$home/config" + seed_ops_inbox_fingerprint "$home" "$state" + + PATH="$fakebin:$PATH" FM_HOME="$home" FM_CONFIG_OVERRIDE="$home/config" FM_STATE_OVERRIDE="$state" \ + FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 FM_HEARTBEAT_MAX=1 "$WATCH" > "$out" & + pid=$! + wait_live "$pid" 15 || { reap "$pid"; fail "watcher exited before heartbeat operations-inbox event: $(cat "$out")"; } + printf 'P0 failure\n' > "$home/ops-inbox/new.event" + wait_for_exit "$pid" 40 || fail "watcher did not wake for a new operations-inbox event on its heartbeat scan" + grep -Fx 'check: ops-inbox changed - inspect the OPS INBOX session-start digest' "$out" >/dev/null \ + || fail "operations-inbox heartbeat wake did not use the actionable check classification: $(cat "$out")" + grep "$(printf '\tcheck\tops-inbox\t')" "$state/.wake-queue" >/dev/null \ + || fail "operations-inbox heartbeat wake was not durably queued as a check" + + pass "a new operations-inbox event wakes on heartbeat when no task is in flight" +} + +test_ops_inbox_fingerprint_distinguishes_same_second_same_size_rewrite() { + local dir home before after + dir=$(make_case ops-inbox-subsecond); home="$dir/home" + mkdir -p "$home/ops-inbox" "$home/config" + printf 'first\n' > "$home/ops-inbox/event" + before=$(fm_ops_inbox_fingerprint "$home" "$home/config") + printf 'other\n' > "$home/ops-inbox/event" + after=$(fm_ops_inbox_fingerprint "$home" "$home/config") + [ "$before" != "$after" ] || fail "same-size operations-inbox rewrite did not change its fingerprint" + pass "operations-inbox fingerprints retain sub-second rewrite resolution" +} + +test_ops_inbox_fingerprint_uses_bounded_one_level_markers() { + local dir home fakebin find_log find_count real_find before repeat after marker + dir=$(make_case ops-inbox-directory-marker); home="$dir/home"; fakebin="$dir/fakebin" + find_log="$dir/find.log"; find_count="$dir/find-count"; real_find=$(command -v find) + mkdir -p "$home/ops-inbox/source" "$home/config" + printf 'first\n' > "$home/ops-inbox/source/event" + cat > "$fakebin/find" <> "\$FM_OPS_INBOX_FIND_LOG" +case " \$* " in + *' -mindepth 1 -maxdepth 1 -print0 '*) + count=\$(cat "\$FM_OPS_INBOX_FIND_COUNT" 2>/dev/null || echo 0) + count=\$((count + 1)) + printf '%s\\n' "\$count" > "\$FM_OPS_INBOX_FIND_COUNT" + if [ "\$count" -eq 2 ]; then + "$real_find" "\$@" | perl -0 -e 'print for reverse <>' + exit "\${PIPESTATUS[0]}" + fi + ;; +esac +exec "$real_find" "\$@" +SH + chmod +x "$fakebin/find" + before=$(PATH="$fakebin:$PATH" FM_OPS_INBOX_FIND_LOG="$find_log" FM_OPS_INBOX_FIND_COUNT="$find_count" fm_ops_inbox_fingerprint "$home" "$home/config") + repeat=$(PATH="$fakebin:$PATH" FM_OPS_INBOX_FIND_LOG="$find_log" FM_OPS_INBOX_FIND_COUNT="$find_count" fm_ops_inbox_fingerprint "$home" "$home/config") + [ "$before" = "$repeat" ] || fail "directory traversal order changed the operations-inbox fingerprint" + printf 'second\n' > "$home/ops-inbox/source/new-event" + after=$(PATH="$fakebin:$PATH" FM_OPS_INBOX_FIND_LOG="$find_log" FM_OPS_INBOX_FIND_COUNT="$find_count" fm_ops_inbox_fingerprint "$home" "$home/config") + [ "$before" != "$after" ] || fail "one-level operations-inbox event did not change its directory-marker fingerprint" + grep -F -- '-mindepth 1 -maxdepth 1 -print0' "$find_log" >/dev/null || fail "fingerprint did not restrict markers to top-level entries" + [ "$(wc -l < "$find_log" | tr -d '[:space:]')" -eq 3 ] || fail "fingerprint performed unexpected operations-inbox scans" + + mkdir -p "$home/ops-inbox/overflow-a" "$home/ops-inbox/overflow-b" "$home/ops-inbox/overflow-c" + marker=$(FM_OPS_INBOX_MARKER_LIMIT=2 fm_ops_inbox_home_marker "$home") + printf '%s\n' "$marker" | grep -Fx '__FM_OPS_INBOX_MARKER_OVERFLOW__:2' >/dev/null \ + || fail "bounded operations-inbox marker did not disclose overflow" + [ "$(printf '%s\n' "$marker" | awk 'END { print NR + 0 }')" -eq 3 ] \ + || fail "bounded operations-inbox marker retained more than its limit" + pass "operations-inbox fingerprints use bounded one-level markers" +} + +test_ops_inbox_external_output_is_bounded_and_timed() { + local dir home command output rc bytes started elapsed escaped_pid + dir=$(make_case ops-inbox-bounded-command); home="$dir/home"; command="$dir/external-inbox" + mkdir -p "$home/config" + cat > "$command" <<'SH' +#!/usr/bin/env bash +while :; do printf '0123456789abcdef'; done +SH + chmod +x "$command" + printf '%s\n' "$command" > "$home/config/ops-inbox-cmd" + output=$(FM_OPS_INBOX_OUTPUT_MAX_BYTES=128 fm_ops_inbox_external_output "$home/config") + rc=$? + bytes=$(printf '%s' "$output" | LC_ALL=C wc -c | tr -d '[:space:]') + [ "$bytes" -le 128 ] || fail "external operations-inbox output exceeded its byte cap ($bytes)" + [ "$rc" -ne 0 ] || fail "capped external operations-inbox command unexpectedly succeeded" + + cat > "$command" <<'SH' +#!/usr/bin/env bash +printf 'external failure\n' +exit 42 +SH + chmod +x "$command" + output=$(fm_ops_inbox_external_output "$home/config") + rc=$? + [ "$output" = 'external failure' ] || fail "external operations-inbox command lost its output" + [ "$rc" -eq 42 ] || fail "external operations-inbox command returned $rc, expected 42" + + cat > "$command" <<'SH' +#!/usr/bin/env bash +kill -TERM "$$" +SH + chmod +x "$command" + output=$(fm_ops_inbox_external_output "$home/config") + rc=$? + [ -z "$output" ] || fail "signalled external operations-inbox command produced unexpected output" + [ "$rc" -eq 143 ] || fail "signalled external operations-inbox command returned $rc, expected 143" + + cat > "$command" <<'SH' +#!/usr/bin/env bash +sleep 3 +SH + chmod +x "$command" + started=$SECONDS + output=$(FM_OPS_INBOX_TIMEOUT=1 fm_ops_inbox_external_output "$home/config") + rc=$? + elapsed=$((SECONDS - started)) + [ -z "$output" ] || fail "timed external operations-inbox command produced unexpected output" + [ "$rc" -eq 124 ] || fail "timed external operations-inbox command returned $rc, expected 124" + [ "$elapsed" -lt 3 ] || fail "timed external operations-inbox command exceeded its deadline (${elapsed}s)" + + cat > "$command" <<'SH' +#!/usr/bin/env bash +sleep 3 & +SH + chmod +x "$command" + started=$SECONDS + output=$(FM_OPS_INBOX_TIMEOUT=1 fm_ops_inbox_external_output "$home/config") + rc=$? + elapsed=$((SECONDS - started)) + [ -z "$output" ] || fail "background-child external command produced unexpected output" + [ "$rc" -eq 124 ] || fail "background-child external command returned $rc, expected 124" + [ "$elapsed" -lt 3 ] || fail "background-child external command bypassed its deadline (${elapsed}s)" + + escaped_pid="$dir/escaped.pid" + cat > "$command" < "$escaped_pid" +SH + chmod +x "$command" + started=$SECONDS + output=$(FM_OPS_INBOX_TIMEOUT=1 fm_ops_inbox_external_output "$home/config") + rc=$? + elapsed=$((SECONDS - started)) + [ -z "$output" ] || fail "setsid-child external command produced unexpected output" + [ "$rc" -eq 124 ] || fail "setsid-child external command returned $rc, expected 124" + [ "$elapsed" -lt 3 ] || fail "setsid-child external command bypassed the parent capture deadline (${elapsed}s)" + kill "$(cat "$escaped_pid")" 2>/dev/null || true + pass "external operations-inbox commands have bounded output and runtime" +} + # --- beacon stays fresh while absorbing ------------------------------------- test_beacon_stays_fresh_while_absorbing() { @@ -1299,6 +1489,11 @@ test_nonterminal_stale_repairs_missing_or_corrupt_timer test_triage_log_size_cap_accepts_spaced_wc_counts test_heartbeat_no_change_absorbed test_heartbeat_backstop_surfaces_unsurfaced_status +test_ops_inbox_new_event_wakes_with_task_in_flight +test_ops_inbox_new_event_wakes_on_heartbeat_without_tasks +test_ops_inbox_fingerprint_distinguishes_same_second_same_size_rewrite +test_ops_inbox_fingerprint_uses_bounded_one_level_markers +test_ops_inbox_external_output_is_bounded_and_timed test_beacon_stays_fresh_while_absorbing test_afk_present_reverts_watcher_to_one_shot test_afk_paused_changed_pane_hands_off_plain_stale