diff --git a/.agents/skills/operational-home-layout/SKILL.md b/.agents/skills/operational-home-layout/SKILL.md index 522ac837f7a..66bfb28ac41 100644 --- a/.agents/skills/operational-home-layout/SKILL.md +++ b/.agents/skills/operational-home-layout/SKILL.md @@ -24,6 +24,7 @@ config/crew-harness crewmate harness override; LOCAL, gitignored; absent or "de config/claude-permission-mode optional one-token permission posture for every Claude worker launch: absent or "bypass" keeps --dangerously-skip-permissions, "auto" launches with --permission-mode auto; LOCAL, gitignored; inherited by secondmate homes; see docs/configuration.md "Claude permission mode" config/claude-account config/pi-account optional per-home worker account pin for Claude and Pi launches; LOCAL, gitignored, not inherited; absent keeps today's ambient account; present refuses a launch unless the pinned account resolves and is signed in (section 4 owns the refusal rule); see docs/configuration.md "Worker account pin" config/crew-dispatch.json optional crewmate dispatch profiles; LOCAL, gitignored; firstmate-maintained but human-editable natural-language rules that choose a per-task harness/model/effort profile (section 4). Inherited by secondmate homes +config/project-capacity optional per-machine count of workers each named project admits at once, read from the root home by every local home; LOCAL, gitignored; see docs/configuration.md "Project capacity" config/secondmate-harness harness the PRIMARY uses to launch SECONDMATE agents, optionally followed by a model and effort token on the same line (" [] []"; section 4); LOCAL, gitignored; absent or "default" harness falls back to config/crew-harness then firstmate's own. The primary's own setting; NOT inherited into secondmate homes (secondmates do not spawn secondmates) config/backlog-backend backlog backend override; LOCAL, gitignored; absent or "tasks-axi" = the configured tasks-axi backend, "manual" = force routine backlog updates to hand-editing; inherited by secondmate homes (section 10) config/backend runtime session-provider backend override for new tasks; LOCAL, gitignored; absent = falls through to runtime auto-detection (the runtime firstmate itself is executing inside), then tmux; tmux is the verified reference backend (docs/tmux-backend.md), herdr has its own required CI lane (docs/herdr-backend.md), while zellij, orca, and cmux remain experimental with no dedicated real-backend CI lane (docs/zellij-backend.md, docs/orca-backend.md, docs/cmux-backend.md) - herdr and cmux can also be selected by runtime auto-detection, zellij and orca never are (always explicit), and codex-app is not accepted; see docs/codex-app-backend.md; inherited by secondmate homes under the primary-authoritative contract in secondmate-provisioning @@ -92,6 +93,7 @@ state/ runtime records and signals; gitignored tool-updates.check.sh generated watched-tool update poll shim and its .check-trust binding; present only after bin/fm-tool-update-check.sh arm; its report record .tool-updates is what keeps one pending update from being reported on every poll mail.check.sh generated received-mail poll shim and its .check-trust binding; present only after bin/fm-mail-check.sh arm; report record .mail-check (mail schema: docs/configuration.md "Mail plane") .mail-seen .mail-woken .mail-retry .mail-retry-pos .mail-turn .mail-seen.lock mail-plane poll cursor, emission journal, transient-fetch retry set, retry-scan position, contended-slot turn flag, and overlapping-poll lock; written only by bin/fm-mail.sh (mail schema: docs/configuration.md "Mail plane") + startup-growth.check.sh generated daily startup-growth poll shim and its .check-trust binding; present only after bin/fm-startup-growth-check.sh arm; its record .startup-growth-check holds the daily gate, the per-file growth baselines, and the last reported finding set, so removing it re-baselines growth silently and repeats a standing finding such as a budget overrun once (docs/configuration.md "Daily startup growth check") pending-replies/ parent-owned secondmate pending-reply records (correlation id, delivery vs reply, recovery, escalation); fm-pending-reply-lib.sh procevent/ registered process-to-event sources, one private record per canonical source id; written only by bin/fm-procevent.sh, and their presence alone keeps supervision required (`process-event-sources` skill) procevent-inbox/ private captured results and their durable handled-acknowledgement markers; source output lives here and never in an event line diff --git a/.agents/skills/project-management/SKILL.md b/.agents/skills/project-management/SKILL.md index d68d5195330..6c39cc871ed 100644 --- a/.agents/skills/project-management/SKILL.md +++ b/.agents/skills/project-management/SKILL.md @@ -84,7 +84,7 @@ The captain's request to create that local project authorizes this local initial Run no-mistakes initialization only for `no-mistakes` and `no-mistakes-prod-only` projects: ```sh -cd projects/ && no-mistakes init && no-mistakes doctor +(cd projects/ && no-mistakes init && no-mistakes doctor) ``` Initialization configures the local gate and does not vendor a no-mistakes skill into the project. diff --git a/.pi/extensions/fm-calm.ts b/.pi/extensions/fm-calm.ts index 3070cb1e826..1c37d179bda 100644 --- a/.pi/extensions/fm-calm.ts +++ b/.pi/extensions/fm-calm.ts @@ -136,6 +136,7 @@ export default function (pi: ExtensionAPI) { // continuations, retries, or compaction that stay inside the same run. let agentRunActive = false; let workingShipShown = false; + let workingShipWidgetDisposed = false; // One animation instance per extension lifetime. Hiding the working widget freezes // this state; the next working period resumes it. session_start resets it so a fresh // Pi session starts at the normal initial position. Never module-global. @@ -143,6 +144,8 @@ export default function (pi: ExtensionAPI) { // Single owner of Calm's working-row presentation choice. The widget is only created // or removed on a real transition, so repeated starts cannot duplicate its timer. + // The slot is shared with standalone Pi Calm; the dispose signal prevents turning + // Firstmate Calm off from clearing a widget that the other extension installed. const applyWorkingPresentation = ( ui: ExtensionUIContext, forceStockVisibility = false, @@ -150,14 +153,23 @@ export default function (pi: ExtensionAPI) { const showShip = agentRunActive && calmPresentationIsActive(); if (showShip !== workingShipShown) { workingShipShown = showShip; - ui.setWidget( - CALM_WORKING_SHIP_WIDGET_KEY, - showShip - ? (tui) => createCalmWorkingShipWidget(tui, workingShipAnimation) - : undefined, - ); - ui.setWorkingVisible(!showShip); - } else if (forceStockVisibility && !showShip) { + if (showShip) { + ui.setWidget(CALM_WORKING_SHIP_WIDGET_KEY, (tui) => { + workingShipWidgetDisposed = false; + const widget = createCalmWorkingShipWidget(tui, workingShipAnimation); + const dispose = widget.dispose; + widget.dispose = () => { + workingShipWidgetDisposed = true; + dispose(); + }; + return widget; + }); + ui.setWorkingVisible(false); + } else if (!workingShipWidgetDisposed) { + ui.setWidget(CALM_WORKING_SHIP_WIDGET_KEY, undefined); + ui.setWorkingVisible(true); + } + } else if (forceStockVisibility && !showShip && !workingShipWidgetDisposed) { ui.setWorkingVisible(true); } }; diff --git a/.pi/extensions/lib/fm-calm-working-ship.ts b/.pi/extensions/lib/fm-calm-working-ship.ts index d8f8ede4695..2b28633bfa0 100644 --- a/.pi/extensions/lib/fm-calm-working-ship.ts +++ b/.pi/extensions/lib/fm-calm-working-ship.ts @@ -44,7 +44,12 @@ const ANSI_FOREGROUND: Record, string> = // Restores the default foreground so color never bleeds into padding or later frames. const RESET = "\u001b[39m"; -export const CALM_WORKING_SHIP_WIDGET_KEY = "firstmate-calm-working-ship"; +// The working-row widget slot is deliberately shared with the standalone Pi Calm +// extension, which installs its boat under the same "calm-working-ship" key. Pi +// replaces widgets under one key, so a session that loads both Calms renders a +// single boat and a session loading either alone is unchanged. Rename the slot +// in both implementations together, or dual-install sessions duplicate the boat. +export const CALM_WORKING_SHIP_WIDGET_KEY = "calm-working-ship"; export type CalmWorkingShipAnimation = Omit & { /** Render one frame that exactly fits `width`, clamping the track to it first. */ diff --git a/AGENTS.md b/AGENTS.md index fcda49000fa..dd1a994d5a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,6 @@ Tracked files hold shared instructions and tooling; `data/` holds durable privat Load `operational-home-layout` when locating, interpreting, or changing Firstmate home, config, data, state, project, or generated runtime paths. - A `state/.status` line is a wake event, not current-state truth; `bin/fm-crew-state.sh` owns current-state reconciliation. Treat `data/captain.md` as the domain-local record of captain preferences, optional `data/captain-shared.md` as the main-authoritative shared captain-preference file for secondmate inheritance, and `data/learnings.md` as curated home-local knowledge, regardless of harness memory. @@ -197,6 +196,7 @@ An unregistered project or absent registry resolves to `no-mistakes` with yolo o Record the resulting mode, `yolo` merge posture, and the one-line reason for any deviation in the backlog item note. Treat file or subsystem overlap as a risk signal rather than an automatic reason to wait, and dispatch isolated work immediately with no concurrency cap when each change can be independently implemented and validated and the selected delivery path can reconcile ordinary rebases or conflicts. +A project's declared machine capacity (`config/project-capacity`) still bounds that dispatch: a spawn beyond it exits 75 without launching, and its item stays queued rather than blocked. Serialize only for a true semantic dependency, shared mutable external state, incompatible concurrent migration, or another concrete condition that makes independent progress or reconciliation unsafe; same-file editing alone is insufficient, and genuine blockers remain durable. Write the task-specific brief under section 11 before spawning. Fill the task subsections according to section 11. @@ -369,7 +369,7 @@ A decision is simply a task held for the captain: create the task with `bin/fm-t When a main-side thread such as a pending captain decision or relay reminder is worth durable tracking, file it as its own work item and hold it through that wrapper. Captain calls discovered by investigations or visual reviews follow `captain-hold-lifecycle`, which owns their completion gate and recorded-answer rules. When the automatic transition gate applies, dispatch and completion move the item themselves - `bin/fm-spawn.sh` and `bin/fm-teardown.sh` own those transitions and refuse rather than report success without them - so what remains yours is filing the item before dispatch, recording decisions, and keeping notes current; `docs/configuration.md` owns gate applicability and the manual-backend exception. -Re-evaluate queued work after every teardown and heartbeat, dispatching items only when dependencies and time gates have cleared. +Re-evaluate queued work after every teardown and heartbeat, and also after a recorded PR-ready handoff when `config/project-capacity` caps that project, dispatching items only when dependencies, time gates, and project capacity have cleared. - `.tasks.toml`, `docs/configuration.md`, and current `tasks-axi --help` own the backlog schema, compatibility, retention, and routine command syntax. - Use compatible `tasks-axi` when the configured backend selects it, always through `bin/fm-tasks-axi.sh` so the call reaches this home's backlog from any directory, and the documented manual path otherwise; keep only the configured recent Done entries. diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index c0cdd0540db..15ae10a5800 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -3487,7 +3487,12 @@ fm_backend_herdr_send_text_submit() { # esac # Native stayed idle. Composer empty is positive delivery (a landed # Claude turn that never flipped agent_status). Proven pending retries. + # A picker that classifies pending must not receive that retry. verdict=$(fm_backend_herdr_composer_state "$target") + if fm_composer_blocking_dialog_noted >/dev/null; then + printf 'unknown' + return 0 + fi case "$verdict" in empty) printf 'empty'; return 0 ;; pending|pending-unproven) ;; @@ -3496,6 +3501,10 @@ fm_backend_herdr_send_text_submit() { # else sleep "$sleep_s" verdict=$(fm_backend_herdr_composer_state "$target") + if fm_composer_blocking_dialog_noted >/dev/null; then + printf 'unknown' + return 0 + fi if [ "$verdict" = pending ] && [ "$raw_status" != working ] \ && [ "$footer_baseline" = idle ] \ && [ "$(fm_backend_herdr_rendered_busy_state "$target")" = busy ]; then diff --git a/bin/fm-backend.sh b/bin/fm-backend.sh index 798bb5c599a..916e3d2e353 100644 --- a/bin/fm-backend.sh +++ b/bin/fm-backend.sh @@ -811,18 +811,43 @@ fm_backend_send_key() { # [expected-label] # fm_backend_send_text_submit: type text once, then submit and verify, # retrying only the submission (never retyping). Echoes the backend's # proof-carrying verdict; callers require exact empty for confirmed delivery. +# A pane that already shows the recognised dialog is refused before any +# adapter types, so that submit neither types the text nor sends Enter. fm_backend_send_text_submit() { # [expected-label] - local backend=$1 + local backend=$1 rc=0 target label dialog shift + target=$1 + label=${6:-} fm_backend_source "$backend" || return 1 + # Every Enter loop below reads the dialog sink, so it must exist before + # any adapter types: a sink that fails here leaves the composer untouched. + fm_composer_dialog_sink_prepare || { + echo "error: the dialog check for a $backend submit could not be recorded" >&2 + return 1 + } + # One composer read after the sink exists and before the adapter types. + # The classify writes the sink; a named dialog means the next Enter would + # answer it. + if [ -n "$label" ]; then + fm_backend_composer_state "$backend" "$target" "$label" >/dev/null || true + else + fm_backend_composer_state "$backend" "$target" >/dev/null || true + fi + if dialog=$(fm_composer_blocking_dialog_noted); then + fm_composer_dialog_sink_release + echo "error: blocked on a prompt: $dialog" >&2 + return 1 + fi case "$backend" in - tmux) fm_backend_tmux_send_text_submit "$@" ;; - herdr) fm_backend_herdr_send_text_submit "$@" ;; - zellij) fm_backend_zellij_send_text_submit "$@" ;; - orca) fm_backend_orca_send_text_submit "$@" ;; - cmux) fm_backend_cmux_send_text_submit "$@" ;; - *) echo "error: no send-text implementation for backend '$backend'" >&2; return 1 ;; + tmux) fm_backend_tmux_send_text_submit "$@" || rc=$? ;; + herdr) fm_backend_herdr_send_text_submit "$@" || rc=$? ;; + zellij) fm_backend_zellij_send_text_submit "$@" || rc=$? ;; + orca) fm_backend_orca_send_text_submit "$@" || rc=$? ;; + cmux) fm_backend_cmux_send_text_submit "$@" || rc=$? ;; + *) echo "error: no send-text implementation for backend '$backend'" >&2; rc=1 ;; esac + fm_composer_dialog_sink_release + return "$rc" } # fm_backend_kill: remove the task's session endpoint. An already-gone target diff --git a/bin/fm-composer-lib.sh b/bin/fm-composer-lib.sh index 38285587980..1e2cb3b8371 100644 --- a/bin/fm-composer-lib.sh +++ b/bin/fm-composer-lib.sh @@ -1671,9 +1671,80 @@ EOF printf '%s\n' "$joined" | LC_ALL=C awk '{$1=$1; printf "%s", $0}' } +# fm_composer_blocking_dialog: name a screen whose next Enter would answer it. +# Prints the name and returns 0 only for the recorded structure of one dialog: +# the heading on its own line, then its selected row alone on a row, with the +# recorded footer as the last non-blank row. A heading buried in a sentence, +# or a last line that only starts with the same words, is not that dialog. +# The strings alone are not enough, because a diff, a note, or a test fixture +# on the pane can quote all of them above a normal composer. A miss returns 1 +# and prints nothing. +# Recorded 2026-10-05 on Claude Code 2.1.289: /exit while a background shell +# is still running opens this picker, and its selected row is Exit and stop tasks. +fm_composer_blocking_dialog() { # -> dialog name + local screen=${1-} + [ -n "$screen" ] || return 1 + if printf '%s\n' "$screen" | fm_composer_strip_ansi | LC_ALL=C awk ' + /^[ \t]*Background work is running[ \t\r]*$/ { heading = 1 } + heading && /^[ \t]*❯ 1\. Exit and stop tasks[ \t\r]*$/ { selected = 1 } + /[^ \t\r]/ { last = $0 } + END { exit !(selected && last ~ /^[ \t]*Enter to confirm · Esc to cancel[ \t\r]*$/) } + '; then + printf '%s' 'Claude background-task exit picker' + return 0 + fi + return 1 +} + +# A command substitution drops a shell variable, and every composer read runs +# inside one. The name is therefore written to FM_COMPOSER_DIALOG_SINK when +# that path is set. The classifier verdict is unchanged. When the sink is +# unset the name would be discarded, so the match is skipped. +fm_composer_note_blocking_dialog() { # + local name= + [ -n "${FM_COMPOSER_DIALOG_SINK:-}" ] || return 1 + if name=$(fm_composer_blocking_dialog "$1"); then + printf '%s' "$name" > "$FM_COMPOSER_DIALOG_SINK" || return 1 + return 0 + fi + : > "$FM_COMPOSER_DIALOG_SINK" || return 1 + return 1 +} + +# fm_composer_blocking_dialog_noted: print the name the latest classify wrote +# to the sink. Returns 1 when the sink is unset or empty. +fm_composer_blocking_dialog_noted() { + [ -n "${FM_COMPOSER_DIALOG_SINK:-}" ] || return 1 + [ -s "$FM_COMPOSER_DIALOG_SINK" ] || return 1 + cat "$FM_COMPOSER_DIALOG_SINK" +} + +# Empty the sink, creating it when the caller has not. Sets +# FM_COMPOSER_DIALOG_OWNED=1 only for a sink this call created, so a caller +# that shares the path can still read the name after the release. +fm_composer_dialog_sink_prepare() { + FM_COMPOSER_DIALOG_OWNED=0 + if [ -z "${FM_COMPOSER_DIALOG_SINK:-}" ]; then + FM_COMPOSER_DIALOG_SINK=$(mktemp "${TMPDIR:-/tmp}/fm-composer-dialog.XXXXXX") || return 1 + FM_COMPOSER_DIALOG_OWNED=1 + return 0 + fi + : > "$FM_COMPOSER_DIALOG_SINK" +} + +fm_composer_dialog_sink_release() { + if [ "${FM_COMPOSER_DIALOG_OWNED:-}" = 1 ]; then + rm -f "$FM_COMPOSER_DIALOG_SINK" + FM_COMPOSER_DIALOG_SINK= + FM_COMPOSER_DIALOG_OWNED=0 + fi +} + fm_composer_classify_screen() { # [cursor_row] [identity] local caps=$1 screen=$2 cy=${3:-} identity=${4:-} local styled=0 cursor=0 has_identity=0 kv plain + # Note the dialog before any early return so a pending picker is still named. + fm_composer_note_blocking_dialog "$screen" || true while IFS= read -r kv; do case "$kv" in styled=1) styled=1 ;; @@ -1795,6 +1866,11 @@ fm_composer_submit_retry_core() { # "$send_key_fn" "$target" Enter "$expected_label" || true sleep "$sleep_s" state=$("$state_fn" "$target" "$expected_label") + # The first Enter can open a picker. A later Enter would confirm it. + if fm_composer_blocking_dialog_noted >/dev/null; then + printf 'unknown' + return 0 + fi case "$state" in pending|pending-unproven) ;; *) printf '%s' "$state"; return 0 ;; diff --git a/bin/fm-control.sh b/bin/fm-control.sh index aa4c9af2004..e5642b9e33b 100755 --- a/bin/fm-control.sh +++ b/bin/fm-control.sh @@ -76,6 +76,9 @@ # worker account pin (bin/fm-worker-account-lib.sh) here, so a pin # that no longer resolves or is signed out refuses before the old # agent stops. +# The same pre-stop refusal applies to this home's worker tool +# exclusions (bin/fm-exclude-tools-lib.sh): a malformed list, or a +# replacement runtime that cannot hide the listed tools. # --note is required for a ship or scout, whose replacement # inherits the local copy but none of the conversation; a # secondmate reconciles its own home's records at startup, so its @@ -176,6 +179,8 @@ DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" . "$SCRIPT_DIR/fm-wake-lib.sh" # shellcheck source=bin/fm-worker-account-lib.sh . "$SCRIPT_DIR/fm-worker-account-lib.sh" +# shellcheck source=bin/fm-exclude-tools-lib.sh +. "$SCRIPT_DIR/fm-exclude-tools-lib.sh" POLL=${FM_CONTROL_POLL:-0.5} SETTLE_WAIT=${FM_CONTROL_SETTLE_WAIT:-5} @@ -200,6 +205,11 @@ control_cleanup() { && declare -F relaunch_rollback >/dev/null 2>&1; then relaunch_rollback || true fi + # Remove the dialog file while the lock is still held: once it is released, + # the next lifecycle command for this task writes the same path. + if [ -n "${FM_COMPOSER_DIALOG_SINK:-}" ]; then + rm -f "$FM_COMPOSER_DIALOG_SINK" + fi if [ "$CONTROL_LOCK_HELD" = 1 ]; then CONTROL_LOCK_HELD=0 fm_lock_release "$CONTROL_LOCK" || true @@ -313,6 +323,11 @@ trap control_cleanup EXIT fm_lock_try_acquire "$CONTROL_LOCK" \ || die "another lifecycle action is already running for task $ID" CONTROL_LOCK_HELD=1 +# do_exit runs in a command substitution. That subshell does not run this +# EXIT trap, so the parent has to hold the path the trap removes. Set it +# only once the lock is held: a process that loses the lock runs the same +# trap, and would remove the file the lock holder is reading. +FM_COMPOSER_DIALOG_SINK=$STATE/$ID.composer-dialog META="$STATE/$ID.meta" if [ ! -f "$META" ]; then case "$RAW_ID" in @@ -390,6 +405,13 @@ require_state_verified_backend() { # die "task $ID runs on the $BACKEND backend, which has no recovery-grade agent-state classifier, so '$1' cannot prove the agent actually stopped; refusing rather than reporting an unproven transition as done" } +# refuse_blocking_prompt: the screen is a dialog a confirming Enter would +# answer. Name it and stop. Do not type Escape or an option: both dismiss +# or choose. +refuse_blocking_prompt() { # + die "task $ID is blocked on a prompt: $1. Refusing to type Enter into it." +} + # rendered_matches : whether any row of the visible viewport matches. # An unreadable viewport is a no, so every caller treats it as missing proof. rendered_matches() { # @@ -548,16 +570,72 @@ do_interrupt() { printf '%s cancel=%s' "$proof" "$cancel" } +# Drop busy_gen from the task record when it still names . +# fm-busy-event.sh owns the sidecar and the record; fm_backlog_atomic_transition +# publish owns the task record. Clearing the line inside the busy writer would +# take the task-record lock that teardown and spawn already hold; the busy +# writer is their child process, so it would wait on a live holder that is +# itself waiting on the child, and neither would ever proceed. +clear_retired_meta_busy_gen() { # + local gen=$1 meta="$STATE/$ID.meta" lock tmp current line + [ -n "$gen" ] || return 0 + [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 + if ! declare -F fm_backlog_atomic_transition >/dev/null 2>&1; then + # shellcheck source=bin/fm-tasks-axi-lib.sh + . "$SCRIPT_DIR/fm-tasks-axi-lib.sh" + # shellcheck source=bin/fm-backlog-transition-lib.sh + . "$SCRIPT_DIR/fm-backlog-transition-lib.sh" + fi + lock=$(fm_meta_lock_path "$meta") || return 1 + fm_lock_acquire_wait "$lock" + current=$(fm_meta_get "$meta" busy_gen) + if [ "$current" != "$gen" ]; then + fm_lock_release "$lock" + return 0 + fi + tmp=$(mktemp "$STATE/.$ID.meta.retire.XXXXXX") || { + fm_lock_release "$lock" + return 1 + } + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + busy_gen=*) ;; + *) + printf '%s\n' "$line" >> "$tmp" || { + rm -f "$tmp" + fm_lock_release "$lock" + return 1 + } + ;; + esac + done < "$meta" || { + rm -f "$tmp" + fm_lock_release "$lock" + return 1 + } + if ! fm_backlog_atomic_transition publish "$tmp" "$meta" "task record" "$STATE"; then + rm -f "$tmp" + fm_lock_release "$lock" + return 1 + fi + fm_lock_release "$lock" +} + retire_busy_incarnation() { + local gen= if [ -f "$STATE/$ID.busy-gen" ]; then - "$SCRIPT_DIR/fm-busy-event.sh" retire "$STATE" "$ID" --current-gen >/dev/null 2>&1 || true + gen=$(fm_busy_current_gen "$STATE" "$ID" 2>/dev/null || true) + if [ -n "$gen" ] \ + && "$SCRIPT_DIR/fm-busy-event.sh" retire "$STATE" "$ID" --gen "$gen" >/dev/null 2>&1; then + clear_retired_meta_busy_gen "$gen" || true + fi fi } # do_exit: stop the running agent, preserving endpoint and worktree. Prints # `already-stopped`, `endpoint-gone`, or `stopped`. do_exit() { - local state cmd hazard verdict composer_state cancel absence interrupt_result=not-needed + local state cmd hazard verdict composer_state cancel absence interrupt_result=not-needed dialog require_state_verified_backend exit state=$(agent_state) case "$state" in @@ -624,8 +702,16 @@ do_exit() { if [ -n "$hazard" ] && rendered_matches "$hazard"; then die "task $ID shows the $HARNESS revert picker, where typed text becomes a search and Enter reverts file changes; refusing to type the $cmd exit command. Close it with $(fm_control_interrupt_key "$HARNESS"), never Enter, then retry '$VERB'" fi + : > "$FM_COMPOSER_DIALOG_SINK" \ + || die "task $ID's dialog check could not be recorded" composer_state=$(fm_backend_composer_state "$BACKEND" "$T" "$LABEL" 2>/dev/null) \ || composer_state=unknown + # The classify that filled the sink ran in a subshell, so read the file + # rather than a function that subshell sourced. + if [ -s "${FM_COMPOSER_DIALOG_SINK:-}" ]; then + dialog=$(cat "$FM_COMPOSER_DIALOG_SINK") + refuse_blocking_prompt "$dialog" + fi case "$composer_state" in empty) ;; pending) @@ -645,7 +731,24 @@ do_exit() { || die "the exit command could not be sent to task $ID on $BACKEND" [ "$verdict" != send-failed ] \ || die "the exit command could not be sent to task $ID on $BACKEND" + # The submitting Enter can open the picker. The agent is still alive, and + # another Enter would confirm the selected row. A dead agent may leave the + # same text behind; that is not a prompt still waiting. + if [ -s "${FM_COMPOSER_DIALOG_SINK:-}" ]; then + dialog=$(cat "$FM_COMPOSER_DIALOG_SINK") + if [ "$(agent_state)" != dead ]; then + refuse_blocking_prompt "$dialog" + fi + fi state=$(wait_agent_state "$EXIT_WAIT" dead) || { + # A submit can return before any read sees the picker: a native busy + # verdict needs no composer read, and a cleared composer can be read + # before the picker renders. Read the screen once more here. + : > "$FM_COMPOSER_DIALOG_SINK" || true + fm_backend_composer_state "$BACKEND" "$T" "$LABEL" >/dev/null 2>&1 || true + if [ -s "$FM_COMPOSER_DIALOG_SINK" ]; then + refuse_blocking_prompt "$(cat "$FM_COMPOSER_DIALOG_SINK")" + fi die "exit-delivered $ID interrupt=$interrupt_result exit-command=delivered agent-state=$state exit=unconfirmed; the agent did not stop within ${EXIT_WAIT}s" } # The incarnation is over: retire its busy wiring so no stale record or @@ -858,6 +961,12 @@ resolve_relaunch_profile() { [ "$account_model" != default ] || account_model= fm_worker_account_select "$TARGET_HARNESS" "${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" \ "$account_model" "$TARGET_HARNESS" >/dev/null || return 1 + # Likewise config/crew-exclude-tools: a malformed file, or a replacement + # runtime that cannot hide the listed tools, refuses here, before the old + # agent stops. Secondmate agents are not covered. + if [ "$KIND" != secondmate ]; then + fm_exclude_tools_check "$TARGET_HARNESS" 0 "${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" >/dev/null || return 1 + fi } # safe_checkpoint: prove, before anything is stopped, that the work a relaunch diff --git a/bin/fm-exclude-tools-lib.sh b/bin/fm-exclude-tools-lib.sh new file mode 100644 index 00000000000..0306a7e8026 --- /dev/null +++ b/bin/fm-exclude-tools-lib.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# fm-exclude-tools-lib.sh - shared parsing and runtime-support checks for +# config/crew-exclude-tools, sourced by bin/fm-spawn.sh and bin/fm-control.sh. +# +# docs/configuration.md "Worker tool exclusions" owns the file format and +# operator contract; bin/fm-spawn.sh's header owns launch-flag mechanics. +# Name verification belongs inside the worker's loaded-tool registry, never +# in an out-of-band server connection from these launch-time checks. + +# fm_exclude_tools_names : print the comma-joined names, empty when +# the file is absent or lists nothing. Non-zero with the reason on stderr when +# the file or an entry is invalid. +fm_exclude_tools_names() { + local config=$1 file line names='' present + file=$config/crew-exclude-tools + present=$(perl -MErrno=ENOENT -e ' + if (lstat $ARGV[0]) { print 1 } + elsif ($! == ENOENT) { print 0 } + else { die "error: cannot inspect config/crew-exclude-tools: $!\n" } + ' -- "$file") || return 1 + [ "$present" = 1 ] || return 0 + if [ ! -f "$file" ] || [ ! -r "$file" ]; then + echo "error: config/crew-exclude-tools must be a readable regular file" >&2 + return 1 + fi + while IFS= read -r line || [ -n "$line" ]; do + line=${line#"${line%%[![:space:]]*}"} + line=${line%"${line##*[![:space:]]}"} + case "$line" in + '' | '#'*) continue ;; + esac + if [ -n "${line//[A-Za-z0-9_.-]/}" ]; then + echo "error: config/crew-exclude-tools has a malformed entry '$line'; expected one tool name per line using only letters, digits, _ . and - (blank lines and # comment lines are allowed)" >&2 + return 1 + fi + names="${names:+$names,}$line" + done <"$file" || { + echo "error: cannot read config/crew-exclude-tools" >&2 + return 1 + } + printf '%s' "$names" +} + +# fm_exclude_tools_check : succeed when +# this launch can honor the home's list (empty list, or a runtime that hides +# tools); otherwise refuse naming the file. Prints the names on stdout. +fm_exclude_tools_check() { + local harness=$1 raw=$2 config=$3 names + names=$(fm_exclude_tools_names "$config") || return 1 + if [ -n "$names" ]; then + if [ "$raw" = 1 ]; then + echo "error: config/crew-exclude-tools lists tools to hide, but a raw launch command cannot hide tools; remove the entries or launch a runtime that supports them (pi, pi-signed)" >&2 + return 1 + fi + case "$harness" in + pi | pi-signed) ;; + *) + echo "error: config/crew-exclude-tools lists tools to hide, but the $harness runtime cannot hide tools, so this launch is refused rather than running with them available; empty the file or launch a runtime that supports exclusion (pi, pi-signed)" >&2 + return 1 + ;; + esac + fi + printf '%s' "$names" +} diff --git a/bin/fm-pr-check.sh b/bin/fm-pr-check.sh index 77d148960a6..f39ea0db1bf 100755 --- a/bin/fm-pr-check.sh +++ b/bin/fm-pr-check.sh @@ -19,6 +19,8 @@ # draft state does not refuse, matching how the head read below is optional. # bin/fm-pr-merge.sh records through this script with FM_PR_CHECK_MERGE=1 and # skips this refusal, because its own merge-time draft refusal is authoritative. +# The recorded pr= also frees the task's place in a declared project capacity +# (bin/fm-project-capacity-lib.sh). # Usage: fm-pr-check.sh set -eu diff --git a/bin/fm-procevent-remote-reply.sh b/bin/fm-procevent-remote-reply.sh index 7d545c0508b..a7d82904aa7 100755 --- a/bin/fm-procevent-remote-reply.sh +++ b/bin/fm-procevent-remote-reply.sh @@ -128,6 +128,7 @@ source_id() { cursor_path() { printf '%s/%s.cursor\n' "$CURSOR_DIR" "$1"; } ingest_receipt_path() { printf '%s/%s.%s.ingested\n' "$CURSOR_DIR" "$1" "$2"; } +retirement_count_path() { printf '%s/%s.retirements\n' "$CURSOR_DIR" "$1"; } mirrored_source_path() { printf '%s/.remote-reply-mirrored-%s\n' "$STATE" "$1"; } read_cursor() { # ; sets CURSOR_OFFSET and CURSOR_HASH @@ -164,6 +165,43 @@ write_cursor() { # mv -f -- "$tmp" "$path" } +# A missing file is zero and is not created. Ingest only reads this. +# Retirement is the one writer, so a crash during ingest cannot change it. +read_retirement_count() { # ; sets RETIREMENT_COUNT + local path count lines + path=$(retirement_count_path "$1") + RETIREMENT_COUNT=0 + [ -e "$path" ] || [ -L "$path" ] || return 0 + [ -f "$path" ] && [ ! -L "$path" ] || die "reply retirement count is unsafe: $path" + lines=$(grep -c '^count=' "$path" 2>/dev/null || true) + [ "$lines" = 1 ] || die "reply retirement count is invalid: $path" + count=$(sed -n 's/^count=//p' "$path") + case "$count" in ''|*[!0-9]*) die "reply retirement count is invalid: $path" ;; esac + RETIREMENT_COUNT=$count +} + +write_retirement_count() { # + local id=$1 count=$2 path tmp + case "$count" in ''|*[!0-9]*) return 1 ;; esac + mkdir -p "$CURSOR_DIR" || return 1 + chmod 700 "$CURSOR_DIR" 2>/dev/null || true + path=$(retirement_count_path "$id") + [ ! -L "$path" ] || return 1 + tmp=$(umask 077; mktemp "$CURSOR_DIR/.retirements.XXXXXX") || return 1 + printf 'count=%s\n' "$count" > "$tmp" || { rm -f -- "$tmp"; return 1; } + chmod 600 "$tmp" || { rm -f -- "$tmp"; return 1; } + if ! mv -f -- "$tmp" "$path"; then + rm -f -- "$tmp" + return 1 + fi +} + +# Twelve characters distinguish breaks in the status line. The cursor keeps +# the full digest the reader uses. +continuity_prefix() { + printf '%.12s' "$CURSOR_HASH" +} + ingest_receipt_matches() { # local path stored actual count path=$(ingest_receipt_path "$1" "$2") @@ -544,7 +582,12 @@ cmd_ingest() { die "result does not continue the current cursor for $id" fi if [ "$class" = continuity-broken ]; then - line="blocked [key=remote-reply-continuity-$id]: remote reply continuity broke for $id ($reason)" + # The same offset, prefix, and retirement count build the same line, so a + # retry appends nothing. Retirement removes the cursor before it records + # the next count, so a later break is a new line even when the restored + # bytes match, and a stop between those steps leaves the count unchanged. + read_retirement_count "$id" + line="blocked [key=remote-reply-continuity-$id]: remote reply continuity broke for $id ($reason) at offset ${CURSOR_OFFSET} prefix $(continuity_prefix) retirements ${RETIREMENT_COUNT}" append_rc=0 if status_event_recorded "$status_file" "$line"; then append_rc=1 @@ -726,7 +769,7 @@ cmd_retire_quiesce_locked() { } cmd_retire_finalize_locked() { - local id=${1:-} force=${2:-} sid path + local id=${1:-} force=${2:-} sid path cursor validate_id "$id" [ -z "$force" ] || [ "$force" = --force ] || die "invalid retirement option: $force" sid=$(source_id "$id") @@ -742,7 +785,16 @@ cmd_retire_finalize_locked() { done fi fi - rm -f -- "$(cursor_path "$id")" + # Remove the cursor first. A stop before the count write leaves that count + # unchanged, so the same break still builds the same line. + cursor=$(cursor_path "$id") + rm -f -- "$cursor" || die "cannot remove remote reply cursor" + if [ -e "$cursor" ] || [ -L "$cursor" ]; then + die "cannot remove remote reply cursor" + fi + read_retirement_count "$id" + write_retirement_count "$id" "$((RETIREMENT_COUNT + 1))" \ + || die "cannot record remote reply retirement" rm -f -- "$CURSOR_DIR/$id".*.ingested rm -f -- "$(fm_pending_reply_remote_channel_watermark_path "$STATE" "$id")" } diff --git a/bin/fm-project-capacity-lib.sh b/bin/fm-project-capacity-lib.sh new file mode 100644 index 00000000000..a184c3d28a9 --- /dev/null +++ b/bin/fm-project-capacity-lib.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# fm-project-capacity-lib.sh - how many workers a project admits at once on this +# machine, and whether a fresh worker spawn still fits. +# +# A project can depend on a machine-local resource that only a few workers can +# use at the same time: a heavy test suite, a local editor stack, a device. +# Firstmate cannot see which part of a worker's life touches that resource, so +# the captain declares how many workers the project admits on this machine, and +# bin/fm-spawn.sh defers a fresh worker beyond that number instead of launching +# it only to spend full-context turns retrying the resource. A deferred task +# keeps its queued backlog item and is dispatched again when a place frees. +# Without a declaration nothing changes and dispatch stays uncapped +# (AGENTS.md section 7). +# +# This file is the single owner of the declaration format, of what holds a +# place, and of the admission verdict. docs/configuration.md "Project capacity" +# is the operator reference, and bin/fm-spawn.sh owns where the check runs. +# +# Declaration: config/project-capacity in the local root Firstmate home (the home +# bin/fm-wake-lib.sh's fm_firstmate_root_home resolves), so every home on this +# machine reads the same number for the same machine's resources. One line per +# project: +# +# is the project's registered name, which is the basename of its +# clone directory, and is a positive integer of at most six digits. +# The capacity is the last whitespace-separated field, so the name before it may +# contain spaces. Blank lines are ignored. A line is a comment when it is `#`, +# when `#` is followed by whitespace, or when it starts with `#` and its last +# field is not an integer. A project whose name begins with `#` is declared by +# writing that `#` immediately against the rest of the name and ending the line +# with the capacity, for example `#repo 2`. A name that is `#`, or that begins +# with `#` followed by a space, is the same spelling as a comment and cannot be +# declared. Any other shape, a project named twice, or an unreadable file makes +# the whole declaration unreadable, and bin/fm-spawn.sh then refuses every fresh +# ship or scout spawn from this machine's homes rather than guessing which limit +# was meant. +# +# Occupancy: a place is held by every task record, in any local Firstmate home on +# this machine (fm_local_firstmate_state_dirs in bin/fm-wake-lib.sh), that +# - is not a secondmate, which is a persistent home rather than a worker, +# - names the same project identity, meaning its project resolves to the same +# shared project lock path (fm_treehouse_project_lock_path), which is keyed +# by the project's resolved origin, so workers in any clone of that origin +# are counted, and +# - has no recorded PR handoff: the pr= line bin/fm-pr-check.sh records when a +# worker's PR is ready, after which the worker waits on review or merge and +# no longer uses local resources. +# A place therefore frees when a PR-based ship records its ready PR, or when any +# task is cleaned up and its record removed. A local-only ship and a scout have +# no recorded handoff and hold their place until cleanup. A worker that is +# steered back into work after its PR handoff is not counted again. +# The declaration is matched by the spawning clone's directory name, so clones +# of one origin share the cap only when they use that same directory name. A +# clone of that origin under a different directory name finds no declaration +# and is not capped, though its workers still count as holders for a +# same-origin clone that is capped. +# A record whose project directory no longer exists cannot be matched and holds +# no place. Remote homes are never walked, because their workers run on another +# machine. +# +# Race safety: bin/fm-spawn.sh evaluates admission while holding the shared +# project lock and keeps holding it until the new task record is published, so +# two concurrent spawns for one project can never both publish from the same +# count. A spawn on an uncapped project can still publish a holder for a capped +# same-origin clone, so every backend takes that lock whenever the declaration +# caps any project; Orca, which otherwise never takes it, is included. Freeing a +# place needs no lock, because removing a record or adding pr= only ever lowers +# the count. +# +# Requires bin/fm-wake-lib.sh (root home, local homes, project lock path), +# bin/fm-secondmate-registry-lib.sh (which the local-homes walk reads), and +# bin/fm-backend.sh (fm_meta_get) to be sourced first. No side effects on source. + +# Exit status of a spawn deferred because the project is at capacity: the +# sysexits "temporary failure" code, so a caller can tell a deferral that leaves +# the task queued from an ordinary failure. +# shellcheck disable=SC2034 # read by bin/fm-spawn.sh after sourcing. +FM_PROJECT_CAPACITY_DEFER_EXIT=75 + +# The config directory holding this machine's declaration: the spawning home's +# own when that home is the local root (so an override of it +# applies), otherwise the root home's config/. +fm_project_capacity_config_dir() { # + local home=$1 config=$2 root home_real + root=$(fm_firstmate_root_home "$home") || return 1 + home_real=$(CDPATH='' cd -- "$home" 2>/dev/null && pwd -P) || return 1 + if [ "$root" = "$home_real" ]; then + printf '%s\n' "$config" + else + printf '%s/config\n' "$root" + fi +} + +# Read the declared capacity for one project. +# Sets FM_PROJECT_CAPACITY_FILE to the declaration path, FM_PROJECT_CAPACITY +# to the project's capacity, or to empty when the project declares none, and +# FM_PROJECT_CAPACITY_ANY to 1 when the declaration caps any project at all. +# Returns 1 with FM_PROJECT_CAPACITY_ERROR when the declaration is unreadable. +fm_project_capacity_lookup() { # + local name=$2 line lineno=0 pname pcap seen='|' + FM_PROJECT_CAPACITY_FILE="$1/project-capacity" + FM_PROJECT_CAPACITY= + FM_PROJECT_CAPACITY_ANY= + FM_PROJECT_CAPACITY_ERROR= + if [ ! -e "$FM_PROJECT_CAPACITY_FILE" ] && [ ! -L "$FM_PROJECT_CAPACITY_FILE" ]; then + return 0 + fi + if [ ! -f "$FM_PROJECT_CAPACITY_FILE" ] || [ ! -r "$FM_PROJECT_CAPACITY_FILE" ]; then + FM_PROJECT_CAPACITY_ERROR="$FM_PROJECT_CAPACITY_FILE is not a readable regular file" + return 1 + fi + while IFS= read -r line || [ -n "$line" ]; do + lineno=$((lineno + 1)) + line=${line%$'\r'} + line=${line#"${line%%[![:space:]]*}"} + line=${line%"${line##*[![:space:]]}"} + # '#' followed by whitespace is always a comment, including one that ends + # with a number. A line that begins with '#' glued to the rest of a name is + # a declaration only when its last field is an integer; any other such line + # stays a comment, so a note does not refuse every spawn. + case "$line" in + '' | '#' | '#'[[:space:]]*) continue ;; + '#'*) + case "${line##*[[:space:]]}" in + *[!0-9]*) continue ;; + esac + ;; + esac + # The capacity is the last field, so the name before it may hold spaces. + pcap=${line##*[[:space:]]} + pname=${line%"$pcap"} + pname=${pname%"${pname##*[![:space:]]}"} + if [ -z "$pname" ]; then + FM_PROJECT_CAPACITY_ERROR="$FM_PROJECT_CAPACITY_FILE line $lineno is not ' '" + FM_PROJECT_CAPACITY= + return 1 + fi + case "$pcap" in + '' | *[!0-9]* | 0*) + FM_PROJECT_CAPACITY_ERROR="$FM_PROJECT_CAPACITY_FILE line $lineno gives $pname a capacity that is not a positive integer" + FM_PROJECT_CAPACITY= + return 1 + ;; + esac + if [ "${#pcap}" -gt 6 ]; then + FM_PROJECT_CAPACITY_ERROR="$FM_PROJECT_CAPACITY_FILE line $lineno gives $pname a capacity longer than six digits" + FM_PROJECT_CAPACITY= + return 1 + fi + case "$seen" in + *"|$pname|"*) + FM_PROJECT_CAPACITY_ERROR="$FM_PROJECT_CAPACITY_FILE line $lineno names $pname a second time" + FM_PROJECT_CAPACITY= + return 1 + ;; + esac + seen="$seen$pname|" + [ "$pname" != "$name" ] || FM_PROJECT_CAPACITY=$pcap + done < "$FM_PROJECT_CAPACITY_FILE" + [ "$seen" = '|' ] || FM_PROJECT_CAPACITY_ANY=1 + return 0 +} + +# Count the task records holding a place in one project's capacity. +# is fm_treehouse_project_lock_path for the project being +# admitted, and is that project's own directory, which matches +# without recomputing its identity. The local homes come from +# fm_local_firstmate_state_dirs . is the task being +# admitted; its own record in is the one this spawn replaces, so +# it is not counted. +# Sets FM_PROJECT_CAPACITY_OCCUPANTS to the count and +# FM_PROJECT_CAPACITY_OCCUPANT_IDS to a comma-separated list of the holders, +# each outside qualified with its home. Returns 1 with +# FM_PROJECT_CAPACITY_ERROR when the local homes cannot be enumerated, or when +# a state directory or task record in them cannot be read, since skipping it +# could undercount the holders. +fm_project_capacity_occupants() { # + local want=$1 own=$2 first=$3 self=$4 state meta kind project lock id label i + local -a cache_dirs cache_locks + FM_PROJECT_CAPACITY_OCCUPANTS=0 + FM_PROJECT_CAPACITY_OCCUPANT_IDS= + FM_PROJECT_CAPACITY_ERROR= + fm_local_firstmate_state_dirs "$first" || { + FM_PROJECT_CAPACITY_ERROR=$FM_LOCAL_FIRSTMATE_ERROR + return 1 + } + cache_dirs=("$own") + cache_locks=("$want") + for state in "${FM_LOCAL_FIRSTMATE_STATES[@]}"; do + if [ -e "$state" ] && { [ ! -d "$state" ] || [ ! -r "$state" ] || [ ! -x "$state" ]; }; then + FM_PROJECT_CAPACITY_ERROR="local Firstmate state directory $state cannot be read" + return 1 + fi + for meta in "$state"/*.meta; do + [ -f "$meta" ] && [ ! -L "$meta" ] || continue + [ "$meta" != "$first/$self.meta" ] || continue + [ -r "$meta" ] || { + FM_PROJECT_CAPACITY_ERROR="task record $meta cannot be read" + return 1 + } + kind=$(fm_meta_get "$meta" kind) + [ "$kind" != secondmate ] || continue + [ -z "$(fm_meta_get "$meta" pr)" ] || continue + project=$(fm_meta_get "$meta" project) + [ -n "$project" ] || continue + lock= + i=0 + while [ "$i" -lt "${#cache_dirs[@]}" ]; do + if [ "${cache_dirs[$i]}" = "$project" ]; then + lock=${cache_locks[$i]} + break + fi + i=$((i + 1)) + done + if [ "$i" -ge "${#cache_dirs[@]}" ]; then + lock=$(fm_treehouse_project_lock_path "$project" 2>/dev/null) || lock= + cache_dirs+=("$project") + cache_locks+=("$lock") + fi + [ -n "$lock" ] && [ "$lock" = "$want" ] || continue + id=$(basename "$meta" .meta) + label=$id + [ "$state" = "$first" ] || label="$id in $(dirname "$state")" + FM_PROJECT_CAPACITY_OCCUPANTS=$((FM_PROJECT_CAPACITY_OCCUPANTS + 1)) + FM_PROJECT_CAPACITY_OCCUPANT_IDS="${FM_PROJECT_CAPACITY_OCCUPANT_IDS:+$FM_PROJECT_CAPACITY_OCCUPANT_IDS, }$label" + done + done + return 0 +} diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 0e58e9c29c0..445d9e13ca0 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Spawn a direct report: a crewmate in a treehouse or Orca worktree, or a # secondmate in its isolated firstmate home. -# Usage: fm-spawn.sh --mode --yolo [--branch-prefix ] [--base-branch ] [--harness |harness|launch-command] [--model ] [--effort ] [--backend ] -# fm-spawn.sh --scout [--base-branch ] [--harness |harness|launch-command] [--model ] [--effort ] [--backend ] +# Usage: fm-spawn.sh --mode --yolo [--branch-prefix ] [--base-branch ] [--harness |harness|launch-command] [--model ] [--effort ] [--backend ] [--herdr-resume-lock-wait] +# fm-spawn.sh --scout [--base-branch ] [--harness |harness|launch-command] [--model ] [--effort ] [--backend ] [--herdr-resume-lock-wait] # fm-spawn.sh [] [--harness |harness|launch-command] [--model ] [--effort ] [--backend ] --secondmate # --mode and --yolo are this task's delivery contract, REQUIRED for every ship # spawn and refused on --scout and --secondmate spawns. Firstmate resolves both @@ -140,14 +140,23 @@ # authority, and every ambiguous recovery stays on the flat fallback after # duplicate-agent risk is independently absent. Treehouse allocation and task # metadata are unchanged. -# A clean projected create or exact resume makes one bounded attempt to hold -# the one session-scoped presentation-order lock (keyed by named session plus -# canonical socket, outside any home's state/) through launch handoff. Lock -# contention warns and falls back to the ordinary flat layout before any -# projection mutation. The exact response-derived new workspace is inserted -# immediately after its owning parent (firstmate or 2ndmate-) contiguous -# child block. Ordering never authorizes lifecycle cleanup, and any -# unavailable, ambiguous, or failed move warns while the spawn continues. +# A clean projected create and an exact resume both hold the one +# session-scoped presentation-order lock (keyed by named session plus +# canonical socket, outside any home's state/) through launch handoff. +# On contention a create makes one bounded attempt and falls back to the +# ordinary flat layout before any projection mutation. A resume refuses by +# default on the same contention (it does not degrade flat; a concurrent +# resume is a hard failure). Pass --herdr-resume-lock-wait to opt that +# resume into waiting for the lock instead, so two concurrent recoveries +# can serialize and each still replace its own exact husk. The flag acts +# only on that fresh ship or scout spawn path: --relaunch reuses the +# recorded endpoint without taking this lock, so the flag has no effect +# there, and a secondmate spawn never projects. Unbounded +# blocking on a third-party session lock is never the default. The exact +# response-derived new workspace is inserted immediately after its owning +# parent (firstmate or 2ndmate-) contiguous child block. Ordering never +# authorizes lifecycle cleanup, and any unavailable, ambiguous, or failed +# move warns while the spawn continues. # Every projected create, prune, and move captures and verifies the named # session's exact active workspace and tab. A detected focus change restores # only that exact tab id; an ambiguous pre-operation snapshot refuses the @@ -173,6 +182,20 @@ # fm_firstmate_root_home resolves, so a home seeded from another machine anchors # that lock itself rather than failing to resolve one; # contention refuses rather than waits. +# Project capacity: when this machine declares how many workers a project +# admits at once (config/project-capacity; bin/fm-project-capacity-lib.sh owns +# the declaration, what holds a place, and the race argument), a fresh ship or +# scout spawn counts the places already held while holding that same +# project-identity lock - taken on every backend whenever the declaration caps +# any project, Orca included, because an uncapped clone's worker still holds a +# place for a capped clone of the same origin - and holds it through metadata +# publication. A spawn that finds +# every place held prints one `deferred:` line and exits 75 before any brief +# render, endpoint, worktree, record, or backlog move exists, so the task stays +# exactly as queued as it was; an unreadable declaration refuses with exit 1. +# A batch reports such a pair as `batch: DEFERRED` and exits 75 when nothing +# else failed. A relaunch and a --secondmate spawn are never counted against +# capacity. # With no harness arg, a crewmate/scout spawn resolves the CREW harness only when # config/crew-dispatch.json is absent. When that file exists, crewmate/scout # spawns require an explicit harness so firstmate cannot silently skip dispatch @@ -333,6 +356,10 @@ # worktree, or record exists and names the accepted values. The file is read # on every spawn and relaunch, so a change reaches the next launch without a # restart, and it is inherited into secondmate homes (bin/fm-config-inherit-lib.sh). +# Worker tool exclusions: +# docs/configuration.md "Worker tool exclusions" owns config/crew-exclude-tools +# and its operator contract. Resolve it with bin/fm-exclude-tools-lib.sh +# before provisioning; __PIEXCLUDE__ below owns the Pi launch substitution. # Worker account pin (config/claude-account, config/pi-account): # Opt-in. With no file, a Claude or Pi launch is unchanged: Claude still # receives this process's own CLAUDE_CONFIG_DIR when it is set, and Pi the @@ -359,6 +386,9 @@ # __PIAPPROVE__ optional --approve on a seeded Pi/pi-signed secondmate when # that executable advertises the flag (empty otherwise; session # trust for the launch cwd only, never a trust.json rewrite) +# __PIEXCLUDE__ optional ` --exclude-tools ''` from +# config/crew-exclude-tools on Pi/pi-signed ship and scout +# launches (supplies its own leading space, empty otherwise) # __PIRESUME__ optional relaunch-only `--session ` that keeps a # Pi replacement on the session the endpoint's runtime already # reports (relaunch_resume_args below owns it; it supplies its @@ -543,6 +573,8 @@ PROJECTS="${FM_PROJECTS_OVERRIDE:-$FM_HOME/projects}" CONFIG="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" # shellcheck source=bin/fm-config-inherit-lib.sh . "$SCRIPT_DIR/fm-config-inherit-lib.sh" +# shellcheck source=bin/fm-exclude-tools-lib.sh +. "$SCRIPT_DIR/fm-exclude-tools-lib.sh" if ! LAUNCH_ENV_ENABLED=$(fm_config_source_present "$CONFIG/launch-env-allowlist"); then exit 1 fi @@ -647,6 +679,8 @@ fm_backlog_directory_present "$STATE" "state directory" || { . "$SCRIPT_DIR/fm-remote-readiness-lib.sh" # shellcheck source=bin/fm-timeout-lib.sh . "$SCRIPT_DIR/fm-timeout-lib.sh" +# shellcheck source=bin/fm-project-capacity-lib.sh +. "$SCRIPT_DIR/fm-project-capacity-lib.sh" # shellcheck source=bin/fm-worker-account-lib.sh . "$SCRIPT_DIR/fm-worker-account-lib.sh" # Fail closed before any fleet mutation: a no-mistakes gate agent must never spawn @@ -676,6 +710,9 @@ BASE_BRANCH= BASE_BRANCH_SET=0 TRACEPARENT_SET=0 RELAUNCH=0 +# Opt-in only: exact-resume presentation-order lock waits instead of refusing. +# Absent/unset keeps upstream refuse-on-contention. See header. +HERDR_RESUME_LOCK_WAIT=0 POS=() want_value= for a in "$@"; do @@ -741,6 +778,7 @@ for a in "$@"; do KIND_SET=1 ;; --relaunch) RELAUNCH=1 ;; + --herdr-resume-lock-wait) HERDR_RESUME_LOCK_WAIT=1 ;; --harness) want_value=harness ;; --harness=*) HARNESS_ARG=${a#--harness=} @@ -1419,14 +1457,30 @@ spawn_abort_cleanup() { } trap spawn_abort_cleanup EXIT -# One bounded lock per live Herdr session/socket, shared across all homes. -# is required so secondmate and primary spawns serialize against the -# same session without writing any other home's state directory. +# One lock per live Herdr session/socket, shared across all homes. +# is required so secondmate and primary spawns serialize against the same +# session without writing any other home's state directory. +# +# Default mode is one BOUNDED attempt. A clean create uses that default and +# falls back to the ordinary flat layout on contention. An exact resume also +# defaults to the bounded attempt and hard-refuses on contention (it does not +# degrade flat). Passing mode `wait` makes this call WAIT for the lock instead +# (`fm_lock_acquire_wait`, the same unbounded-wait idiom this file already uses +# for its other fleet-shared locks). Only the recovery path under the explicit +# --herdr-resume-lock-wait opt-in passes `wait`, so unbounded blocking on a +# third-party session lock never becomes the default for every caller. +# Dead-owner reclaim inside `fm_lock_try_acquire` still bounds a wait against a +# holder that crashed mid-hold. spawn_herdr_presentation_order_lock_acquire() { - local session=${1:-} attempt lock_path + local session=${1:-} mode=${2:-} attempt lock_path [ -n "$session" ] || session=$(fm_backend_herdr_session) lock_path=$(fm_backend_herdr_presentation_session_lock_path "$session") || return 1 HERDR_PRESENTATION_ORDER_LOCK="$lock_path" + if [ "$mode" = wait ]; then + fm_lock_acquire_wait "$HERDR_PRESENTATION_ORDER_LOCK" + HERDR_PRESENTATION_ORDER_LOCK_HELD=1 + return 0 + fi attempt=0 while [ "$attempt" -lt 50 ]; do if fm_lock_try_acquire "$HERDR_PRESENTATION_ORDER_LOCK"; then @@ -1502,6 +1556,7 @@ if [ "${#POS[@]}" -gt 0 ] && [ "${POS[0]}" != "$idpart" ] && case "$idpart" in * [ "$YOLO_SET" -eq 0 ] || shared_args+=(--yolo "$YOLO") [ "$BRANCH_PREFIX_SET" -eq 0 ] || shared_args+=(--branch-prefix "$BRANCH_PREFIX") [ "$BASE_BRANCH_SET" -eq 0 ] || shared_args+=(--base-branch "$BASE_BRANCH") + [ "$HERDR_RESUME_LOCK_WAIT" -eq 0 ] || shared_args+=(--herdr-resume-lock-wait) for pair in "${POS[@]}"; do case "$pair" in *=*) : ;; @@ -1515,16 +1570,17 @@ if [ "${#POS[@]}" -gt 0 ] && [ "${POS[0]}" != "$idpart" ] && case "$idpart" in * echo "error: batch dispatch does not support --secondmate; spawn each secondmate explicitly" >&2 rc=2 continue - elif [ "$KIND" = scout ]; then - if FM_SPAWN_NO_GUARD=1 "$FM_ROOT/bin/fm-spawn.sh" "${pair%%=*}" "${pair#*=}" "${shared_args[@]+"${shared_args[@]}"}" --scout; then :; else - echo "batch: FAILED to spawn ${pair%%=*} (${pair#*=})" >&2 - rc=1 - fi - else - if FM_SPAWN_NO_GUARD=1 "$FM_ROOT/bin/fm-spawn.sh" "${pair%%=*}" "${pair#*=}" "${shared_args[@]+"${shared_args[@]}"}"; then :; else - echo "batch: FAILED to spawn ${pair%%=*} (${pair#*=})" >&2 - rc=1 - fi + fi + pair_args=("${pair%%=*}" "${pair#*=}" "${shared_args[@]+"${shared_args[@]}"}") + [ "$KIND" != scout ] || pair_args+=(--scout) + pair_rc=0 + FM_SPAWN_NO_GUARD=1 "$FM_ROOT/bin/fm-spawn.sh" "${pair_args[@]}" || pair_rc=$? + if [ "$pair_rc" -eq "$FM_PROJECT_CAPACITY_DEFER_EXIT" ]; then + echo "batch: DEFERRED ${pair%%=*} (${pair#*=}) - its project is at capacity, so it stays queued" >&2 + [ "$rc" -ne 0 ] || rc=$FM_PROJECT_CAPACITY_DEFER_EXIT + elif [ "$pair_rc" -ne 0 ]; then + echo "batch: FAILED to spawn ${pair%%=*} (${pair#*=})" >&2 + rc=1 fi done exit "$rc" @@ -2080,7 +2136,7 @@ launch_template() { ;; opencode) printf '%s' 'OPENCODE_CONFIG_CONTENT='\''{"permission":{"*":"allow"}__EFFORTFLAG__}'\'' opencode __MODELFLAG__--prompt "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' ;; pi | pi-signed) - printf '%s' '__PIBIN____PITUIMODE____PIAPPROVE____PIRESUME__' + printf '%s' '__PIBIN____PITUIMODE____PIAPPROVE____PIEXCLUDE____PIRESUME__' if [ "$kind" = secondmate ]; then printf '%s' ' __MODELFLAG____EFFORTFLAG__-e __PITURNEND__ -e __PIWATCH__ "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' else @@ -2329,6 +2385,13 @@ if [ "$KIND" = secondmate ] && [ "$HARNESS" = rovo ]; then exit 1 fi +# config/crew-exclude-tools (header above): refuse before worker provisioning +# if this launch cannot honor the list. Secondmate agents are not covered. +EXCLUDE_TOOLS= +if [ "$KIND" != secondmate ]; then + EXCLUDE_TOOLS=$(fm_exclude_tools_check "$HARNESS" "$RAW_LAUNCH" "$CONFIG") || exit 1 +fi + case "$HARNESS" in devin) DEVIN_BIN=$(command -v devin) || { @@ -2355,6 +2418,9 @@ pi | pi-signed) PI_APPROVE=' --approve' fi LAUNCH=${LAUNCH//__PIAPPROVE__/$PI_APPROVE} + PI_EXCLUDE= + [ -z "$EXCLUDE_TOOLS" ] || PI_EXCLUDE=" --exclude-tools $(shell_quote "$EXCLUDE_TOOLS")" + LAUNCH=${LAUNCH//__PIEXCLUDE__/$PI_EXCLUDE} LAUNCH="FM_PI_HARNESS=$HARNESS $LAUNCH" ;; cursor) @@ -3050,17 +3116,52 @@ else WT="" BRIEF="$DATA/$ID/brief.md" fi -if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then +# Project capacity admission (bin/fm-project-capacity-lib.sh owns the +# declaration, what holds a place, and why this is race-safe). A fresh worker +# for a project whose declared capacity is already held is deferred here, before +# any brief render, endpoint, worktree, record, or backlog move exists, so the +# deferral leaves the task exactly as queued as it was. A relaunch replaces a +# worker that already holds a place, and a secondmate is not a worker. +SPAWN_PROJECT_CAPACITY= +SPAWN_PROJECT_CAPACITY_ANY= +if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ]; then + SPAWN_CAPACITY_CONFIG=$(fm_project_capacity_config_dir "$FM_HOME" "$CONFIG") || { + echo "error: could not resolve the root Firstmate home that declares project capacity for $PROJ_ABS" >&2 + exit 1 + } + if ! fm_project_capacity_lookup "$SPAWN_CAPACITY_CONFIG" "$(basename "$PROJ_ABS")"; then + echo "error: spawn refused: the project capacity declaration is unreadable ($FM_PROJECT_CAPACITY_ERROR); fix it so the captain's worker limits are known (docs/configuration.md \"Project capacity\")" >&2 + exit 1 + fi + SPAWN_PROJECT_CAPACITY=$FM_PROJECT_CAPACITY + SPAWN_PROJECT_CAPACITY_ANY=$FM_PROJECT_CAPACITY_ANY +fi +if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && + { [ "$BACKEND" != orca ] || [ -n "$SPAWN_PROJECT_CAPACITY_ANY" ]; }; then SPAWN_TREEHOUSE_PROJECT_LOCK=$(fm_treehouse_project_lock_path "$PROJ_ABS") || { echo "error: could not resolve the shared Treehouse project lock for $PROJ_ABS" >&2 exit 1 } if ! fm_lock_try_acquire "$SPAWN_TREEHOUSE_PROJECT_LOCK"; then - echo "error: another Treehouse slot allocation or return is in progress for $PROJ_ABS; refusing to race it" >&2 + if [ "$BACKEND" = orca ]; then + echo "error: another spawn or cleanup holds the shared project lock for $PROJ_ABS; refusing to race its capacity admission" >&2 + else + echo "error: another Treehouse slot allocation or return is in progress for $PROJ_ABS; refusing to race it" >&2 + fi exit 1 fi SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=1 fi +if [ -n "$SPAWN_PROJECT_CAPACITY" ]; then + if ! fm_project_capacity_occupants "$SPAWN_TREEHOUSE_PROJECT_LOCK" "$PROJ_ABS" "$STATE" "$ID"; then + echo "error: spawn refused: project $(basename "$PROJ_ABS") declares a capacity of $SPAWN_PROJECT_CAPACITY, but this machine's task records cannot all be read to count it ($FM_PROJECT_CAPACITY_ERROR)" >&2 + exit 1 + fi + if [ "$FM_PROJECT_CAPACITY_OCCUPANTS" -ge "$SPAWN_PROJECT_CAPACITY" ]; then + echo "deferred: project $(basename "$PROJ_ABS") admits $SPAWN_PROJECT_CAPACITY worker(s) at once on this machine ($FM_PROJECT_CAPACITY_FILE) and $FM_PROJECT_CAPACITY_OCCUPANTS already hold a place ($FM_PROJECT_CAPACITY_OCCUPANT_IDS); task $ID was not launched and its backlog item stays queued - dispatch it again once one of them records its ready PR or is cleaned up" >&2 + exit "$FM_PROJECT_CAPACITY_DEFER_EXIT" + fi +fi [ -f "$BRIEF" ] || { echo "error: task $ID has no brief at inaccessible data path $BRIEF" >&2 exit 1 @@ -3712,10 +3813,19 @@ else echo "error: herdr presentation recovery could not ensure its exact named session" >&2 exit 1 } - spawn_herdr_presentation_order_lock_acquire "$HERDR_SES" || { - echo "error: herdr presentation recovery could not acquire its session lock; refusing a concurrent resume" >&2 - exit 1 - } + # Refuse-by-default on contention. Wait only when the caller opted in + # with --herdr-resume-lock-wait (see header). + if [ "$HERDR_RESUME_LOCK_WAIT" = 1 ]; then + spawn_herdr_presentation_order_lock_acquire "$HERDR_SES" wait || { + echo "error: herdr presentation recovery could not resolve its session lock" >&2 + exit 1 + } + else + spawn_herdr_presentation_order_lock_acquire "$HERDR_SES" || { + echo "error: herdr presentation recovery could not acquire its session lock; refusing a concurrent resume" >&2 + exit 1 + } + fi if [ -e "$STATE/$ID.meta" ] || [ -L "$STATE/$ID.meta" ]; then herdr_projection_existing_meta_allows_flat "$STATE/$ID.meta" || exit 1 fi @@ -4675,6 +4785,10 @@ EOF // tool calls) and stays a wake NOTIFICATION touch for the watcher, never // current-state truth. import { execFile } from "node:child_process"; +import { appendFileSync } from "node:fs"; +const excludeTools = "$EXCLUDE_TOOLS".split(",").filter(Boolean); +const excludeFile = $(perl -MJSON::PP -MEncode=decode_utf8 -e 'print encode_json(decode_utf8($ARGV[0]))' -- "$CONFIG/crew-exclude-tools"); +const statusFile = $(perl -MJSON::PP -MEncode=decode_utf8 -e 'print encode_json(decode_utf8($ARGV[0]))' -- "$STATE/$ID.status"); const busyEvent = (state: string, event: string) => new Promise((resolve) => { execFile("$FM_ROOT/bin/fm-busy-event.sh", [ @@ -4683,7 +4797,22 @@ const busyEvent = (state: string, event: string) => ], () => resolve()); }); export default function (pi: any) { - pi.on("agent_start", () => busyEvent("busy", "agent-start")); + let checkedExclusions = false; + pi.on("agent_start", async () => { + await busyEvent("busy", "agent-start"); + // Verify only this worker's registry, never connect servers from Firstmate. + // Check before actions so the warning cannot supersede this turn's terminal status. + if (!checkedExclusions && excludeTools.length) { + const loaded = new Set(pi.getAllTools().map((tool: any) => tool.name)); + const unmatched = excludeTools.filter((name) => !loaded.has(name)); + if (unmatched.length) { + appendFileSync(statusFile, "note [at=" + Math.floor(Date.now() / 1000) + "]: warning: " + excludeFile + + " unmatched exclusion entries (unverified: absent from the worker's loaded-tool registry; excluded tools or unavailable servers cannot be verified): " + + unmatched.join(", ") + "\n"); + } + checkedExclusions = true; + } + }); pi.on("agent_settled", (_event: any, ctx: any) => { if (ctx && typeof ctx.isIdle === "function" && !ctx.isIdle()) return; return busyEvent("idle", "agent-settled"); diff --git a/bin/fm-startup-growth-check.sh b/bin/fm-startup-growth-check.sh new file mode 100755 index 00000000000..3543666d727 --- /dev/null +++ b/bin/fm-startup-growth-check.sh @@ -0,0 +1,417 @@ +#!/usr/bin/env bash +# fm-startup-growth-check.sh - daily cheap growth check for startup memory and instruction surfaces. +# +# Usage: +# fm-startup-growth-check.sh [check] +# fm-startup-growth-check.sh arm +# fm-startup-growth-check.sh disarm +# fm-startup-growth-check.sh --help +# +# `check` evaluates at most once every 86400 seconds, one daily evaluation. +# Polls inside that interval only read this check's small state record and stay +# silent. +# +# A due evaluation uses metadata only: regular-file safety checks plus stat(1) +# byte sizes. It does not run the startup digest, bootstrap, network checks, +# model calls, repository refreshes, /stow, or full preference/learning +# rereads. The budget total, its verdict, and its secondmate exception come +# from `bin/fm-startup-memory-budget.sh report`, the single owner of +# config/startup-memory-budget, and are never re-derived here. data/projects.md +# and data/secondmates.md are printed in full by every session start too, so +# they are watched for prompt growth without entering that budget total. +# The tracked set is the startup entrypoints session start executes directly +# plus the agent instruction files, not every script and library the startup +# path reaches; those bytes are code/instruction size, not LLM prompt cost. +# +# A secondmate home is never notified about the primary-owned +# data/captain-shared.md it cannot edit: the owner suppresses the budget overrun +# it causes alone, and this check suppresses its per-file growth there while +# still recording the observation. +# +# Growth is measured against a retained per-file baseline rather than only +# against the previous evaluation, so accumulation that stays under one day's +# threshold is still caught. A surface seen for the first time is baselined +# silently, including the first content of an optional file that was absent when +# the check started; an established baseline survives the file disappearing and +# coming back. Reporting a file rebases its baseline to the reported size, so +# accepted growth then stays silent. The thresholds are fixed: +# 2048 bytes for tracked startup/instruction files +# 250 estimated tokens, ceil(bytes / 3), for printed startup memory files +# Budget overrun is always meaningful. +# +# A due evaluation also removes the empty temporary records a killed +# evaluation can leave in state/: only files matching its own mint pattern +# that are empty and untouched for an hour, never a record with bytes in it. +# +# `arm` writes state/startup-growth.check.sh and binds its bytes with +# fm-check-register.sh so the existing watcher slow-check cadence invokes the +# daily gate. `disarm` removes the shim, trust binding, and report record. +set -u +export LC_ALL=C + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +CONFIG_DIR="${FM_CONFIG_OVERRIDE:-$FM_HOME/config}" +DATA_DIR="${FM_DATA_OVERRIDE:-$FM_HOME/data}" +CHECK_ID=startup-growth +CHECK_SHIM="$STATE/$CHECK_ID.check.sh" +CHECK_TRUST="$STATE/$CHECK_ID.check-trust" +RECORD="$STATE/.startup-growth-check" +RECORD_SCHEMA_LINE=$'schema\tfm-startup-growth-check-v1' +REGISTER_BIN="$SCRIPT_DIR/fm-check-register.sh" +BUDGET_BIN="$SCRIPT_DIR/fm-startup-memory-budget.sh" + +# shellcheck source=bin/fm-pr-lib.sh +. "$SCRIPT_DIR/fm-pr-lib.sh" +# shellcheck source=bin/fm-startup-memory-budget-lib.sh +. "$SCRIPT_DIR/fm-startup-memory-budget-lib.sh" +# shellcheck source=bin/fm-line-cap-lib.sh +. "$SCRIPT_DIR/fm-line-cap-lib.sh" +# shellcheck source=bin/fm-check-lib.sh +. "$SCRIPT_DIR/fm-check-lib.sh" + +usage() { + sed -n '2,48{s/^# \{0,1\}//;p;}' "$0" +} + +fail() { + printf 'fm-startup-growth-check: %s\n' "$1" >&2 + exit 1 +} + +now_epoch() { + case "${FM_STARTUP_GROWTH_NOW:-}" in + ''|*[!0-9]*) date +%s ;; + *) printf '%s\n' "$FM_STARTUP_GROWTH_NOW" ;; + esac +} + +INTERVAL=86400 +BYTE_THRESHOLD=2048 +TOKEN_THRESHOLD=250 +MAX_LINE=1000 +ORPHAN_GRACE=3600 +ORPHAN_SWEEP_LIMIT=64 +PRIMARY_OWNED_MEMORY= +if [ -e "$FM_HOME/.fm-secondmate-home" ] || [ -L "$FM_HOME/.fm-secondmate-home" ]; then + PRIMARY_OWNED_MEMORY=data/captain-shared.md +fi + +file_size() { + if [ "$(uname)" = Darwin ]; then + /usr/bin/stat -f %z "$1" 2>/dev/null + else + stat -c %s "$1" 2>/dev/null + fi +} + +file_mtime() { + if [ "$(uname)" = Darwin ]; then + /usr/bin/stat -f %m "$1" 2>/dev/null + else + stat -c %Y "$1" 2>/dev/null + fi +} + +# A kill landing between mktemp(1) and the traps that own the temporary record +# leaves an empty scratch file nothing else would ever remove. A due +# evaluation sweeps those, bounded on every axis: only the mint pattern, only +# empty regular files, only ones untouched for ORPHAN_GRACE seconds, and at +# most ORPHAN_SWEEP_LIMIT per evaluation. A concurrent evaluation's live +# scratch is minutes younger than that grace, and a scratch carrying any +# record bytes is never a candidate, so neither published baselines nor work in +# flight can be removed here. +sweep_orphan_records() { # + local now=$1 scratch mtime swept=0 + for scratch in "$STATE"/.startup-growth-check.??????; do + [ "$swept" -lt "$ORPHAN_SWEEP_LIMIT" ] || break + [ -f "$scratch" ] && [ ! -L "$scratch" ] && [ ! -s "$scratch" ] || continue + mtime=$(file_mtime "$scratch") || continue + case "$mtime" in ''|*[!0-9]*) continue ;; esac + [ $((now - mtime)) -ge "$ORPHAN_GRACE" ] || continue + rm -f -- "$scratch" || true + swept=$((swept + 1)) + done +} + +append_finding() { + if [ -z "$FINDINGS" ]; then + FINDINGS=$1 + else + FINDINGS="$FINDINGS; $1" + fi +} + +stat_surface() { # + local kind=$1 display=$2 path=$3 absence_ok=$4 bytes tokens prev_baseline baseline delta presence=present + if [ ! -e "$path" ] && [ ! -L "$path" ]; then + bytes=0 + presence=absent + [ "$absence_ok" = yes ] || append_finding "missing $kind $display" + elif [ -L "$path" ] || [ ! -f "$path" ]; then + bytes=0 + presence=unsafe + append_finding "unsafe $kind $display" + else + bytes=$(file_size "$path") || true + case "$bytes" in + ''|*[!0-9]*) + bytes=0 + presence=unreadable + append_finding "unreadable $kind $display" + ;; + esac + fi + + prev_baseline=$(awk -F '\t' -v p="$display" '$1 == p { print $5; found=1; exit } END { if (!found) print "" }' "$OLD_RECORD" 2>/dev/null || true) + case "$prev_baseline" in + ''|*[!0-9]*) prev_baseline= ;; + esac + + if [ "$presence" != present ]; then + baseline=${prev_baseline:--} + elif [ -z "$prev_baseline" ] || [ "$bytes" -le "$prev_baseline" ]; then + baseline=$bytes + else + baseline=$prev_baseline + delta=$((bytes - baseline)) + case "$kind" in + memory|printed-memory) + tokens=$(fm_startup_memory_estimated_tokens_for_bytes "$delta") || tokens=0 + if [ "$tokens" -ge "$TOKEN_THRESHOLD" ]; then + baseline=$bytes + [ "$display" = "$PRIMARY_OWNED_MEMORY" ] \ + || append_finding "$kind growth $display +${tokens} estimated_tokens (+${delta} bytes, total ${bytes} bytes)" + fi + ;; + tracked) + if [ "$delta" -ge "$BYTE_THRESHOLD" ]; then + append_finding "tracked startup surface growth $display +${delta} bytes (total ${bytes} bytes)" + baseline=$bytes + fi + ;; + esac + fi + + printf '%s\t%s\t%s\t%s\t%s\n' "$display" "$kind" "$presence" "$bytes" "$baseline" >> "$NEW_RECORD" || exit 1 +} + +write_record_atomically() { + local tmp=$1 dest=$2 state_device + [ -d "$STATE" ] && [ ! -L "$STATE" ] || return 1 + state_device=$(fm_pr_file_device "$STATE") || return 1 + fm_pr_regular_destination_on_device_or_absent "$dest" "$state_device" || return 1 + mv -f -- "$tmp" "$dest" +} + +record_usable() { + local line + [ -f "$RECORD" ] && [ ! -L "$RECORD" ] || return 1 + IFS= read -r line < "$RECORD" || return 1 + [ "$line" = "$RECORD_SCHEMA_LINE" ] +} + +read_last_eval() { + record_usable || return 0 + awk -F '\t' '$1 == "last_eval" { print $2; exit }' "$RECORD" 2>/dev/null || true +} + +check_due() { + local now last age + now=$(now_epoch) + last=$(read_last_eval) + case "$last" in + ''|*[!0-9]*) printf '%s\n' "$now"; return 0 ;; + esac + age=$((now - last)) + if [ "$age" -lt 0 ] || [ "$age" -ge "$INTERVAL" ]; then + printf '%s\n' "$now" + return 0 + fi + return 1 +} + +evaluate_budget() { + local report line reason valid=yes budget='' total='' status='' exception='' + if ! report=$(FM_HOME="$FM_HOME" FM_CONFIG_OVERRIDE="$CONFIG_DIR" FM_DATA_OVERRIDE="$DATA_DIR" \ + "$BUDGET_BIN" report 2>&1); then + reason=${report##*startup-memory-budget: } + append_finding "startup memory budget unavailable owner=bin/fm-startup-memory-budget.sh reason=${reason//$'\n'/ }" + return 0 + fi + while IFS= read -r line; do + case "$line" in + effective_budget_tokens=*) budget=${line#*=} ;; + total_estimated_tokens=*) total=${line#*=} ;; + budget_status=*) status=${line#*=} ;; + exception=*) exception=${line#*=} ;; + esac + done < <(printf '%s\n' "$report") + case "$budget:$total" in + *[!0-9:]*|:*|*:) valid=no ;; + esac + case "$status" in + within-budget|over-budget) ;; + *) valid=no ;; + esac + case "$exception" in + ''|primary-owned-shared-file-alone-exceeds-budget) ;; + *) valid=no ;; + esac + if [ "$valid" = no ]; then + append_finding "startup memory budget unavailable owner=bin/fm-startup-memory-budget.sh reason=unparseable report" + return 0 + fi + printf '%s\t%s\t%s\t%s\t%s\n' memory_budget "$budget" "$total" "$status" "$exception" >> "$NEW_RECORD" || exit 1 + [ "$status" = over-budget ] && [ -z "$exception" ] || return 0 + append_finding "startup memory budget overrun total_estimated_tokens=$total budget=$budget owner=bin/fm-startup-memory-budget.sh" +} + +run_check() { + local now reported_previous + if ! now=$(check_due); then + return 0 + fi + [ -d "$STATE" ] && [ ! -L "$STATE" ] || fail "state directory is unavailable" + sweep_orphan_records "$now" + OLD_RECORD=$RECORD + record_usable || OLD_RECORD=/dev/null + reported_previous=$(awk -F '\t' '$1 == "reported" { print substr($0, index($0, "\t") + 1); exit }' "$OLD_RECORD" 2>/dev/null || true) + NEW_RECORD=$(mktemp "$STATE/.startup-growth-check.XXXXXX") || exit 1 + trap 'rm -f -- "${NEW_RECORD:-}"' EXIT + trap 'rm -f -- "${NEW_RECORD:-}"; exit 1' HUP INT TERM + FINDINGS= + printf '%s\n' "$RECORD_SCHEMA_LINE" > "$NEW_RECORD" || exit 1 + printf '%s\t%s\n' last_eval "$now" >> "$NEW_RECORD" || exit 1 + + stat_surface tracked AGENTS.md "$FM_ROOT/AGENTS.md" no + stat_surface tracked CLAUDE.md "$FM_ROOT/CLAUDE.md" yes + stat_surface tracked bin/fm-session-start.sh "$FM_ROOT/bin/fm-session-start.sh" no + stat_surface tracked bin/fm-bootstrap.sh "$FM_ROOT/bin/fm-bootstrap.sh" no + stat_surface tracked bin/fm-supervision-instructions.sh "$FM_ROOT/bin/fm-supervision-instructions.sh" no + stat_surface printed-memory data/projects.md "$DATA_DIR/projects.md" yes + stat_surface printed-memory data/secondmates.md "$DATA_DIR/secondmates.md" yes + stat_surface memory data/captain.md "$DATA_DIR/captain.md" yes + stat_surface memory data/captain-shared.md "$DATA_DIR/captain-shared.md" yes + stat_surface memory data/learnings.md "$DATA_DIR/learnings.md" yes + + evaluate_budget + + if [ -n "$FINDINGS" ]; then + if [ "$FINDINGS" != "$reported_previous" ]; then + fm_cap_line "startup-growth: $FINDINGS" "$MAX_LINE" + fi + printf '%s\t%s\n' reported "$FINDINGS" >> "$NEW_RECORD" || exit 1 + fi + write_record_atomically "$NEW_RECORD" "$RECORD" || fail "could not publish report record" + NEW_RECORD= +} + +SHIM_TMP= +ARM_BACKUP= + +shim_write() { # + local want=$1 device=$2 + fm_pr_regular_destination_on_device_or_absent "$CHECK_SHIM" "$device" || return 1 + if [ -e "$CHECK_SHIM" ] && [ "$(fm_pr_file_mode "$CHECK_SHIM")" = 700 ] \ + && [ "$(cat "$CHECK_SHIM" 2>/dev/null)" = "$want" ]; then + return 0 + fi + SHIM_TMP=$(umask 077; mktemp "$STATE/.startup-growth-check-shim.XXXXXX" 2>/dev/null) || return 1 + if ! printf '%s\n' "$want" > "$SHIM_TMP" \ + || ! chmod 0700 "$SHIM_TMP" \ + || ! fm_pr_private_file_valid "$SHIM_TMP" 700 "$device" \ + || ! fm_pr_regular_destination_on_device_or_absent "$CHECK_SHIM" "$device" \ + || ! mv -f -- "$SHIM_TMP" "$CHECK_SHIM"; then + rm -f -- "$SHIM_TMP" + SHIM_TMP= + return 1 + fi + SHIM_TMP= + fm_pr_private_file_valid "$CHECK_SHIM" 700 "$device" +} + +shim_backup() { # + local device=$1 tmp + tmp=$(umask 077; mktemp "$STATE/.startup-growth-check-shim.XXXXXX" 2>/dev/null) || return 1 + if ! cat "$CHECK_SHIM" > "$tmp" 2>/dev/null \ + || ! chmod 0700 "$tmp" \ + || ! fm_pr_private_file_valid "$tmp" 700 "$device"; then + rm -f -- "$tmp" + return 1 + fi + printf '%s\n' "$tmp" +} + +arm_rollback() { + [ -z "$SHIM_TMP" ] || rm -f -- "$SHIM_TMP" + SHIM_TMP= + if [ -n "$ARM_BACKUP" ]; then + mv -f -- "$ARM_BACKUP" "$CHECK_SHIM" 2>/dev/null || rm -f -- "$ARM_BACKUP" + ARM_BACKUP= + if fm_custom_check_registered "$STATE" "$CHECK_ID"; then + return 0 + fi + fi + rm -f -- "$CHECK_SHIM" "$CHECK_TRUST" +} + +arm_failed() { # + trap - HUP INT TERM + arm_rollback + fail "$1" +} + +arm() { + local state_device home want + [ -d "$STATE" ] && [ ! -L "$STATE" ] || fail "state directory is unavailable" + case "$FM_HOME" in + /*) home=$FM_HOME ;; + *) home=$(CDPATH='' cd -- "$FM_HOME" 2>/dev/null && pwd -P) || fail "cannot resolve FM_HOME $FM_HOME" ;; + esac + state_device=$(fm_pr_file_device "$STATE") || fail "state directory is unavailable" + want=$(printf '%s\n' \ + '#!/usr/bin/env bash' \ + "export FM_HOME=$(printf '%q' "$home")" \ + "exec $(printf '%q' "$SCRIPT_DIR/fm-startup-growth-check.sh") check") + ARM_BACKUP= + if [ -f "$CHECK_SHIM" ] && [ ! -L "$CHECK_SHIM" ]; then + ARM_BACKUP=$(shim_backup "$state_device") || fail "could not save the existing check shim" + fi + trap 'arm_failed "arming was interrupted"' HUP INT TERM + shim_write "$want" "$state_device" || arm_failed "check shim path is unavailable" + FM_HOME="$home" "$REGISTER_BIN" "$CHECK_ID" >/dev/null || arm_failed "could not register the check shim" + trap - HUP INT TERM + [ -z "$ARM_BACKUP" ] || rm -f -- "$ARM_BACKUP" + ARM_BACKUP= + printf 'armed: state/%s.check.sh\n' "$CHECK_ID" +} + +disarm() { + rm -f -- "$CHECK_SHIM" "$CHECK_TRUST" "$RECORD" + printf 'disarmed: state/%s.check.sh\n' "$CHECK_ID" +} + +case "${1:-check}" in + check) + [ "$#" -le 1 ] || { usage >&2; exit 2; } + run_check + ;; + arm) + [ "$#" -eq 1 ] || { usage >&2; exit 2; } + arm + ;; + disarm) + [ "$#" -eq 1 ] || { usage >&2; exit 2; } + disarm + ;; + -h|--help|help) + usage + ;; + *) + usage >&2 + exit 2 + ;; +esac diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index fd1751d2207..2871a44b7f6 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -2359,50 +2359,15 @@ teardown_live_slot_path() { canonical_existing_dir "$WT" } +# Every local Firstmate state directory whose records can name a pool slot this +# task's slot might also be; bin/fm-wake-lib.sh's fm_local_firstmate_state_dirs +# owns the walk and what it refuses. collect_local_firstmate_states() { - local record_state=$1 root home reg line child known existing i=0 - local -a homes - TREEHOUSE_OWNER_STATES=("$record_state") - root=$(fm_firstmate_root_home "$FM_HOME") || { - echo "REFUSED: cannot resolve the root Firstmate home; nothing was changed" >&2 + fm_local_firstmate_state_dirs "$1" || { + echo "REFUSED: $FM_LOCAL_FIRSTMATE_ERROR; nothing was changed" >&2 return 1 } - homes=("$root") - while [ "$i" -lt "${#homes[@]}" ]; do - home=${homes[$i]} - i=$((i + 1)) - known=0 - for existing in "${TREEHOUSE_OWNER_STATES[@]}"; do - [ "$existing" != "$home/state" ] || known=1 - done - [ "$known" = 1 ] || TREEHOUSE_OWNER_STATES+=("$home/state") - reg="$home/data/secondmates.md" - [ ! -e "$reg" ] && [ ! -L "$reg" ] && continue - [ -f "$reg" ] && [ ! -L "$reg" ] || { - echo "REFUSED: local Firstmate registry is unsafe at $reg; nothing was changed" >&2 - return 1 - } - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in - "- "*) - secondmate_registry_parse_line "$line" || { - echo "REFUSED: malformed local Firstmate registry entry in $reg; nothing was changed" >&2 - return 1 - } - [ "$SECONDMATE_REGISTRY_REMOTE" -eq 0 ] || continue - child=$(canonical_existing_dir "$SECONDMATE_REGISTRY_HOME") || { - echo "REFUSED: registered local Firstmate home is unavailable: $SECONDMATE_REGISTRY_HOME; nothing was changed" >&2 - return 1 - } - known=0 - for existing in "${homes[@]}"; do - [ "$existing" != "$child" ] || known=1 - done - [ "$known" = 1 ] || homes+=("$child") - ;; - esac - done < "$reg" - done + TREEHOUSE_OWNER_STATES=("${FM_LOCAL_FIRSTMATE_STATES[@]}") } require_exclusive_worktree_slot_record() { diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index b58a1aa04b9..720e43fc355 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -394,6 +394,7 @@ family_for_basename() { fm-trace-context-spawn.test.sh|fm-spawn-worktree-settle.test.sh|\ fm-spawn-compact-adviser-disable.test.sh|\ fm-spawn-compact-adviser-disable-remote.test.sh|\ + fm-project-capacity.test.sh|\ fm-teardown-endpoint-safety.test.sh) printf '%s\n' backend-dispatch ;; diff --git a/bin/fm-timeout-lib.sh b/bin/fm-timeout-lib.sh index a785ad8b793..18a793a87ac 100644 --- a/bin/fm-timeout-lib.sh +++ b/bin/fm-timeout-lib.sh @@ -221,11 +221,13 @@ fm_exec_timed() { # exit 125 fi owner=${FM_EXEC_TIMED_OWNER_PID:-$$} - [ "$owner" != "$BASHPID" ] || owner=$PPID unset FM_EXEC_TIMED_OWNER_PID if command -v perl >/dev/null 2>&1; then exec perl -MPOSIX=WNOHANG,setpgid -MTime::HiRes=time -e ' - my ($bound, $grace, $owner) = (shift, shift, shift); + my ($bound, $grace, $owner, $shell_parent) = (shift, shift, shift, shift); + # exec preserves the shell PID, including in Bash 3.2 subshells where + # BASHPID is unavailable. Keep the pre-exec parent for startup races. + $owner = $shell_parent if $owner == $$; my $parent = getppid(); my ($pid, $pending, $kill_at, $timed_out) = (0, "", 0, 0); for my $sig (qw(TERM INT HUP)) { @@ -272,7 +274,7 @@ fm_exec_timed() { # } select undef, undef, undef, 0.05; } - ' -- "$seconds" "$grace" "$owner" "$@" + ' -- "$seconds" "$grace" "$owner" "$PPID" "$@" elif command -v timeout >/dev/null 2>&1; then exec timeout -k "$grace" "$seconds" "$@" elif command -v gtimeout >/dev/null 2>&1; then diff --git a/bin/fm-tmux-lib.sh b/bin/fm-tmux-lib.sh index f031e65870b..f7cb21f2c84 100755 --- a/bin/fm-tmux-lib.sh +++ b/bin/fm-tmux-lib.sh @@ -250,6 +250,11 @@ fm_tmux_submit_enter_core() { # [baseline-idle tmux send-keys -t "$target" Enter 2>/dev/null || true sleep "$sleep_s" state=$(fm_tmux_composer_state "$target") + # The first Enter can open a picker. A later Enter would confirm it. + if fm_composer_blocking_dialog_noted >/dev/null; then + printf 'unknown' + return 0 + fi case "$state" in pending|pending-unproven) ;; unknown) diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index bb470423666..f1176cc374d 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -1436,10 +1436,10 @@ fm_task_set_lock_path() { # # the walk at the current home, which is the correct answer rather than an # error: the parent lives on another machine, so its filesystem can neither hold # nor be observed by a lock taken here, and a remote-seeded home is itself the -# top of the local tree that bin/fm-teardown.sh's collect_local_firstmate_states -# enumerates (that walk already skips remote registry entries for the same -# reason). Refusing a remote binding instead made every operation anchored here -# fail closed inside a remote secondmate home and its local descendants. +# top of the local tree that fm_local_firstmate_state_dirs below enumerates +# (that walk already skips remote registry entries for the same reason). +# Refusing a remote binding instead made every operation anchored here fail +# closed inside a remote secondmate home and its local descendants. # # Everything else still fails closed: an unreadable or malformed binding, an # unreachable local parent, a cycle, and a chain deeper than the bound. @@ -1468,7 +1468,72 @@ fm_firstmate_root_home() { printf '%s\n' "$home" } -# The one lock serializing Treehouse slot allocation and return for a project. +# Every Firstmate state directory on THIS machine whose task records can share a +# machine-local resource with : itself, then the local +# root home and each local secondmate home registered below it, walked through +# every data/secondmates.md breadth-first. Remote registry entries are skipped, +# because their workers run on another machine. +# +# Sets FM_LOCAL_FIRSTMATE_STATES to that list, first and without +# duplicates however each directory is spelled. Returns 1 with +# FM_LOCAL_FIRSTMATE_ERROR naming what could not be proved - an unresolvable +# root, an unsafe or malformed registry, or an unavailable registered local +# home - so a caller refuses rather than treating an unreadable home as one +# with no tasks. Requires bin/fm-secondmate-registry-lib.sh to be sourced first. +# shellcheck disable=SC2034 # FM_LOCAL_FIRSTMATE_ERROR is read by callers. +fm_local_firstmate_state_dirs() { # + local first=$1 root home reg line child known existing i=0 + local -a homes + FM_LOCAL_FIRSTMATE_STATES=("$first") + FM_LOCAL_FIRSTMATE_ERROR= + root=$(fm_firstmate_root_home "$FM_HOME") || { + FM_LOCAL_FIRSTMATE_ERROR="cannot resolve the root Firstmate home" + return 1 + } + homes=("$root") + while [ "$i" -lt "${#homes[@]}" ]; do + home=${homes[$i]} + i=$((i + 1)) + known=0 + for existing in "${FM_LOCAL_FIRSTMATE_STATES[@]}"; do + if [ "$existing" = "$home/state" ] || [ "$existing" -ef "$home/state" ]; then + known=1 + fi + done + [ "$known" = 1 ] || FM_LOCAL_FIRSTMATE_STATES+=("$home/state") + reg="$home/data/secondmates.md" + [ ! -e "$reg" ] && [ ! -L "$reg" ] && continue + [ -f "$reg" ] && [ ! -L "$reg" ] || { + FM_LOCAL_FIRSTMATE_ERROR="local Firstmate registry is unsafe at $reg" + return 1 + } + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + "- "*) + secondmate_registry_parse_line "$line" || { + FM_LOCAL_FIRSTMATE_ERROR="malformed local Firstmate registry entry in $reg" + return 1 + } + [ "$SECONDMATE_REGISTRY_REMOTE" -eq 0 ] || continue + child=$([ -d "$SECONDMATE_REGISTRY_HOME" ] && + CDPATH='' cd -- "$SECONDMATE_REGISTRY_HOME" 2>/dev/null && pwd -P) || { + FM_LOCAL_FIRSTMATE_ERROR="registered local Firstmate home is unavailable: $SECONDMATE_REGISTRY_HOME" + return 1 + } + known=0 + for existing in "${homes[@]}"; do + [ "$existing" != "$child" ] || known=1 + done + [ "$known" = 1 ] || homes+=("$child") + ;; + esac + done < "$reg" + done +} + +# The one lock serializing Treehouse slot allocation and return for a project, +# and project capacity admission (bin/fm-project-capacity-lib.sh), which a +# fresh spawn evaluates under it on every backend. # # It is anchored in the local root home's state directory so that every home on # this machine that can reach the same pool - the root, and each secondmate home diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index dcd1f7c4933..16b7845dfb2 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -549,7 +549,8 @@ inbox_steer_check() { # return 0 ;; esac - tail40=$(fm_backend_capture "$backend" "$w" 40 "$(window_label "$w")" 2>/dev/null) || tail40= + watcher_capture "$backend" "$w" 40 "$(window_label "$w")" || WATCHER_CAPTURE= + tail40=$WATCHER_CAPTURE if window_is_busy "$w" "$tail40"; then [ "$verb" != retry ] || return 0 if ! count=$(fm_task_inbox_record_busy "$STATE" "$task" "$rec"); then @@ -763,7 +764,8 @@ signal_turnend_panes_churned() { # ... [ "$hash_bytes" = 32 ] || return 1 prev=$(cat "$hash_file" 2>/dev/null) || return 1 [[ $prev =~ ^[0-9a-f]{32}$ ]] || return 1 - now=$(fm_backend_capture "$backend" "$w" 40 "$label" 2>/dev/null) || return 1 + watcher_capture "$backend" "$w" 40 "$label" || return 1 + now=$WATCHER_CAPTURE [ -n "$now" ] || return 1 [ "$(printf '%s' "$now" | hash_pane)" != "$prev" ] || return 1 churned_keys+=("$key") @@ -869,27 +871,29 @@ secondmate_in_active_turn() { # local w=$1 idle=$2 tail40 [ -n "$w" ] || return 1 [ "$idle" -lt "$BUSY_TURN_MAX_SECS" ] || return 1 - tail40=$(fm_backend_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" 2>/dev/null) || return 1 + watcher_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" || return 1 + tail40=$WATCHER_CAPTURE window_is_busy "$w" "$tail40" } -# First token of the semantic busy classification for : busy, idle, -# unknown, or dead. Capture failure and a missing window are unknown, never -# idle. Empty inbox and a fresh watcher beacon are not consulted. +# Set SECONDMATE_BUSY_CLASS to the first token of the semantic busy +# classification for : busy, idle, unknown, or dead. Capture failure and +# a missing window are unknown, never idle. Empty inbox and a fresh watcher +# beacon are not consulted. Call it directly, never in $(...), so its pane +# capture runs in the watcher's own shell. +SECONDMATE_BUSY_CLASS= secondmate_busy_class() { # local w=$1 task meta tail40 verdict + SECONDMATE_BUSY_CLASS=unknown task=$(window_to_task "$w" "$STATE") meta="$STATE/$task.meta" if [ -z "$w" ] || [ -z "$task" ] || [ ! -f "$meta" ]; then - printf 'unknown' return 0 fi - tail40=$(fm_backend_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" 2>/dev/null) || { - printf 'unknown' - return 0 - } + watcher_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" || return 0 + tail40=$WATCHER_CAPTURE verdict=$(fm_busy_classify_meta "$meta" "$task" "$STATE" "$tail40") - printf '%s' "${verdict%% *}" + SECONDMATE_BUSY_CLASS=${verdict%% *} } # 0 iff a child ring is authorized: exact idle, a live agent, and a composer @@ -899,7 +903,8 @@ secondmate_busy_class() { # secondmate_idle_ring_safe() { # local w=$1 backend agent_state cstate [ -n "$w" ] || return 1 - [ "$(secondmate_busy_class "$w")" = idle ] || return 1 + secondmate_busy_class "$w" + [ "$SECONDMATE_BUSY_CLASS" = idle ] || return 1 backend=$(window_backend "$w") agent_state=$(fm_backend_agent_state "$backend" "$w" 2>/dev/null || true) [ "$agent_state" = alive ] || return 1 @@ -2138,8 +2143,10 @@ fm_active_check_stop() { # parse of the next command substitution, the body then fails to parse ("trap: # line 2: unexpected EOF while looking for matching `)'", or nothing at all), # and the signal is consumed, so a stop request could leave this watcher -# polling forever while its stopper waits (fixed upstream in bash 5.3). INT -# keeps its trap because bash ignores a direct SIGINT while a child runs. +# polling forever while its stopper waits (fixed upstream in bash 5.3). Bash +# 3.2 holds HUP and TERM until a running command substitution's child exits, so +# fm_backend_capture pane reads go through watcher_capture instead. INT keeps +# its trap because bash ignores a direct SIGINT while a child runs. watcher_stop_signals() { trap - HUP TERM trap 'exit 1' INT @@ -2176,6 +2183,50 @@ run_check_capture() { fm_check_output_cleanup } +FM_CAPTURE_OUTPUT= +WATCHER_CAPTURE= + +fm_capture_output_cleanup() { + [ -z "$FM_CAPTURE_OUTPUT" ] || rm -f -- "$FM_CAPTURE_OUTPUT" + FM_CAPTURE_OUTPUT= +} + +# watcher_capture: fm_backend_capture into WATCHER_CAPTURE with its exit status. +# The read runs as a waited background process group rather than inside $(...), +# so a stop is not held for a blocked read (watcher_stop_signals). The group is +# recorded like a check's, so watcher_cleanup stops a read still in flight. +watcher_capture() { # [expected-label] + local rc pgid + fm_capture_output_cleanup + WATCHER_CAPTURE= + FM_CAPTURE_OUTPUT=$(mktemp "$STATE/.fm-capture-output.XXXXXX") || return 1 + FM_CHECK_SIGNAL_PENDING= + trap 'FM_CHECK_SIGNAL_PENDING=1' HUP INT TERM + # The group's stderr is redirected before the fork: bash 3.2 on macOS can + # print a harmless "child setpgid ... Operation not permitted" race from the + # child before the command's own redirections apply. + set -m + { fm_backend_capture "$@" < /dev/null > "$FM_CAPTURE_OUTPUT" & } 2>/dev/null + FM_ACTIVE_CHECK_PID=$! + FM_ACTIVE_CHECK_PGID=$FM_ACTIVE_CHECK_PID + set +m + watcher_stop_signals + [ -z "$FM_CHECK_SIGNAL_PENDING" ] || exit 1 + pgid=$(ps -o pgid= -p "$FM_ACTIVE_CHECK_PID" 2>/dev/null | tr -d '[:space:]') + if [ -n "$pgid" ] && [ "$pgid" != "$FM_ACTIVE_CHECK_PGID" ]; then + fm_active_check_stop || true + fm_capture_output_cleanup + return 1 + fi + wait "$FM_ACTIVE_CHECK_PID" + rc=$? + FM_ACTIVE_CHECK_PID= + fm_active_check_stop || { fm_capture_output_cleanup; return 1; } + WATCHER_CAPTURE=$(cat "$FM_CAPTURE_OUTPUT" 2>/dev/null || true) + fm_capture_output_cleanup + return "$rc" +} + # 0 when any signaled status file carries a captain-relevant event in the bytes # appended since this watcher last classified it. The start offset is the # classified-position field in that file's .seen-* marker, and fm-classify-lib.sh's @@ -2539,6 +2590,7 @@ watcher_cleanup() { fi fm_active_check_stop || cleanup_status=1 fm_check_output_cleanup + fm_capture_output_cleanup fm_custom_check_snapshot_cleanup if [ "$owns_lock" -eq 1 ] \ && ! fm_recovery_transition "$WATCHER_DOWNTIME_MARKER" "$transition" "$WATCH_LOCK" \ @@ -3016,7 +3068,8 @@ EOF if [ "$kind" = secondmate ] && ! status_is_paused_or_captain_held "$last"; then continue fi - tail40=$(fm_backend_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" 2>/dev/null) || continue + watcher_capture "$(window_backend "$w")" "$w" 40 "$(window_label "$w")" || continue + tail40=$WATCHER_CAPTURE h=$(printf '%s' "$tail40" | hash_pane) hf="$STATE/.hash-$key" cf="$STATE/.count-$key" diff --git a/docs/agent-control.md b/docs/agent-control.md index c949a13ba96..280cb900de7 100644 --- a/docs/agent-control.md +++ b/docs/agent-control.md @@ -50,6 +50,9 @@ muse is the one verified adapter that restores the cancelled prompt back into it The clear is refused before anything is sent when the recorded backend cannot deliver it. `exit` reads the composer's state before typing the exit command and requires the exact `empty` verdict; a `pending` verdict refuses by naming the pending text, and any other verdict (`unknown`, `pending-unproven`, or an unreadable read) refuses as not proven empty, matching the fail-safe contract every other consumer that can overwrite composer input follows. +`exit` also refuses, naming the dialog as `blocked on a prompt`, when the screen shows a recognised dialog that a further Enter would answer, whether the dialog was open before the exit command was typed or the submitting Enter opened it; it sends no Escape and chooses no option, so closing the dialog is left to the operator. +A stopped agent whose pane still shows the dialog text is not refused. +[`fm_composer_blocking_dialog`](../bin/fm-composer-lib.sh) owns the recognised set, which today is only Claude's background-task exit picker; [its verification record](verification/runtime-backends.md#claude-background-task-exit-picker) lists the dialogs that are not covered. **Teardown and discard are not verbs and will not become verbs.** `exit` stops an agent and preserves everything else. @@ -71,6 +74,7 @@ A relaunch does take one session reference when the endpoint's own runtime recor A recorded raw-command basename that differs from its resolved adapter cannot reproduce the command actually running, so relaunch refuses before the checkpoint unless the caller passes an explicit `--harness` to choose the replacement runtime deliberately. A harness change resets model and effort unless they are named too, because a model chosen for one adapter does not transfer to another. A Claude or Pi replacement must also pass the home's [worker account pin](configuration.md#worker-account-pin-configclaude-account-configpi-account), so a pin that no longer resolves or is signed out refuses before the old agent stops. + Ship and scout replacements also pass the [worker tool exclusion checks](configuration.md#worker-tool-exclusions-configcrew-exclude-tools) at this step. 2. **Safe checkpoint.** The recorded worktree must exist and be a worktree root; its head and dirty state are recorded. For a `kind=secondmate` task, the home's identity marker must match and its child records must be readable, so a relaunch can never strand child work behind an unreadable home. diff --git a/docs/calm-mode-feasibility.md b/docs/calm-mode-feasibility.md index 89e84796271..f1515eef7bd 100644 --- a/docs/calm-mode-feasibility.md +++ b/docs/calm-mode-feasibility.md @@ -154,10 +154,10 @@ Calm replaces Pi's stock working row with a small animated boat while Calm is on This path uses only public extension API and patches nothing: `ExtensionUIContext.setWorkingVisible(false)` hides the stock row, and `setWidget()` installs a temporary component factory above the editor. Pi's documented custom working-indicator frames are static and width-blind, so they cannot own responsive geometry; a widget component receives `render(width)` and can. -`.pi/extensions/fm-calm.ts` remains the sole owner of the presentation choice and the only caller of `setWorkingVisible()`, while `.pi/extensions/lib/fm-calm-working-ship.ts` owns Pi's ANSI painting and the widget over the sprite geometry, bounce track, cadences, and freeze/resume state in `.claude/mods/firstmate-calm/lib/fm-calm-working-ship-sprite.ts`, the harness-neutral core the Claude Code mod also draws from (reached from the Pi tree through a tracked symlink, because Claude Code refuses a hooks-module import from outside the plugin folder). +Within Firstmate Calm, `.pi/extensions/fm-calm.ts` remains the sole owner of its presentation choice and the only caller of `setWorkingVisible()`, while `.pi/extensions/lib/fm-calm-working-ship.ts` owns Pi's ANSI painting and the widget over the sprite geometry, bounce track, cadences, and freeze/resume state in `.claude/mods/firstmate-calm/lib/fm-calm-working-ship-sprite.ts`, the harness-neutral core the Claude Code mod also draws from (reached from the Pi tree through a tracked symlink, because Claude Code refuses a hooks-module import from outside the plugin folder). Visibility follows `agent_start` through `agent_settled` rather than turns or tool calls. Pi emits `agent_settled` from a `finally` block once a run will not continue automatically, so retries, automatic continuations, queued follow-ups, and compaction inside one run never remove the boat, while settle, abort, and failure all reach the same cleanup. -Repeated `agent_start` events inside one run are idempotent, and Pi disposes the previous component before installing a replacement under the same key and when it clears extension widgets, so the frame timer cannot duplicate or outlive the widget. +Repeated `agent_start` events inside one run are idempotent, and both Calm extensions use the shared `calm-working-ship` widget key, so Pi disposes the previous component before installing a replacement under that key and when it clears extension widgets, while Firstmate Calm's disposal signal prevents its cleanup from clearing a standalone Calm widget that replaced its own widget. Pi's above-editor widget container reserves one spacer row whether or not a widget is present, so removing the boat leaves no residual blank row. The sprite is two rows when the usable width admits the complete hull: an asymmetric three-cell `◿│◣` sail centered over a five-cell `╲▁▁▁╱` hull that sits inside the water row rather than adding a third row. diff --git a/docs/calm.md b/docs/calm.md index 1c979465ef5..c2130db72cc 100644 --- a/docs/calm.md +++ b/docs/calm.md @@ -32,6 +32,7 @@ Streaming text and the genuine reply that ends a response remain visible. While Calm is active and an agent run is under way, Calm hides Pi's built-in `Working...` row and shows a small two-row animated boat in its place. No separate Calm status row is added. +The boat claims the same Pi working-row widget slot as the standalone Pi Calm extension, so a session that loads both Calms shows one boat rather than two, and turning Firstmate Calm off does not clear the standalone boat. While Calm is off, Pi's stock working row is left exactly as Pi renders it. The boat looks like this: diff --git a/docs/configuration.md b/docs/configuration.md index 16691d9e685..d7882eea205 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,7 +9,7 @@ Start with the directory layout, then use the setting reference for the behavior | --- | --- | | Firstmate's code, private files, or project location | [FM_HOME](#fm_home) and [operational home layout](#operational-home-layout-and-state) | | Task windows and worker tools | [Runtime backend](#runtime-backend-configbackend--fm_backend) and [harness support](#harness-support) | -| Worker permissions, accounts, or environment | [Claude permission mode](#claude-permission-mode-configclaude-permission-mode), [worker account pin](#worker-account-pin-configclaude-account-configpi-account), and [worker launch environment](#worker-launch-environment-configlaunch-env-allowlist) | +| Worker permissions, accounts, or environment | [Claude permission mode](#claude-permission-mode-configclaude-permission-mode), [worker account pin](#worker-account-pin-configclaude-account-configpi-account), [worker tool exclusions](#worker-tool-exclusions-configcrew-exclude-tools), and [worker launch environment](#worker-launch-environment-configlaunch-env-allowlist) | | Backlog, preferences, and memory | [Backlog backend](#backlog-backend-taskstoml--configbacklog-backend), [captain preferences](#captain-preferences-datacaptainmd--datacaptain-sharedmd), and [startup memory budget](#startup-memory-budget-configstartup-memory-budget) | | Supervision and presentation | [Pi supervision branch](#pi-supervision-branch), [supervision host](#supervision-host-configsupervision-host), and [Calm preference](#calm-preference-configcalm) | | Persistent secondmates | [Secondmate routes](#secondmate-routes-datasecondmatesmd) | @@ -682,6 +682,28 @@ The internal [`/stow` skill](../.agents/skills/stow/SKILL.md) owns curation and The helper's header owns exact parsing, publication, and report output mechanics. +### Daily startup growth check + +A home can arm a lightweight daily growth monitor with `bin/fm-startup-growth-check.sh arm`. +It writes `state/startup-growth.check.sh` and binds it through the existing authenticated watcher-check mechanism, so no extra daemon or scheduler is installed. +Registering it is a reason to watch on the same terms as the [watched-tool check](#watched-tool-updates-configwatched-toolsjson), so an armed home keeps needing a watcher after its last task is torn down. +Use `bin/fm-startup-growth-check.sh disarm` to remove the check and its local report record. + +The check evaluates at most once per day and stays silent when nothing meaningful changed. +A due evaluation uses file metadata and byte sizes before any content inspection: it asks `bin/fm-startup-memory-budget.sh report` for the budget verdict over `data/captain.md`, `data/captain-shared.md`, and `data/learnings.md`, watches the `data/projects.md` and `data/secondmates.md` that session start also prints in full for growth without entering that budget total, and separately watches the tracked startup/instruction owner files described by the script header. +`bin/fm-startup-memory-budget.sh` remains the sole owner of the budget total and its verdict, so the check never re-derives either: when that owner annotates an overrun caused by the primary-owned `data/captain-shared.md` alone, a secondmate home is not woken about an overrun it cannot act on. +A secondmate home is likewise not notified about per-file growth of that same primary-owned `data/captain-shared.md`, which it receives read-only; the growth is still observed and recorded, and a primary home reports it normally. +Those tracked bytes are code and instruction-surface size, not prompt-memory cost. +The check does not run session-start, bootstrap, network checks, model calls, repository refreshes, `/stow`, or full preference/learnings rereads. + +Growth is measured against a per-file baseline retained in the check's own state record, so accumulation that stays under one day's threshold is still caught once it adds up; reporting a file rebases its baseline to the reported size, so accepted growth then stays silent. +A surface observed for the first time is baselined silently, including the first content of an optional file that did not exist yet when the check was armed, and an established baseline survives that file disappearing and coming back. +The fixed growth thresholds are inspectable in the script header: 2048 bytes for tracked startup/instruction files and 250 estimated tokens for the printed startup-memory files. +Budget overrun, unsafe or unreadable inputs, missing required tracked owner files, or material growth are reported once and deduplicated until the finding changes or clears; the report line is delivered before the check advances its own record, so a state-publication failure can repeat a finding but never swallow one. +That one line goes out through the shared per-line digest cut, so an over-long finding set carries the repo's `[truncated]` marker instead of ending mid-finding, while deduplication keeps comparing the full uncapped set. +Older bulk learning files remain reference-only; this monitor neither loads nor merges them. +A reported review need is only a recommendation, not cleanup authority. + ## Stow pass horizon (config/stow-pass-horizon) `config/stow-pass-horizon` is an optional local, gitignored presence flag that opts this home in to the pass-count decay horizon in the internal [`/stow` skill](../.agents/skills/stow/SKILL.md). @@ -862,6 +884,43 @@ The file is a captain-wide safety preference, so it is inherited into secondmate The [Claude adapter reference](../.agents/skills/harness-adapters/references/harness/claude.md) records the permission-mode observations and the distinct startup dialogs. +## Worker tool exclusions (config/crew-exclude-tools) + +The optional local, gitignored `config/crew-exclude-tools` hides named tools from this home's ship and scout workers, for example to keep an MCP server's write tools out of reach while its read tools stay available. +The contract is runtime-neutral: a runtime must support hiding the listed tool names or refuse the launch, and a non-empty list is never silently ignored. +With no file, or a file with no entries, every launch on every runtime is unchanged. + +Create the file with one tool name per line, such as `mcp____` for an MCP tool. +Blank lines and lines beginning with `#` are allowed, and surrounding whitespace on a line is trimmed; a trailing comment on an entry line is not allowed. +The file is read from this home's own configuration directory on every launch, so a change reaches the next worker or relaunch without a restart. +It is not in the inherited configuration set, so no other home, including a secondmate home, receives it; create the file in each home that wants it. +It does not apply to a secondmate's own agent, which neither reads nor refuses on it. + +### Runtime support + +| Runtime | With a non-empty list | +| --- | --- | +| `pi`, `pi-signed` | Hides listed tool names, MCP tools included, on every ship and scout spawn and relaunch. | +| Every other runtime, and a raw launch command | The launch refuses with an error naming `config/crew-exclude-tools`, because that runtime has no verified way to hide tools. | + +A relaunch validates the list and the replacement runtime's support before stopping the running worker, so an exclusion-list refusal preserves the running agent. + +### Validation + +An entry may use only `A-Z`, `a-z`, `0-9`, `_`, `.`, and `-`. +An entry with any other character, including internal whitespace, a comma, or a `*`, refuses the launch and names the offending entry. +An unreadable or nonregular file, or a path inspection error, also refuses and names the configuration file. +For a new worker, these checks run before its endpoint, local copy, or task record is created; Firstmate never launches with a partial list. +Only exact tool names are accepted, not wildcard patterns. +Firstmate checks syntax and runtime support before launch but never runs `pi mcp list` or otherwise connects to servers to validate names. +When its first agent run starts, after Pi's startup tool-loading boundary, the worker extension compares the launch's exclusion list with its own loaded-tool registry and appends a timestamped warning note to `state/.status` naming the configuration file and every unmatched entry for the supervisor. +The check runs before worker actions so it cannot supersede a terminal status emitted during the turn. +An unmatched entry is reported as **unverified**, not valid: Pi versions that omit excluded tools from the registry cannot distinguish a correct exclusion from a typo, and a server that has not connected cannot verify its tools either. +Names present in the registry produce no report; unknown or unverified names do not refuse the launch. +Each relaunch installs a fresh worker extension with the home's current list, so the replacement performs the same check. + +[`bin/fm-exclude-tools-lib.sh`](../bin/fm-exclude-tools-lib.sh) implements parsing and pre-launch validation for this contract; [`bin/fm-spawn.sh`](../bin/fm-spawn.sh)'s header owns the launch-flag mechanics. + ## Worker account pin (config/claude-account, config/pi-account) A home that mixes accounts for one runner, such as a work login and a personal one, can pin the account its own Claude and Pi workers launch on. @@ -1017,6 +1076,29 @@ When stripping is enabled, the hooks directory is read-only, so a hook manager r The flag is a home-wide attribution choice, so it is inherited into secondmate homes under the [`secondmate-provisioning`](../.agents/skills/secondmate-provisioning/SKILL.md) inherited-local-material contract and a secondmate's own workers keep AI trailers too. Per-machine Cursor `cli-config.json` attribution-off is not this contract: it does not travel with Firstmate, defaults back to on when unset, and only feeds the CLI's request to the server, so it suppresses the trailer rather than preventing it. +## Project capacity (config/project-capacity) + +The optional local, gitignored `config/project-capacity` tells Firstmate how many workers a project can run at once on this machine, for a project whose machine-local resource - a heavy test suite, a local editor stack, a device - only serves a few workers at a time. +Without it, dispatch stays uncapped as `AGENTS.md` section 7 describes, and a surplus worker is launched only to spend full-context turns waiting for the resource. +The file lives in the machine's root Firstmate home, so every local secondmate home reads the same limit, and it holds one line per project: + +```text +# heavy suite serves two workers +my-project 2 +``` + +The name is the project's registered name, which is its clone directory name and may contain spaces, and the number, the last field on the line, is a positive integer. +A line that is only `#`, or that begins with `#` followed by whitespace, is a comment, as is a `#` line whose last field is not an integer. +A project name may begin with `#` when that `#` is written immediately against the rest of the name and the line ends with the project's capacity. +A name that is `#`, or that begins with `#` and a space, cannot be declared, because that line is a comment. +A place is held by every ship or scout on that project in the root home or any local secondmate home registered under it, including one working in a separate clone of the same origin, until its ready PR is recorded or it is cleaned up; a local-only ship or a scout holds its place until cleanup. +The declaration is matched by the spawning clone's directory name, so clones of the same origin share the cap only when they use that same directory name. +A clone of that origin under a different directory name finds no declaration and is not capped, though its workers are still counted as holders for a same-origin clone that is capped. +When every place is held, `bin/fm-spawn.sh` launches nothing, creates no record, leaves the backlog item queued, prints one `deferred:` line naming the holders, and exits 75, so Firstmate dispatches the item again once a place frees. +A malformed or unreadable file refuses every fresh ship or scout spawn until it is fixed, rather than guessing the intended limit, and so does a local home's state directory or task record that cannot be read while counting a capped project's holders. +Firstmate cannot see which part of a worker's life uses the resource, so the number bounds whole workers from launch to handoff, and the tightest resource every worker needs should decide it. +[`bin/fm-project-capacity-lib.sh`](../bin/fm-project-capacity-lib.sh) owns the file format, what holds a place, and why concurrent spawns cannot both take the last one. + ## Crew dispatch profiles (config/crew-dispatch.json) `config/crew-dispatch.json` is an optional local, gitignored file containing natural-language rules that firstmate reads before dispatching a crewmate or scout. diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index d9a7ee19721..1cfab87a63c 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -457,8 +457,13 @@ Any of these preserves the candidate and lets session startup continue with at m - A failed journal publication or projected workspace create stops that spawn instead of falling back flat. So a Herdr create failure surfaces as a spawn failure in every Herdr home, rather than only in homes that opted in. Every earlier degradation on the fresh projected-create path (no session server, contended presentation lock, absent or ambiguous parent) still warns and continues flat. -- Recovery of an existing presentation journal deliberately refuses the spawn when the shared presentation lock is contended, rather than falling back flat. - Default-on makes that refusal reachable in any Herdr home. +- Recovery of an existing presentation journal refuses by default when the shared presentation lock is contended, rather than falling back flat. + Pass `fm-spawn.sh --herdr-resume-lock-wait` to opt that recovery into waiting for the lock instead, so concurrent recoveries can serialize. + The flag applies to a fresh ship or scout spawn that recovers a journal. + The multi-task path forwards the flag to each per-pair spawn. + `fm-spawn.sh --relaunch` and `--secondmate` take no exact-resume presentation-order lock, so the flag has no effect there. + Dead-owner reclaim still stops the wait when a holder crashed. + Unbounded blocking on the session lock is never the default. - Existing layouts are not force-renamed or rearranged. - Missing or ambiguous restart bindings fall back to the ordinary home workspace while the old projection remains untouched. - Crashes, lost responses, failed exact-pane cleanup, or human renames can leave quarantined spaces. diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index cb725a7b65c..980fbceaf8c 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -588,6 +588,12 @@ The [process-to-event operating contract](configuration.md#process-to-event-sour The source log is never truncated or consumed. A shortened or changed prefix stops the relay and surfaces a continuity failure instead of silently resetting the cursor. +The failure appends one `blocked` line to the parent status stream, which opens a decision. +The line records the reason, the reader position (the cursor offset and the first 12 characters of the prefix hash), and the retirement count (how many times the route has been retired). +Reading the same break again, with the cursor where it was and the count unchanged, appends nothing. +A later break at a different reader position, or after another retirement, appends a new `blocked` line and opens the decision again. +A line written before that position was recorded does not match, so the next break appends the new line once. + ### SSH exit 255 and unavailable homes An SSH exit status of 255 always means transport failure or unknown remote completion. diff --git a/docs/scripts.md b/docs/scripts.md index 109305abc4a..46c4d87a735 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -136,6 +136,7 @@ The shared no-mistakes gate lifecycle boundary is summarized in [architecture.md | `fm-check-unregister.sh` | Retire a custom watcher check and its trust binding by validated task id | | `fm-check-lib.sh` | Validate custom-check registrations and prepare private execution snapshots | | `fm-tool-update-check.sh` | Report watched tooling with an update available, and updates installed but left inert by PATH order | +| `fm-startup-growth-check.sh` | Daily metadata-only growth check for startup memory and tracked startup/instruction surfaces | | `fm-pr-lib.sh` | Own canonical task and PR validation plus private atomic PR-poll publication, merge-notification identity, and retirement | | `fm-pr-gitea-lib.sh` | Own the Gitea/Forgejo instance allow-list (`config/gitea-instances`), tea login derivation, and pull request forge read | | `fm-pr-poll.sh` | Provide the byte-static watcher program for validated pull-request, merge-request, and Gerrit-change poll sidecars | diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index f45033ef120..7fe1ba7df17 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -1264,6 +1264,24 @@ FM_HERDR_SUBMIT_CONFIRM_LIVE=1 tests/fm-herdr-submit-confirm-live-e2e.test.sh ok - live Herdr submit confirm: Claude Code (2.1.283 (Claude Code)) on herdr 0.9.0 proves and submits a typed /exit behind its command popup ``` +### Claude background-task exit picker + +Measured 2026-10-05 against Claude Code 2.1.289 in an isolated tmux session. +The Herdr lab was not running, so the Herdr path is covered by the existing fakes. +Typing `/exit` while a background shell is still running opens a picker whose selected row is "Exit and stop tasks" and whose footer is "Enter to confirm · Esc to cancel". +That screen still classifies as pending, the same verdict as unsubmitted composer text. +A second Enter would confirm the selected row. +The picker is recognised by its recorded structure only: the heading on its own line, then the selected row alone on its row, with `Enter to confirm · Esc to cancel` as the last non-blank row. +The same strings quoted above a normal composer, as a diff, this note, or a test fixture shows them, are not a picker. +Submit retries now stop after the Enter that opened the picker and report unknown. +A typed submit to a pane that already shows the picker types nothing and sends no Enter. +Exit reports that the worker is blocked on the Claude background-task exit picker and does not type another Enter. +A submit can return before any read sees the picker, so exit reads the screen once more when its wait for the agent to stop times out, and names the picker there too. +Exit does not report a stopped agent whose pane still shows the picker text as blocked on a prompt. +The watcher does not read the picker: a pane parked on it keeps the ordinary stale triage. +No recorded screen was available for a model-downgrade confirmation, an MCP approval, or a Claude exit confirmation other than this picker, so those dialogs are not covered. +Refusing an Enter that would confirm a dialog restores an existing safety path, so it is not gated behind a flag. + ### Prune and respawn The real label-collision reproduction is owned by: @@ -1406,7 +1424,10 @@ ok - real Herdr lab validation completed on Herdr 0.8.0 with the default-session ``` The projected spawn in that run used the historical empty opt-in file, so a home that had already enabled the projection keeps it without any migration step. -One concurrent cross-home recovery case refused under contention on a loaded machine and passed on an immediate rerun; recovery-path presentation lock contention is a deliberate hard refusal rather than a flat fallback, which default-on now makes reachable from any Herdr home. +One concurrent cross-home recovery case refused under contention on a loaded machine and passed on an immediate rerun; recovery-path presentation lock contention remains a deliberate hard refusal by default rather than a flat fallback, which default-on makes reachable from any Herdr home. +Callers that need concurrent recoveries to serialize can pass `fm-spawn.sh --herdr-resume-lock-wait`. +The flag applies to a fresh ship or scout spawn, and the multi-task path forwards it to each per-pair spawn. +It has no effect on `--relaunch` and `--secondmate`, because those paths take no exact-resume presentation-order lock. That run measured the default-on projection on Herdr 0.8.0 only, while the focus-flash regression below was last run on 0.7.5 before the flip, so neither run covered a defective release under default-on projection; the version floor and the focus-flash suite's Part C close that gap. The restored-shell session-start cleanup ran on 2026-07-24 against Herdr 0.7.5 protocol 17: diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index f14717596b4..b82ccf2e712 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -402,7 +402,7 @@ Only the watcher process touches `state/.last-watcher-beat`. No helper process can make a wedged watcher appear healthy. An arm whose own script path sits under a disposable no-mistakes validation checkout (`.no-mistakes/worktrees/`) refuses with the typed failure line before touching any state, because a watcher started there outlives the validation step and keeps writing the real home's state from a checkout about to be deleted. Once per poll the watcher checks that its home, its state directory, and its own code root still exist, and exits with a logged reason when one is gone, scoped to itself alone, so a torn-down temporary home or a discarded checkout never leaves an orphan watcher behind. -The watcher uses bash's native fatal handling for HUP and TERM, including during a blocked poll, so both run its EXIT cleanup. +The watcher uses bash's native fatal handling for HUP and TERM, including during a blocked check or a blocked `fm_backend_capture` pane read, so both run its EXIT cleanup and stop that read. `watcher_stop_signals` in `bin/fm-watch.sh` owns the signal-handling rationale. The EXIT cleanup bounds its wait for `state/.watcher-down.lock` while persisting recovery state with `FM_WATCHER_CLEANUP_LOCK_BOUND` (default 2 seconds). Only positive decimal integers are accepted, including leading-zero forms such as `08`; empty, non-numeric, and zero values (including `00`) fall back to 2 seconds. @@ -468,6 +468,7 @@ They also prove that a legacy or handoff-phase watcher marker from an absent rep `tests/fm-watch-triage.test.sh` proves TERM stops a watcher blocked inside a poll's pane capture and still releases its lock and records an acknowledgeable stop. It also exercises a single TERM with a live foreign downtime-marker lock holder, retained stale singleton and subsequent arm-style recovery, including decimal `08` and zero `00` cleanup bounds. It checks that a newly appended keyed decision is classified without rereading earlier status bytes, so signal handling can return to the watcher's beacon refresh even when the status history is long. +`tests/fm-wake-queue.test.sh` proves TERM likewise stops a watcher blocked in the drain-ring idle check's pane capture. `tests/fm-watcher-lock.test.sh` covers: diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index 4aef0afc349..1be3a2054ea 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -8,6 +8,8 @@ set -u ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" HERDR_LAB_HELPER=${HERDR_LAB_HELPER:-$ROOT/bin/fm-herdr-lab.sh} +# shellcheck source=bin/fm-timeout-lib.sh +. "$ROOT/bin/fm-timeout-lib.sh" fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } pass() { printf 'ok - %s\n' "$1"; } @@ -278,13 +280,18 @@ export HERDR_SESSION="$HERDR_LAB_SESSION" HERDR_LAB_SESSION LAB_READY=0 RECORDED_WORKTREES="" LOCK_CONTENTION_OWNER_PID= +LOCK_REFUSE_HOLDER_PID= +LOCK_WAIT_HOLDER_PID= cleanup_all() { - local wt - if [ -n "$LOCK_CONTENTION_OWNER_PID" ]; then - kill "$LOCK_CONTENTION_OWNER_PID" 2>/dev/null || true - wait "$LOCK_CONTENTION_OWNER_PID" 2>/dev/null || true - LOCK_CONTENTION_OWNER_PID= - fi + local wt pid + for pid in "$LOCK_CONTENTION_OWNER_PID" "$LOCK_REFUSE_HOLDER_PID" "$LOCK_WAIT_HOLDER_PID"; do + [ -n "$pid" ] || continue + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + done + LOCK_CONTENTION_OWNER_PID= + LOCK_REFUSE_HOLDER_PID= + LOCK_WAIT_HOLDER_PID= while IFS= read -r wt; do [ -n "$wt" ] || continue [ -d "$wt" ] || continue @@ -406,10 +413,14 @@ Verify projected workspace behavior for $id. EOF } -spawn_task() { # +spawn_task() { # [extra fm-spawn args...]; SPAWN_DEADLINE_SECONDS bounds the run local id=$1 home=$2 project=$3 + shift 3 + local -a deadline_cmd=() + [ -z "${SPAWN_DEADLINE_SECONDS:-}" ] || deadline_cmd=(fm_run_timed "$SPAWN_DEADLINE_SECONDS") FM_GATE_REFUSE_BYPASS=1 FM_SPAWN_NO_GUARD=1 FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ - "$ROOT/bin/fm-spawn.sh" "$id" "$project" "sh -c 'while :; do sleep 60; done'" --mode no-mistakes --yolo off --backend herdr + ${deadline_cmd[@]+"${deadline_cmd[@]}"} "$ROOT/bin/fm-spawn.sh" "$id" "$project" "sh -c 'while :; do sleep 60; done'" \ + --mode no-mistakes --yolo off --backend herdr "$@" } finish_concurrent_spawn() { # @@ -1349,6 +1360,129 @@ teardown_task "$BRAVO_WAVE_ID" "$SECOND_HOME_B" > "$TMP_ROOT/bravo-wave-teardown "$REAL_TREEHOUSE" return --force "$BRAVO_WAVE_NEW_WT" >/dev/null 2>&1 || true pass "real Herdr lab: concurrent cross-home recoveries replace exact husks under one session lock with no focus drift" +# Exact-resume presentation-lock contention refuses by default. Hold the +# shared session lock from an unrelated process past the bounded-retry window +# and assert the default resume hard-refuses without the opt-in flag. +LOCK_REFUSE_ID=lock-refuse-resume-r1 +mkdir -p "$HOME_DIR/data/$LOCK_REFUSE_ID" +write_ship_brief "$HOME_DIR" "$LOCK_REFUSE_ID" 'Resume lock-refuse fixture.' +spawn_task "$LOCK_REFUSE_ID" "$HOME_DIR" "$RECOVERY_PROJECT_DIR" > "$TMP_ROOT/lock-refuse-first.out" 2> "$TMP_ROOT/lock-refuse-first.err" \ + || fail "lock-refuse recovery fixture failed: $(cat "$TMP_ROOT/lock-refuse-first.err")" +LOCK_REFUSE_META="$HOME_DIR/state/$LOCK_REFUSE_ID.meta" +LOCK_REFUSE_OLD_WT=$(remember_meta_worktree "$LOCK_REFUSE_META") +LOCK_REFUSE_OLD_PANE=$(grep '^herdr_pane_id=' "$LOCK_REFUSE_META" | cut -d= -f2-) +PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" stop "$HERDR_LAB_SESSION" >/dev/null \ + || fail "could not stop the isolated session for resume lock-refuse" +PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" provision "$HERDR_LAB_SESSION" \ + || fail "could not reprovision the isolated session for resume lock-refuse" + +LOCK_REFUSE_READY="$TMP_ROOT/lock-refuse-ready" +LOCK_REFUSE_HOLD_SECONDS=15 +LOCK_REFUSE_PATH=$(session_presentation_lock_path) \ + || fail "could not resolve session lock for resume lock-refuse" +ROOT="$ROOT" READY="$LOCK_REFUSE_READY" HOLD="$LOCK_REFUSE_HOLD_SECONDS" LOCK="$LOCK_REFUSE_PATH" bash -c ' + . "$ROOT/bin/fm-wake-lib.sh" + fm_lock_try_acquire "$LOCK" || exit 1 + : > "$READY" + sleep "$HOLD" + fm_lock_release "$LOCK" +' & +LOCK_REFUSE_HOLDER_PID=$! +while [ ! -e "$LOCK_REFUSE_READY" ] && kill -0 "$LOCK_REFUSE_HOLDER_PID" 2>/dev/null; do sleep 0.01; done +[ -e "$LOCK_REFUSE_READY" ] || fail "could not hold the session presentation lock for resume lock-refuse" + +LOCK_REFUSE_FOCUS=$(focus_snapshot) +if spawn_task "$LOCK_REFUSE_ID" "$HOME_DIR" "$RECOVERY_PROJECT_DIR" \ + > "$TMP_ROOT/lock-refuse-resume.out" 2> "$TMP_ROOT/lock-refuse-resume.err"; then + LOCK_REFUSE_STATUS=0 +else + LOCK_REFUSE_STATUS=$? +fi +if [ "$LOCK_REFUSE_STATUS" -eq 0 ]; then + kill "$LOCK_REFUSE_HOLDER_PID" 2>/dev/null || true + wait "$LOCK_REFUSE_HOLDER_PID" 2>/dev/null || true + fail "default resumed identity succeeded under session lock contention instead of refusing: $(cat "$TMP_ROOT/lock-refuse-resume.out")" +fi +wait "$LOCK_REFUSE_HOLDER_PID" || fail "resume lock-refuse lock holder failed" +LOCK_REFUSE_HOLDER_PID= +[ "$LOCK_REFUSE_STATUS" -ne 0 ] \ + || fail "default resumed identity returned success under contention" +grep -F "refusing a concurrent resume" "$TMP_ROOT/lock-refuse-resume.err" >/dev/null 2>&1 \ + || fail "default resume under contention did not refuse with the concurrent-resume message: $(cat "$TMP_ROOT/lock-refuse-resume.err")" +# Fixture metadata and husk must be unchanged after the refused resume. +[ "$(grep '^herdr_pane_id=' "$LOCK_REFUSE_META" | cut -d= -f2-)" = "$LOCK_REFUSE_OLD_PANE" ] \ + || fail "refused resume mutated the recorded pane id" +assert_focus_is "$LOCK_REFUSE_FOCUS" "resume lock-refuse" +# Leave the journal/meta in place so the opt-in wait path below can resume the +# same identity after another stop/reprovision cycle. +pass "real Herdr lab: default resumed identity refuses session lock contention" + +# With --herdr-resume-lock-wait, the same exact resume WAITS for session lock +# contention rather than treating a short bounded window as fatal. Hold the +# shared session lock from an unrelated process for a duration well past any +# plausible bounded-retry window so the assertion below is deterministic +# rather than a race that could pass by luck on a fast machine. +LOCK_WAIT_ID=$LOCK_REFUSE_ID +LOCK_WAIT_META=$LOCK_REFUSE_META +LOCK_WAIT_OLD_WT=$LOCK_REFUSE_OLD_WT +LOCK_WAIT_WSID=$(grep '^herdr_workspace_id=' "$LOCK_WAIT_META" | cut -d= -f2-) +LOCK_WAIT_OLD_PANE=$LOCK_REFUSE_OLD_PANE +PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" stop "$HERDR_LAB_SESSION" >/dev/null \ + || fail "could not stop the isolated session for resume lock-wait" +PATH="$HERDR_ORIGINAL_PATH" "$HERDR_LAB_HELPER" provision "$HERDR_LAB_SESSION" \ + || fail "could not reprovision the isolated session for resume lock-wait" + +LOCK_WAIT_READY="$TMP_ROOT/lock-wait-ready" +LOCK_WAIT_HOLD_SECONDS=30 +LOCK_WAIT_PATH=$(session_presentation_lock_path) \ + || fail "could not resolve session lock for resume lock-wait" +ROOT="$ROOT" READY="$LOCK_WAIT_READY" HOLD="$LOCK_WAIT_HOLD_SECONDS" LOCK="$LOCK_WAIT_PATH" bash -c ' + . "$ROOT/bin/fm-wake-lib.sh" + fm_lock_try_acquire "$LOCK" || exit 1 + : > "$READY" + sleep "$HOLD" + fm_lock_release "$LOCK" +' & +LOCK_WAIT_HOLDER_PID=$! +while [ ! -e "$LOCK_WAIT_READY" ] && kill -0 "$LOCK_WAIT_HOLDER_PID" 2>/dev/null; do sleep 0.01; done +[ -e "$LOCK_WAIT_READY" ] || fail "could not hold the session presentation lock for resume lock-wait" + +LOCK_WAIT_DEADLINE_SECONDS=$((LOCK_WAIT_HOLD_SECONDS + 60)) +LOCK_WAIT_FOCUS=$(focus_snapshot) +LOCK_WAIT_START=$(date +%s) +if SPAWN_DEADLINE_SECONDS=$LOCK_WAIT_DEADLINE_SECONDS \ + spawn_task "$LOCK_WAIT_ID" "$HOME_DIR" "$RECOVERY_PROJECT_DIR" --herdr-resume-lock-wait \ + > "$TMP_ROOT/lock-wait-resume.out" 2> "$TMP_ROOT/lock-wait-resume.err"; then + LOCK_WAIT_STATUS=0 +else + LOCK_WAIT_STATUS=$? +fi +LOCK_WAIT_ELAPSED=$(( $(date +%s) - LOCK_WAIT_START )) +wait "$LOCK_WAIT_HOLDER_PID" || fail "resume lock-wait lock holder failed" +LOCK_WAIT_HOLDER_PID= +if [ "$LOCK_WAIT_STATUS" -eq 124 ]; then + fail "opt-in resumed recovery hung for over ${LOCK_WAIT_DEADLINE_SECONDS}s instead of waiting out a ${LOCK_WAIT_HOLD_SECONDS}s session lock hold" +fi +[ "$LOCK_WAIT_STATUS" -eq 0 ] \ + || fail "opt-in resumed identity refused instead of waiting out session lock contention: $(cat "$TMP_ROOT/lock-wait-resume.err")" +[ "$LOCK_WAIT_ELAPSED" -ge $((LOCK_WAIT_HOLD_SECONDS - 5)) ] \ + || fail "opt-in resumed recovery returned after ${LOCK_WAIT_ELAPSED}s, too soon to have genuinely waited out a ${LOCK_WAIT_HOLD_SECONDS}s hold" +LOCK_WAIT_NEW_WT=$(remember_meta_worktree "$LOCK_WAIT_META") +[ "$(grep '^herdr_workspace_id=' "$LOCK_WAIT_META" | cut -d= -f2-)" = "$LOCK_WAIT_WSID" ] \ + || fail "opt-in resume lock-wait flattened the task into a different workspace" +LOCK_WAIT_NEW_PANE=$(grep '^herdr_pane_id=' "$LOCK_WAIT_META" | cut -d= -f2-) +[ "$LOCK_WAIT_NEW_PANE" != "$LOCK_WAIT_OLD_PANE" ] \ + || fail "opt-in resume lock-wait reused the old husk pane" +if lab pane get "$LOCK_WAIT_OLD_PANE" >/dev/null 2>&1; then + fail "opt-in resume lock-wait left the old husk pane behind" +fi +assert_focus_is "$LOCK_WAIT_FOCUS" "resume lock-wait" +teardown_task "$LOCK_WAIT_ID" "$HOME_DIR" > "$TMP_ROOT/lock-wait-teardown.out" 2> "$TMP_ROOT/lock-wait-teardown.err" \ + || fail "resume lock-wait fixture teardown failed: $(cat "$TMP_ROOT/lock-wait-teardown.err")" +"$REAL_TREEHOUSE" return --force "$LOCK_WAIT_OLD_WT" >/dev/null 2>&1 || true +"$REAL_TREEHOUSE" return --force "$LOCK_WAIT_NEW_WT" >/dev/null 2>&1 || true +pass "real Herdr lab: --herdr-resume-lock-wait waits out session lock contention instead of refusing" + # Seed a legacy old-format primary projection and a flat secondmate tab; correction must not migrate them. LEGACY_OUT=$(lab workspace create --cwd "$PROJECT_DIR" --label "firstmate/legacy-seed · p:AbCdEfGhIjKlMnOpQrStUv" --no-focus) \ || fail "could not seed a legacy old-format presentation space" diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 0911e981bcc..01d0a7015f8 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -126,6 +126,14 @@ herdr_submit_claude_prefix() { # printf ' \xe2\x9d\xaf %s\n' "$text" > "$resp/4.out" } +# herdr_submit_preflight_prefix: fm_backend_send_text_submit reads the composer +# once before the adapter types. That read is call 1 and shows an empty +# composer, so every adapter call moves one slot later. +herdr_submit_preflight_prefix() { # + herdr_submit_shift "$1" 1 + printf ' \xe2\x9d\xaf\n' > "$1/1.out" +} + # make_herdr_server_env_fakebin: a stateful server stub that records only the # long-lived server launch environment, then reports the server as running. make_herdr_server_env_fakebin() { # -> echoes fakebin dir @@ -4431,6 +4439,67 @@ test_send_text_submit_popup_autocomplete_requires_second_enter() { pass "fm_backend_herdr_send_text_submit: a slash-command popup's placeholder fill on Enter #1 never flips agent_status to working, so it does not short-circuit as submitted; Enter #2 is retried and lands it" } +test_send_text_submit_refuses_confirming_enter_on_exit_picker() { + local dir log resp fb out enter_count + dir="$TMP_ROOT/submit-exit-picker"; mkdir -p "$dir/responses" "$dir/tmp"; log="$dir/log"; resp="$dir/responses"; : > "$log" + herdr_submit_claude_prefix "$resp" "/exit" + printf '{"result":{"agent":{"agent":"claude","agent_status":"idle"}}}\n' > "$resp/5.out" + printf '{"result":{"agent":{"agent":"claude","agent_status":"idle"}}}\n' > "$resp/7.out" + printf '%s\n' \ + 'Background work is running' \ + '❯ 1. Exit and stop tasks' \ + 'The following will stop when you exit:' \ + 'shell · sleep 300' \ + ' 2. Move to background and exit' \ + ' 3. Stay' \ + 'Enter to confirm · Esc to cancel' > "$resp/8.out" + herdr_submit_preflight_prefix "$resp" + fb=$(make_herdr_fakebin "$dir") + out=$( PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" FM_BACKEND_HERDR_SUBMIT_POLLS=1 \ + TMPDIR="$dir/tmp" \ + bash -c '. "$0/bin/fm-backend.sh"; fm_backend_send_text_submit herdr default:w1:p2 "/exit" 3 0.01 0.01' "$ROOT" ) + enter_count=$(grep -c $'\x1f''pane'$'\x1f''send-keys'$'\x1f''w1:p2'$'\x1f''enter' "$log" || true) + [ "$out" = unknown ] || fail "the exit picker should stop the retry as unknown, got '$out'; log: $(cat "$log")" + [ "$enter_count" -eq 1 ] || fail "the exit picker should get one Enter, got $enter_count; log: $(cat "$log")" + [ -z "$(ls -A "$dir/tmp")" ] || fail "the submit left its dialog record behind: $(ls -A "$dir/tmp")" + pass "fm_backend_herdr_send_text_submit: the Claude background-task exit picker gets no confirming Enter" +} + +# Herdr can report `blocked` for a picker the submitting Enter opened. The +# submit then reports delivery with no composer read, so the picker is named +# only by the caller's next composer read, the one fm-control exit takes when +# its wait for the agent to stop times out. +test_blocked_submit_leaves_the_exit_picker_to_the_next_composer_read() { + local dir log resp fb out enter_count + dir="$TMP_ROOT/submit-blocked-picker"; mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log" + herdr_submit_claude_prefix "$resp" "/exit" + printf '{"result":{"agent":{"agent":"claude","agent_status":"idle"}}}\n' > "$resp/5.out" + printf '{"result":{"agent":{"agent":"claude","agent_status":"blocked"}}}\n' > "$resp/7.out" + printf '%s\n' \ + 'Background work is running' \ + '❯ 1. Exit and stop tasks' \ + 'The following will stop when you exit:' \ + 'shell · sleep 300' \ + ' 2. Move to background and exit' \ + ' 3. Stay' \ + 'Enter to confirm · Esc to cancel' > "$resp/8.out" + herdr_submit_preflight_prefix "$resp" + fb=$(make_herdr_fakebin "$dir") + out=$( PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" FM_BACKEND_HERDR_SUBMIT_POLLS=1 \ + FM_COMPOSER_DIALOG_SINK="$dir/sink" \ + bash -c '. "$0/bin/fm-backend.sh" + verdict=$(fm_backend_send_text_submit herdr default:w1:p2 "/exit" 3 0.01 0.01) + printf "%s|%s|" "$verdict" "$(cat "$FM_COMPOSER_DIALOG_SINK")" + fm_backend_composer_state herdr default:w1:p2 >/dev/null + cat "$FM_COMPOSER_DIALOG_SINK"' "$ROOT" ) + enter_count=$(grep -c $'\x1f''pane'$'\x1f''send-keys'$'\x1f''w1:p2'$'\x1f''enter' "$log" || true) + [ "$out" = 'empty||Claude background-task exit picker' ] \ + || fail "a blocked submit should report delivery unnamed and the next composer read should name the picker, got '$out'; log: $(cat "$log")" + [ "$enter_count" -eq 1 ] || fail "a blocked submit should send one Enter, got $enter_count; log: $(cat "$log")" + [ -f "$dir/sink" ] || fail "a submit must not remove a dialog record its caller owns" + pass "fm_backend_send_text_submit (herdr): a picker behind a blocked verdict is named by the caller's next composer read" +} + test_send_text_submit_confirms_blocked_after_enter() { local dir log resp fb out enter_count dir="$TMP_ROOT/submit-blocked"; mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log" @@ -5954,6 +6023,8 @@ test_send_text_submit_detects_landed_send test_send_text_submit_detects_swallowed_enter test_send_text_submit_replays_literal_send_stderr test_send_text_submit_popup_autocomplete_requires_second_enter +test_send_text_submit_refuses_confirming_enter_on_exit_picker +test_blocked_submit_leaves_the_exit_picker_to_the_next_composer_read test_send_text_submit_confirms_blocked_after_enter test_send_text_submit_preexisting_working_pending_is_queued_enter test_send_text_submit_preexisting_working_does_not_confirm_failed_enter diff --git a/tests/fm-calm-pi-extension.test.sh b/tests/fm-calm-pi-extension.test.sh index 1e8a52c52e3..83dc8a42bbe 100755 --- a/tests/fm-calm-pi-extension.test.sh +++ b/tests/fm-calm-pi-extension.test.sh @@ -2843,13 +2843,17 @@ TS return 1 } - wait_for_geometry_transition() { - local file=$1 transient_text=$2 final_text=$3 attempt=0 saw_transient=0 + # Pi's "Reloading..." box is a single intermediate frame, so no polling + # interval can be guaranteed to sample it on a loaded machine. Wait instead + # for the durable status row Pi appends to the transcript once the reload has + # completed and the chat has been rebuilt: it is absent before the reload and + # never appears when the reload fails. + wait_for_geometry_reload() { + local file=$1 reloaded_text=$2 final_text=$3 attempt=0 while [ "$attempt" -lt 600 ]; do capture_geometry_viewport "$file" || true - if grep -Fq "$transient_text" "$file" 2>/dev/null; then - saw_transient=1 - elif [ "$saw_transient" -eq 1 ] && grep -Fq "$final_text" "$file" 2>/dev/null; then + if grep -Fq "$reloaded_text" "$file" 2>/dev/null && + grep -Fq "$final_text" "$file" 2>/dev/null; then return 0 fi sleep 0.01 @@ -2904,9 +2908,9 @@ TS tmux -L "$TMUX_SOCKET" send-keys -t "$TMUX_SESSION" -l '/reload' tmux -L "$TMUX_SOCKET" send-keys -t "$TMUX_SESSION" Enter - wait_for_geometry_transition \ + wait_for_geometry_reload \ "$snapshot" \ - "Reloading keybindings, extensions, skills, prompts, themes, and context files..." \ + "Reloaded keybindings, extensions, skills, prompts, themes, and context files" \ "CALM_GEOMETRY_FINAL" \ || fail "Pi Calm hidden-block geometry E2E did not complete the /reload viewport transition" assert_geometry_gap "$snapshot" "reloaded native Calm transcript" @@ -2962,7 +2966,7 @@ TS } test_working_ship_geometry_and_lifecycle() { - local fixture out status version + local fixture out status version standalone_ship if ! command -v node >/dev/null 2>&1 || ! command -v npm >/dev/null 2>&1; then echo "skip: node or npm not found for Pi Calm working-ship test" return 0 @@ -2974,6 +2978,17 @@ test_working_ship_geometry_and_lifecycle() { version=$(node -p "require('$PI_PACKAGE_DIR/package.json').version") record_pi_version_evidence "$version" "Pi Calm working-ship assumptions" + # The standalone Pi Calm extension is a separate project that installs its own boat + # in the same Pi working-row slot, so the dual-install check below reads its real + # module when it is installed: a rename on either side then renders two boats and + # fails there, instead of passing against a key this test invented. The pinned slot + # contract inside the program covers a machine without that extension. + standalone_ship=${FM_STANDALONE_CALM_SHIP:-} + if [ -z "$standalone_ship" ] && [ -f "${HOME:-}/.pi/agent/extensions/calm/lib/working-ship.ts" ]; then + standalone_ship=${HOME:-}/.pi/agent/extensions/calm/lib/working-ship.ts + fi + [ -f "$standalone_ship" ] || standalone_ship= + fixture="$TMP_ROOT/working-ship" mkdir -p "$fixture/home" "$fixture/lib" "$fixture/node_modules/@earendil-works" cp "$EXT" "$fixture/fm-calm.ts" @@ -2991,7 +3006,7 @@ test_working_ship_geometry_and_lifecycle() { ln -s "$PI_PACKAGE_DIR/node_modules/typebox" "$fixture/node_modules/typebox" printf '%s\n' '{"type":"module"}' >"$fixture/package.json" - out=$(cd "$fixture" && EXT="$fixture/fm-calm.ts" FM_HOME="$fixture/home" PI_PACKAGE_DIR="$PI_PACKAGE_DIR" node --input-type=module 2>&1 <<'JS' + out=$(cd "$fixture" && EXT="$fixture/fm-calm.ts" FM_HOME="$fixture/home" PI_PACKAGE_DIR="$PI_PACKAGE_DIR" STANDALONE_CALM_SHIP="$standalone_ship" node --input-type=module 2>&1 <<'JS' import { pathToFileURL } from "node:url"; const packageRoot = process.env.PI_PACKAGE_DIR; @@ -3649,6 +3664,39 @@ const reset = () => { }; const shipWidget = () => ui.widgets.get(CALM_WORKING_SHIP_WIDGET_KEY); +let standaloneDisposed = false; +const standaloneWidget = { + render: () => ["standalone boat"], + dispose: () => { standaloneDisposed = true; }, +}; +const firstmateWidget = { + render: () => ["firstmate boat"], + dispose: () => {}, +}; +// The standalone Pi Calm extension installs its boat in the same Pi working-row +// widget slot. Where that extension is installed, the slot key comes from its own +// module, so a rename on either side registers two widgets and fails here rather +// than passing against a key this test invented; the pinned slot is the shared +// contract both implementations must keep. +const STANDALONE_SLOT = "calm-working-ship"; +let standaloneSlot = STANDALONE_SLOT; +if (process.env.STANDALONE_CALM_SHIP) { + const standaloneShip = await import( + `${pathToFileURL(process.env.STANDALONE_CALM_SHIP).href}?standalone=${Date.now()}` + ); + standaloneSlot = standaloneShip.CALM_WORKING_SHIP_WIDGET_KEY; +} +ui.setWidget(standaloneSlot, () => standaloneWidget); +ui.setWidget(CALM_WORKING_SHIP_WIDGET_KEY, () => firstmateWidget); +const renderedDualInstallWidgets = [...ui.widgets.values()].map((widget) => widget.render(80)); +check( + standaloneDisposed && + renderedDualInstallWidgets.length === 1 && + renderedDualInstallWidgets[0][0] === "firstmate boat", + `dual Calm install rendered ${renderedDualInstallWidgets.length} working widgets instead of one`, +); +ui.setWidget(standaloneSlot, undefined); + // --- Calm off leaves Pi's stock working behavior completely untouched ------------- await fire("session_start", { reason: "startup" }); reset(); @@ -3856,6 +3904,25 @@ check(liveTimers === 1, "a later run did not use the boat after an idle Calm tog await fire("agent_settled"); check(liveTimers === 0, "the later run did not clean up"); +await fire("agent_start"); +let survivingStandaloneDisposed = false; +const survivingStandaloneWidget = { + render: () => ["standalone boat"], + dispose: () => { survivingStandaloneDisposed = true; }, +}; +ui.setWidget(standaloneSlot, () => survivingStandaloneWidget); +reset(); +await calmCommand.handler("", ctx); +check( + !survivingStandaloneDisposed && + ui.widgets.size === 1 && + ui.widgets.get(standaloneSlot) === survivingStandaloneWidget && + ui.widgetOps.length === 0 && + ui.workingVisible.length === 0, + "turning Firstmate Calm off cleared or exposed the standalone working ship", +); +ui.setWidget(standaloneSlot, undefined); + // --- The visual-only widget never touches session, transcript, or export data ------ check( sessionWrites.length === 0, @@ -3870,6 +3937,11 @@ JS [ "$status" -eq 0 ] || fail "Pi Calm working-ship checks failed: $out" [ -z "$out" ] || fail "Pi Calm working-ship test printed output: $out" pass "Pi Calm working ship keeps its centered two-row asymmetric Unicode boat inside a deterministic long-wave trough, paints all water standard blue and the whole boat standard yellow with balanced resets, keeps ANSI-stripped width exact, reverses cleanly at both edges and every width, clamps visible and hidden resizes, falls back deterministically when narrow, freezes and resumes across settle/start without hidden-time jumps or duplicate timers, resets only on a fresh session, and leaves Calm-off visibility untouched" + if [ -n "$standalone_ship" ]; then + pass "Pi Calm dual-install coverage read the installed standalone Pi Calm extension's own working-ship slot from $standalone_ship" + else + pass "SKIP: no standalone Pi Calm extension is installed, so dual-install coverage used the pinned shared-slot contract; set FM_STANDALONE_CALM_SHIP to check one" + fi } # The rendered-DOM assertions below depend on a real browser, so the render step diff --git a/tests/fm-composer-lib.test.sh b/tests/fm-composer-lib.test.sh index 4dbddf14b82..83ce4599472 100755 --- a/tests/fm-composer-lib.test.sh +++ b/tests/fm-composer-lib.test.sh @@ -1080,3 +1080,158 @@ test_queued_enter_verdict_does_not_convert_other_states() { test_queued_enter_verdict_busy_pending_is_empty test_queued_enter_verdict_idle_pending_stays_pending test_queued_enter_verdict_does_not_convert_other_states + +# The selected row sits on cursor row 1 so a tmux read whose cursor is that +# row, and a cursorless read, both still see unsubmitted text. +exit_picker_screen() { + printf '%s\n' \ + 'Background work is running' \ + '❯ 1. Exit and stop tasks' \ + 'The following will stop when you exit:' \ + 'shell · sleep 300' \ + ' 2. Move to background and exit' \ + ' 3. Stay' \ + 'Enter to confirm · Esc to cancel' +} + +fm_test_picker_send() { + printf 'Enter\n' >> "$FM_TEST_PICKER_ENTERS" +} + +fm_test_picker_state() { + fm_composer_classify_screen 'styled=1' "$FM_TEST_PICKER_SCREEN" 1 +} + +test_background_exit_picker_stays_pending_and_blocks_retry() { + local screen out rc sink enters + screen=$(exit_picker_screen) + out=$(fm_composer_blocking_dialog "$screen"); rc=$? + [ "$rc" -eq 0 ] || fail "the recorded picker should match" + [ "$out" = 'Claude background-task exit picker' ] || fail "dialog name was '$out'" + out=$(fm_composer_blocking_dialog 'Background work is running'); rc=$? + [ "$rc" -eq 1 ] || fail "a heading alone must not match" + [ -z "$out" ] || fail "a miss must print nothing, got '$out'" + out=$(fm_composer_blocking_dialog "$(printf '%s\n' 'Background work is running' 'Exit and stop tasks')"); rc=$? + [ "$rc" -eq 1 ] || fail "two of the three strings must not match" + out=$(fm_composer_blocking_dialog "$(printf '%s\n' "$screen" '' '')"); rc=$? + [ "$rc" -eq 0 ] || fail "blank rows below the footer should still match" + sink=$(mktemp) + FM_COMPOSER_DIALOG_SINK=$sink + out=$(fm_composer_classify_screen 'styled=1' "$screen" 1) + [ "$out" = pending ] || fail "cursor on the selected row should stay pending, got '$out'" + [ "$(cat "$sink")" = 'Claude background-task exit picker' ] || fail "classify should note the dialog, got '$(cat "$sink")'" + out=$(fm_composer_classify_screen 'styled=1' "$screen") + [ "$out" = pending ] || fail "a styled cursorless picker should stay pending, got '$out'" + unset FM_COMPOSER_DIALOG_SINK + rm -f "$sink" + FM_TEST_PICKER_SCREEN=$screen + FM_TEST_PICKER_ENTERS=$(mktemp) + : > "$FM_TEST_PICKER_ENTERS" + fm_composer_dialog_sink_prepare || fail "the dialog sink could not be prepared" + sink=$FM_COMPOSER_DIALOG_SINK + out=$(fm_composer_submit_retry_core fm_test_picker_send fm_test_picker_state win 3 0) + fm_composer_dialog_sink_release + [ ! -e "$sink" ] || fail "the release should remove a sink that prepare created" + [ -z "${FM_COMPOSER_DIALOG_SINK:-}" ] || fail "the release should unset a sink that prepare created" + enters=$(grep -c '^Enter$' "$FM_TEST_PICKER_ENTERS" || true) + [ "$out" = unknown ] || fail "a picker must stop the retry as unknown, got '$out'" + [ "$enters" -eq 1 ] || fail "a picker must receive one Enter, got $enters" + rm -f "$FM_TEST_PICKER_ENTERS" + unset FM_TEST_PICKER_SCREEN FM_TEST_PICKER_ENTERS + pass "the Claude background-task exit picker stays pending and receives no confirming Enter" +} + +# The picker's own text, shown the way a worker pane shows it when it prints +# this repository's diff, verification note, or a test fixture: quoted above a +# normal composer. No picker is open, so the next Enter confirms nothing. +quoted_exit_picker_screen() { + printf '%s\n' \ + '● Here is the fixture the test uses:' \ + "+ 'Background work is running' \\" \ + "+ '❯ 1. Exit and stop tasks' \\" \ + "+ 'Enter to confirm · Esc to cancel'" \ + ' The selected row is "❯ 1. Exit and stop tasks" and the footer is "Enter to confirm · Esc to cancel".' \ + 'Background work is running' \ + '❯ 1. Exit and stop tasks' \ + 'Enter to confirm · Esc to cancel' \ + '' \ + '╭──────────────╮' \ + '│ > next steer │' \ + '╰──────────────╯' +} + +test_dialog_heading_and_footer_must_be_the_recorded_lines() { + local screen out rc + screen=$(printf '%s\n' \ + 'The fixture mentions Background work is running in a sentence' \ + '❯ 1. Exit and stop tasks' \ + 'Enter to confirm · Esc to cancel') + out=$(fm_composer_blocking_dialog "$screen"); rc=$? + [ "$rc" -eq 1 ] || fail "a heading buried in a sentence must not match" + [ -z "$out" ] || fail "a miss must print nothing, got '$out'" + screen=$(printf '%s\n' \ + 'Background work is running' \ + '❯ 1. Exit and stop tasks' \ + 'Enter to confirm the deployment') + out=$(fm_composer_blocking_dialog "$screen"); rc=$? + [ "$rc" -eq 1 ] || fail "a last line that only starts with the confirm words must not match" + [ -z "$out" ] || fail "a miss must print nothing, got '$out'" + pass "a buried heading or a different last line is not the exit picker" +} + +test_dialog_note_skips_the_match_when_no_sink_is_set() { + local screen out rc before after + screen=$(exit_picker_screen) + unset FM_COMPOSER_DIALOG_SINK + out=$(fm_composer_note_blocking_dialog "$screen"); rc=$? + [ "$rc" -eq 1 ] || fail "a note without a sink should return 1, got $rc" + [ -z "$out" ] || fail "a note without a sink should print nothing, got '$out'" + [ -z "${FM_COMPOSER_DIALOG_SINK:-}" ] || fail "a note without a sink must not create one" + out=$(fm_composer_classify_screen 'styled=1' "$screen" 1) + [ "$out" = pending ] || fail "classify without a sink should stay pending, got '$out'" + trap 'true' RETURN + before=$(trap -p RETURN) + fm_composer_dialog_sink_prepare || fail "the dialog sink could not be prepared" + fm_composer_dialog_sink_release + after=$(trap -p RETURN) + trap - RETURN + [ "$before" = "$after" ] || fail "release replaced the caller RETURN trap: $after" + pass "a dialog note without a sink skips the match, and release leaves a caller RETURN trap" +} + +test_quoted_exit_picker_text_is_not_a_dialog() { + local screen out rc sink enters + screen=$(quoted_exit_picker_screen) + out=$(fm_composer_blocking_dialog "$screen"); rc=$? + [ "$rc" -eq 1 ] || fail "picker text quoted above a normal composer must not match" + [ -z "$out" ] || fail "a miss must print nothing, got '$out'" + out=$(fm_composer_blocking_dialog "$(printf '%s\n' \ + 'Background work is running' \ + "+ '❯ 1. Exit and stop tasks' \\" \ + 'Enter to confirm · Esc to cancel')"); rc=$? + [ "$rc" -eq 1 ] || fail "a selected row that is not alone on its row must not match" + out=$(fm_composer_blocking_dialog "$(printf '%s\n' \ + '❯ 1. Exit and stop tasks' \ + 'Background work is running' \ + 'Enter to confirm · Esc to cancel')"); rc=$? + [ "$rc" -eq 1 ] || fail "a selected row above the heading must not match" + FM_TEST_PICKER_SCREEN=$screen + FM_TEST_PICKER_ENTERS=$(mktemp) + : > "$FM_TEST_PICKER_ENTERS" + fm_composer_dialog_sink_prepare || fail "the dialog sink could not be prepared" + sink=$FM_COMPOSER_DIALOG_SINK + out=$(fm_composer_submit_retry_core fm_test_picker_send fm_test_picker_state win 3 0) + [ ! -s "$sink" ] || fail "quoted picker text must not be noted as a dialog, got '$(cat "$sink")'" + fm_composer_dialog_sink_release + enters=$(grep -c '^Enter$' "$FM_TEST_PICKER_ENTERS" || true) + [ "$out" = pending ] || fail "quoted picker text must keep the ordinary pending verdict, got '$out'" + [ "$enters" -eq 3 ] || fail "quoted picker text must keep the ordinary Enter retries, got $enters" + rm -f "$FM_TEST_PICKER_ENTERS" + unset FM_TEST_PICKER_SCREEN FM_TEST_PICKER_ENTERS + pass "picker text quoted above a normal composer is not read as a live picker" +} + +test_background_exit_picker_stays_pending_and_blocks_retry +test_dialog_heading_and_footer_must_be_the_recorded_lines +test_dialog_note_skips_the_match_when_no_sink_is_set +test_quoted_exit_picker_text_is_not_a_dialog diff --git a/tests/fm-contributions.test.sh b/tests/fm-contributions.test.sh index 9af32fc2950..3c71ee9132e 100755 --- a/tests/fm-contributions.test.sh +++ b/tests/fm-contributions.test.sh @@ -1020,7 +1020,13 @@ test_arm_plumbs_a_configured_budget_into_the_check_shim() { wrap_forge "$home" mutate_record "$home" delivery '.records[0].checked_at="2026-09-15T08:00:00Z"' cp "$home/data/delivery/contributions.json" "$home/prior.json" + # Freeze the clock: an unfrozen one can tick past the one-second budget + # before the first forge call, so nothing is ever observed. + /bin/date +%s > "$home/forge/clock" printf 'hang\n' > "$home/forge/fault" + # Freeze the clock. An unfrozen one-second budget can tick past before the + # first forge call, so the generated check never writes forge/calls. + /bin/date +%s > "$home/forge/clock" if [ "$mode" = configured ]; then with_home "$home" env FM_CONTRIBUTIONS_BUDGET=1 "$ROOT/bin/fm-contributions.sh" arm >/dev/null \ || fail 'arm with a configured budget failed' diff --git a/tests/fm-control-relaunch.test.sh b/tests/fm-control-relaunch.test.sh index 1242dc26bfe..6400bd33b4a 100755 --- a/tests/fm-control-relaunch.test.sh +++ b/tests/fm-control-relaunch.test.sh @@ -814,6 +814,54 @@ test_worker_account_pin_follows_the_relaunch() { pass "fm-control relaunch: the replacement follows the home's current worker account pin" } +test_pi_exclude_tools_follow_the_relaunch() { + local dir out rc id=rl-pi-excl + dir=$(new_case pi-exclude "$id") + add_ship_task "$dir" "$id" pi + printf pi > "$dir/fake/command" + printf pi > "$dir/fake/becomes" + printf '#!/usr/bin/env bash\nprintf "Options: --tui-mode\\n"\n' > "$dir/fakebin/pi" + chmod +x "$dir/fakebin/pi" + mkdir -p "$dir/home/config" + printf '%s\n' '# hide writes' 'mcp__srv__writeTool' 'mcp__srv__adminTool' > "$dir/home/config/crew-exclude-tools" + out=$(run_control "$dir" "$id" relaunch --note "keep exclusions"); rc=$? + expect_code 0 "$rc" "a Pi relaunch with exclusions should succeed"$'\n'"$out" + assert_contains "$(cat "$dir/fake/literal")" "--exclude-tools 'mcp__srv__writeTool,mcp__srv__adminTool'" \ + "the relaunched Pi worker must keep the home's tool exclusions" + rm "$dir/home/config/crew-exclude-tools" + : > "$dir/fake/literal" + printf pi > "$dir/fake/command" + out=$(run_control "$dir" "$id" relaunch --note "exclusions removed"); rc=$? + expect_code 0 "$rc" "a Pi relaunch after the file is removed should succeed"$'\n'"$out" + assert_not_contains "$(cat "$dir/fake/literal")" "--exclude-tools" \ + "a relaunch without the file must launch with no exclusions" + pass "fm-control relaunch: a Pi replacement keeps the home's tool exclusions" +} + +test_exclude_tools_refusals_happen_before_the_agent_stops() { + local dir out rc id=rl-excl-refuse + dir=$(new_case excl-refuse "$id") + add_ship_task "$dir" "$id" pi + printf pi > "$dir/fake/command" + printf pi > "$dir/fake/becomes" + printf '#!/usr/bin/env bash\nprintf "Options: --tui-mode\\n"\n' > "$dir/fakebin/pi" + chmod +x "$dir/fakebin/pi" + mkdir -p "$dir/home/config" + printf '%s\n' 'two words' > "$dir/home/config/crew-exclude-tools" + out=$(run_control "$dir" "$id" relaunch --note "bad list"); rc=$? + expect_code 1 "$rc" "a malformed exclusion list must refuse the relaunch" + assert_contains "$out" "config/crew-exclude-tools has a malformed entry" "refusal must name the entry" + [ "$(cat "$dir/fake/command")" = pi ] || fail "a malformed exclusion list stopped the running agent" + [ ! -s "$dir/fake/literal" ] || fail "a refused relaunch sent lifecycle input" + printf '%s\n' 'mcp__srv__writeTool' > "$dir/home/config/crew-exclude-tools" + out=$(run_control "$dir" "$id" relaunch --harness codex --note "switch runtime"); rc=$? + expect_code 1 "$rc" "relaunching onto a runtime that cannot hide tools must refuse" + assert_contains "$out" "config/crew-exclude-tools" "refusal must name the config file" + [ "$(cat "$dir/fake/command")" = pi ] || fail "an unhonorable exclusion list stopped the running agent" + [ ! -s "$dir/fake/literal" ] || fail "a refused relaunch sent lifecycle input" + pass "fm-control relaunch: exclusion-list refusals happen before the running agent stops" +} + test_explicit_model_wins_over_the_recorded_one() { local dir out rc dir=$(new_case explicit rl7) @@ -1627,6 +1675,7 @@ test_concurrent_relaunch_is_refused() { i=$((i + 1)) done [ -e "$lock" ] || { kill "$holder" 2>/dev/null; fail "could not stage a held control lock"; } + printf 'held\n' > "$dir/home/state/rl19.composer-dialog" out=$(run_control "$dir" rl19 relaunch --note "concurrent"); rc=$? kill "$holder" 2>/dev/null || true wait "$holder" 2>/dev/null || true @@ -1635,6 +1684,8 @@ test_concurrent_relaunch_is_refused() { "the refusal should name the concurrent action" [ "$(cat "$dir/fake/command")" = claude ] \ || fail "a refused concurrent relaunch must not stop the agent" + [ "$(cat "$dir/home/state/rl19.composer-dialog" 2>/dev/null)" = held ] \ + || fail "a refused concurrent relaunch must not remove the lock holder's dialog file" pass "fm-control relaunch: two control actions on one task serialize instead of interleaving" } @@ -2386,6 +2437,57 @@ test_relaunch_moves_a_drifted_item_back_in_flight() { pass "relaunch heals an item that drifted out of In flight while the task stayed live" } +test_exit_and_relaunch_remove_the_dialog_file() { + local dir out rc + dir=$(new_case dialog-file-exit rl70) + add_ship_task "$dir" rl70 claude + out=$(run_control "$dir" rl70 exit); rc=$? + expect_code 0 "$rc" "exit should stop the agent"$'\n'"$out" + [ ! -e "$dir/home/state/rl70.composer-dialog" ] \ + || fail "exit should remove the dialog file" + + dir=$(new_case dialog-file-relaunch rl71) + add_ship_task "$dir" rl71 claude + out=$(run_control "$dir" rl71 relaunch --note "replace the agent"); rc=$? + expect_code 0 "$rc" "relaunch should replace the agent"$'\n'"$out" + [ ! -e "$dir/home/state/rl71.composer-dialog" ] \ + || fail "relaunch should remove the dialog file" + pass "fm-control removes the dialog file after exit and after relaunch" +} + +# The lock release removes paths at or under the control lock with rm, so a +# recording rm sees the state directory at the moment of release without a +# second overlapping command. +test_exit_removes_the_dialog_file_before_releasing_the_lock() { + local dir out rc lock sink trace + dir=$(new_case dialog-file-order rl72) + add_ship_task "$dir" rl72 claude + lock="$dir/home/state/.control-rl72.lock" + sink="$dir/home/state/rl72.composer-dialog" + trace="$dir/fake/rm-trace" + cat > "$dir/fakebin/rm" <> "$trace" + break + ;; + esac +done +exec "$(command -v rm)" "\$@" +SH + chmod +x "$dir/fakebin/rm" + out=$(run_control "$dir" rl72 exit); rc=$? + expect_code 0 "$rc" "exit should stop the agent"$'\n'"$out" + [ ! -e "$lock" ] || fail "exit should release the control lock" + [ "$(tail -n 1 "$trace" 2>/dev/null)" = absent ] \ + || fail "the dialog file must be gone when the control lock is released, got: $(cat "$trace" 2>/dev/null)" + pass "fm-control exit removes the dialog file before it releases the control lock" +} + +test_exit_and_relaunch_remove_the_dialog_file +test_exit_removes_the_dialog_file_before_releasing_the_lock test_same_harness_relaunch_keeps_identity_and_reuses_the_endpoint test_relaunch_refuses_before_exit_when_the_composer_holds_pending_text test_relaunch_refuses_before_exit_when_the_composer_state_is_unproven @@ -2403,6 +2505,8 @@ test_same_harness_relaunch_keeps_the_profile_axes test_native_ultra_relaunch_preserves_profile_and_rejects_before_stop test_signed_out_worker_account_pin_refuses_before_stop test_worker_account_pin_follows_the_relaunch +test_pi_exclude_tools_follow_the_relaunch +test_exclude_tools_refusals_happen_before_the_agent_stops test_explicit_model_wins_over_the_recorded_one test_relaunch_onto_an_unverified_harness_is_refused test_prior_harness_turnend_registry_entry_is_cleared diff --git a/tests/fm-control.test.sh b/tests/fm-control.test.sh index 832c3fd7a49..fcffbc36dd0 100755 --- a/tests/fm-control.test.sh +++ b/tests/fm-control.test.sh @@ -175,6 +175,18 @@ case "${1:-}" in done printf 'fakepane\n'; exit 0 ;; capture-pane) + if [ -f "$D/after-enter" ] && [ -f "$D/keys" ] && grep -qx Enter "$D/keys"; then + # after-enter-late holds how many captures after Enter still show the + # ordinary pane, for a screen that renders after the submit has read it. + late=0 + [ ! -f "$D/after-enter-late" ] || late=$(cat "$D/after-enter-late") + if [ "$late" -gt 0 ]; then + printf '%s' "$((late - 1))" > "$D/after-enter-late" + else + cat "$D/after-enter" + exit 0 + fi + fi if [ -f "$D/devin" ]; then devin_screen "$(cat "$D/devin")"; elif [ -f "$D/pane" ]; then cat "$D/pane"; else printf '╭────╮\n│ │\n╰────╯\n'; fi exit 0 ;; list-windows) @@ -838,6 +850,77 @@ test_busy_agent_is_interrupted_before_the_exit_command() { pass "fm-control exit: a busy agent receives interrupt delivery before the exit command" } +exit_picker_screen() { + printf '%s\n' \ + 'Background work is running' \ + '❯ 1. Exit and stop tasks' \ + 'The following will stop when you exit:' \ + 'shell · sleep 300' \ + ' 2. Move to background and exit' \ + ' 3. Stay' \ + 'Enter to confirm · Esc to cancel' +} + +test_exit_refuses_an_open_background_picker() { + local dir out rc + dir=$(new_case open-picker) + add_task "$dir" t1 claude + alive_as "$dir" claude + exit_picker_screen > "$dir/fake/pane" + out=$(run_control "$dir" t1 exit); rc=$? + expect_code 1 "$rc" "an open exit picker should refuse"$'\n'"$out" + assert_contains "$out" "blocked on a prompt: Claude background-task exit picker" \ + "the refusal should name the dialog" + assert_not_contains "$out" "Esc" "the refusal must not name a dismissal key" + [ ! -s "$dir/fake/literal" ] || fail "an open picker must not be typed into" + [ ! -s "$dir/fake/keys" ] || fail "an open picker must receive no keys" + pass "fm-control exit: an already-open background-task picker is not typed into" +} + +test_exit_refuses_the_confirming_enter() { + local dir out rc enters + dir=$(new_case confirm-picker) + add_task "$dir" t1 claude + alive_as "$dir" claude + exit_picker_screen > "$dir/fake/after-enter" + out=$(env FM_FAKE_NEVER_DIES=1 PATH="$dir/fakebin:$PATH" FM_HOME="$dir/home" \ + FM_FAKE_DIR="$dir/fake" FM_CONTROL_POLL=0.01 FM_CONTROL_EXIT_WAIT=0.05 \ + "$CONTROL" t1 exit 2>&1); rc=$? + expect_code 1 "$rc" "the confirming Enter should refuse"$'\n'"$out" + assert_contains "$out" "blocked on a prompt: Claude background-task exit picker" \ + "the refusal should name the dialog" + assert_not_contains "$out" "Esc" "the refusal must not name a dismissal key" + [ "$(literals "$dir")" = /exit ] || fail "the exit command should still be typed, got '$(literals "$dir")'" + enters=$(grep -c '^Enter$' "$dir/fake/keys" || true) + [ "$enters" -eq 1 ] || fail "only the submitting Enter should be sent, got $enters" + pass "fm-control exit: the Enter that opens the background-task picker is not followed by a confirming Enter" +} + +# The submit reads a cleared composer before the picker renders, so it reports +# delivery and no read inside it sees the picker. Exit's own read after the +# stop wait times out must still name the dialog. +test_exit_names_a_picker_that_renders_after_the_submit() { + local dir out rc enters + dir=$(new_case late-picker) + add_task "$dir" t1 claude + alive_as "$dir" claude + exit_picker_screen > "$dir/fake/after-enter" + printf '1' > "$dir/fake/after-enter-late" + out=$(env FM_FAKE_NEVER_DIES=1 PATH="$dir/fakebin:$PATH" FM_HOME="$dir/home" \ + FM_FAKE_DIR="$dir/fake" FM_CONTROL_POLL=0.01 FM_CONTROL_EXIT_WAIT=0.05 \ + "$CONTROL" t1 exit 2>&1); rc=$? + expect_code 1 "$rc" "a picker that renders after the submit should refuse"$'\n'"$out" + [ "$(cat "$dir/fake/after-enter-late")" = 0 ] \ + || fail "the submit should have read the ordinary pane once after Enter" + assert_contains "$out" "blocked on a prompt: Claude background-task exit picker" \ + "the refusal should name the dialog" + assert_not_contains "$out" "did not stop within" \ + "a recognised picker must not fall back to the generic timeout message" + enters=$(grep -c '^Enter$' "$dir/fake/keys" || true) + [ "$enters" -eq 1 ] || fail "only the submitting Enter should be sent, got $enters" + pass "fm-control exit: a picker that renders after the submit returned is named when the stop wait times out" +} + test_idle_agent_is_not_interrupted() { local dir out rc gen dir=$(new_case idle) @@ -853,6 +936,23 @@ test_idle_agent_is_not_interrupted() { pass "fm-control exit: an idle agent goes straight to its exit command" } +test_exit_drops_meta_busy_gen_with_the_sidecar() { + local dir out rc gen changed + dir=$(new_case codex-retire) + add_task "$dir" t1 codex + alive_as "$dir" codex + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$dir/home/state" t1) + printf 'busy_gen=%s\n' "$gen" >> "$dir/home/state/t1.meta" + grep -v '^busy_gen=' "$dir/home/state/t1.meta" > "$dir/expected.meta" + out=$(run_control "$dir" t1 exit); rc=$? + expect_code 0 "$rc" "exiting a codex agent should succeed"$'\n'"$out" + [ ! -e "$dir/home/state/t1.busy-gen" ] && [ ! -e "$dir/home/state/t1.busy-state" ] \ + || fail "exit should retire the busy sidecar and record" + changed=$(diff "$dir/expected.meta" "$dir/home/state/t1.meta") \ + || fail "exit should drop only busy_gen from the task record:"$'\n'"$changed" + pass "fm-control exit: retiring a codex incarnation drops busy_gen with the sidecar" +} + test_interrupt_without_acknowledgement_preserves_busy_state() { local dir gen before after out rc dir=$(new_case unconfirmed) @@ -864,6 +964,8 @@ test_interrupt_without_acknowledgement_preserves_busy_state() { out=$(run_control "$dir" t1 interrupt); rc=$? expect_code 0 "$rc" "an interrupt without acknowledgement should still deliver"$'\n'"$out" after=$(cat "$dir/home/state/t1.busy-state") + grep -q "^busy_gen=$gen$" "$dir/home/state/t1.meta" \ + || fail "an interrupt must leave the task record's busy_gen in place" [ "$after" = "$before" ] || fail "an unconfirmed interrupt must preserve adapter-owned busy state" assert_contains "$out" "verified=agent-alive cancel=unconfirmed" \ "the result should distinguish delivery proof from unconfirmed cancellation" @@ -955,6 +1057,8 @@ test_agent_that_does_not_stop_fails_closed() { || fail "a stubborn busy agent should receive its interrupt sequence" [ "$(literals "$dir")" = /exit ] \ || fail "a stubborn busy agent should receive its exit command" + grep -q "^busy_gen=$gen$" "$dir/home/state/t1.meta" \ + || fail "a failed exit must leave busy_gen in the task record" pass "fm-control exit: a stubborn agent reports delivered input and an unconfirmed exit" } @@ -1100,6 +1204,10 @@ test_interrupt_refuses_when_no_agent_runs test_ambiguous_endpoint_refuses test_busy_agent_is_interrupted_before_the_exit_command test_idle_agent_is_not_interrupted +test_exit_drops_meta_busy_gen_with_the_sidecar +test_exit_refuses_an_open_background_picker +test_exit_refuses_the_confirming_enter +test_exit_names_a_picker_that_renders_after_the_submit test_interrupt_without_acknowledgement_preserves_busy_state test_muse_interrupt_confirms_adapter_acknowledgement test_interrupt_revalidates_agent_after_acknowledgement_wait diff --git a/tests/fm-project-capacity.test.sh b/tests/fm-project-capacity.test.sh new file mode 100755 index 00000000000..bc64b8283b1 --- /dev/null +++ b/tests/fm-project-capacity.test.sh @@ -0,0 +1,642 @@ +#!/usr/bin/env bash +# Behavior tests for project capacity admission: a project that declares how +# many workers it admits at once on this machine never gets a fresh worker +# launched beyond that number (bin/fm-project-capacity-lib.sh owns the +# contract; bin/fm-spawn.sh runs the check). +# +# Every case drives the real bin/fm-spawn.sh against a real project clone with +# an origin, fake tmux and treehouse binaries that log every call, and a real +# markdown backlog when tasks-axi is installed. A deferred spawn is judged by +# what it left behind - no task record, no rendered launch brief, no endpoint, +# no worktree allocation, and a backlog item still queued - never by wording +# alone. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +unset TASKS_AXI_BACKEND || : + +SPAWN="$ROOT/bin/fm-spawn.sh" +TEARDOWN="$ROOT/bin/fm-teardown.sh" +TMP_ROOT=$(fm_test_tmproot fm-project-capacity) +DEFER_EXIT=75 +HAVE_TASKS_AXI=0 +command -v tasks-axi >/dev/null 2>&1 && HAVE_TASKS_AXI=1 + +# --- fixture ---------------------------------------------------------------- + +write_brief() { # + mkdir -p "$1/data/$2" + cat > "$1/data/$2/brief.md" < [task-id...] + local home=$1 id + shift + mkdir -p "$home/state" "$home/config" "$home/data" "$home/projects" + touch "$home/state/.last-watcher-beat" + printf '%s\n' codex > "$home/config/crew-harness" + if [ "$HAVE_TASKS_AXI" = 1 ]; then + printf '%s\n' '# Backlog' '' '## In flight' '' '## Queued' '' '## Done' \ + > "$home/data/backlog.md" + cat > "$home/.tasks.toml" <<'EOF' +backend = "markdown" + +[markdown] +path = "data/backlog.md" +EOF + fi + for id in "$@"; do + write_brief "$home" "$id" + add_item "$home" "$id" + done +} + +# A case: one home, a project clone with an origin, and fake tmux/treehouse +# that record every call so a deferral can be proved to have created nothing. +make_case() { # [task-id...] + local name=$1 case_dir fakebin + shift + case_dir="$TMP_ROOT/$name" + mkdir -p "$case_dir" + fakebin=$(fm_fakebin "$case_dir") + : > "$case_dir/calls.log" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$FM_FAKE_CALL_LOG" +case "$*" in + *"#{pane_current_path}"*) + # A spawn that is told to hold waits here, after its capacity check and + # while it still holds the project lock, until the test releases it. + if [ -n "${FM_FAKE_HOLD:-}" ]; then + : > "$FM_FAKE_HOLD.reached" + i=0 + while [ ! -f "$FM_FAKE_HOLD.release" ]; do + i=$((i + 1)) + [ "$i" -lt 400 ] || exit 1 + sleep 0.05 + done + fi + printf '%s\n' "${FM_FAKE_PANE_PATH:-}" + exit 0 + ;; +esac +case "${1:-}" in display-message) printf 'firstmate\n' ;; esac +exit 0 +SH + cat > "$fakebin/treehouse" <<'SH' +#!/usr/bin/env bash +printf 'treehouse %s\n' "$*" >> "$FM_FAKE_CALL_LOG" +exit 0 +SH + chmod +x "$fakebin/tmux" "$fakebin/treehouse" + fm_fake_exit0 "$fakebin" gh gh-axi no-mistakes + fm_git_init_commit "$case_dir/project" + fm_git_add_origin "$case_dir/project" "$case_dir/project.origin.git" + fm_git_init_commit "$case_dir/other-project" + fm_git_add_origin "$case_dir/other-project" "$case_dir/other-project.origin.git" + make_home "$case_dir/home" "$@" + printf '%s\n' "$case_dir" +} + +add_item() { # + [ "$HAVE_TASKS_AXI" = 1 ] || return 0 + tasks-axi add "$2" "item for $2" --kind ship --file "$1/data/backlog.md" >/dev/null +} + +row_state() { # + tasks-axi show "$2" --file "$1/data/backlog.md" 2>/dev/null | + sed -n 's/^ state: *//p' | head -1 +} + +declare_capacity() { # ... + local home=$1 + shift + printf '%s\n' "$@" > "$home/config/project-capacity" +} + +# A live worker already on a project: the record shape bin/fm-spawn.sh +# publishes, with its backlog item In flight so cleanup can close it. +write_live() { # [extra-line...] + local home=$1 id=$2 project=$3 + shift 3 + fm_write_meta "$home/state/$id.meta" \ + "window=firstmate:fm-$id" \ + "endpoint_task_id=$id" \ + "worktree=$home/absent-worktree-$id" \ + "project=$project" \ + "harness=codex" \ + "kind=ship" \ + "mode=no-mistakes" \ + "yolo=off" \ + "spawn_gen=s-$id" \ + "$@" + if [ "$HAVE_TASKS_AXI" = 1 ]; then + add_item "$home" "$id" + tasks-axi start "$id" --file "$home/data/backlog.md" >/dev/null + fi +} + +# One isolated worktree per spawn, so every admitted launch has its own copy. +new_worktree() { # + git -C "$1/project" worktree add --quiet -b "wt-$2" "$1/wt-$2" + printf '%s\n' "$1/wt-$2" +} + +run_spawn() { # + local case_dir=$1 home=$2 pane=$3 + shift 3 + FM_ROOT_OVERRIDE='' FM_HOME="$home" \ + FM_STATE_OVERRIDE='' FM_DATA_OVERRIDE='' FM_PROJECTS_OVERRIDE='' FM_CONFIG_OVERRIDE='' \ + FM_SPAWN_NO_GUARD=1 TMUX="fake,1,0" FM_BACKEND=tmux \ + FM_FAKE_PANE_PATH="$pane" FM_FAKE_CALL_LOG="$case_dir/calls.log" \ + PATH="$case_dir/fakebin:$PATH" \ + "$SPAWN" "$@" 2>&1 +} + +spawn_ship() { # [pane-path] + local case_dir=$1 id=$2 pane=${3:-} + [ -n "$pane" ] || pane=$(new_worktree "$case_dir" "$id") + run_spawn "$case_dir" "$case_dir/home" "$pane" "$id" "$case_dir/project" --mode no-mistakes --yolo off +} + +# Everything a deferred spawn must not have created for ; +# is worktree_list taken before the spawn. +assert_nothing_created() { # + local case_dir=$1 home=$2 id=$3 before=$4 worktrees=$5 after + assert_absent "$home/state/$id.meta" "a deferred spawn published a task record for $id" + assert_absent "$home/data/$id/launch-brief.md" "a deferred spawn rendered a launch brief for $id" + after=$(call_count "$case_dir") + [ "$after" -eq "$before" ] || + fail "a deferred spawn touched the terminal or worktree pool for $id: $(tail -n +"$((before + 1))" "$case_dir/calls.log")" + assert_equals "$worktrees" "$(worktree_list "$case_dir")" "a deferred spawn left a git worktree for $id" + if [ "$HAVE_TASKS_AXI" = 1 ]; then + [ "$(row_state "$home" "$id")" = queued ] || + fail "a deferred spawn moved $id's backlog item: $(row_state "$home" "$id")" + fi +} + +call_count() { wc -l < "$1/calls.log" | tr -d ' '; } + +worktree_list() { git -C "$1/project" worktree list --porcelain; } + +# --- cases ------------------------------------------------------------------ + +# The reported incident's shape before any declaration: the project is already +# busy, and nothing caps a further launch. Absent a declaration that stays true. +test_undeclared_capacity_keeps_dispatch_uncapped() { + local case_dir home out rc=0 + case_dir=$(make_case undeclared task-c) + home="$case_dir/home" + write_live "$home" live-a "$case_dir/project" + write_live "$home" live-b "$case_dir/project" + out=$(spawn_ship "$case_dir" task-c) || rc=$? + expect_code 0 "$rc" "an undeclared project refused a spawn: $out" + assert_contains "$out" "spawned task-c" "an undeclared project did not launch the worker" + assert_present "$home/state/task-c.meta" "an undeclared project's spawn published no record" + pass "a project with no declared capacity keeps today's uncapped dispatch" +} + +test_available_capacity_admits_the_worker() { + local case_dir home out rc=0 + case_dir=$(make_case available task-c) + home="$case_dir/home" + declare_capacity "$home" "# heavy suite serves two workers" "project 2" "other-project 1" + write_live "$home" live-a "$case_dir/project" + out=$(spawn_ship "$case_dir" task-c) || rc=$? + expect_code 0 "$rc" "a spawn with a free place was refused: $out" + assert_contains "$out" "spawned task-c" "a spawn with a free place did not launch" + assert_not_contains "$out" "deferred:" "a spawn with a free place reported a deferral" + if [ "$HAVE_TASKS_AXI" = 1 ]; then + [ "$(row_state "$home" task-c)" = in_flight ] || fail "an admitted spawn did not move its item In flight" + fi + pass "a spawn is admitted while its project still has a free place" +} + +test_exhausted_capacity_defers_without_leaving_anything_behind() { + local case_dir home out rc=0 before worktrees + case_dir=$(make_case exhausted task-c) + home="$case_dir/home" + declare_capacity "$home" "project 2" + write_live "$home" live-a "$case_dir/project" + write_live "$home" live-b "$case_dir/project" + before=$(call_count "$case_dir") + worktrees=$(worktree_list "$case_dir") + out=$(spawn_ship "$case_dir" task-c "$case_dir/unused") || rc=$? + expect_code "$DEFER_EXIT" "$rc" "a spawn beyond capacity was not deferred: $out" + assert_contains "$out" "deferred: project project admits 2 worker(s) at once on this machine ($home/config/project-capacity) and 2 already hold a place (live-a, live-b)" \ + "the deferral did not name the capacity and its holders" + assert_contains "$out" "task task-c was not launched and its backlog item stays queued" \ + "the deferral did not say the task stays queued" + assert_nothing_created "$case_dir" "$home" task-c "$before" "$worktrees" + pass "a spawn beyond capacity is deferred before any record, brief, endpoint, worktree, or backlog move exists" +} + +# The capacity is the last field, so a project whose clone directory name holds +# spaces can be declared. An indented '#' line is still a comment. +test_spaced_project_name_is_declared() { + local case_dir home spaced out rc=0 + case_dir=$(make_case spaced task-c) + home="$case_dir/home" + spaced="$case_dir/my heavy project" + git clone -q "$(git -C "$case_dir/project" remote get-url origin)" "$spaced" + declare_capacity "$home" " # my heavy project 9" "my heavy project 1" "project 5" + write_live "$home" live-a "$spaced" + out=$(run_spawn "$case_dir" "$home" "$case_dir/unused" task-c "$spaced" --mode no-mistakes --yolo off) || rc=$? + expect_code "$DEFER_EXIT" "$rc" "a project whose name holds spaces was not capped by its declaration: $out" + assert_contains "$out" "deferred: project my heavy project admits 1 worker(s) at once" \ + "the deferral did not use the spaced project's declared capacity" + assert_absent "$home/state/task-c.meta" "the deferred spaced-name spawn published a record" + pass "a project name with spaces is declared by taking the capacity from the last field" +} + +# A clone directory may be named with a leading '#'. That name is declared when +# the '#' is written against the rest of the name and the line ends with the +# capacity. A '#' followed by whitespace stays a comment even when the line +# ends with a number, and a '#' note that is not a capacity stays a comment. +test_hash_prefixed_project_name_is_declared() { + local case_dir home hashed spaced out rc=0 wt + case_dir=$(make_case hash-name task-c task-d) + home="$case_dir/home" + hashed="$case_dir/#hash-project" + spaced="$case_dir/# serves" + git clone -q "$(git -C "$case_dir/project" remote get-url origin)" "$hashed" + git clone -q "$(git -C "$case_dir/other-project" remote get-url origin)" "$spaced" + declare_capacity "$home" "# serves 2" "#not-a-capacity" "#hash-project 1" + write_live "$home" live-a "$hashed" + out=$(run_spawn "$case_dir" "$home" "$case_dir/unused" task-c "$hashed" --mode no-mistakes --yolo off) || rc=$? + expect_code "$DEFER_EXIT" "$rc" "a project whose name begins with # was not capped by its declaration: $out" + assert_contains "$out" "deferred: project #hash-project admits 1 worker(s) at once" \ + "the deferral did not use the hash-prefixed project's declared capacity" + assert_absent "$home/state/task-c.meta" "the deferred hash-prefixed spawn published a record" + + write_live "$home" live-b "$spaced" + write_live "$home" live-c "$spaced" + git -C "$spaced" worktree add --quiet -b wt-d "$case_dir/wt-d" + wt="$case_dir/wt-d" + rc=0 + out=$(run_spawn "$case_dir" "$home" "$wt" task-d "$spaced" --mode no-mistakes --yolo off) || rc=$? + expect_code 0 "$rc" "a '#' comment that ends with a number was read as a capacity: $out" + assert_contains "$out" "spawned task-d" "a project whose declaration line is a comment did not stay uncapped" + pass "a project name beginning with # is declared, and a # comment stays a comment" +} + +# A home reached through a symlink is still one home: its workers hold one +# place each, not one per spelling of its state directory. +test_symlinked_home_counts_each_worker_once() { + local case_dir home out rc=0 + case_dir=$(make_case symlinked-home task-c) + home="$case_dir/home" + ln -s "$home" "$case_dir/home-link" + declare_capacity "$home" "project 2" + write_live "$home" live-a "$case_dir/project" + out=$(run_spawn "$case_dir" "$case_dir/home-link" "$(new_worktree "$case_dir" task-c)" \ + task-c "$case_dir/project" --mode no-mistakes --yolo off) || rc=$? + expect_code 0 "$rc" "a spawn through a symlinked home counted its own worker twice: $out" + assert_contains "$out" "spawned task-c" "a spawn through a symlinked home did not launch" + pass "a home reached through a symlink counts each of its workers once" +} + +# A fresh spawn that restarts an existing task id replaces that task's own +# record, so the record does not hold a place against it. Any other task on the +# project still sees that record as a holder. +test_restart_does_not_count_its_own_record() { + local case_dir home out rc=0 + case_dir=$(make_case restart task-c task-d) + home="$case_dir/home" + declare_capacity "$home" "project 1" + fm_write_meta "$home/state/task-c.meta" \ + "window=firstmate:fm-task-c" \ + "project=$case_dir/project" \ + "kind=ship" + out=$(spawn_ship "$case_dir" task-c) || rc=$? + assert_not_contains "$out" "deferred:" "a restart was deferred by its own record" + [ "$rc" -ne "$DEFER_EXIT" ] || fail "a restart exited with the deferral code: $out" + rc=0 + out=$(spawn_ship "$case_dir" task-d "$case_dir/unused") || rc=$? + expect_code "$DEFER_EXIT" "$rc" "another task ignored the restarted task's place: $out" + assert_contains "$out" "1 already hold a place (task-c)" "another task did not count the restarted task" + pass "a restart of a task id does not count that task's own record" +} + +# A place frees when a worker records its ready PR and when a task is cleaned +# up; each release admits exactly one more worker. +test_release_frees_a_place() { + local case_dir home out rc=0 + case_dir=$(make_case release task-c task-d) + home="$case_dir/home" + declare_capacity "$home" "project 2" + write_live "$home" live-a "$case_dir/project" + write_live "$home" live-b "$case_dir/project" + out=$(spawn_ship "$case_dir" task-c "$case_dir/unused") || rc=$? + expect_code "$DEFER_EXIT" "$rc" "the full project admitted a worker: $out" + + # PR handoff: the line bin/fm-pr-check.sh records for a ready PR. + printf 'pr=%s\n' "https://github.com/o/r/pull/7" >> "$home/state/live-a.meta" + rc=0 + out=$(spawn_ship "$case_dir" task-c) || rc=$? + expect_code 0 "$rc" "a recorded PR handoff did not free a place: $out" + rc=0 + out=$(spawn_ship "$case_dir" task-d "$case_dir/unused") || rc=$? + expect_code "$DEFER_EXIT" "$rc" "one freed place admitted two workers: $out" + assert_contains "$out" "2 already hold a place (live-b, task-c)" "the new worker did not take the freed place" + + # Cleanup: the real teardown removes the record, which frees its place. + rc=0 + out=$(FM_ROOT_OVERRIDE='' FM_HOME="$home" FM_STATE_OVERRIDE='' FM_DATA_OVERRIDE='' FM_CONFIG_OVERRIDE='' \ + FM_FAKE_CALL_LOG="$case_dir/calls.log" PATH="$case_dir/fakebin:$PATH" \ + "$TEARDOWN" live-b 2>&1) || rc=$? + expect_code 0 "$rc" "cleanup of a live worker failed: $out" + assert_absent "$home/state/live-b.meta" "cleanup left the worker's record" + rc=0 + out=$(spawn_ship "$case_dir" task-d) || rc=$? + expect_code 0 "$rc" "cleanup did not free a place: $out" + pass "a recorded PR handoff or a cleanup each frees exactly one place" +} + +# Only workers on the same project identity hold places: a secondmate record, a +# record for another project, and a scout on another project are ignored, while +# a scout and a legacy record without kind= on this project count. +test_occupancy_counts_only_this_projects_workers() { + local case_dir home out rc=0 + case_dir=$(make_case occupancy task-c) + home="$case_dir/home" + declare_capacity "$home" "project 3" + write_live "$home" scout-a "$case_dir/project" + sed -i.bak 's/^kind=ship$/kind=scout/' "$home/state/scout-a.meta" && rm -f "$home/state/scout-a.meta.bak" + fm_write_meta "$home/state/legacy-b.meta" "window=firstmate:fm-legacy-b" "project=$case_dir/project" + write_live "$home" other-c "$case_dir/other-project" + fm_write_meta "$home/state/mate-d.meta" "window=remote:mate-d" "project=$case_dir/project" "kind=secondmate" + out=$(spawn_ship "$case_dir" task-c) || rc=$? + expect_code 0 "$rc" "records outside this project's workers took a place: $out" + rc=0 + write_brief "$home" task-e + add_item "$home" task-e + out=$(spawn_ship "$case_dir" task-e "$case_dir/unused") || rc=$? + expect_code "$DEFER_EXIT" "$rc" "the third worker on the project was not counted: $out" + assert_contains "$out" "3 already hold a place (legacy-b, scout-a, task-c)" \ + "occupancy counted the wrong records" + pass "only this project's ship and scout records hold places" +} + +# Capacity belongs to the machine: a local secondmate home's workers on a +# separate clone of the same origin count, the declaration comes from the root +# home even when the secondmate spawns, and a remote home's workers never count. +test_capacity_is_shared_by_every_local_home() { + local case_dir root mate out rc=0 mate_project + case_dir=$(make_case machine) + root="$case_dir/home" + mate="$case_dir/mate" + make_home "$mate" task-m + mate_project="$mate/projects/project" + git clone -q "$(git -C "$case_dir/project" remote get-url origin)" "$mate_project" + printf '%s\n' schema=fm-secondmate-parent.v1 route=local "parent_home=$root" > "$mate/.fm-secondmate-parent" + printf -- '- mate - a local mate (home: %s; scope: project work; projects: project; added 2026-09-01)\n' "$mate" \ + > "$root/data/secondmates.md" + printf -- '- far - a remote mate (host: far.example; root: /srv/fm; home: %s/remote-home; scope: other; projects: project; added 2026-09-01)\n' "$case_dir" \ + >> "$root/data/secondmates.md" + mkdir -p "$case_dir/remote-home/state" + fm_write_meta "$case_dir/remote-home/state/far-x.meta" "window=w" "project=$case_dir/project" "kind=ship" + declare_capacity "$root" "project 1" + write_live "$root" live-a "$case_dir/project" + git -C "$mate_project" worktree add --quiet -b wt-m "$case_dir/wt-m" + out=$(run_spawn "$case_dir" "$mate" "$case_dir/wt-m" task-m "$mate_project" --mode no-mistakes --yolo off) || rc=$? + expect_code "$DEFER_EXIT" "$rc" "a local secondmate launched past the machine's capacity: $out" + assert_contains "$out" "admits 1 worker(s) at once on this machine ($root/config/project-capacity) and 1 already hold a place (live-a in $root)" \ + "the secondmate did not count the root home's worker against the root's declaration" + assert_absent "$mate/state/task-m.meta" "the deferred secondmate spawn published a record" + printf 'pr=%s\n' "https://github.com/o/r/pull/8" >> "$root/state/live-a.meta" + rc=0 + out=$(run_spawn "$case_dir" "$mate" "$case_dir/wt-m" task-m "$mate_project" --mode no-mistakes --yolo off) || rc=$? + expect_code 0 "$rc" "the remote home's worker, or a handed-off one, took the machine's place: $out" + pass "every local home shares one declared capacity per project origin, and remote homes do not count" +} + +# A local home's state directory or task record that cannot be read could hide a +# holder, so admission refuses instead of counting without it. +test_unreadable_holders_refuse_admission() { + local case_dir root mate out rc=0 + if [ "$(id -u)" = 0 ]; then + printf 'ok - skipped the unreadable-holder case (root reads files regardless of mode)\n' + return 0 + fi + case_dir=$(make_case unreadable-holders task-c) + root="$case_dir/home" + mate="$case_dir/mate" + make_home "$mate" + printf '%s\n' schema=fm-secondmate-parent.v1 route=local "parent_home=$root" > "$mate/.fm-secondmate-parent" + printf -- '- mate - a local mate (home: %s; scope: project work; projects: project; added 2026-09-01)\n' "$mate" \ + > "$root/data/secondmates.md" + declare_capacity "$root" "project 2" + + write_live "$root" live-a "$case_dir/project" + chmod 000 "$root/state/live-a.meta" + out=$(spawn_ship "$case_dir" task-c "$case_dir/unused") || rc=$? + chmod 600 "$root/state/live-a.meta" + expect_code 1 "$rc" "an unreadable task record did not refuse admission: $out" + assert_contains "$out" "task record $root/state/live-a.meta cannot be read" \ + "the refusal did not name the unreadable task record" + assert_absent "$root/state/task-c.meta" "a spawn published past an unreadable task record" + + chmod 000 "$mate/state" + rc=0 + out=$(spawn_ship "$case_dir" task-c "$case_dir/unused") || rc=$? + chmod 755 "$mate/state" + expect_code 1 "$rc" "an unreadable local state directory did not refuse admission: $out" + assert_contains "$out" "local Firstmate state directory $mate/state cannot be read" \ + "the refusal did not name the unreadable state directory" + assert_absent "$root/state/task-c.meta" "a spawn published past an unreadable state directory" + + rc=0 + out=$(spawn_ship "$case_dir" task-c) || rc=$? + expect_code 0 "$rc" "a readable machine did not admit the worker: $out" + pass "an unreadable state directory or task record refuses admission rather than undercounting" +} + +# Two spawns racing for the last place: the one holding the project lock +# publishes, the other cannot publish while it waits and is deferred afterwards. +test_concurrent_spawns_cannot_both_take_the_last_place() { + local case_dir home out rc=0 hold i wt + case_dir=$(make_case concurrent task-a task-b) + home="$case_dir/home" + declare_capacity "$home" "project 1" + hold="$case_dir/hold" + wt=$(new_worktree "$case_dir" task-a) + FM_FAKE_HOLD="$hold" spawn_ship "$case_dir" task-a "$wt" > "$case_dir/a.out" 2>&1 & + i=0 + while [ ! -f "$hold.reached" ]; do + i=$((i + 1)) + [ "$i" -lt 400 ] || fail "the first spawn never reached its launch: $(cat "$case_dir/a.out")" + sleep 0.05 + done + out=$(spawn_ship "$case_dir" task-b "$case_dir/unused") || rc=$? + [ "$rc" -ne 0 ] || fail "a concurrent spawn launched while the last place was being taken: $out" + assert_absent "$home/state/task-b.meta" "a concurrent spawn published past the last place" + : > "$hold.release" + wait || true + assert_contains "$(cat "$case_dir/a.out")" "spawned task-a" "the lock-holding spawn did not finish" + rc=0 + out=$(spawn_ship "$case_dir" task-b "$case_dir/unused") || rc=$? + expect_code "$DEFER_EXIT" "$rc" "the retried spawn was not deferred once the place was taken: $out" + assert_absent "$home/state/task-b.meta" "the retried spawn published past capacity" + pass "concurrent spawns can never both take the last place" +} + +# A spawn that fails after admission removes nothing it did not create and +# leaves no record, so it holds no place afterwards. +test_failed_spawn_after_admission_holds_no_place() { + local case_dir home out rc=0 + case_dir=$(make_case failed task-a task-b) + home="$case_dir/home" + declare_capacity "$home" "project 1" + printf '%s\n' '# Task' "## Captain's intent" '{TASK}' '' '## Firstmate spec' 'x' > "$home/data/task-a/brief.md" + out=$(spawn_ship "$case_dir" task-a "$case_dir/unused") || rc=$? + [ "$rc" -ne 0 ] && [ "$rc" -ne "$DEFER_EXIT" ] || fail "an invalid brief did not fail after admission (exit $rc): $out" + assert_contains "$out" "still contains {TASK}" "the spawn did not fail where expected" + assert_absent "$home/state/task-a.meta" "the failed spawn left a record" + rc=0 + out=$(spawn_ship "$case_dir" task-b) || rc=$? + expect_code 0 "$rc" "a failed spawn kept holding the only place: $out" + pass "a spawn that fails after admission leaves no record and holds no place" +} + +test_unreadable_declaration_refuses_every_spawn() { + local case_dir home out rc label body before worktrees + case_dir=$(make_case unreadable task-c) + home="$case_dir/home" + while IFS='|' read -r label body; do + [ -n "$label" ] || continue + printf '%b' "$body" > "$home/config/project-capacity" + before=$(call_count "$case_dir") + worktrees=$(worktree_list "$case_dir") + rc=0 + out=$(spawn_ship "$case_dir" task-c "$case_dir/unused") || rc=$? + expect_code 1 "$rc" "$label: an unreadable declaration did not refuse: $out" + assert_contains "$out" "the project capacity declaration is unreadable" "$label: the refusal did not name the declaration" + assert_nothing_created "$case_dir" "$home" task-c "$before" "$worktrees" + done <<'ROWS' +missing capacity|project\n +zero capacity|project 0\n +non-numeric capacity|other-project two\n +trailing text|project 2 # suite\n +named twice|project 2\nproject 3\n +ROWS + rm -f "$home/config/project-capacity" + mkdir "$home/config/project-capacity" + rc=0 + out=$(spawn_ship "$case_dir" task-c "$case_dir/unused") || rc=$? + expect_code 1 "$rc" "a declaration that is not a file did not refuse: $out" + pass "an unreadable declaration refuses every fresh spawn rather than guessing the limit" +} + +test_batch_reports_a_deferred_pair() { + local case_dir home out rc=0 + case_dir=$(make_case batch task-c) + home="$case_dir/home" + declare_capacity "$home" "project 1" + write_live "$home" live-a "$case_dir/project" + out=$(run_spawn "$case_dir" "$home" "$case_dir/unused" "task-c=$case_dir/project" --mode no-mistakes --yolo off) || rc=$? + expect_code "$DEFER_EXIT" "$rc" "a batch whose only pair was deferred did not exit with the deferral status: $out" + assert_contains "$out" "batch: DEFERRED task-c ($case_dir/project) - its project is at capacity, so it stays queued" \ + "the batch did not report the deferral" + assert_not_contains "$out" "batch: FAILED" "the batch reported a deferral as a failure" + pass "a batch reports a capacity deferral as deferred, not failed" +} + +# Orca owns its own worktrees and never takes the Treehouse allocation lock, so +# a declared capacity is what makes an Orca spawn take the shared project lock, +# even from an uncapped clone of a capped origin whose worker would still hold a +# place: it refuses while another holder has it, and defers at capacity before +# asking Orca for anything but its runtime status. +test_orca_spawn_is_admitted_under_the_shared_project_lock() { + local case_dir home out out2 rc=0 rc2 holder i + command -v node >/dev/null 2>&1 || { + printf 'ok - skipped the Orca capacity case (node, which the Orca status check needs, is not installed)\n' + return 0 + } + case_dir=$(make_case orca task-o) + home="$case_dir/home" + cat > "$case_dir/fakebin/orca" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = status ]; then + printf '{"ok":true,"result":{"runtime":{"reachable":true,"state":"ready"}}}\n' + exit 0 +fi +printf 'orca %s\n' "$*" >> "$FM_FAKE_CALL_LOG" +exit 1 +SH + chmod +x "$case_dir/fakebin/orca" + git clone -q "$(git -C "$case_dir/project" remote get-url origin)" "$case_dir/project-2" + declare_capacity "$home" "project 1" + + # shellcheck disable=SC2016 # expanded by the holder's own shell + FM_HOME="$home" FM_STATE_OVERRIDE='' bash -c ' + . "$1/bin/fm-wake-lib.sh" + lock=$(fm_treehouse_project_lock_path "$2") || exit 1 + fm_lock_try_acquire "$lock" || exit 1 + : > "$3.held" + while [ ! -f "$3.release" ]; do sleep 0.05; done + fm_lock_release "$lock" + ' _ "$ROOT" "$case_dir/project" "$case_dir/holder" & + holder=$! + i=0 + while [ ! -f "$case_dir/holder.held" ]; do + i=$((i + 1)) + [ "$i" -lt 200 ] || fail "the lock holder never took the project lock" + sleep 0.05 + done + out=$(run_spawn "$case_dir" "$home" "$case_dir/unused" task-o "$case_dir/project" \ + --backend orca --mode no-mistakes --yolo off) || rc=$? + rc2=0 + out2=$(run_spawn "$case_dir" "$home" "$case_dir/unused" task-o "$case_dir/project-2" \ + --backend orca --mode no-mistakes --yolo off) || rc2=$? + : > "$case_dir/holder.release" + wait "$holder" || true + expect_code 1 "$rc" "an Orca spawn ignored a held project lock: $out" + assert_contains "$out" "another spawn or cleanup holds the shared project lock for $case_dir/project; refusing to race its capacity admission" \ + "the Orca spawn did not refuse on the shared project lock" + expect_code 1 "$rc2" "an Orca spawn from an uncapped same-origin clone ignored the held project lock: $out2" + assert_contains "$out2" "another spawn or cleanup holds the shared project lock for $case_dir/project-2" \ + "the uncapped same-origin clone's Orca spawn did not refuse on the shared project lock" + assert_absent "$home/state/task-o.meta" "the uncapped clone's Orca spawn published a record while the lock was held" + assert_no_grep "orca " "$case_dir/calls.log" "the Orca spawn asked Orca for more than its runtime status" + + write_live "$home" live-a "$case_dir/project" + rc=0 + out=$(run_spawn "$case_dir" "$home" "$case_dir/unused" task-o "$case_dir/project" \ + --backend orca --mode no-mistakes --yolo off) || rc=$? + expect_code "$DEFER_EXIT" "$rc" "an Orca spawn beyond capacity was not deferred: $out" + assert_absent "$home/state/task-o.meta" "the deferred Orca spawn published a record" + assert_no_grep "orca " "$case_dir/calls.log" "the deferred Orca spawn created an Orca worktree" + pass "an Orca spawn takes the shared project lock whenever a same-origin clone is capped and defers before creating anything" +} + +test_undeclared_capacity_keeps_dispatch_uncapped +test_available_capacity_admits_the_worker +test_exhausted_capacity_defers_without_leaving_anything_behind +test_spaced_project_name_is_declared +test_hash_prefixed_project_name_is_declared +test_symlinked_home_counts_each_worker_once +test_release_frees_a_place +test_restart_does_not_count_its_own_record +test_occupancy_counts_only_this_projects_workers +test_capacity_is_shared_by_every_local_home +test_unreadable_holders_refuse_admission +test_concurrent_spawns_cannot_both_take_the_last_place +test_failed_spawn_after_admission_holds_no_place +test_unreadable_declaration_refuses_every_spawn +test_batch_reports_a_deferred_pair +test_orca_spawn_is_admitted_under_the_shared_project_lock diff --git a/tests/fm-remote-reply.test.sh b/tests/fm-remote-reply.test.sh index f5e8aef0c5d..92f6d9e73d0 100755 --- a/tests/fm-remote-reply.test.sh +++ b/tests/fm-remote-reply.test.sh @@ -1024,6 +1024,15 @@ assert_absent "$PARENT/state/procevent/$SID.source" "continuity break was re-arm remote_env "$ADAPTER" ingest ios "$RESULT_TWELVE" >/dev/null 2>&1 || true [ "$(grep -cF 'blocked [key=remote-reply-continuity-ios]' "$PARENT/state/ios.status")" -eq 1 ] \ || fail "continuity replay duplicated the escalation" +first_offset=$(sed -n 's/^offset=//p' "$PARENT/state/remote-replies/ios.cursor") +first_hash=$(sed -n 's/^prefix_sha256=//p' "$PARENT/state/remote-replies/ios.cursor" | tr 'A-F' 'a-f') +first_prefix=$(printf '%.12s' "$first_hash") +assert_grep "at offset ${first_offset} prefix ${first_prefix} retirements 0" "$PARENT/state/ios.status" \ + "continuity break did not record the reader position" +assert_no_grep "prefix ${first_hash}" "$PARENT/state/ios.status" \ + "continuity break recorded the full prefix hash" +assert_absent "$PARENT/state/remote-replies/ios.retirements" \ + "a route that has never been retired gained a retirement count" status_line_at_epoch "$(grep -F 'blocked [key=remote-reply-continuity-ios]' "$PARENT/state/ios.status")" >/dev/null \ || fail "new continuity escalation has unknown emission time" if [ "${FM_TEST_EVIDENCE:-0}" = 1 ]; then @@ -1064,8 +1073,229 @@ remote_env "$ADAPTER" handle ios "$GEN" "$RESULT_TWELVE" >/dev/null 2>&1 || [ "$ || fail "pending continuity result could not be acknowledged after retirement refusal" remote_env "$ADAPTER" retire ios >/dev/null assert_absent "$PARENT/state/remote-replies/ios.cursor" "adapter retirement left its cursor" +recorded_retirements=$(cat "$PARENT/state/remote-replies/ios.retirements" 2>/dev/null || true) +[ "$recorded_retirements" = count=1 ] \ + || fail "adapter retirement did not record its count (got: ${recorded_retirements:-absent})" assert_absent "$PARENT/state/remote-replies/ios.caught-up" \ "adapter retirement left a caught-up watermark a later route could inherit" pass "remote reply retirement quiesces and refuses unhandled captured results" +# Empty the remote log under the committed cursor and handle the break the +# next blocking source reports. Sets RESULT_BREAK. +break_repaired_route() { #