From 7e505dfa8c290b580ab72797d9a8b22769019e1c Mon Sep 17 00:00:00 2001 From: Adam Gardner Date: Thu, 1 Oct 2026 00:29:30 +0800 Subject: [PATCH 1/7] fix(bin): bind crew-state run attribution to the task incarnation A refresh task that reuses an existing PR branch at an unchanged head read the earlier task's finished no-mistakes run as its own result, so fm-crew-state.sh reported "done - run passed: PR open" while the new worker was still resolving conflicts. - fm-spawn.sh records task_started= on a fresh spawn and carries it across relaunches; fm-nm-run-lib.sh owns the incarnation rule (fm_nm_run_in_incarnation, from the ULID run id's creation time, and a creation-minute bound on the runs ledger). A run created before the task started is never attributed, and the ignored run is named on the line that answers instead. - A run record states only what the run did. A merged claim needs the task's recorded pr= identity checked against the forge or a merge receipt, every PR disposition names its source, and a run PR that differs from the recorded one is reported as a conflict. - Run-step wording never presents green: a passed override names its exception, and a ci log reporting no CI checks reads as a wait. - A worker status line written after a finished run started is shown beside the run-step verdict with its source. - A deliberately aborted run names the abort and never reads as a failure. --- .agents/skills/ship-landing/SKILL.md | 2 +- bin/fm-crew-state.sh | 347 +++++++++++++++++--------- bin/fm-nm-run-lib.sh | 53 +++- bin/fm-spawn.sh | 22 +- docs/architecture.md | 8 +- docs/gerrit-change-watch.md | 2 +- tests/fm-control-relaunch.test.sh | 21 ++ tests/fm-crew-state.test.sh | 359 ++++++++++++++++++++++++--- tests/fm-trace-context-spawn.test.sh | 22 ++ 9 files changed, 676 insertions(+), 160 deletions(-) diff --git a/.agents/skills/ship-landing/SKILL.md b/.agents/skills/ship-landing/SKILL.md index fe17397f3d0..0815947b8a9 100644 --- a/.agents/skills/ship-landing/SKILL.md +++ b/.agents/skills/ship-landing/SKILL.md @@ -9,7 +9,7 @@ metadata: # Ship landing For PR-based ship tasks, the ready signal depends on mode: `no-mistakes` reports `done [at=]: PR checks green` after CI is green, while `direct-PR` reports `done [at=]: PR ` after opening the PR, each only for a non-draft PR; a lane that deliberately holds a draft declares a wait instead, and `bin/fm-pr-check.sh` refuses to arm merge monitoring on a draft. -Run `bin/fm-pr-check.sh ` with the URL copied from that ready signal or the resolved checks-green `fm-crew-state.sh` line - it records `pr=` and the forge's `pr_head=` when available in the task's meta and arms the watcher's merge poll. +Run `bin/fm-pr-check.sh ` with the URL copied from that ready signal or the resolved ready-for-review `fm-crew-state.sh` line - it records `pr=` and the forge's `pr_head=` when available in the task's meta and arms the watcher's merge poll. `bin/fm-dod-lib.sh` owns the named-head gate on that ready signal: a ship `done:` whose named head exists only in the worker's disposable copy is not ready (`bin/fm-crew-state.sh` reports blocked, `bin/fm-pr-check.sh` refuses to register, and a secondmate does not publish that done upstream). That blocked reading is the gate working, not a stuck worker, so steer the worker on the commit the refusal names rather than waiting. A direct-PR worker pushes that commit to its PR branch, and a local-only worker commits it on its ship branch. diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index 26e5b2d26cb..1f4f5f567c0 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -16,8 +16,13 @@ # The determinism lives entirely here - run-step / pane / log reads, fixed # mapping logic, and terminal passed-run PR detail from bounded evidence only, # with no heuristics and no LLM. -# For a terminal passed no-mistakes run, a matching merge-poll retirement +# A run record states only what that run did; it never claims a forge outcome +# or "green" itself. For a terminal passed no-mistakes run, the PR disposition +# is read for the task's recorded pr= identity: a matching merge-poll retirement # receipt is local merged evidence; otherwise a 5s-bounded forge read is tried. +# Each disposition names its source. Without a recorded pr=, the run's own PR +# may be read for open or closed but never proves a merge, and a run PR that +# differs from the recorded one is a conflict (pr_disposition owns the rule). # FM_CREW_STATE_NO_FORGE=1 keeps the receipt read but skips the forge fallback. # An absent or unreadable PR identity yields an honest unknown, never an # optimistic merged claim. @@ -39,6 +44,14 @@ # active or terminal (from `axi status`, or the coarse `no-mistakes runs` # fallback)? Branch name alone is not enough: a historical run on a reused # branch whose head was rewritten or diverged must not be attributed. +# Code identity alone is not enough either: every route first requires the +# run to belong to this task's incarnation, created at or after the task's +# task_started= (else spawn_gen=) epoch (fm_nm_run_in_incarnation in +# bin/fm-nm-run-lib.sh). A refresh task that reuses an existing PR branch at +# an unchanged head otherwise reads the earlier task's finished run as its +# own result. When the branch's newest run predates the task, no run is +# attributed, current pane or status evidence answers, and that line names +# the ignored run. # A run EXECUTING on this crew's branch (pending, running, fixing or ci - # the detail-object vocabulary, which carries all four; the selected route # re-reads it by id and the legacy route passes the same detail SHAPE, and @@ -97,27 +110,37 @@ # (the id-addressed detail read carries step words the overview does not), # awaiting_approval/fix_review -> parked (with gate findings), terminal # passed/checks-passed/passed-with-override/passed-with-skips -> done, -# failed -> failed, cancelled -> unknown (no verdict unless the green +# failed -> failed, cancelled -> unknown (no verdict unless the passed-CI # delivery safeguard below applies). A cancelled outcome takes precedence # over an interrupted step's failed status or outstanding gate findings; -# it does not rewrite historical events or backlog records. +# it does not rewrite historical events or backlog records. Its recorded +# reason stays visible, so a deliberate abort reads as one and never as a +# failure (cancelled_run_detail). # passed-with-override is a passing outcome # carrying an explicitly approved Test or CI exception (no-mistakes' own -# vocabulary), read identically to a clean passed. passed-with-skips is +# vocabulary), read as done like a clean passed but with the exception +# and its reason kept visible, never as green. passed-with-skips is # also a passing outcome (publication or CI verification was # automatically skipped, no-mistakes' own vocabulary), read as done but # with that skip kept visible in the detail, unlike a clean passed. # EXCEPT: while # the active step is ci, `axi status` alone cannot tell "still waiting on -# checks" from "checks green, waiting on merge" (see nm_ci_checks_state) - -# a check of the full ci-step log overrides working -> done once checks read -# green, so a green PR is never silently read as still-validating. And a +# checks" from "checks passed, waiting on merge" (see nm_ci_checks_state) - +# a check of the full ci-step log overrides working -> done once the log +# reports checks passed, so a ready PR is never silently read as +# still-validating. A log reporting no CI checks at all stays working as a +# wait: CI that has not run is never green. And a # terminal failed or cancelled run whose only unfinished step is the ci # monitor, after every substantive step completed (an explicitly skipped -# rebase is allowed) and the ci log's last marker reads checks green, +# rebase is allowed) and the ci log's last marker reports checks passed, # also reads done only when the bounded forge read confirms the PR is -# open (held-for-merge) or merged. Closed, missing, unreadable, or skipped -# forge evidence leaves the original failed or unknown classification. +# open (held-for-merge) or, for the recorded pr=, merged. Closed, missing, +# unreadable, or skipped forge evidence leaves the original failed or +# unknown classification. +# A finished run's record is not the worker's current word: when the +# resolved status declaration was written after the run started and maps to +# a different state, it is shown beside the run-step verdict with its +# source. # A monitor whose only remaining job is to observe a merge decision must not # convert the absence of that decision into a failure verdict # (nm_reclassify_failed_run_as_held_green). In the @@ -203,10 +226,13 @@ FM_CREW_STATE_RUNS_LIMIT=${FM_CREW_STATE_RUNS_LIMIT:-200} case "$FM_CREW_STATE_RUNS_LIMIT" in ''|*[!0-9]*) FM_CREW_STATE_RUNS_LIMIT=200 ;; esac SEP=' · ' -# Emit the one canonical line and exit 0. Detail is optional. +# Emit the one canonical line and exit 0. Detail is optional. EMIT_NOTE, when +# set, is appended as its own component to whichever line answers. +EMIT_NOTE="" emit() { # [detail] local line="state: $1${SEP}source: $2" [ -n "${3:-}" ] && line="$line${SEP}$3" + [ -n "$EMIT_NOTE" ] && line="$line${SEP}$EMIT_NOTE" printf '%s\n' "$line" exit 0 } @@ -224,6 +250,18 @@ KIND=$(meta_value kind) HARNESS=$(meta_value harness) REMOTE_HOST=$(meta_value remote_host) [ -n "$KIND" ] || KIND=ship +# The task incarnation's start epoch (bin/fm-spawn.sh records task_started= on a +# fresh spawn and carries it across relaunches). A record without it falls back +# to its spawn_gen= epoch; neither leaves attribution unbounded. +TASK_START=$(meta_value task_started) +case "$TASK_START" in + ''|*[!0-9]*) + TASK_START=$(meta_value spawn_gen) + TASK_START=${TASK_START#s} + TASK_START=${TASK_START%%.*} + case "$TASK_START" in ''|*[!0-9]*) TASK_START='' ;; esac + ;; +esac # A torn-down (or never-created) worktree has no current state to read. A # remote secondmate's recorded worktree is a path on ITS host, so the local @@ -411,94 +449,112 @@ change_read_record_bounded() { # FM_PR_RECORD_MERGED=$merged } -passed_pr_detail() { - local provider url host path number owner repo raw_pr state_lc +# The PR disposition a finished run is reported with, as PR_DISPOSITION +# (open|merged|closed|other|unknown) and PR_DISPOSITION_TEXT, which always names +# where the claim came from. A run record never claims a forge outcome itself: +# the PR is read from the forge (or a merge-poll retirement receipt, which is +# the merge poll's own forge observation), and a merged claim needs the task's +# recorded pr= identity. The run's own PR field may identify the PR only while +# no pr= is recorded, and then never proves a merge; a run PR that differs from +# the recorded one is a conflict, never either PR's state. +pr_disposition() { + local provider url host path number raw_pr label state_lc run_url='' + PR_DISPOSITION=unknown raw_pr=$(strip_quotes "$(nm_field pr)") if fm_pr_url_parse "$raw_pr"; then + run_url=$FM_PR_URL provider=$FM_PR_PROVIDER url=$FM_PR_URL host=$FM_PR_HOST path=$FM_PR_PATH number=$FM_PR_NUMBER - elif fm_pr_metadata_identity_parse "$META"; then + label='run PR' + fi + if fm_pr_metadata_identity_parse "$META"; then + if [ -n "$run_url" ] && { [ "$FM_PR_META_PROVIDER" != "$provider" ] || [ "$FM_PR_META_HOST" != "$host" ] \ + || [ "$FM_PR_META_PATH" != "$path" ] || [ "$FM_PR_META_NUMBER" != "$number" ]; }; then + PR_DISPOSITION_TEXT="PR state unknown (run PR $run_url differs from recorded PR $FM_PR_META_URL)" + return + fi provider=$FM_PR_META_PROVIDER url=$FM_PR_META_URL host=$FM_PR_META_HOST path=$FM_PR_META_PATH number=$FM_PR_META_NUMBER - else - printf 'run passed: PR state unknown (no PR identity)' + label='recorded PR' + elif [ -z "$run_url" ]; then + PR_DISPOSITION_TEXT='PR state unknown (no PR identity)' return fi - if fm_pr_poll_retirement_receipt_valid "$STATE" "$ID" \ + if [ "$label" = 'recorded PR' ] \ + && fm_pr_poll_retirement_receipt_valid "$STATE" "$ID" \ && [ "$FM_PR_RETIRE_PROVIDER" = "$provider" ] \ && [ "$FM_PR_RETIRE_URL" = "$url" ] \ && [ "$FM_PR_RETIRE_HOST" = "$host" ] \ && [ "$FM_PR_RETIRE_PATH" = "$path" ] \ && [ "$FM_PR_RETIRE_NUMBER" = "$number" ]; then - printf 'run passed: PR merged' + PR_DISPOSITION=merged + PR_DISPOSITION_TEXT='recorded PR merged (merge receipt)' return fi if [ "${FM_CREW_STATE_NO_FORGE:-0}" = 1 ]; then - printf 'run passed: PR state unknown (forge read skipped)' + PR_DISPOSITION_TEXT="$label state unknown (forge read skipped)" return fi case "$provider" in - github) - owner=${path%%/*} - repo=${path#*/} - if ! pr_read_record_bounded "$owner" "$repo" "$number"; then - printf 'run passed: PR state unknown (unreadable)' - return - fi - if [ "$FM_PR_RECORD_MERGED" = true ]; then - printf 'run passed: PR merged' - return - fi - state_lc=$(printf '%s' "$FM_PR_RECORD_STATE" | tr '[:upper:]' '[:lower:]') - case "$state_lc" in - open) printf 'run passed: PR open' ;; - closed) printf 'run passed: PR closed' ;; - *) printf 'run passed: PR state %s' "$state_lc" ;; - esac - ;; - gitlab) - if ! mr_read_record_bounded "$host" "$path" "$number"; then - printf 'run passed: PR state unknown (unreadable)' - return - fi - if [ "$FM_PR_RECORD_MERGED" = true ]; then - printf 'run passed: PR merged' - return - fi - state_lc=$(printf '%s' "$FM_PR_RECORD_STATE" | tr '[:upper:]' '[:lower:]') - case "$state_lc" in - open|opened) printf 'run passed: PR open' ;; - closed) printf 'run passed: PR closed' ;; - *) printf 'run passed: PR state %s' "$state_lc" ;; - esac - ;; - gerrit) - if ! change_read_record_bounded "$host" "$number"; then - printf 'run passed: PR state unknown (unreadable)' - return - fi - if [ "$FM_PR_RECORD_MERGED" = true ]; then - printf 'run passed: PR merged' - return - fi - # Gerrit spells an open change NEW and a closed one ABANDONED. - state_lc=$(printf '%s' "$FM_PR_RECORD_STATE" | tr '[:upper:]' '[:lower:]') - case "$state_lc" in - new) printf 'run passed: PR open' ;; - abandoned) printf 'run passed: PR closed' ;; - *) printf 'run passed: PR state %s' "$state_lc" ;; - esac - ;; + github) pr_read_record_bounded "${path%%/*}" "${path#*/}" "$number" ;; + gitlab) mr_read_record_bounded "$host" "$path" "$number" ;; + gerrit) change_read_record_bounded "$host" "$number" ;; + *) false ;; + esac || { + PR_DISPOSITION_TEXT="$label state unknown (unreadable)" + return + } + if [ "$FM_PR_RECORD_MERGED" = true ]; then + if [ "$label" = 'recorded PR' ]; then + PR_DISPOSITION=merged + PR_DISPOSITION_TEXT='recorded PR merged (forge)' + else + PR_DISPOSITION_TEXT='run PR merge unconfirmed (no recorded PR identity)' + fi + return + fi + # GitLab spells an open merge request opened; Gerrit spells an open change NEW + # and a closed one ABANDONED. + state_lc=$(printf '%s' "$FM_PR_RECORD_STATE" | tr '[:upper:]' '[:lower:]') + case "$state_lc" in + open|opened|new) PR_DISPOSITION=open; PR_DISPOSITION_TEXT="$label open (forge)" ;; + closed|abandoned) PR_DISPOSITION=closed; PR_DISPOSITION_TEXT="$label closed (forge)" ;; + *) PR_DISPOSITION=other; PR_DISPOSITION_TEXT="$label state $state_lc (forge)" ;; + esac +} + +# The finished passing run's detail: what the run did, then the PR disposition +# with its source. An override is an approved exception, never green. +passed_run_detail() { # + local reason + pr_disposition + case "$1" in + passed-with-override) + reason=$(strip_quotes "$(nm_field ci_override_reason)") + printf 'run passed-with-override (approved exception, not green%s): %s' "${reason:+: $reason}" "$PR_DISPOSITION_TEXT" ;; + passed-with-skips) + printf 'run passed-with-skips: %s (publication/CI verification skipped)' "$PR_DISPOSITION_TEXT" ;; *) - printf 'run passed: PR state unknown (unreadable: %s)' "$url" - ;; + printf 'run passed: %s' "$PR_DISPOSITION_TEXT" ;; + esac +} + +# A cancelled run carries no verdict; its recorded reason says whether it was a +# deliberate abort, which is never a failure, or another cancellation. +cancelled_run_detail() { + local error + error=$(strip_quotes "$(nm_field error)") + case "$error" in + "cancelled: aborted by user") printf 'run cancelled: no verdict (deliberately aborted, not a failure)' ;; + "cancelled: "?*) printf 'run cancelled: no verdict (%s)' "${error#cancelled: }" ;; + *) printf 'run cancelled: no verdict' ;; esac } # Finding count from a findings[N]{...} table header; empty when none. @@ -715,15 +771,16 @@ nm_run_activity_is_recent() { } # 0 when a terminal failed or cancelled run ended at the ci monitor and the -# ci log's last recognized marker reads checks green. Requires the exact +# ci log's last recognized marker reports CI checks passed. Requires the exact # shape, all on positive evidence: a steps[] table where every step completed # except `ci` failed/cancelled and an optional skipped rebase (any other -# non-completed step disqualifies), plus nm_ci_checks_state=green (a genuinely red -# check, or an unreadable ci log, cannot prove delivery). This is the -# orphaned-CI-monitor gap (2026-09-05 jr-voice): a run held for a captain -# merge decision polls until the shared daemon restarts under it and marks -# the run failed, although GitHub's own check state - the actual shippability -# authority - is green and every substantive step completed. +# non-completed step disqualifies), plus nm_ci_checks_state=passed (a genuinely +# red check, no reported checks, or an unreadable ci log cannot prove +# delivery). This is the orphaned-CI-monitor gap (2026-09-05 jr-voice): a +# run held for a captain merge decision polls until the shared daemon restarts +# under it and marks the run failed, although GitHub's own check state - the +# actual shippability authority - is green and every substantive step +# completed. nm_failed_run_is_green_held_ci() { local rows row rest step status saw_ci_failed rows=$(nm_steps_rows) @@ -751,18 +808,18 @@ nm_failed_run_is_green_held_ci() { $rows EOF [ "$saw_ci_failed" = 1 ] || return 1 - [ "$(nm_ci_checks_state)" = green ] + [ "$(nm_ci_checks_state)" = passed ] } -# Apply the header's terminal-delivery safeguard. The earlier green log cannot +# Apply the header's terminal-delivery safeguard. The earlier passed log cannot # prove current PR disposition: a subsequent close can itself end the monitor. -nm_reclassify_failed_run_as_held_green() { +nm_reclassify_failed_run_as_held_green() { # nm_failed_run_is_green_held_ci || return 1 - local disposition pr_url - disposition=$(passed_pr_detail) - case "$disposition" in - "run passed: PR open") RUN_DETAIL="checks green: PR held for merge (ci monitor ended)" ;; - "run passed: PR merged") RUN_DETAIL="checks green: PR merged (ci monitor ended)" ;; + local pr_url + pr_disposition + case "$PR_DISPOSITION" in + open) RUN_DETAIL="PR held for merge: run $1 at the ci monitor after its ci log reported CI checks passed; $PR_DISPOSITION_TEXT" ;; + merged) RUN_DETAIL="run $1 at the ci monitor after its ci log reported CI checks passed; $PR_DISPOSITION_TEXT" ;; *) return 1 ;; esac RUN_STATE="done" @@ -852,14 +909,17 @@ nm_effective_ci_step_status() { # ~/.no-mistakes/logs/*/ci.log on the installed v1.32.2 binary, including the # actual PR #252 run). Reads the ci step's log via `axi logs --full` and scans # it for the MOST RECENT recognized marker (the log is append-only/chronological, -# so the last match is current): green with nothing red after it means CI is -# green right now, still only waiting on merge/close. +# so the last match is current) and prints passed, none-reported, not-ready, or +# unknown. passed with nothing red after it means the run saw its checks pass, +# still only waiting on merge/close. none-reported is kept apart from passed: +# CI that has not run (a fork's workflow runs held for maintainer approval +# report no checks at all) is a wait, never green. # "base branch advanced (..), re-arming CI monitor timeout" is deliberately NOT # a marker: the monitor logs a checks state only when that state changes, and a # base advance re-arms only its idle timeout without clearing readiness, so the -# green marker before it is still current (no-mistakes' own ci-log parser +# passed marker before it is still current (no-mistakes' own ci-log parser # ignores the line the same way, v1.32.2 through v1.79.0). Reading it as -# not-ready held a green PR at working for as long as main kept advancing. +# not-ready held a passed PR at working for as long as main kept advancing. nm_ci_checks_state() { local run_id ci_log marker run_id=$(strip_quotes "$(nm_field id)") @@ -870,7 +930,8 @@ nm_ci_checks_state() { | grep -E 'CI checks passed|no CI checks reported - still monitoring|no CI checks reported yet|checks failed|issues detected|CI checks running' \ | tail -1) case "$marker" in - *"checks passed"*|*"no CI checks reported - still monitoring"*) printf 'green' ;; + *"checks passed"*) printf 'passed' ;; + *"no CI checks reported - still monitoring"*) printf 'none-reported' ;; *"no CI checks reported yet"*|*"checks failed"*|*"issues detected"*|*"CI checks running"*) printf 'not-ready' ;; *) printf 'unknown' ;; esac @@ -924,6 +985,17 @@ NM_DAEMON_ANSWER="" RUN_DEAD_DAEMON="" COARSE_STATUS="" SELECTED_RUN_ID="" +# The id of this branch's newest run when it is not this task incarnation's own +# (fm_nm_run_in_incarnation in bin/fm-nm-run-lib.sh). It is named on whichever +# current-evidence line answers instead, so the ignored record stays visible. +RUN_BEFORE_TASK="" +note_unattributed_run() { # + if fm_nm_run_created_epoch "$1" >/dev/null; then + EMIT_NOTE="run $1 predates this task: not attributed" + else + EMIT_NOTE="run $1 creation time unreadable: not attributed" + fi +} # Scouts and secondmates never drive a no-mistakes validation of their own # worktree, so skip the lookup for them and read state from pane/log directly. if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && command -v no-mistakes >/dev/null 2>&1; then @@ -952,6 +1024,17 @@ if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && command -v no-mistakes >/dev/n ;; selected\|*) IFS='|' read -r _ selected_id selected_status candidate_ids <<< "$run_choice" + # The newest same-branch run is from before this task started: no run of + # this incarnation exists, so current evidence below answers. + if ! fm_nm_run_in_incarnation "$selected_id" "$TASK_START"; then + note_unattributed_run "$selected_id" + RUN_BEFORE_TASK=$selected_id + run_choice=earlier + fi + ;; + esac + case "$run_choice" in + selected\|*) RUN_OUT=$(fm_nm_run_checked "$WT" "$NM_TIMEOUT" axi status --run "$selected_id") \ || emit unknown run-step "selected run unreadable; run ids: $candidate_ids" if [ "$(strip_quotes "$(nm_field id)")" != "$selected_id" ] \ @@ -971,7 +1054,7 @@ if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && command -v no-mistakes >/dev/n HAVE_RUN=1 elif [ -z "$(fm_nm_resolve_commit "$WT" "$(strip_quotes "$(nm_field head)")")" ]; then if fm_nm_run_is_active "$RUN_OUT" \ - && [ "$(fm_nm_runs_status_for_worktree "$WT" "$CREW_BRANCH" "$(nm_runs_list)" "$(strip_quotes "$(nm_field head)")")" = running ]; then + && [ "$(fm_nm_runs_status_for_worktree "$WT" "$CREW_BRANCH" "$(nm_runs_list)" "$(strip_quotes "$(nm_field head)")" "$TASK_START")" = running ]; then # The anchor PROVED code identity; only liveness can still fail, so # a dead daemon is reported as such rather than as an identity # failure, and a parked run keeps its gate and findings. @@ -986,8 +1069,17 @@ if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && command -v no-mistakes >/dev/n SELECTED_RUN_ID=$selected_id ;; esac - if [ "$HAVE_RUN" = 0 ] && [ -z "$SELECTED_RUN_ID" ]; then + if [ "$HAVE_RUN" = 0 ] && [ -z "$SELECTED_RUN_ID" ] && [ -z "$RUN_BEFORE_TASK" ]; then run_branch=$(strip_quotes "$(nm_field branch)") + # This branch's current record is from before this task started, so no + # newer run of this incarnation exists for the ledger to find either. + if [ -n "$run_branch" ] && [ "$run_branch" = "$CREW_BRANCH" ] \ + && ! fm_nm_run_in_incarnation "$(strip_quotes "$(nm_field id)")" "$TASK_START"; then + RUN_BEFORE_TASK=$(strip_quotes "$(nm_field id)") + note_unattributed_run "$RUN_BEFORE_TASK" + fi + fi + if [ "$HAVE_RUN" = 0 ] && [ -z "$SELECTED_RUN_ID" ] && [ -z "$RUN_BEFORE_TASK" ]; then # Head equality, the pipeline-owned parked-run exemption, or executing # regardless of head: a live run on this branch is current even after a # rebase, and while the pipeline owns this branch a parked run binds @@ -1003,7 +1095,7 @@ if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && command -v no-mistakes >/dev/n # Without run ids, contradictory liveness cannot prove precedence. # A live replacement also needs an id-addressed status read: a bare # "running" row cannot tell working from waiting at a gate. - ledger_status=$(fm_nm_runs_status_for_worktree "$WT" "$CREW_BRANCH" "$(nm_runs_list)") + ledger_status=$(fm_nm_runs_status_for_worktree "$WT" "$CREW_BRANCH" "$(nm_runs_list)" "" "$TASK_START") if fm_nm_run_is_active "$RUN_OUT"; then if [ "$(fm_nm_run_status_class "$ledger_status")" = terminal ]; then emit unknown run-step "run records disagree; run ids: $(strip_quotes "$(nm_field id)"), competing identity unavailable" @@ -1025,7 +1117,7 @@ if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && command -v no-mistakes >/dev/n # `[ -n "$RUN_OUT" ]`: an empty/timed-out primary call means the CLI # itself did not respond, so retrying it immediately with a second # bounded call would just double the wait for no better answer. - COARSE_STATUS=$(fm_nm_runs_status_for_worktree "$WT" "$CREW_BRANCH" "$(nm_runs_list)") + COARSE_STATUS=$(fm_nm_runs_status_for_worktree "$WT" "$CREW_BRANCH" "$(nm_runs_list)" "" "$TASK_START") if [ -n "$COARSE_STATUS" ]; then HAVE_RUN=1 # A branch-matching answer the strict rule rejected is this branch's @@ -1087,16 +1179,15 @@ if [ "$HAVE_RUN" = 1 ]; then if [ -n "$outcome" ]; then case "$outcome" in - passed|passed-with-override) RUN_STATE="done"; RUN_DETAIL=$(passed_pr_detail) ;; - passed-with-skips) RUN_STATE="done"; RUN_DETAIL="$(passed_pr_detail) (publication/CI verification skipped)" ;; - checks-passed) RUN_STATE="done"; RUN_DETAIL="checks green: PR ready for review" ;; + passed|passed-with-override|passed-with-skips) RUN_STATE="done"; RUN_DETAIL=$(passed_run_detail "$outcome") ;; + checks-passed) RUN_STATE="done"; RUN_DETAIL="run reports CI checks passed: PR ready for review" ;; failed) - if nm_reclassify_failed_run_as_held_green; then :; else + if nm_reclassify_failed_run_as_held_green failed; then :; else RUN_STATE=failed; RUN_DETAIL="run failed" fi ;; cancelled) - if nm_reclassify_failed_run_as_held_green; then :; else - RUN_STATE=unknown; RUN_DETAIL="run cancelled: no verdict" + if nm_reclassify_failed_run_as_held_green cancelled; then :; else + RUN_STATE=unknown; RUN_DETAIL=$(cancelled_run_detail) fi ;; *) RUN_STATE=unknown; RUN_DETAIL="outcome: $outcome" ;; esac @@ -1124,12 +1215,12 @@ if [ "$HAVE_RUN" = 1 ]; then running|fixing) RUN_STATE=working; RUN_DETAIL="validating ($status)" ;; completed) RUN_STATE="done"; RUN_DETAIL="run completed" ;; failed) - if nm_reclassify_failed_run_as_held_green; then :; else + if nm_reclassify_failed_run_as_held_green failed; then :; else RUN_STATE=failed; RUN_DETAIL="run failed" fi ;; cancelled) - if nm_reclassify_failed_run_as_held_green; then :; else - RUN_STATE=unknown; RUN_DETAIL="run cancelled: no verdict" + if nm_reclassify_failed_run_as_held_green cancelled; then :; else + RUN_STATE=unknown; RUN_DETAIL=$(cancelled_run_detail) fi ;; "") RUN_STATE=working; RUN_DETAIL="run active" ;; *) RUN_STATE=working; RUN_DETAIL="run active ($status)" ;; @@ -1139,14 +1230,19 @@ if [ "$HAVE_RUN" = 1 ]; then case "$CI_STEP_STATUS" in running) CI_LOG_STATE=$(nm_ci_checks_state) - if [ "$CI_LOG_STATE" = green ]; then - RUN_STATE="done" - RUN_DETAIL="checks green: PR ready for review (still monitoring for merge/close)" - # The run's own PR URL makes this reading actionable even when - # the worker never reported it and no pr= was recorded. - ci_pr_url=$(strip_quotes "$(nm_field pr)") - [ -z "$ci_pr_url" ] || RUN_DETAIL="$RUN_DETAIL: $ci_pr_url" - fi + case "$CI_LOG_STATE" in + passed) + RUN_STATE="done" + RUN_DETAIL="run ci log reports CI checks passed: PR ready for review (still monitoring for merge/close)" + # The run's own PR URL makes this reading actionable even when + # the worker never reported it and no pr= was recorded. + ci_pr_url=$(strip_quotes "$(nm_field pr)") + [ -z "$ci_pr_url" ] || RUN_DETAIL="$RUN_DETAIL: $ci_pr_url" + ;; + none-reported) + RUN_DETAIL="ci running: no CI checks reported (waiting for checks, not green)" + ;; + esac ;; fixing) CI_LOG_STATE=not-ready @@ -1168,7 +1264,9 @@ if [ "$HAVE_RUN" = 1 ]; then elif [ "$CI_STEP_STATUS" = fixing ]; then CI_LOG_STATE=not-ready fi - if [ "$CI_LOG_STATE" != not-ready ]; then + if [ "$CI_LOG_STATE" = none-reported ]; then + emit_ship_status_done "run still monitoring PR; run ci log: no CI checks reported (not green)" + elif [ "$CI_LOG_STATE" != not-ready ]; then emit_ship_status_done "run still monitoring PR" fi fi @@ -1221,6 +1319,27 @@ if [ "$HAVE_RUN" = 1 ]; then ;; esac + # A finished run's record says what that run did, not what the worker is + # doing now. When the worker's own resolved status declaration (the classifier + # owns which line that is) was written after the run started and disagrees + # with it, both are shown with their sources rather than one silently + # outranking the other: the run record carries no local last-update time, so + # neither witness can be proven newer than the other's final word. + case "$LOG_VERB" in + ''|needs-decision|blocked) ;; + *) + if [ "$RUN_SOURCE" = full ] && [ -z "$RUN_DEAD_DAEMON" ] && ! fm_nm_run_is_active "$RUN_OUT"; then + LOG_TIP_STATE=$(map_log_state "$LOG_LINE") + if [ "$LOG_TIP_STATE" != unknown ] && [ "$LOG_TIP_STATE" != "$RUN_STATE" ] \ + && run_created=$(fm_nm_run_created_epoch "$(strip_quotes "$(nm_field id)")") \ + && log_at=$(status_line_at_epoch "$LOG_LINE") \ + && [ "$log_at" -ge "$run_created" ]; then + RUN_DETAIL="$RUN_DETAIL${SEP}status-log since run start: $LOG_TIP_STATE: $(status_line_note "$LOG_LINE")" + fi + fi + ;; + esac + [ -z "$SELECTED_RUN_ID" ] || RUN_DETAIL="$RUN_DETAIL${SEP}run: $SELECTED_RUN_ID" emit "$RUN_STATE" run-step "$RUN_DETAIL" fi diff --git a/bin/fm-nm-run-lib.sh b/bin/fm-nm-run-lib.sh index f0e4e83b2ee..10ccb6b572e 100644 --- a/bin/fm-nm-run-lib.sh +++ b/bin/fm-nm-run-lib.sh @@ -112,6 +112,43 @@ fm_nm_head_matches_worktree() { # git -C "$wt" merge-base --is-ancestor "$local_full" "$run_full" 2>/dev/null } +# Task incarnation: code identity alone cannot tell this task's run from one an +# earlier task finished on the same branch at the same head, which is exactly +# what a refresh task reusing an existing PR branch sees. A run created before +# the task's start epoch is therefore never the task's result. no-mistakes run +# ids are ULIDs, whose first ten Crockford base32 characters encode the creation +# time in milliseconds (the same instant as the database's created_at). +# Prints the creation epoch in whole seconds; nonzero when $1 is not a ULID. +fm_nm_run_created_epoch() { # + local id alphabet=0123456789ABCDEFGHJKMNPQRSTVWXYZ ms=0 i prefix + id=$(printf '%s' "${1:-}" | tr '[:lower:]' '[:upper:]') + [ "${#id}" -eq 26 ] || return 1 + case "$id" in [0-7]*) ;; *) return 1 ;; esac + case "$id" in *[!0-9ABCDEFGHJKMNPQRSTVWXYZ]*) return 1 ;; esac + for ((i = 0; i < 10; i++)); do + prefix=${alphabet%%"${id:i:1}"*} + ms=$((ms * 32 + ${#prefix})) + done + printf '%s' $((ms / 1000)) +} + +# 0 when run $1 belongs to the task incarnation that started at epoch $2: no +# recorded start (no bound), or a creation time at or after it. A run whose +# creation time cannot be read cannot be proven to be the task's own and is +# refused whenever a start is recorded. +fm_nm_run_in_incarnation() { # + local created + [ -n "${2:-}" ] || return 0 + created=$(fm_nm_run_created_epoch "$1") || return 1 + [ "$created" -ge "$2" ] +} + +# Local "YYYY-MM-DD HH:MM" of epoch $1, the resolution and timezone of the +# `no-mistakes runs` creation column (the CLI runs in this process's timezone). +fm_nm_epoch_ledger_minute() { # + date -r "$1" '+%Y-%m-%d %H:%M' 2>/dev/null || date -d "@$1" '+%Y-%m-%d %H:%M' 2>/dev/null +} + # Liveness class of a recorded ledger status word. # The coarse `no-mistakes runs` ledger emits database status words; an # `axi status` run object reports its terminal result through its own outcome @@ -440,13 +477,22 @@ fm_nm_run_is_executing() { # # name matches. The one live bind is the EXECUTING record on the `axi status` # route (fm_nm_run_is_executing above), which the caller pairs with its own # liveness evidence. +# When optional task start epoch $5 is supplied, the answering row must have been +# created in or after that start's minute (fm_nm_run_in_incarnation above owns +# the rule): a newest row from before the task started is an earlier task's +# history, so it prints nothing. The anchor row is exempt; it proves code +# identity only. # Read-only: git reads resolve objects in place; custody never changes. -fm_nm_runs_status_for_worktree() { # [expected-head] - local wt=$1 branch=$2 list=$3 expected_head=${4:-} +fm_nm_runs_status_for_worktree() { # [expected-head] [task-start-epoch] + local wt=$1 branch=$2 list=$3 expected_head=${4:-} task_start=${5:-} start_minute='' local local_full row_full row st br sha day clock pr extra year_num month_num day_num max_day pending_st='' local decided='' local_full=$(git -C "$wt" rev-parse HEAD 2>/dev/null) || return 0 [ -n "$list" ] || return 0 + if [ -n "$task_start" ]; then + start_minute=$(fm_nm_epoch_ledger_minute "$task_start") || return 0 + [ -n "$start_minute" ] || return 0 + fi while IFS= read -r row; do row=$(fm_nm_trim "$row") [ -n "$row" ] || continue @@ -486,6 +532,9 @@ fm_nm_runs_status_for_worktree() { # [ex fi break fi + if [ -n "$start_minute" ] && [[ "$day $clock" < "$start_minute" ]]; then + break + fi if [ -n "$expected_head" ]; then case "$expected_head" in *[!A-Fa-f0-9]*|'') break ;; esac [ "${#expected_head}" -ge 7 ] && [ "${#expected_head}" -le 40 ] || break diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 5f9c6ecc950..48cb9bdf495 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -451,6 +451,10 @@ # success line and state/.meta omit them. # Every fresh spawn or relaunch records a new spawn_gen= incarnation token so durable # consumers can distinguish a replacement worker that reuses the same task id. +# A fresh spawn also records task_started=, which a relaunch carries forward +# (derived from the prior spawn_gen= when an older record lacks it), so +# bin/fm-crew-state.sh never attributes a validation run created before the task +# started, such as an earlier task's run on a reused PR branch. # When the home session's frozen trace-context decision is enabled (see # docs/configuration.md and bin/fm-trace-context-lib.sh), the meta also records # one W3C traceparent= carrier, the same value injected into the pane as @@ -4842,7 +4846,20 @@ fi META_WINDOW=$T [ "$BACKEND" = orca ] && META_WINDOW=$W -SPAWN_GEN="s$(date +%s).${BASHPID:-$$}.$RANDOM" +SPAWN_EPOCH=$(date +%s) +SPAWN_GEN="s$SPAWN_EPOCH.${BASHPID:-$$}.$RANDOM" +TASK_STARTED=$SPAWN_EPOCH +if [ "$RELAUNCH" -eq 1 ]; then + RELAUNCH_TASK_STARTED=$(fm_meta_get "$RELAUNCH_META" task_started) + case "$RELAUNCH_TASK_STARTED" in + ''|*[!0-9]*) + RELAUNCH_TASK_STARTED=$(fm_meta_get "$RELAUNCH_META" spawn_gen) + RELAUNCH_TASK_STARTED=${RELAUNCH_TASK_STARTED#s} + RELAUNCH_TASK_STARTED=${RELAUNCH_TASK_STARTED%%.*} + ;; + esac + case "$RELAUNCH_TASK_STARTED" in ''|*[!0-9]*) ;; *) TASK_STARTED=$RELAUNCH_TASK_STARTED ;; esac +fi SPAWN_META_PATH="$STATE/$ID.meta" if [ "$SPAWN_META_LOCK_HELD" != 1 ]; then SPAWN_META_LOCK=$(fm_meta_lock_path "$STATE/$ID.meta") || exit 1 @@ -4859,7 +4876,7 @@ SPAWN_META_PATH=$SPAWN_META_TMP preserve_relaunch_meta() { awk -F= ' BEGIN { - split("window endpoint_task_id worktree project harness kind mode yolo branch tasktmp model effort account account_provider busy_gen spawn_gen traceparent backend herdr_session herdr_workspace_id herdr_tab_id herdr_pane_id zellij_session zellij_tab_id zellij_pane_id orca_worktree_id terminal cmux_workspace_id cmux_surface_id home projects control_relaunch_tx", keys, " ") + split("window endpoint_task_id worktree project harness kind mode yolo branch tasktmp model effort account account_provider busy_gen spawn_gen task_started traceparent backend herdr_session herdr_workspace_id herdr_tab_id herdr_pane_id zellij_session zellij_tab_id zellij_pane_id orca_worktree_id terminal cmux_workspace_id cmux_surface_id home projects control_relaunch_tx", keys, " ") for (i in keys) owned[keys[i]] = 1 } !($1 in owned) @@ -4884,6 +4901,7 @@ preserve_relaunch_meta() { [ -z "$WORKER_ACCOUNT_PROVIDER" ] || echo "account_provider=$WORKER_ACCOUNT_PROVIDER" [ -z "${BUSY_GEN:-}" ] || echo "busy_gen=$BUSY_GEN" echo "spawn_gen=$SPAWN_GEN" + echo "task_started=$TASK_STARTED" # Default-off writes no traceparent= line. # backend= is written only for a non-default (non-tmux) backend, so the # default path's meta stays byte-identical (absent backend= means tmux; diff --git a/docs/architecture.md b/docs/architecture.md index 4b2b6f9cbfe..fc29ef1c5f7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -123,11 +123,13 @@ The owned-append ledger only decides whether growth wakes this home; it never re `bin/fm-crew-state.sh ` is the cheap current-state read for an actionable heartbeat review: it attributes an active or terminal no-mistakes run under the shared run-attribution contract, then keeps that run-step authoritative even if the pane has closed, except that a `blocked:` event reporting a refused or missing daemon socket outranks a potentially stale active run record only while that socket-down declaration is itself the log's latest recognized event, since any later event, including another `blocked:` one, means the crew moved on. For other daemon, timeout, or unreachability claims, a running or fixing run with recent pipeline-reported activity supersedes the event and names reattachment as the recovery instead of surfacing a false block. [`bin/fm-nm-run-lib.sh`](../bin/fm-nm-run-lib.sh) owns branch, head, and pipeline-custody attribution, plus complete same-branch run selection, optional inventory lookup, and ambiguity reporting. +Every route first binds a run to the task's own incarnation: a run created before the task's recorded start is never its result, so a refresh task that reuses an existing PR branch at an unchanged head cannot read the earlier task's finished run as its own. A run executing on the crew's own branch is current regardless of head, because the pipeline rebases that branch and commits its fix rounds in its own checkout, so reading an older run that still matches the local head would report a working crew as failed; every other run, parked or terminal, still binds on head equality or ancestry, or on the pipeline's own custody attribution while it owns the branch, and that head-free live bind is withdrawn once an explicit `daemon status` probe answers that the daemon is down. [`tests/fm-crew-state.test.sh`](../tests/fm-crew-state.test.sh) covers run selection; its [capture provenance and live-evidence limits](../tests/captures/no-mistakes-v1.70.1/README.md) distinguish recorded inputs from composed scenarios. -During no-mistakes' `ci` monitor phase, it also reads the full ci step log because `axi status` reports both "still waiting on checks" and "checks green, waiting on merge" as `ci,running`. -The most recent recognized ci log marker wins, so checks-green monitoring reports done while a later failed-check, checks-running, or issue marker returns the crew to working; a base-branch timeout re-arm is not a marker because it leaves readiness unchanged. -`bin/fm-crew-state.sh` owns the evidence guard that recognizes ended CI monitors after green checks, including cancelled runs and skipped rebase steps; a passed run alone never proves a forge merge. +During no-mistakes' `ci` monitor phase, it also reads the full ci step log because `axi status` reports both "still waiting on checks" and "checks passed, waiting on merge" as `ci,running`. +The most recent recognized ci log marker wins, so monitoring whose log reports checks passed reads done while a later failed-check, checks-running, or issue marker returns the crew to working; a log reporting no CI checks at all stays a wait, and a base-branch timeout re-arm is not a marker because it leaves readiness unchanged. +`bin/fm-crew-state.sh` owns the evidence guard that recognizes ended CI monitors after passed checks, including cancelled runs and skipped rebase steps. +A run record states only what that run did and is never presented as green; a merged claim comes only from the task's recorded `pr=` identity checked against the forge or a merge receipt, and a worker status line written after a finished run started is shown beside it with its source. In the coarse runs-ledger fallback, which has no steps table and no ci log, a terminal failed record whose daemon an explicit `daemon status` probe proves down reports unknown as unverified instead: an instrument failure must never read as work failure. The same instrument rule covers the ledger-anchored continuation of a selected run whose head this copy cannot resolve: once the probe answers down, that still-executing record reports unknown as unverified, while a run parked at a gate keeps its gate and findings because an open decision stays open when the instrument dies, and a `needs-decision` or `blocked` event the crew observed first hand stays open with the unverified record named as the reason rather than superseded by it. Only when no matching run exists does it consult semantic busy state; exact busy reports working, exact idle permits fallback to the log's resolved current declaration - the newest decision the fold still holds open, otherwise a declared wait still standing after later resolved lines for other keys, otherwise the latest recognized event - when its verb maps to a recognized run-state, and unknown or a dead pane stays unknown instead of trusting a stale log. diff --git a/docs/gerrit-change-watch.md b/docs/gerrit-change-watch.md index fe65422ed0e..4d405722dc2 100644 --- a/docs/gerrit-change-watch.md +++ b/docs/gerrit-change-watch.md @@ -128,6 +128,6 @@ The server permitting self-approval is what makes this a policy boundary rather - A no-mistakes done naming a Gerrit change is also refused unless the copy holds a passed pipeline's result: refused when the run's outcome is missing or not a pass, while the run reports `recover_custody` or `continue_active_run`, when HEAD's tree differs from the pipeline head's, or when the run cannot be read, and accepted once recovered even after the Change-Id stamp rewrote the branch's messages. - A `published for review` done whose URL is not a canonical Gerrit change URL is refused, even when a remote-tracking ref holds HEAD. -`tests/fm-crew-state.test.sh` pins the crew-state read with no server either: a passed run whose change is open reports `PR open`, an abandoned one `PR closed`, a merged one `PR merged`, and an unreadable record or one naming another change reports an honest unknown rather than a merge. +`tests/fm-crew-state.test.sh` pins the crew-state read with no server either: a passed run whose recorded change is open reports `recorded PR open`, an abandoned one `recorded PR closed`, a merged one `recorded PR merged`, and an unreadable record or one naming another change reports an honest unknown rather than a merge. Refresh this record by rerunning those suites, and rerun the transcripts above after a `gerrit-axi` upgrade. diff --git a/tests/fm-control-relaunch.test.sh b/tests/fm-control-relaunch.test.sh index 1242dc26bfe..c6d4bc90c06 100755 --- a/tests/fm-control-relaunch.test.sh +++ b/tests/fm-control-relaunch.test.sh @@ -490,6 +490,26 @@ test_relaunch_preserves_durable_task_metadata() { pass "fm-control relaunch: durable task metadata survives replacement launch publication" } +# A relaunch continues the same task, so its validation runs stay attributable: +# the task start carries forward, and a record older than task_started= derives +# it from the spawn generation it was first launched with. +test_relaunch_carries_the_task_start() { + local dir out rc + dir=$(new_case task-start rl43) + add_ship_task "$dir" rl43 claude + printf '%s\n' 'spawn_gen=s1700000000.1.1' >> "$dir/home/state/rl43.meta" + out=$(run_control "$dir" rl43 relaunch --note "continuing after a stall"); rc=$? + expect_code 0 "$rc" "a relaunch of a legacy record should succeed"$'\n'"$out" + [ "$(meta_field "$dir" rl43 task_started)" = 1700000000 ] \ + || fail "a legacy record must derive its task start from its prior spawn generation" + [ "$(meta_field "$dir" rl43 spawn_gen)" != s1700000000.1.1 ] || fail "a relaunch must mint a new spawn generation" + out=$(run_control "$dir" rl43 relaunch --note "continuing again"); rc=$? + expect_code 0 "$rc" "a second relaunch should succeed"$'\n'"$out" + [ "$(meta_field "$dir" rl43 task_started)" = 1700000000 ] || fail "the task start must survive every relaunch" + [ "$(grep -c '^task_started=' "$dir/home/state/rl43.meta")" = 1 ] || fail "the task start must be recorded once" + pass "fm-control relaunch: the task start carries across relaunches" +} + test_relaunch_serializes_concurrent_durable_metadata_publication() { local dir control_pid link_pid rc i=0 traceparent prepare launch_release waiting ready release dir=$(new_case metadata-race rl28) @@ -2391,6 +2411,7 @@ test_relaunch_refuses_before_exit_when_the_composer_holds_pending_text test_relaunch_refuses_before_exit_when_the_composer_state_is_unproven test_relaunch_from_linked_home_preserves_recorded_worktree test_relaunch_preserves_durable_task_metadata +test_relaunch_carries_the_task_start test_relaunch_serializes_concurrent_durable_metadata_publication test_disabled_relaunch_clears_prior_trace_context test_relaunch_appends_the_progress_note_to_the_instructions diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index bf7d7bc5306..90205231fbd 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -1186,7 +1186,7 @@ EOF local out; out=$(run_crew_state "$d" feat-cigreen) assert_contains "$out" "state: done" "green ci-monitor run -> done" assert_contains "$out" "source: run-step" "green ci-monitor -> run-step source" - assert_contains "$out" "checks green" "green ci-monitor detail mentions checks green" + assert_contains "$out" "run ci log reports CI checks passed" "the ci-monitor detail states what the run's log reported" assert_not_contains "$out" "state: working" "green ci-monitor must not read as still validating" pass "ci-monitoring run with checks already green surfaces done" } @@ -1202,12 +1202,15 @@ test_top_level_ci_checks_green_surfaces_done() { local out; out=$(run_crew_state "$d" feat-topcigreen) assert_contains "$out" "state: done" "top-level ci with green log -> done" assert_contains "$out" "source: run-step" "top-level ci green -> run-step source" - assert_contains "$out" "checks green" "top-level ci green detail mentions checks green" + assert_contains "$out" "run ci log reports CI checks passed" "the top-level ci detail states what the run's log reported" assert_not_contains "$out" "state: working" "top-level ci green must not stay working" pass "top-level ci status uses ci log green marker" } -test_ci_monitoring_no_checks_terminal_surfaces_done() { +# CI that has not run - a fork's workflow runs held for maintainer approval +# report no checks at all - is a wait, never green. The worker's own ci-ready +# claim still answers from its status log, with the run's reading beside it. +test_ci_monitoring_no_checks_reads_waiting_not_green() { reset_fakes local d; d=$(new_case ci-nochecks) make_repo_on_branch "$d/wt" fm/feat-cinochecks @@ -1216,9 +1219,15 @@ test_ci_monitoring_no_checks_terminal_surfaces_done() { FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cinochecks)" FM_FAKE_CI_LOGS="no CI checks reported - still monitoring until merged or closed" local out; out=$(run_crew_state "$d" feat-cinochecks) - assert_contains "$out" "state: done" "terminal no-checks ci-monitor run -> done" - assert_contains "$out" "checks green" "terminal no-checks ci-monitor detail mentions checks green" - pass "terminal no-checks ci-monitor marker surfaces done" + assert_contains "$out" "state: working" "a no-checks ci monitor is still waiting: $out" + assert_contains "$out" "no CI checks reported (waiting for checks, not green)" "the wait is named, not green" + assert_not_contains "$out" "checks green" "no reported checks must not read as checks green" + + printf 'done: PR https://github.com/o/r/pull/2 checks green\n' > "$d/state/feat-cinochecks.status" + out=$(run_crew_state "$d" feat-cinochecks) + assert_contains "$out" "source: status-log" "the worker's ci-ready claim answers from its own log: $out" + assert_contains "$out" "run ci log: no CI checks reported (not green)" "the run's reading is shown beside it" + pass "a no-checks ci-monitor marker reads as waiting, not green" } # The monitor logs a checks state only when it changes, and a base-branch @@ -1241,7 +1250,7 @@ EOF local out; out=$(run_crew_state "$d" feat-cirearm) assert_contains "$out" "state: done" "a base-advance re-arm after green keeps the PR green" assert_contains "$out" "source: run-step" "re-armed green monitoring stays run-step sourced" - assert_contains "$out" "checks green: PR ready for review" "re-armed green monitoring reads held for merge" + assert_contains "$out" "CI checks passed: PR ready for review" "re-armed green monitoring reads held for merge" assert_contains "$out" "https://github.com/o/r/pull/2" "the held-for-merge reading names the run's PR" assert_not_contains "$out" "state: working" "a re-arm line must not read as checks not ready" pass "base-advance re-arm after green stays checks green" @@ -1268,7 +1277,7 @@ test_ci_monitoring_green_before_log_tail_stays_green() { local out; out=$(run_crew_state "$d" feat-citail) assert_contains "$out" "state: done" "a green marker older than the log tail still reads green" assert_contains "$out" "source: run-step" "the full-log green reading stays run-step sourced" - assert_contains "$out" "checks green: PR ready for review" "the full-log reading is held for merge" + assert_contains "$out" "CI checks passed: PR ready for review" "the full-log reading is held for merge" assert_contains "$out" "https://github.com/o/r/pull/2" "the full-log reading names the run's PR" assert_not_contains "$out" "state: working" "a truncated ci log must not hide a green PR" pass "a green marker before the ci log tail still surfaces done" @@ -1405,12 +1414,12 @@ test_terminal_passed() { local d; d=$(new_case passed) make_repo_on_branch "$d/wt" fm/feat-d make_fakebin "$d" >/dev/null - fm_write_meta "$d/state/feat-d.meta" "window=fm:fm-feat-d" "worktree=$d/wt" "kind=ship" + fm_write_meta "$d/state/feat-d.meta" "window=fm:fm-feat-d" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" FM_FAKE_AXI_STATUS="$(run_passed fm/feat-d)" local out; out=$(run_crew_state "$d" feat-d) assert_contains "$out" "state: done" "passed run -> done" assert_contains "$out" "source: run-step" "passed -> run-step source" - assert_contains "$out" "run passed: PR merged" "passed run reports merged only after the PR record says merged" + assert_contains "$out" "run passed: recorded PR merged" "passed run reports merged only after the PR record says merged" assert_not_contains "$out" "merged/closed" "passed merged PR must not keep the old ambiguous label" pass "terminal passed run is authoritative" } @@ -1420,12 +1429,13 @@ test_terminal_passed_with_override() { local d; d=$(new_case passed-with-override) make_repo_on_branch "$d/wt" fm/feat-override make_fakebin "$d" >/dev/null - fm_write_meta "$d/state/feat-override.meta" "window=fm:fm-feat-override" "worktree=$d/wt" "kind=ship" + fm_write_meta "$d/state/feat-override.meta" "window=fm:fm-feat-override" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" FM_FAKE_AXI_STATUS="$(run_passed_with_override fm/feat-override)" local out; out=$(run_crew_state "$d" feat-override) assert_contains "$out" "state: done" "passed-with-override run -> done, not unknown" assert_contains "$out" "source: run-step" "passed-with-override -> run-step source" - assert_contains "$out" "run passed: PR merged" "passed-with-override run reports merged only after the PR record says merged" + assert_contains "$out" "run passed-with-override (approved exception, not green: live checks not all passed: Lint (fail)): recorded PR merged (forge)" \ + "passed-with-override names its exception and reports merged only after the recorded PR says merged" assert_not_contains "$out" "state: unknown" "passed-with-override must not fall through to unknown" assert_not_contains "$out" "outcome: passed-with-override" "passed-with-override must not surface as a raw unmapped outcome detail" pass "terminal passed-with-override run reads done like a clean pass" @@ -1436,12 +1446,12 @@ test_terminal_passed_with_skips() { local d; d=$(new_case passed-with-skips) make_repo_on_branch "$d/wt" fm/feat-skips make_fakebin "$d" >/dev/null - fm_write_meta "$d/state/feat-skips.meta" "window=fm:fm-feat-skips" "worktree=$d/wt" "kind=ship" + fm_write_meta "$d/state/feat-skips.meta" "window=fm:fm-feat-skips" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" FM_FAKE_AXI_STATUS="$(run_passed_with_skips fm/feat-skips)" local out; out=$(run_crew_state "$d" feat-skips) assert_contains "$out" "state: done" "passed-with-skips run -> done, not unknown" assert_contains "$out" "source: run-step" "passed-with-skips -> run-step source" - assert_contains "$out" "run passed: PR merged" "passed-with-skips run reports merged only after the PR record says merged" + assert_contains "$out" "run passed-with-skips: recorded PR merged" "passed-with-skips run reports merged only after the recorded PR says merged" assert_contains "$out" "publication/CI verification skipped" "passed-with-skips keeps the skip visible, unlike a clean pass" assert_not_contains "$out" "state: unknown" "passed-with-skips must not fall through to unknown" assert_not_contains "$out" "outcome: passed-with-skips" "passed-with-skips must not surface as a raw unmapped outcome detail" @@ -1465,7 +1475,7 @@ test_terminal_passed_uses_matching_retirement_receipt_without_forge() { FM_FAKE_AXI_STATUS="$(run_passed_no_pr fm/feat-dreceipt)" out=$(run_crew_state "$d" feat-dreceipt) assert_contains "$out" "state: done" "passed run with retired PR receipt -> done" - assert_contains "$out" "run passed: PR merged" "matching retirement receipt is local merged evidence" + assert_contains "$out" "run passed: recorded PR merged" "matching retirement receipt is local merged evidence" [ ! -s "$read_log" ] || fail "matching retirement receipt still attempted a forge read" pass "terminal passed run uses matching retirement receipt without forge" } @@ -1485,13 +1495,13 @@ test_terminal_passed_no_forge_switch_skips_read_but_keeps_receipt() { FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dnoforge "$url")" out=$(FM_CREW_STATE_NO_FORGE=1 run_crew_state "$d" feat-dnoforge) - assert_contains "$out" "run passed: PR state unknown (forge read skipped)" "no-forge mode reports skipped read" + assert_contains "$out" "run passed: recorded PR state unknown (forge read skipped)" "no-forge mode reports skipped read" assert_not_contains "$out" "PR merged" "no-forge mode without a receipt must not report merged" [ ! -s "$read_log" ] || fail "no-forge mode invoked a forge read" seed_retired_pr_receipt "$d/state" feat-dnoforge "$url" out=$(FM_CREW_STATE_NO_FORGE=1 run_crew_state "$d" feat-dnoforge) - assert_contains "$out" "run passed: PR merged" "no-forge mode still trusts a matching retirement receipt" + assert_contains "$out" "run passed: recorded PR merged" "no-forge mode still trusts a matching retirement receipt" [ ! -s "$read_log" ] || fail "no-forge mode with a receipt invoked a forge read" pass "terminal passed no-forge mode preserves local receipt evidence" } @@ -1508,13 +1518,13 @@ test_terminal_passed_with_open_pr_does_not_claim_merged() { FM_FAKE_AXI_STATUS="$(run_passed fm/feat-dopen)" local out; out=$(run_crew_state "$d" feat-dopen) assert_contains "$out" "state: done" "passed run with open PR -> done" - assert_contains "$out" "run passed: PR open" "open PR state is named" + assert_contains "$out" "run passed: recorded PR open" "open PR state is named" assert_not_contains "$out" "merged/closed" "open PR must not get the old merged/closed label" assert_not_contains "$out" "PR merged" "open PR must not be reported merged" pass "terminal passed run with open PR does not claim merged" } -test_terminal_passed_run_pr_overrides_stale_metadata() { +test_terminal_passed_conflicting_run_pr_claims_neither_state() { reset_fakes local d; d=$(new_case passed-stale-meta) make_repo_on_branch "$d/wt" fm/feat-dstale @@ -1527,10 +1537,11 @@ test_terminal_passed_run_pr_overrides_stale_metadata() { FM_FAKE_PR_48_MERGED=false FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dstale https://github.com/o/r/pull/48)" local out; out=$(run_crew_state "$d" feat-dstale) - assert_contains "$out" "state: done" "passed run with stale task metadata -> done" - assert_contains "$out" "run passed: PR open" "run PR identity outranks stale task metadata" - assert_not_contains "$out" "PR merged" "stale merged metadata must not report merged" - pass "terminal passed run PR overrides stale task metadata" + assert_contains "$out" "state: done" "passed run with a conflicting recorded PR -> done" + assert_contains "$out" "run passed: PR state unknown (run PR https://github.com/o/r/pull/48 differs from recorded PR https://github.com/o/r/pull/47)" \ + "a run PR that differs from the recorded PR is a conflict, never either PR's state" + assert_not_contains "$out" "PR merged" "the recorded PR's merge must not be claimed for another PR's run" + pass "terminal passed run with a conflicting PR identity claims neither state" } test_terminal_passed_without_readable_pr_identity_reports_unknown() { @@ -1562,7 +1573,7 @@ test_terminal_passed_with_open_gitlab_mr_does_not_claim_merged() { FM_FAKE_GLAB_STATE=opened FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgitlabopen https://git.example.com/group/subgroup/repo/-/merge_requests/9)" out=$(run_crew_state "$d" feat-dgitlabopen) - assert_contains "$out" "run passed: PR open" "open GitLab MR state is named" + assert_contains "$out" "run passed: recorded PR open" "open GitLab MR state is named" assert_not_contains "$out" "PR merged" "open GitLab MR must not be reported merged" assert_grep 'git.example.com|mr view 9 -R https://git.example.com/group/subgroup/repo -F json' "$read_log" \ "GitLab MR read uses the parsed host and project URL" @@ -1580,7 +1591,7 @@ test_terminal_passed_with_merged_gitlab_mr_reports_merged() { FM_FAKE_GLAB_STATE=merged FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgitlabmerged https://gitlab.com/group/repo/-/merge_requests/10)" out=$(run_crew_state "$d" feat-dgitlabmerged) - assert_contains "$out" "run passed: PR merged" "merged GitLab MR is reported merged" + assert_contains "$out" "run passed: recorded PR merged" "merged GitLab MR is reported merged" pass "terminal passed run reads merged GitLab MR state" } @@ -1595,7 +1606,7 @@ test_terminal_passed_with_failed_gitlab_read_reports_unknown() { FM_FAKE_GLAB_READ_FAIL=1 FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgitlabunknown https://gitlab.com/group/repo/-/merge_requests/11)" out=$(run_crew_state "$d" feat-dgitlabunknown) - assert_contains "$out" "run passed: PR state unknown (unreadable)" "failed GitLab read is honest unknown" + assert_contains "$out" "run passed: recorded PR state unknown (unreadable)" "failed GitLab read is honest unknown" assert_not_contains "$out" "PR merged" "failed GitLab read must not be reported merged" pass "terminal passed run handles failed GitLab read" } @@ -1615,7 +1626,7 @@ test_terminal_passed_with_open_gerrit_change_does_not_claim_merged() { FM_FAKE_GERRIT_STATUS=NEW FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgerritopen "$url")" out=$(run_crew_state "$d" feat-dgerritopen) - assert_contains "$out" "run passed: PR open" "open Gerrit change state is named" + assert_contains "$out" "run passed: recorded PR open" "open Gerrit change state is named" assert_not_contains "$out" "PR merged" "open Gerrit change must not be reported merged" assert_grep 'show 4201 --host review.internal --json' "$read_log" \ "Gerrit read addresses the change by number and explicit host" @@ -1638,12 +1649,12 @@ test_terminal_passed_with_merged_gerrit_change_reports_merged() { # gerrit.canonicalWebUrl returns, so the merge is reported off the change # number the read was addressed by rather than off a URL the server may # never compose. - assert_contains "$out" "run passed: PR merged" "merged Gerrit change is reported merged" + assert_contains "$out" "run passed: recorded PR merged" "merged Gerrit change is reported merged" # An abandoned change is this report's closed, and is never merged. FM_FAKE_GERRIT_STATUS=ABANDONED out=$(run_crew_state "$d" feat-dgerritmerged) - assert_contains "$out" "run passed: PR closed" "abandoned Gerrit change is reported closed" + assert_contains "$out" "run passed: recorded PR closed" "abandoned Gerrit change is reported closed" assert_not_contains "$out" "PR merged" "abandoned Gerrit change must not be reported merged" pass "terminal passed run reads merged and abandoned Gerrit change state" } @@ -1660,7 +1671,7 @@ test_terminal_passed_with_unreadable_gerrit_change_reports_unknown() { FM_FAKE_GERRIT_READ_FAIL=1 FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgerritunknown "$url")" out=$(run_crew_state "$d" feat-dgerritunknown) - assert_contains "$out" "run passed: PR state unknown (unreadable)" "failed Gerrit read is honest unknown" + assert_contains "$out" "run passed: recorded PR state unknown (unreadable)" "failed Gerrit read is honest unknown" assert_not_contains "$out" "PR merged" "failed Gerrit read must not be reported merged" # A record naming another change can never answer for this one, however the @@ -1671,7 +1682,7 @@ test_terminal_passed_with_unreadable_gerrit_change_reports_unknown() { FM_FAKE_GERRIT_CHANGE=4203 FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgerritunknown "$url")" out=$(run_crew_state "$d" feat-dgerritunknown) - assert_contains "$out" "run passed: PR state unknown (unreadable)" "mismatched Gerrit record is honest unknown" + assert_contains "$out" "run passed: recorded PR state unknown (unreadable)" "mismatched Gerrit record is honest unknown" assert_not_contains "$out" "PR merged" "another change's merged record must not report merged" pass "terminal passed run handles an unreadable or mismatched Gerrit read" } @@ -1720,7 +1731,7 @@ test_cancelled_delivery_and_skipped_rebase() { assert_not_contains "$out" "PR merged" "$scenario: terminal record cannot prove a merge" if [ "$scenario" != passed ]; then assert_contains "$out" "https://github.com/o/r/pull/203" "$scenario: delivery identity retained" - assert_contains "$out" "checks green" "$scenario: retain positive CI evidence" + assert_contains "$out" "ci log reported CI checks passed" "$scenario: retain positive CI evidence" assert_contains "$out" "held for merge" "$scenario: delivery awaits merge" fi pass "$scenario: terminal delivery reports only observed evidence" @@ -1762,6 +1773,8 @@ test_terminal_green_delivery_disposition() { case "$disposition" in no-identity) FM_FAKE_AXI_STATUS=$(printf '%s\n' "$FM_FAKE_AXI_STATUS" | sed '/^[[:space:]]*pr:/d') ;; merged) + # A merge is claimed only for the task's recorded PR identity. + printf 'pr=%s\n' "$url" >> "$d/state/delivery.meta" FM_FAKE_PR_STATE=MERGED FM_FAKE_PR_MERGED=true FM_FAKE_PR_STATE_AXI=merged @@ -1949,7 +1962,8 @@ test_terminal_failed_ci_orphan_after_green_reads_done() { local d; d=$(new_case failed-ci-orphan) make_repo_on_branch "$d/wt" fm/feat-ci-orphan make_fakebin "$d" >/dev/null - fm_write_meta "$d/state/feat-ci-orphan.meta" "window=fm:fm-feat-ci-orphan" "worktree=$d/wt" "kind=ship" + fm_write_meta "$d/state/feat-ci-orphan.meta" "window=fm:fm-feat-ci-orphan" "worktree=$d/wt" "kind=ship" \ + "pr=https://github.com/o/r/pull/203" FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan fm/feat-ci-orphan)" FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed daemon shutting down" @@ -1966,7 +1980,8 @@ test_terminal_failed_ci_orphan_status_only_reads_done() { local d; d=$(new_case failed-ci-orphan-status-only) make_repo_on_branch "$d/wt" fm/feat-ci-orphan2 make_fakebin "$d" >/dev/null - fm_write_meta "$d/state/feat-ci-orphan2.meta" "window=fm:fm-feat-ci-orphan2" "worktree=$d/wt" "kind=ship" + fm_write_meta "$d/state/feat-ci-orphan2.meta" "window=fm:fm-feat-ci-orphan2" "worktree=$d/wt" "kind=ship" \ + "pr=https://github.com/o/r/pull/203" FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan_status_only fm/feat-ci-orphan2)" FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed daemon shutting down" @@ -4106,7 +4121,7 @@ EOF assert_not_contains "$out" 'state: unknown' 'a green PR in merge monitoring is never unknown' assert_contains "$out" 'state: done' 'a green PR in merge monitoring reads done' assert_contains "$out" 'source: run-step' 'the green reading comes from the selected run' - assert_contains "$out" 'checks green: PR ready for review' 'the reading is held for the merge decision' + assert_contains "$out" 'CI checks passed: PR ready for review' 'the reading is held for the merge decision' assert_contains "$out" 'https://github.com/o/r/pull/2' 'the reading names the PR to ask about' pass 'a linked worktree green PR in merge monitoring reads held for merge' } @@ -5533,7 +5548,7 @@ test_gate_block_parked_not_superseded test_ci_ready_done_log_beats_monitoring_run test_ci_monitoring_checks_green_surfaces_done test_top_level_ci_checks_green_surfaces_done -test_ci_monitoring_no_checks_terminal_surfaces_done +test_ci_monitoring_no_checks_reads_waiting_not_green test_ci_monitoring_green_then_rearm_stays_green test_ci_monitoring_green_before_log_tail_stays_green test_ci_monitoring_no_checks_yet_stays_working @@ -5549,7 +5564,7 @@ test_terminal_passed_with_skips test_terminal_passed_uses_matching_retirement_receipt_without_forge test_terminal_passed_no_forge_switch_skips_read_but_keeps_receipt test_terminal_passed_with_open_pr_does_not_claim_merged -test_terminal_passed_run_pr_overrides_stale_metadata +test_terminal_passed_conflicting_run_pr_claims_neither_state test_terminal_passed_without_readable_pr_identity_reports_unknown test_terminal_passed_with_open_gitlab_mr_does_not_claim_merged test_terminal_passed_with_merged_gitlab_mr_reports_merged @@ -5687,4 +5702,274 @@ test_newer_failed_run_is_not_hidden_by_older_live_run test_unverifiable_run_selection_reports_unknown test_legacy_conflicting_run_records_report_unknown +# --- Task incarnation, forge claims, and run-step wording --------------------- +# A refresh task reuses an existing upstream PR branch at an unchanged head, so a +# run an earlier task finished on that branch matches by branch and head alone. +# Run attribution is bound to this task's own incarnation: a run created before +# the task was first spawned is never this task's result. + +# A ULID run id created at : no-mistakes run ids are ULIDs, whose +# leading ten characters encode the creation time in milliseconds. +ulid_at() { # -> run id + local ms=$(($1 * 1000)) alphabet=0123456789ABCDEFGHJKMNPQRSTVWXYZ time='' i + for i in 1 2 3 4 5 6 7 8 9 10; do + time="${alphabet:$((ms % 32)):1}$time" + ms=$((ms / 32)) + done + printf '%sABCDEFGHJKMNPQRS' "$time" +} + +# The ledger's creation column, in the same local time the real CLI prints. +ledger_minute() { # + date -r "$1" '+%Y-%m-%d %H:%M' 2>/dev/null || date -d "@$1" '+%Y-%m-%d %H:%M' +} + +# One selected-route case: the branch's only run is , finished passed at +# the worktree's own head, exactly the shape an earlier task leaves behind. +make_incarnation_case() { # ... + local name=$1 run_id=$2 d short + shift 2 + reset_fakes + d=$(new_case "$name") + make_repo_on_branch "$d/wt" fm/refresh + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/refresh.meta" "window=fm:fm-refresh" "worktree=$d/wt" "kind=ship" "harness=claude" "$@" + short=$(git -C "$d/wt" rev-parse --short=8 HEAD) + FM_FAKE_AXI_STATUS="$(run_passed_with_override fm/refresh | sed "s/01RUN/$run_id/")" + FM_FAKE_AXI_STATUS_RUN=$FM_FAKE_AXI_STATUS + FM_FAKE_AXI_HOME="count: 1 of 1 total +runs[1]{id,branch,status,head,pr}: + \"$run_id\",fm/refresh,completed,$short,\"https://github.com/o/r/pull/1\"" + FM_FAKE_PR_STATE=OPEN + FM_FAKE_PR_MERGED=false +} + +test_refresh_task_does_not_inherit_an_earlier_run() { + local now d out gen + now=$(date +%s) + make_incarnation_case refresh-busy "$(ulid_at $((now - 3600)))" "task_started=$now" + d=$TMP_ROOT/refresh-busy + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$d/state" refresh) + "$ROOT/bin/fm-busy-event.sh" apply "$d/state" refresh busy --gen "$gen" \ + --source claude-hook --event user-prompt-submit + out=$(run_crew_state "$d" refresh) + assert_not_contains "$out" 'state: done' "an earlier task's finished run must not read as this task's result: $out" + assert_contains "$out" 'state: working' 'the refresh worker is still working' + assert_contains "$out" 'source: pane' 'current pane evidence answers instead of the earlier run' + assert_contains "$out" 'predates this task' 'the ignored run is named with its reason' + pass 'a refresh task does not inherit an earlier task run on the same branch and head' +} + +test_refresh_task_status_log_answers_over_an_earlier_run() { + local now d out + now=$(date +%s) + make_incarnation_case refresh-idle "$(ulid_at $((now - 3600)))" "task_started=$now" + d=$TMP_ROOT/refresh-idle + arm_idle_record "$d/state" refresh >/dev/null + printf 'working [at=%s]: resolving merge conflicts\n' "$now" > "$d/state/refresh.status" + out=$(run_crew_state "$d" refresh) + assert_contains "$out" 'state: working' "the worker's own status answers: $out" + assert_contains "$out" 'source: status-log' 'no run of this incarnation exists yet' + assert_contains "$out" 'resolving merge conflicts' 'the current work detail is preserved' + pass "a refresh task's status log answers over an earlier task run" +} + +test_run_created_after_task_start_is_attributed() { + local now d out + now=$(date +%s) + make_incarnation_case own-run "$(ulid_at $((now + 60)))" "task_started=$now" + d=$TMP_ROOT/own-run + out=$(run_crew_state "$d" refresh) + assert_contains "$out" 'state: done' "this task's own finished run answers: $out" + assert_contains "$out" 'source: run-step' 'the own run keeps run-step attribution' + pass "a run created after the task started is this task's run" +} + +test_relaunch_keeps_the_task_incarnation() { + local now d out + now=$(date +%s) + # task_started survives a relaunch; spawn_gen is the relaunch's own token. + make_incarnation_case relaunched "$(ulid_at $((now - 600)))" \ + "task_started=$((now - 3600))" "spawn_gen=s$now.1.1" + d=$TMP_ROOT/relaunched + out=$(run_crew_state "$d" refresh) + assert_contains "$out" 'source: run-step' "a run from before a relaunch is still this task's run: $out" + pass 'a relaunched task keeps runs from its earlier incarnation' +} + +test_spawn_generation_bounds_a_task_without_a_start_record() { + local now d out + now=$(date +%s) + make_incarnation_case spawn-gen-bound "$(ulid_at $((now - 3600)))" "spawn_gen=s$now.1.1" + d=$TMP_ROOT/spawn-gen-bound + out=$(run_crew_state "$d" refresh) + assert_not_contains "$out" 'source: run-step' "the spawn generation bounds an older record: $out" + assert_contains "$out" 'predates this task' 'the ignored run is named' + pass 'the spawn generation bounds attribution when no start record exists' +} + +test_unreadable_run_creation_time_is_not_attributed_to_a_bounded_task() { + local now d out + now=$(date +%s) + make_incarnation_case opaque-id 01RUN "task_started=$now" + d=$TMP_ROOT/opaque-id + out=$(run_crew_state "$d" refresh) + assert_not_contains "$out" 'source: run-step' "an unprovable creation time cannot claim this task's result: $out" + pass 'a run whose creation time is unreadable is not attributed to a bounded task' +} + +test_legacy_route_does_not_inherit_an_earlier_run() { + local now d out + now=$(date +%s) + make_incarnation_case legacy-refresh "$(ulid_at $((now - 3600)))" "task_started=$now" + d=$TMP_ROOT/legacy-refresh + # No overview table: the CLI answers the bare current-branch record. + FM_FAKE_AXI_HOME= + FM_FAKE_RUNS_LIST=" completed fm/refresh $(git -C "$d/wt" rev-parse --short=8 HEAD) $(ledger_minute $((now - 3600))) https://github.com/o/r/pull/1" + out=$(run_crew_state "$d" refresh) + assert_not_contains "$out" 'source: run-step' "the legacy record route must not inherit the earlier run: $out" + assert_contains "$out" 'predates this task' 'the ignored run is named' + pass 'the legacy route does not inherit an earlier task run' +} + +test_coarse_ledger_does_not_inherit_an_earlier_run() { + local now d out short + now=$(date +%s) + make_incarnation_case coarse-refresh 01OTHER "task_started=$now" + d=$TMP_ROOT/coarse-refresh + short=$(git -C "$d/wt" rev-parse --short=8 HEAD) + FM_FAKE_AXI_HOME= + FM_FAKE_AXI_STATUS="$(run_running fm/another)" + FM_FAKE_RUNS_LIST=" completed fm/refresh $short $(ledger_minute $((now - 3600))) https://github.com/o/r/pull/1" + out=$(run_crew_state "$d" refresh) + assert_not_contains "$out" 'source: run-step' "an earlier ledger row must not answer for this task: $out" + FM_FAKE_RUNS_LIST=" completed fm/refresh $short $(ledger_minute $((now + 120))) https://github.com/o/r/pull/1" + out=$(run_crew_state "$d" refresh) + assert_contains "$out" 'source: run-step' "a ledger row from this task still answers: $out" + pass 'the coarse ledger does not inherit an earlier task run' +} + +# A merged claim must come from the recorded pr= identity checked against the +# forge; the run's own PR field never proves a merge. +test_merged_claim_requires_the_recorded_pr_identity() { + local d out + reset_fakes + d=$(new_case merged-identity) + make_repo_on_branch "$d/wt" fm/merged-identity + make_fakebin "$d" >/dev/null + FM_FAKE_AXI_STATUS="$(run_passed fm/merged-identity)" + FM_FAKE_PR_STATE=MERGED + FM_FAKE_PR_MERGED=true + + fm_write_meta "$d/state/mi.meta" "window=fm:fm-mi" "worktree=$d/wt" "kind=ship" + out=$(run_crew_state "$d" mi) + assert_not_contains "$out" 'PR merged' "a run PR without a recorded identity must not prove a merge: $out" + assert_contains "$out" 'no recorded PR identity' 'the missing recorded identity is named' + + fm_write_meta "$d/state/mi.meta" "window=fm:fm-mi" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/2" + FM_FAKE_PR_1_STATE=MERGED + FM_FAKE_PR_1_MERGED=true + FM_FAKE_PR_2_STATE=OPEN + FM_FAKE_PR_2_MERGED=false + export FM_FAKE_PR_1_STATE FM_FAKE_PR_1_MERGED FM_FAKE_PR_2_STATE FM_FAKE_PR_2_MERGED + out=$(run_crew_state "$d" mi) + assert_not_contains "$out" 'PR merged' "another PR's merge must not be claimed for this task: $out" + assert_contains "$out" 'differs from recorded PR' 'the identity conflict is named' + + fm_write_meta "$d/state/mi.meta" "window=fm:fm-mi" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" + out=$(run_crew_state "$d" mi) + assert_contains "$out" 'recorded PR merged (forge)' "the recorded PR's forge merge is reported with its source: $out" + unset FM_FAKE_PR_1_STATE FM_FAKE_PR_1_MERGED FM_FAKE_PR_2_STATE FM_FAKE_PR_2_MERGED + pass 'a merged claim requires the recorded PR identity' +} + +# The run-step reports what the run did; "green" is never presented from a run +# record, and CI that has not run reads as waiting. Saying "not green" is allowed. +assert_no_green_claim() { # + local rest=${1//not green/} + case "$rest" in *green*) fail "$2 (green claimed): $1" ;; esac +} + +test_run_step_wording_never_presents_green() { + local d out + reset_fakes + d=$(new_case green-wording) + make_repo_on_branch "$d/wt" fm/green + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/green.meta" "window=fm:fm-green" "worktree=$d/wt" "kind=ship" + + FM_FAKE_AXI_STATUS="$(run_passed fm/green | sed 's/outcome: passed/outcome: checks-passed/')" + out=$(run_crew_state "$d" green) + assert_contains "$out" 'state: done' "checks-passed still finishes the task: $out" + assert_no_green_claim "$out" 'a checks-passed run record is not presented as green' + assert_contains "$out" 'run reports CI checks passed' 'the run outcome is stated as what the run reported' + + FM_FAKE_AXI_STATUS="$(run_top_level_ci fm/green)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + out=$(run_crew_state "$d" green) + assert_contains "$out" 'state: done' "a passed ci log is ready for review: $out" + assert_no_green_claim "$out" 'a ci log reading is not presented as green' + + + FM_FAKE_CI_LOGS= + FM_FAKE_AXI_STATUS="$(run_passed_with_override fm/green | sed 's/ci_override_reason: .*/ci_override_reason: "live checks for https:\/\/github.com\/o\/r\/pull\/1: no checks reported"/')" + out=$(run_crew_state "$d" green) + assert_no_green_claim "$out" 'an override for unreported checks is never green' + assert_contains "$out" 'no checks reported' 'the override reason is kept visible' + pass 'run-step wording never presents green' +} + +# A worker's status line written after a finished run started is shown with its +# source beside the run-step record, so the worker can correct a stale reading. +test_newer_status_line_is_shown_beside_a_finished_run() { + local now d out + now=$(date +%s) + make_incarnation_case newer-status "$(ulid_at $((now - 600)))" "task_started=$((now - 3600))" + d=$TMP_ROOT/newer-status + printf 'working [at=%s]: rebasing after upstream conflict\n' "$now" > "$d/state/refresh.status" + out=$(run_crew_state "$d" refresh) + assert_contains "$out" 'source: run-step' "the finished run is still reported: $out" + assert_contains "$out" 'status-log since run start: working: rebasing after upstream conflict' \ + 'the newer worker status is shown with its source' + + printf 'working [at=%s]: setting up before validation\n' "$((now - 1200))" > "$d/state/refresh.status" + out=$(run_crew_state "$d" refresh) + assert_not_contains "$out" 'setting up before validation' "a status line older than the run is superseded: $out" + pass 'a newer status line is shown beside a finished run' +} + +test_deliberate_abort_is_distinct_from_failure() { + local d out + reset_fakes + d=$(new_case abort-distinct) + make_repo_on_branch "$d/wt" fm/abort + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/abort.meta" "window=fm:fm-abort" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_failed fm/abort | sed 's/outcome: failed/outcome: cancelled/; s/status: completed/status: cancelled/') +error: \"cancelled: aborted by user\"" + out=$(run_crew_state "$d" abort) + assert_not_contains "$out" 'state: failed' "a deliberate abort is not a failure: $out" + assert_contains "$out" 'deliberately aborted, not a failure' 'the abort is named as deliberate' + + FM_FAKE_AXI_STATUS="$(run_failed fm/abort | sed 's/outcome: failed/outcome: cancelled/; s/status: completed/status: cancelled/') +error: \"cancelled: superseded by new push\"" + out=$(run_crew_state "$d" abort) + assert_contains "$out" 'superseded by new push' "a superseded cancellation keeps its reason: $out" + assert_not_contains "$out" 'deliberately aborted' 'a superseded run is not called an abort' + pass 'a deliberate abort is distinct from a failure' +} + +test_refresh_task_does_not_inherit_an_earlier_run +test_refresh_task_status_log_answers_over_an_earlier_run +test_run_created_after_task_start_is_attributed +test_relaunch_keeps_the_task_incarnation +test_spawn_generation_bounds_a_task_without_a_start_record +test_unreadable_run_creation_time_is_not_attributed_to_a_bounded_task +test_legacy_route_does_not_inherit_an_earlier_run +test_coarse_ledger_does_not_inherit_an_earlier_run +test_merged_claim_requires_the_recorded_pr_identity +test_run_step_wording_never_presents_green +test_newer_status_line_is_shown_beside_a_finished_run +test_deliberate_abort_is_distinct_from_failure + echo "all fm-crew-state tests passed" diff --git a/tests/fm-trace-context-spawn.test.sh b/tests/fm-trace-context-spawn.test.sh index 23291116e42..00c7d5c9fca 100755 --- a/tests/fm-trace-context-spawn.test.sh +++ b/tests/fm-trace-context-spawn.test.sh @@ -317,6 +317,27 @@ test_disabled_writes_and_injects_neither() { pass "disabled: neither traceparent= in meta nor a TRACEPARENT export is produced" } +# The same default-off fresh spawn records the task's incarnation start beside its +# first spawn generation (bin/fm-spawn.sh header), which bin/fm-crew-state.sh uses +# so a validation run created before the task existed is never the task's result. +test_fresh_spawn_records_the_task_start() { + local rec out status meta before after started gen + rec=$(make_spawn_case task-start) + read_case_record "$rec" + before=$(date +%s) + out=$(run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$CASE_ID" "$PROJ_DIR") + status=$? + after=$(date +%s) + expect_code 0 "$status" "fresh spawn should succeed"$'\n'"$out" + meta="$HOME_DIR/state/$CASE_ID.meta" + started=$(grep '^task_started=' "$meta" | cut -d= -f2-) + gen=$(grep '^spawn_gen=' "$meta" | cut -d= -f2-) + case "$started" in ''|*[!0-9]*) fail "a fresh spawn did not record task_started: '$started'" ;; esac + [ "$started" -ge "$before" ] && [ "$started" -le "$after" ] || fail "task_started $started is not the spawn time" + [ "${gen#s"$started".}" != "$gen" ] || fail "task_started and spawn_gen disagree on a fresh spawn: $started vs $gen" + pass "fresh spawn: the task start is recorded with the first spawn generation" +} + test_failed_delivery_omits_metadata_and_still_launches() { local rec out status meta rec=$(make_spawn_case tc-send-failure) @@ -603,6 +624,7 @@ test_secondmate_carrier_and_snapshot_share_one_decision() { test_enabled_records_and_injects_identical_carrier_before_launch test_disabled_writes_and_injects_neither +test_fresh_spawn_records_the_task_start test_failed_delivery_omits_metadata_and_still_launches test_unsafe_delivery_refuses_to_append_launch test_failed_metadata_append_unsets_carrier_and_still_launches From ed9ae7614b383793b38f1d7e42e45bfa007d781e Mon Sep 17 00:00:00 2001 From: Adam Gardner Date: Thu, 1 Oct 2026 00:38:39 +0800 Subject: [PATCH 2/7] no-mistakes(review): Word run-reported checks pass as required checks not verified --- bin/fm-crew-state.sh | 6 +++--- tests/fm-crew-state.test.sh | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index 1f4f5f567c0..d661d20dd27 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -818,7 +818,7 @@ nm_reclassify_failed_run_as_held_green() { # local pr_url pr_disposition case "$PR_DISPOSITION" in - open) RUN_DETAIL="PR held for merge: run $1 at the ci monitor after its ci log reported CI checks passed; $PR_DISPOSITION_TEXT" ;; + open) RUN_DETAIL="PR held for merge: run $1 at the ci monitor after its ci log reported CI checks passed (required checks not verified); $PR_DISPOSITION_TEXT" ;; merged) RUN_DETAIL="run $1 at the ci monitor after its ci log reported CI checks passed; $PR_DISPOSITION_TEXT" ;; *) return 1 ;; esac @@ -1180,7 +1180,7 @@ if [ "$HAVE_RUN" = 1 ]; then if [ -n "$outcome" ]; then case "$outcome" in passed|passed-with-override|passed-with-skips) RUN_STATE="done"; RUN_DETAIL=$(passed_run_detail "$outcome") ;; - checks-passed) RUN_STATE="done"; RUN_DETAIL="run reports CI checks passed: PR ready for review" ;; + checks-passed) RUN_STATE="done"; RUN_DETAIL="run reports CI checks passed (required checks not verified)" ;; failed) if nm_reclassify_failed_run_as_held_green failed; then :; else RUN_STATE=failed; RUN_DETAIL="run failed" @@ -1233,7 +1233,7 @@ if [ "$HAVE_RUN" = 1 ]; then case "$CI_LOG_STATE" in passed) RUN_STATE="done" - RUN_DETAIL="run ci log reports CI checks passed: PR ready for review (still monitoring for merge/close)" + RUN_DETAIL="run ci log reports CI checks passed (required checks not verified; still monitoring for merge/close)" # The run's own PR URL makes this reading actionable even when # the worker never reported it and no pr= was recorded. ci_pr_url=$(strip_quotes "$(nm_field pr)") diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index 90205231fbd..05fe84b3c10 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -1250,7 +1250,7 @@ EOF local out; out=$(run_crew_state "$d" feat-cirearm) assert_contains "$out" "state: done" "a base-advance re-arm after green keeps the PR green" assert_contains "$out" "source: run-step" "re-armed green monitoring stays run-step sourced" - assert_contains "$out" "CI checks passed: PR ready for review" "re-armed green monitoring reads held for merge" + assert_contains "$out" "CI checks passed (required checks not verified" "re-armed green monitoring reads held for merge" assert_contains "$out" "https://github.com/o/r/pull/2" "the held-for-merge reading names the run's PR" assert_not_contains "$out" "state: working" "a re-arm line must not read as checks not ready" pass "base-advance re-arm after green stays checks green" @@ -1277,7 +1277,7 @@ test_ci_monitoring_green_before_log_tail_stays_green() { local out; out=$(run_crew_state "$d" feat-citail) assert_contains "$out" "state: done" "a green marker older than the log tail still reads green" assert_contains "$out" "source: run-step" "the full-log green reading stays run-step sourced" - assert_contains "$out" "CI checks passed: PR ready for review" "the full-log reading is held for merge" + assert_contains "$out" "CI checks passed (required checks not verified" "the full-log reading is held for merge" assert_contains "$out" "https://github.com/o/r/pull/2" "the full-log reading names the run's PR" assert_not_contains "$out" "state: working" "a truncated ci log must not hide a green PR" pass "a green marker before the ci log tail still surfaces done" @@ -1733,6 +1733,7 @@ test_cancelled_delivery_and_skipped_rebase() { assert_contains "$out" "https://github.com/o/r/pull/203" "$scenario: delivery identity retained" assert_contains "$out" "ci log reported CI checks passed" "$scenario: retain positive CI evidence" assert_contains "$out" "held for merge" "$scenario: delivery awaits merge" + assert_contains "$out" "required checks not verified" "$scenario: the ci log pass is not verified against required checks" fi pass "$scenario: terminal delivery reports only observed evidence" ) || failures=$((failures + 1)) @@ -4121,7 +4122,7 @@ EOF assert_not_contains "$out" 'state: unknown' 'a green PR in merge monitoring is never unknown' assert_contains "$out" 'state: done' 'a green PR in merge monitoring reads done' assert_contains "$out" 'source: run-step' 'the green reading comes from the selected run' - assert_contains "$out" 'CI checks passed: PR ready for review' 'the reading is held for the merge decision' + assert_contains "$out" 'CI checks passed (required checks not verified' 'the reading is held for the merge decision' assert_contains "$out" 'https://github.com/o/r/pull/2' 'the reading names the PR to ask about' pass 'a linked worktree green PR in merge monitoring reads held for merge' } @@ -5903,12 +5904,16 @@ test_run_step_wording_never_presents_green() { assert_contains "$out" 'state: done' "checks-passed still finishes the task: $out" assert_no_green_claim "$out" 'a checks-passed run record is not presented as green' assert_contains "$out" 'run reports CI checks passed' 'the run outcome is stated as what the run reported' + assert_contains "$out" 'required checks not verified' 'a run-reported checks pass says the required checks were not verified' + assert_not_contains "$out" 'ready for review' 'a run-reported checks pass is not presented as ready for review' FM_FAKE_AXI_STATUS="$(run_top_level_ci fm/green)" FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" out=$(run_crew_state "$d" green) - assert_contains "$out" 'state: done' "a passed ci log is ready for review: $out" + assert_contains "$out" 'state: done' "a passed ci log still finishes the task: $out" assert_no_green_claim "$out" 'a ci log reading is not presented as green' + assert_contains "$out" 'run ci log reports CI checks passed (required checks not verified' 'a ci log pass says the required checks were not verified' + assert_not_contains "$out" 'ready for review' 'a ci log pass is not presented as ready for review' FM_FAKE_CI_LOGS= From fd2402754e6d478b0b290553e43df626fb381a5a Mon Sep 17 00:00:00 2001 From: Adam Gardner Date: Thu, 1 Oct 2026 00:49:44 +0800 Subject: [PATCH 3/7] no-mistakes(document): Align ship-landing crew-state line wording with checks-passed output --- .agents/skills/ship-landing/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.agents/skills/ship-landing/SKILL.md b/.agents/skills/ship-landing/SKILL.md index 0815947b8a9..af0a32d0cb7 100644 --- a/.agents/skills/ship-landing/SKILL.md +++ b/.agents/skills/ship-landing/SKILL.md @@ -9,13 +9,13 @@ metadata: # Ship landing For PR-based ship tasks, the ready signal depends on mode: `no-mistakes` reports `done [at=]: PR checks green` after CI is green, while `direct-PR` reports `done [at=]: PR ` after opening the PR, each only for a non-draft PR; a lane that deliberately holds a draft declares a wait instead, and `bin/fm-pr-check.sh` refuses to arm merge monitoring on a draft. -Run `bin/fm-pr-check.sh ` with the URL copied from that ready signal or the resolved ready-for-review `fm-crew-state.sh` line - it records `pr=` and the forge's `pr_head=` when available in the task's meta and arms the watcher's merge poll. +Run `bin/fm-pr-check.sh ` with the URL copied from that ready signal or the resolved done `fm-crew-state.sh` line - it records `pr=` and the forge's `pr_head=` when available in the task's meta and arms the watcher's merge poll. `bin/fm-dod-lib.sh` owns the named-head gate on that ready signal: a ship `done:` whose named head exists only in the worker's disposable copy is not ready (`bin/fm-crew-state.sh` reports blocked, `bin/fm-pr-check.sh` refuses to register, and a secondmate does not publish that done upstream). That blocked reading is the gate working, not a stuck worker, so steer the worker on the commit the refusal names rather than waiting. A direct-PR worker pushes that commit to its PR branch, and a local-only worker commits it on its ship branch. A no-mistakes worker re-validates it with /no-mistakes so the pipeline stays the one publisher; it never pushes from its copy. In no-mistakes mode the earlier `done [at=]: {summary}` is the pipeline handoff and is not gated. -Tell the captain the PR's full `https://...` URL copied from the worker's ready line, the resolved checks-green crew-state line, or the task's `pr=` metadata, a concise outcome summary, and the no-mistakes risk level when applicable. +Tell the captain the PR's full `https://...` URL copied from the worker's ready line, the resolved done crew-state line, or the task's `pr=` metadata, a concise outcome summary, and the no-mistakes risk level when applicable. A captain instruction to merge is explicit authority; `yolo` is the only standing routine merge authority. For any custom `state/.check.sh` you write yourself, keep it an ordinary single-link mode-`0700` file, print one line only when firstmate should wake, print nothing otherwise, finish before `FM_CHECK_TIMEOUT`, then bind its current bytes with `bin/fm-check-register.sh ` before the watcher may execute it. Retire a custom check only through `bin/fm-check-unregister.sh ` (or `bin/fm-teardown.sh` for a spawned task); never hand-compose an `rm` with `$STATE`/`$ID`. From ef782818692c63960764b8c3f1131d80dc167534 Mon Sep 17 00:00:00 2001 From: Adam Gardner Date: Thu, 1 Oct 2026 01:04:35 +0800 Subject: [PATCH 4/7] =?UTF-8?q?no-mistakes(ci):=20Fixed=20ci-3=20(Greptile?= =?UTF-8?q?=20finding:=20runs=20from=20the=20same=20minute=20cross=20the?= =?UTF-8?q?=20task=20boundary),=20as=20the=20user=20instructed.=20The=20in?= =?UTF-8?q?variant:=20a=20row=20in=20the=20coarse=20`no-mistakes=20runs`?= =?UTF-8?q?=20ledger=20answers=20for=20a=20task=20only=20when=20its=20crea?= =?UTF-8?q?tion=20minute=20proves=20the=20row=20was=20created=20after=20th?= =?UTF-8?q?e=20task=20started.=20The=20only=20exception=20is=20the=20head-?= =?UTF-8?q?anchor=20row,=20which=20proves=20code=20identity=20only.=20Minu?= =?UTF-8?q?tes=20are=20compared=20in=20one=20place,=20`fm=5Fnm=5Fruns=5Fst?= =?UTF-8?q?atus=5Ffor=5Fworktree`=20in=20bin/fm-nm-run-lib.sh,=20and=20all?= =?UTF-8?q?=20three=20coarse-ledger=20callers=20in=20bin/fm-crew-state.sh?= =?UTF-8?q?=20go=20through=20it.=20At=20that=20one=20place=20I=20changed?= =?UTF-8?q?=20`[[=20"$day=20$clock"=20<=20"$start=5Fminute"=20]]`=20to=20`?= =?UTF-8?q?!=20[[=20"$start=5Fminute"=20<=20"$day=20$clock"=20]]`.=20Now?= =?UTF-8?q?=20a=20row=20created=20in=20the=20task's=20start=20minute=20pri?= =?UTF-8?q?nts=20nothing.=20The=20anchor-row=20check=20runs=20before=20thi?= =?UTF-8?q?s=20bound,=20so=20it=20stays=20exempt=20as=20before.=20I=20upda?= =?UTF-8?q?ted=20the=20function=20comment=20to=20match.=20Tests:=20test=5F?= =?UTF-8?q?coarse=5Fledger=5Fdoes=5Fnot=5Finherit=5Fan=5Fearlier=5Frun=20n?= =?UTF-8?q?ow=20also=20asserts=20that=20a=20completed=20row=20in=20the=20s?= =?UTF-8?q?tart=20minute=20at=20the=20same=20head=20does=20not=20give=20`s?= =?UTF-8?q?ource:=20run-step`.=20Without=20the=20fix,=20this=20assertion?= =?UTF-8?q?=20failed=20with=20`state:=20done=20=C2=B7=20source:=20run-step?= =?UTF-8?q?=20=C2=B7=20run=20completed`.=20The=20new=20test=20test=5Fcoars?= =?UTF-8?q?e=5Fledger=5Fanchor=5Frow=5Fin=5Fstart=5Fminute=5Fstill=5Fancho?= =?UTF-8?q?rs=20pins=20that=20a=20head-anchor=20row=20in=20the=20start=20m?= =?UTF-8?q?inute=20still=20anchors=20an=20active=20continuation=20that=20f?= =?UTF-8?q?ollows=20it=20(run-step,=20working).=20The=20full=20tests/fm-cr?= =?UTF-8?q?ew-state.test.sh=20suite=20passes.=20Shellcheck=20shows=20no=20?= =?UTF-8?q?new=20findings.=20I=20did=20not=20change=20ci-1=20or=20ci-2,=20?= =?UTF-8?q?because=20the=20user=20declined=20them.=20The=20failing=20Grept?= =?UTF-8?q?ile=20check=20is=20a=20review=20bot,=20and=20its=20log=20was=20?= =?UTF-8?q?not=20available?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/fm-nm-run-lib.sh | 10 +++++----- tests/fm-crew-state.test.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/bin/fm-nm-run-lib.sh b/bin/fm-nm-run-lib.sh index 10ccb6b572e..29554296068 100644 --- a/bin/fm-nm-run-lib.sh +++ b/bin/fm-nm-run-lib.sh @@ -478,10 +478,10 @@ fm_nm_run_is_executing() { # # route (fm_nm_run_is_executing above), which the caller pairs with its own # liveness evidence. # When optional task start epoch $5 is supplied, the answering row must have been -# created in or after that start's minute (fm_nm_run_in_incarnation above owns -# the rule): a newest row from before the task started is an earlier task's -# history, so it prints nothing. The anchor row is exempt; it proves code -# identity only. +# created in a minute after that start's minute (fm_nm_run_in_incarnation above +# owns the rule): a newest row from before the task started is an earlier task's +# history, and a row from the start's own minute cannot prove it is not, so +# either prints nothing. The anchor row is exempt; it proves code identity only. # Read-only: git reads resolve objects in place; custody never changes. fm_nm_runs_status_for_worktree() { # [expected-head] [task-start-epoch] local wt=$1 branch=$2 list=$3 expected_head=${4:-} task_start=${5:-} start_minute='' @@ -532,7 +532,7 @@ fm_nm_runs_status_for_worktree() { # [ex fi break fi - if [ -n "$start_minute" ] && [[ "$day $clock" < "$start_minute" ]]; then + if [ -n "$start_minute" ] && ! [[ "$start_minute" < "$day $clock" ]]; then break fi if [ -n "$expected_head" ]; then diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index 05fe84b3c10..8baef16b0d1 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -5844,12 +5844,38 @@ test_coarse_ledger_does_not_inherit_an_earlier_run() { FM_FAKE_RUNS_LIST=" completed fm/refresh $short $(ledger_minute $((now - 3600))) https://github.com/o/r/pull/1" out=$(run_crew_state "$d" refresh) assert_not_contains "$out" 'source: run-step' "an earlier ledger row must not answer for this task: $out" + FM_FAKE_RUNS_LIST=" completed fm/refresh $short $(ledger_minute "$now") https://github.com/o/r/pull/1" + out=$(run_crew_state "$d" refresh) + assert_not_contains "$out" 'source: run-step' "a row from the task's start minute cannot prove it is this task's: $out" FM_FAKE_RUNS_LIST=" completed fm/refresh $short $(ledger_minute $((now + 120))) https://github.com/o/r/pull/1" out=$(run_crew_state "$d" refresh) assert_contains "$out" 'source: run-step' "a ledger row from this task still answers: $out" pass 'the coarse ledger does not inherit an earlier task run' } +# The start-minute bound applies to the answering row only: the head-anchor row +# proves code identity, so it still anchors from the task's start minute. +test_coarse_ledger_anchor_row_in_start_minute_still_anchors() { + local now d out + now=$(date +%s) + reset_fakes + d=$(new_case coarse-anchor-minute) + make_repo_on_branch "$d/wt" fm/feat-anchor-minute + mint_unfetched_fix_head "$d/wt" >/dev/null + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/anchor-minute.meta" "window=fm:fm-anchor-minute" "worktree=$d/wt" "kind=ship" "task_started=$now" + FM_FAKE_AXI_STATUS="$(run_running fm/other-crew)" + FM_FAKE_RUNS_LIST="$(cat < Date: Tue, 13 Oct 2026 21:51:45 +1100 Subject: [PATCH 5/7] no-mistakes(test): Scratch file deletion denied; incarnation tests pass --- tests/.tmp-crew-subset.test.sh | 5581 ++++++++++++++++++++++++++++++++ 1 file changed, 5581 insertions(+) create mode 100644 tests/.tmp-crew-subset.test.sh diff --git a/tests/.tmp-crew-subset.test.sh b/tests/.tmp-crew-subset.test.sh new file mode 100644 index 00000000000..2dd31b5d930 --- /dev/null +++ b/tests/.tmp-crew-subset.test.sh @@ -0,0 +1,5581 @@ +#!/usr/bin/env bash +# Behavior tests for bin/fm-crew-state.sh - the deterministic crew-current-state +# helper. +# +# The status file (state/.status) is a best-effort append-only EVENT LOG, so +# `tail -1` of it reports the last event, not the current state. fm-crew-state +# reads the AUTHORITATIVE source (a matching no-mistakes run-step, else the +# semantic busy-state contract) and reconciles the possibly-stale log against it. These +# cases pin every branch of that logic, hermetically, over real throwaway git +# repos with a fake `no-mistakes` (run-step source) and a fake `tmux` (pane +# source): +# (a) active run-step is authoritative -> run-step +# (b) needs-decision/blocked log + resumed run = SUPERSEDED -> run-step +# (b2) blocked log claiming the daemon/timeout while the run is fixing with +# fresh activity = superseded BECAUSE THE RUN IS ALIVE; the same claim +# a genuine socket-refusal claim over a stale or terminal run record +# remains blocked, and an ordinary blocked log over a live run keeps the generic +# superseded reading +# (c) genuine parked run + needs-decision log = NOT superseded -> run-step +# (d) terminal run-step (passed/failed) is authoritative -> run-step +# (d2) terminal failed run whose only failure is an orphaned ci monitor +# after checks read green -> done +# (d3) cancelled green deliveries retain done, skipped rebase is allowed; +# other cancellations read unknown without a false fleet contradiction +# (e) cross-branch attribution: this branch's own run found via list lookup +# (e2) multiple runs: creation order preserves newer failures, replacement +# gates retain their run identity, and competing live runs read unknown +# (e3) an older live sibling with an unfetched head cannot hide a newer failure +# (f) no run + semantic busy -> pane +# (g) no run + semantic idle falls to the status-log verb -> status-log +# (h) dead pane: no run -> unknown/none; with a run -> run-step (not the shell) +# (i) kind=scout skips the run lookup -> pane/status-log +# (j) torn-down worktree / missing meta -> unknown/none +# (k) crew_is_provably_working end-to-end over the REAL helper (not a canned +# fake fm-crew-state.sh verdict): cross-branch attribution via the runs +# list -> absorbed; genuinely no run anywhere + idle pane -> surfaced. +# This is the direct regression pair for the 2026-07-02 herdr incident, +# proving the watcher's own absorb-only-when-provably-working predicate +# benefits from the fix in both directions. +# (l) coarse runs-ledger fallback: a terminal failed record with the daemon +# provably down (explicit daemon-status probe fails) reads unknown - +# "unverified", never failed; the same record with the daemon up stays +# failed. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=/dev/null +. "$ROOT/bin/fm-classify-lib.sh" +# shellcheck source=/dev/null +. "$ROOT/bin/fm-pr-lib.sh" + +CREW_STATE="$ROOT/bin/fm-crew-state.sh" +TMP_ROOT=$(fm_test_tmproot fm-crew-state) +fm_git_identity fmtest fmtest@example.invalid + +# A real git repo checked out on , so the helper's branch attribution +# (git symbolic-ref) resolves like it would for a live crew worktree. +# Stamp origin/main at the current HEAD so a later ship done: is not refused +# solely for being a fixture with no remote-tracking refs; tests that need an +# unpreserved named head point those refs at a different commit. +make_repo_on_branch() { # + local dir=$1 branch=$2 + mkdir -p "$dir" + git -C "$dir" init -q + git -C "$dir" commit -q --allow-empty -m init + git -C "$dir" checkout -q -b "$branch" + git -C "$dir" update-ref refs/remotes/origin/main "$(git -C "$dir" rev-parse HEAD)" + # Real worktree HEAD for run head-binding (fixtures read FM_FAKE_RUN_HEAD). + FM_FAKE_RUN_HEAD=$(git -C "$dir" rev-parse HEAD) + export FM_FAKE_RUN_HEAD +} + +# A fakebin with a fake `no-mistakes` (serves the env-driven run output) and a +# fake `tmux` (serves a busy or idle pane). The fake no-mistakes mirrors the real +# command surface the helper uses: `axi` (the identity overview), `axi status`, +# and `axi status --run ` (the +# `axi` surface - no runs-listing subcommand exists under it, verified against +# the real CLI), and the actual top-level run-listing command, `no-mistakes +# runs --limit N`, which is plain text - no run id, no quoting - serving +# FM_FAKE_RUNS_LIST verbatim. +make_fakebin() { # -> echoes fakebin path + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/no-mistakes" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-}" in + axi) + shift + if [ "$#" = 0 ]; then + printf '%s\n' "${FM_FAKE_AXI_HOME:-${FM_FAKE_AXI_STATUS:-}}" + exit "${FM_FAKE_AXI_HOME_ERROR:-0}" + fi + case "${1:-}" in + status) + shift + if [ "${1:-}" = --run ]; then + printf '%s\n' "${FM_FAKE_AXI_STATUS_RUN:-}" + exit "${FM_FAKE_AXI_STATUS_RUN_ERROR:-0}" + else + printf '%s\n' "${FM_FAKE_AXI_STATUS:-}" + exit "${FM_FAKE_AXI_STATUS_ERROR:-0}" + fi ;; + logs) + shift + # The real CLI prints only the last 40 log lines ("lines: 40 of N + # total (tail)", verified against v1.79.0) unless --full asks for the + # whole log, so a marker older than that is invisible to a plain read. + full=0 + for arg in "$@"; do + [ "$arg" = --full ] && full=1 + done + if [ "$full" = 1 ]; then + printf '%s\n' "${FM_FAKE_CI_LOGS:-}" + else + printf '%s\n' "${FM_FAKE_CI_LOGS:-}" | tail -40 + fi ;; + esac + ;; + runs) + printf '%s\n' "${FM_FAKE_RUNS_LIST:-}" ;; + daemon) + # FM_FAKE_DAEMON_DOWN: the explicit down-probe fails, as the real + # `no-mistakes daemon status` does when the daemon is not running. + # FM_FAKE_DAEMON_TIMEOUT: the probe does not answer at all, which is what + # the bounded call reports as 124 when `timeout` kills a slow daemon status. + [ -z "${FM_FAKE_DAEMON_PROBE_LOG:-}" ] || printf 'probe\n' >> "$FM_FAKE_DAEMON_PROBE_LOG" + [ "${FM_FAKE_DAEMON_TIMEOUT:-0}" = 1 ] && exit 124 + [ "${FM_FAKE_DAEMON_DOWN:-0}" = 1 ] && exit 1 + printf '%s\n' 'daemon running (pid 4242)' + exit 0 ;; +esac +exit 0 +SH + cat > "$fb/gh" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-} ${2:-}" in + "api graphql") + [ -z "${FM_FAKE_PR_READ_LOG:-}" ] || printf 'gh\n' >> "$FM_FAKE_PR_READ_LOG" + number=1 + for arg in "$@"; do + case "$arg" in + number=*) number=${arg#number=} ;; + esac + done + case "$number" in *[!0-9]*|'') number=1 ;; esac + state=${FM_FAKE_PR_STATE:-MERGED} + merged=${FM_FAKE_PR_MERGED:-true} + eval "state=\${FM_FAKE_PR_${number}_STATE:-\$state}" + eval "merged=\${FM_FAKE_PR_${number}_MERGED:-\$merged}" + [ "${FM_FAKE_PR_READ_FAIL:-0}" = 1 ] && exit 1 + printf 'state=%s\nmerged=%s\n' "$state" "$merged" + exit 0 ;; +esac +exit 1 +SH + cat > "$fb/gh-axi" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-} ${2:-}" in + "pr view") + [ -z "${FM_FAKE_PR_READ_LOG:-}" ] || printf 'gh-axi\n' >> "$FM_FAKE_PR_READ_LOG" + [ "${FM_FAKE_PR_READ_FAIL:-0}" = 1 ] && exit 1 + printf 'pull_request:\n number: %s\n state: %s\n' "${3:-1}" "${FM_FAKE_PR_STATE_AXI:-merged}" + exit 0 ;; +esac +exit 1 +SH + cat > "$fb/glab" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-} ${2:-}" in + "mr view") + [ -z "${FM_FAKE_GLAB_READ_LOG:-}" ] || printf '%s|%s\n' "${GITLAB_HOST:-}" "$*" >> "$FM_FAKE_GLAB_READ_LOG" + [ "${FM_FAKE_GLAB_READ_FAIL:-0}" = 1 ] && exit 1 + printf '{"state":"%s"}\n' "${FM_FAKE_GLAB_STATE:-merged}" + exit 0 ;; +esac +exit 1 +SH + # tea answers its login list with one login for git.example.org and a pull + # request read through "tea api" with the configured state; a read failure + # prints the error body with exit 0, as the real tea api does. + cat > "$fb/tea" <<'SH' +#!/usr/bin/env bash +set -u +if [ "${1:-} ${2:-}" = "login list" ]; then + printf '[\n {\n "name": "fj",\n "url": "https://git.example.org"\n }\n]\n' + exit 0 +fi +[ "${1:-}" = api ] || exit 1 +[ -z "${FM_FAKE_TEA_READ_LOG:-}" ] || printf '%s\n' "$*" >> "$FM_FAKE_TEA_READ_LOG" +if [ "${FM_FAKE_TEA_READ_FAIL:-0}" = 1 ]; then + printf '{"message":"not found"}\n' + exit 0 +fi +printf '{"state":"%s","merged":%s,"head":{"sha":"abc"}}\n' \ + "${FM_FAKE_TEA_STATE:-closed}" "${FM_FAKE_TEA_MERGED:-true}" +SH + cat > "$fb/gerrit-axi" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-}" in + show) + [ -z "${FM_FAKE_GERRIT_READ_LOG:-}" ] || printf '%s\n' "$*" >> "$FM_FAKE_GERRIT_READ_LOG" + [ "${FM_FAKE_GERRIT_READ_FAIL:-0}" = 1 ] && exit 1 + # url defaults to null, the shape a server whose gerrit.canonicalWebUrl is + # unset returns, so every case here reads a record that carries no URL. + printf '{"ok":true,"op":"show","changes":[{"change":%s,"subject":"fixture change","status":"%s","url":%s}]}\n' \ + "${FM_FAKE_GERRIT_CHANGE:-${2:-0}}" "${FM_FAKE_GERRIT_STATUS:-MERGED}" \ + "${FM_FAKE_GERRIT_URL_JSON:-null}" + exit 0 ;; +esac +exit 1 +SH + cat > "$fb/tmux" <<'SH' +#!/usr/bin/env bash +set -u +# FM_FAKE_TMUX_MISSING: the window is authoritatively gone - every addressed +# call fails, but the session inventory still answers successfully and simply +# omits the window, which is what proves absence. +# FM_FAKE_TMUX_UNREADABLE: tmux itself cannot answer - it fails to execute (a +# trimmed PATH) or errors non-definitively - so even the inventory fails, with +# a message that is NOT one of the definitive no-session/no-server/no-socket +# responses that fm_backend_tmux_agent_state owns as death. +[ "${FM_FAKE_TMUX_UNREADABLE:-0}" = 1 ] && { printf 'no current client\n' >&2; exit 1; } +case "${1:-}" in + list-windows) + # A successful but empty inventory: it omits the crew's window, so absence + # is proved by the answer rather than by an addressed call failing. Only + # reached once display-message has already failed. + ;; + display-message) + [ "${FM_FAKE_TMUX_MISSING:-0}" = 1 ] && exit 1 + printf '%%1\n' ;; + capture-pane) + [ "${FM_FAKE_TMUX_MISSING:-0}" = 1 ] && exit 1 + if [ "${FM_FAKE_BUSY:-0}" = 1 ]; then printf 'work in progress\n%s\n' "${FM_FAKE_BUSY_TEXT:-esc to interrupt}" + else printf 'all quiet\n> \n'; fi ;; +esac +exit 0 +SH + cat > "$fb/herdr" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-}" in + status) + [ "${2:-}" = --json ] && { + printf '{"client":{"version":"0.7.1","protocol":14},"server":{"running":true}}\n' + exit 0 + } ;; + server) + exit 0 ;; + pane) + case "${2:-}" in + read) + [ "${FM_FAKE_HERDR_MISSING:-0}" = 1 ] && exit 1 + [ "${FM_FAKE_HERDR_READ_FAIL:-0}" = 1 ] && exit 1 + if [ "${FM_FAKE_HERDR_BUSY:-0}" = 1 ]; then printf 'work in progress\nesc to interrupt\n' + else printf 'all quiet\n> \n'; fi + exit 0 ;; + get) + if [ "${FM_FAKE_HERDR_MISSING:-0}" = 1 ]; then + printf '{"error":{"code":"pane_not_found","message":"no such pane"}}\n' + exit 1 + fi + printf '{"result":{"pane":{"pane_id":"%s"}}}\n' "${3:-}" + exit 0 ;; + process-info) + # The process-level view a registration is verified against (#4115): + # `agent` puts a live claude in the foreground, `shell` a bare zsh whose + # pid is the test script itself (a real, long-lived process with no + # harness descendant, so the adapter's real process-table walk finds + # it), and anything else answers nothing (unreadable). + pane=""; args=("$@"); for ((i=0; i<${#args[@]}; i++)); do [ "${args[$i]}" = --pane ] && pane=${args[$((i+1))]:-}; done + case "${FM_FAKE_HERDR_PROCESS:-agent}" in + agent) printf '{"result":{"type":"pane_process_info","process_info":{"pane_id":"%s","shell_pid":%s,"foreground_process_group_id":424242,"foreground_processes":[{"pid":424242,"name":"claude","argv0":"claude"}]}}}\n' "$pane" "${FM_FAKE_HERDR_SHELL_PID:-$PPID}" ;; + shell) printf '{"result":{"type":"pane_process_info","process_info":{"pane_id":"%s","shell_pid":%s,"foreground_process_group_id":%s,"foreground_processes":[{"pid":%s,"name":"zsh","argv0":"zsh","argv":["-zsh"]}]}}}\n' "$pane" "${FM_FAKE_HERDR_SHELL_PID:-$PPID}" "${FM_FAKE_HERDR_SHELL_PID:-$PPID}" "${FM_FAKE_HERDR_SHELL_PID:-$PPID}" ;; + esac + exit 0 ;; + esac ;; + agent) + case "${2:-}" in + get) + if [ "${FM_FAKE_HERDR_HUSK:-0}" = 1 ]; then + printf '{"error":{"code":"agent_not_found","message":"no agent in pane"}}\n' + exit 0 + fi + [ -n "${FM_FAKE_HERDR_AGENT_STATUS:-}" ] || exit 1 + printf '{"result":{"agent":{"agent_status":"%s"}}}\n' "$FM_FAKE_HERDR_AGENT_STATUS" + exit 0 ;; + esac ;; +esac +exit 0 +SH + chmod +x "$fb/no-mistakes" "$fb/gh" "$fb/gh-axi" "$fb/glab" "$fb/tea" "$fb/gerrit-axi" "$fb/tmux" "$fb/herdr" + printf '%s\n' "$fb" +} + +make_no_timeout_toolbin() { # -> echoes toolbin path + local dir=$1 tb="$1/notimeoutbin" tool real + mkdir -p "$tb" + for tool in bash git grep sed head cut tail dirname perl; do + real=$(command -v "$tool" || true) + [ -n "$real" ] || fail "missing tool for no-timeout path: $tool" + ln -s "$real" "$tb/$tool" + done + printf '%s\n' "$tb" +} + +# Run the helper for one case dir. FM_FAKE_* env (run output, busy flag) are read +# from the caller's environment by the fakes above. +run_crew_state() { # + PATH="$1/fakebin:$PATH" FM_STATE_OVERRIDE="$1/state" "$CREW_STATE" "$2" +} + +new_case() { # -> echoes case dir with an empty state/ + local d="$TMP_ROOT/$1" + mkdir -p "$d/state" + printf '%s\n' "$d" +} + +arm_idle_record() { # + local state=$1 id=$2 gen + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$state" "$id") + "$ROOT/bin/fm-busy-event.sh" apply "$state" "$id" idle --gen "$gen" \ + --source claude-hook --event stop +} + +# Clear the fake-driver vars and (re-)mark them exported, so the per-test plain +# assignments below stay exported into the fakes without an `export VAR=$(...)` +# command-substitution assignment (SC2155). +reset_fakes() { + NM_HOME="$TMP_ROOT/no-mistakes-unused" + export NM_HOME + FM_FAKE_AXI_STATUS="" + FM_FAKE_AXI_STATUS_ERROR=0 + FM_FAKE_AXI_HOME="" + FM_FAKE_AXI_HOME_ERROR=0 + FM_FAKE_AXI_STATUS_RUN_ERROR=0 + FM_FAKE_AXI_STATUS_RUN="" + FM_FAKE_RUNS_LIST="" + FM_FAKE_BUSY=0 + FM_FAKE_BUSY_TEXT= + FM_FAKE_TMUX_MISSING=0 + FM_FAKE_TMUX_UNREADABLE=0 + FM_FAKE_HERDR_BUSY=0 + FM_FAKE_HERDR_MISSING=0 + FM_FAKE_HERDR_READ_FAIL=0 + FM_FAKE_HERDR_HUSK=0 + FM_FAKE_HERDR_AGENT_STATUS="" + FM_FAKE_HERDR_PROCESS=agent + FM_FAKE_HERDR_SHELL_PID=$$ + FM_FAKE_CI_LOGS="" + FM_FAKE_DAEMON_DOWN=0 + FM_FAKE_DAEMON_TIMEOUT=0 + FM_FAKE_DAEMON_PROBE_LOG= + FM_FAKE_PR_STATE=MERGED + FM_FAKE_PR_MERGED=true + FM_FAKE_PR_READ_FAIL=0 + FM_FAKE_PR_READ_LOG= + FM_FAKE_PR_STATE_AXI=merged + FM_FAKE_GLAB_STATE=merged + FM_FAKE_GLAB_READ_FAIL=0 + FM_FAKE_GLAB_READ_LOG= + FM_FAKE_TEA_STATE=closed + FM_FAKE_TEA_MERGED=true + FM_FAKE_TEA_READ_FAIL=0 + FM_FAKE_TEA_READ_LOG= + FM_FAKE_GERRIT_STATUS=MERGED + FM_FAKE_GERRIT_CHANGE= + FM_FAKE_GERRIT_URL_JSON= + FM_FAKE_GERRIT_READ_FAIL=0 + FM_FAKE_GERRIT_READ_LOG= + unset FM_FAKE_PR_47_STATE FM_FAKE_PR_47_MERGED FM_FAKE_PR_48_STATE FM_FAKE_PR_48_MERGED + export FM_FAKE_AXI_STATUS FM_FAKE_AXI_STATUS_RUN FM_FAKE_RUNS_LIST FM_FAKE_BUSY FM_FAKE_BUSY_TEXT FM_FAKE_TMUX_MISSING FM_FAKE_TMUX_UNREADABLE + export FM_FAKE_HERDR_BUSY FM_FAKE_HERDR_MISSING FM_FAKE_HERDR_READ_FAIL FM_FAKE_HERDR_HUSK FM_FAKE_HERDR_AGENT_STATUS FM_FAKE_HERDR_PROCESS FM_FAKE_HERDR_SHELL_PID FM_FAKE_CI_LOGS + export FM_FAKE_DAEMON_DOWN FM_FAKE_DAEMON_TIMEOUT FM_FAKE_DAEMON_PROBE_LOG FM_FAKE_AXI_HOME + export FM_FAKE_AXI_HOME_ERROR FM_FAKE_AXI_STATUS_RUN_ERROR FM_FAKE_AXI_STATUS_ERROR + export FM_FAKE_PR_STATE FM_FAKE_PR_MERGED FM_FAKE_PR_READ_FAIL FM_FAKE_PR_READ_LOG FM_FAKE_PR_STATE_AXI + export FM_FAKE_GLAB_STATE FM_FAKE_GLAB_READ_FAIL FM_FAKE_GLAB_READ_LOG + export FM_FAKE_TEA_STATE FM_FAKE_TEA_MERGED FM_FAKE_TEA_READ_FAIL FM_FAKE_TEA_READ_LOG + export FM_FAKE_GERRIT_STATUS FM_FAKE_GERRIT_CHANGE FM_FAKE_GERRIT_URL_JSON + export FM_FAKE_GERRIT_READ_FAIL FM_FAKE_GERRIT_READ_LOG + export FM_FAKE_PR_47_STATE FM_FAKE_PR_47_MERGED FM_FAKE_PR_48_STATE FM_FAKE_PR_48_MERGED +} + +seed_retired_pr_receipt() { # + local state=$1 id=$2 url=$3 template provider host path number + template="$ROOT/bin/fm-pr-poll.sh" + fm_pr_url_parse "$url" || fail "retirement fixture URL was invalid" + provider=$FM_PR_PROVIDER + host=$FM_PR_HOST + path=$FM_PR_PATH + number=$FM_PR_NUMBER + fm_pr_poll_prepare "$state" "$id" "$provider" "$url" "$host" "$path" "$number" "$template" \ + || fail "could not prepare retirement fixture" + fm_pr_poll_publish_prepared || fail "could not publish retirement fixture" + fm_pr_poll_snapshot_capture "$state" "$id" "$template" || fail "could not snapshot retirement fixture" + fm_pr_poll_retirement_publish "$state" "$id" "$template" merged \ + || fail "could not publish retirement receipt" +} + +# --- run-object fixtures (TOON, as `no-mistakes axi status` emits) ----------- + +run_running() { # + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat < + cat </dev/null + fm_write_meta "$d/state/feat-a.meta" "window=fm:fm-feat-a" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_running fm/feat-a)" + local out; out=$(run_crew_state "$d" feat-a) + assert_contains "$out" "state: working" "active run -> working" + assert_contains "$out" "source: run-step" "active run -> run-step source" + assert_contains "$out" "validating (running)" "active run reports the step" + pass "active run-step is authoritative" +} + +# (b) needs-decision log + a resumed (running/fixing) run = SUPERSEDED +test_stale_needs_decision_superseded() { + reset_fakes + local d; d=$(new_case superseded) + make_repo_on_branch "$d/wt" fm/feat-b + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-b.meta" "window=fm:fm-feat-b" "worktree=$d/wt" "kind=ship" + printf 'working: started\nneeds-decision: pick A or B\n' > "$d/state/feat-b.status" + FM_FAKE_AXI_STATUS="$(run_fixing fm/feat-b)" + local out; out=$(run_crew_state "$d" feat-b) + assert_contains "$out" "state: working" "resumed run -> working despite needs-decision log" + assert_contains "$out" "source: run-step" "resumed run -> run-step source" + assert_contains "$out" "superseded" "stale needs-decision log flagged superseded" + pass "stale needs-decision over active run is superseded" +} + +# blocked log + a resumed run is also superseded +test_stale_blocked_superseded() { + reset_fakes + local d; d=$(new_case superseded-blocked) + make_repo_on_branch "$d/wt" fm/feat-bb + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-bb.meta" "window=fm:fm-feat-bb" "worktree=$d/wt" "kind=ship" + printf 'blocked: waiting on review answer\n' > "$d/state/feat-bb.status" + FM_FAKE_AXI_STATUS="$(run_running fm/feat-bb)" + local out; out=$(run_crew_state "$d" feat-bb) + assert_contains "$out" "state: working" "resumed run -> working despite blocked log" + assert_contains "$out" "superseded" "stale blocked log flagged superseded" + pass "stale blocked over active run is superseded" +} + +# A crew whose drive call timed out or was killed by its harness command limit +# routinely blocks claiming the pipeline died. The daemon accepts `respond` +# immediately and runs the fix round in the background, so such a claim over a +# run that is fixing WITH fresh activity is contradicted by the run itself: the +# supervisor answer is to steer a reattach, not to escalate a dead pipeline. +test_daemon_claim_over_live_run_reads_run_alive() { + reset_fakes + local d; d=$(new_case daemon-claim-live) + make_repo_on_branch "$d/wt" fm/feat-dl + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dl.meta" "window=fm:fm-feat-dl" "worktree=$d/wt" "kind=ship" + printf 'blocked: no-mistakes daemon unreachable, drive run: read response: i/o timeout\n' \ + > "$d/state/feat-dl.status" + FM_FAKE_AXI_STATUS="$(run_fixing_active_recent fm/feat-dl)" + local out; out=$(run_crew_state "$d" feat-dl) + assert_contains "$out" "state: working" "live run beats the crew's death claim" + assert_contains "$out" "source: run-step" "live run -> run-step source" + assert_contains "$out" "run alive" "daemon claim over a live run is named as run alive" + assert_contains "$out" "reattach" "the reading names the reattach steer" + assert_not_contains "$out" "superseded by active run" \ + "the daemon claim gets the sharper reading, not the generic one" + pass "daemon/timeout blocked claim over a live fixing run reads as run alive" +} + +# A genuine refused socket outranks the persisted fixing record, which can +# survive after the daemon exits. +test_socket_refusal_over_stale_fixing_run_reports_blocked() { + reset_fakes + local d; d=$(new_case daemon-socket-refused) + make_repo_on_branch "$d/wt" fm/feat-dq + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dq.meta" "window=fm:fm-feat-dq" "worktree=$d/wt" "kind=ship" + printf 'blocked: no-mistakes daemon socket refused connections\n' \ + > "$d/state/feat-dq.status" + FM_FAKE_AXI_STATUS="$(run_fixing_active_quiet fm/feat-dq)" + local out; out=$(run_crew_state "$d" feat-dq) + assert_contains "$out" "state: blocked" "socket refusal outranks a stale fixing record" + assert_contains "$out" "source: status-log" "socket refusal remains status-log evidence" + assert_contains "$out" "socket refused connections" "socket failure is preserved" + assert_not_contains "$out" "run alive" "stale fixing record is not reported alive" + + # Exercise the exact alternate wordings emitted by the generated crew rule. + printf 'blocked: no-mistakes daemon socket refuses connections\n' \ + > "$d/state/feat-dq.status" + out=$(run_crew_state "$d" feat-dq) + assert_contains "$out" "state: blocked" "socket-refuses wording outranks a stale fixing record" + assert_not_contains "$out" "state: working" "socket-refuses wording cannot be suppressed by a stale active record" + + printf 'blocked: no-mistakes daemon socket is missing\n' \ + > "$d/state/feat-dq.status" + out=$(run_crew_state "$d" feat-dq) + assert_contains "$out" "state: blocked" "missing socket outranks a stale fixing record" + assert_contains "$out" "source: status-log" "missing socket remains status-log evidence" + assert_not_contains "$out" "state: working" "missing socket cannot be suppressed by a stale active record" + pass "socket refusal or missing socket over a stale fixing run reports blocked" +} + +# A terminal run record can be the final persisted state after the daemon exits. +# Positive socket-failure evidence must not be discarded merely because that +# attributed record no longer has an active status. +test_socket_refusal_over_terminal_run_reports_blocked() { + reset_fakes + local d; d=$(new_case daemon-socket-refused-terminal) + make_repo_on_branch "$d/wt" fm/feat-dqt + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dqt.meta" "window=fm:fm-feat-dqt" "worktree=$d/wt" "kind=ship" + printf 'blocked: no-mistakes daemon socket refused connections\n' \ + > "$d/state/feat-dqt.status" + FM_FAKE_AXI_STATUS="$(run_failed fm/feat-dqt)" + local out; out=$(run_crew_state "$d" feat-dqt) + assert_contains "$out" "state: blocked" "socket refusal outranks a terminal run record" + assert_contains "$out" "source: status-log" "terminal run cannot suppress socket-failure evidence" + assert_not_contains "$out" "state: failed" "terminal run state is not emitted over socket-failure evidence" + pass "socket refusal over a terminal attributed run reports blocked" +} + +# The socket-down override is evidence about the log's CURRENT tip, not a latch: +# once the crew appends any later event the attributed run is the better witness. +test_socket_refusal_override_expires_when_the_crew_moves_on() { + reset_fakes + local d out + d=$(new_case daemon-socket-refused-superseded) + make_repo_on_branch "$d/wt" fm/feat-ds + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ds.meta" "window=fm:fm-feat-ds" "worktree=$d/wt" "kind=ship" + printf 'blocked: no-mistakes daemon socket is missing\n' > "$d/state/feat-ds.status" + FM_FAKE_AXI_STATUS="$(run_fixing_active_recent fm/feat-ds)" + out=$(run_crew_state "$d" feat-ds) + assert_contains "$out" "state: blocked" "socket-down as the latest event still outranks a live run" + assert_contains "$out" "source: status-log" "the override remains status-log evidence" + assert_contains "$out" "daemon socket down despite attributed run record" "the override names its reason" + + printf 'working: reattached and continuing\n' >> "$d/state/feat-ds.status" + out=$(run_crew_state "$d" feat-ds) + assert_contains "$out" "state: working" "a later working event hands the reading back to the live run" + assert_contains "$out" "source: run-step" "the superseded override no longer emits status-log state" + assert_not_contains "$out" "daemon socket down despite attributed run record" \ + "a stale socket-down blocker cannot override a live run forever" + + # The later event does not have to be one the decision fold accepts. A blocked + # line on a reserved key whose note does not speak that namespace is folded as + # ordinary status, so the socket-down blocker stays the reconciled declaration + # while the tip of the log has moved on; the override reads the tip, not the + # declaration, so the stale daemon evidence stays retired. + printf 'blocked: no-mistakes daemon socket is missing\nblocked [key=pending-reply-t3]: still waiting on the answer\n' \ + > "$d/state/feat-ds.status" + out=$(run_crew_state "$d" feat-ds) + assert_contains "$out" "state: working" "a later unfolded blocked event also hands the reading back to the run" + assert_contains "$out" "source: run-step" "the retired override emits no status-log state" + assert_not_contains "$out" "daemon socket down despite attributed run record" \ + "an unrelated later blocker cannot republish stale socket-down evidence" + pass "socket-down evidence outranks a live run only while it is the log's latest event" +} + +# And the claim half: an ordinary blocked line over the same live run keeps the +# generic reading, so the sharper one cannot fire on every superseded block. +test_ordinary_blocked_over_live_run_keeps_plain_superseded() { + reset_fakes + local d; d=$(new_case ordinary-blocked-live) + make_repo_on_branch "$d/wt" fm/feat-ob + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ob.meta" "window=fm:fm-feat-ob" "worktree=$d/wt" "kind=ship" + printf 'blocked: database upload failed with broken pipe\n' > "$d/state/feat-ob.status" + FM_FAKE_AXI_STATUS="$(run_fixing_active_recent fm/feat-ob)" + local out; out=$(run_crew_state "$d" feat-ob) + assert_contains "$out" "state: working" "ordinary blocked log over an active run -> working" + assert_contains "$out" "superseded by active run" "ordinary blocked keeps the generic reading" + assert_not_contains "$out" "run alive" "broken pipe is not a pipeline-unreachable alias" + pass "broken-pipe blocker over a live run keeps the plain superseded reading" +} + +# The genuine daemon-down case still reaches the supervisor as blocked: the +# socket refused connections and no run is executing anywhere. +test_genuine_daemon_down_reports_blocked() { + reset_fakes + local d; d=$(new_case daemon-down) + make_repo_on_branch "$d/wt" fm/feat-dd + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dd.meta" "window=fm:fm-feat-dd" "worktree=$d/wt" "kind=ship" "harness=claude" + printf 'blocked: no-mistakes daemon socket refused connections\n' > "$d/state/feat-dd.status" + FM_FAKE_AXI_STATUS="" + FM_FAKE_BUSY=0 + arm_idle_record "$d/state" feat-dd + local out; out=$(run_crew_state "$d" feat-dd) + assert_contains "$out" "state: blocked" "a genuine daemon-down claim with no run stays blocked" + assert_contains "$out" "source: status-log" "no run -> status-log source" + assert_not_contains "$out" "run alive" "nothing is alive to report" + pass "genuine daemon-down blocked line still reports blocked" +} + +# (c) genuine parked run + needs-decision log AGREE -> parked, NOT superseded +test_genuine_parked_not_superseded() { + reset_fakes + local d; d=$(new_case parked) + make_repo_on_branch "$d/wt" fm/feat-c + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-c.meta" "window=fm:fm-feat-c" "worktree=$d/wt" "kind=ship" + printf 'needs-decision: review gate\n' > "$d/state/feat-c.status" + FM_FAKE_AXI_STATUS="$(run_parked fm/feat-c)" + local out; out=$(run_crew_state "$d" feat-c) + assert_contains "$out" "state: parked" "genuine parked run -> parked" + assert_contains "$out" "source: run-step" "parked -> run-step source" + assert_contains "$out" "2 finding(s)" "parked includes gate finding count" + assert_contains "$out" "ask-user" "parked surfaces ask-user finding" + assert_not_contains "$out" "superseded" "agreeing parked+needs-decision not flagged stale" + pass "genuine parked run is not flagged superseded" +} + +# Which HUMAN owes a parked gate its answer is the distinction the watcher's +# wedge deferral rests on, so the component that carries it must come from the +# findings table's `action` column and from nothing else. Both directions, plus +# the counterexample a text match would have accepted. +test_parked_human_decision_comes_from_the_action_column() { + local d out + reset_fakes + d=$(new_case parked-ask-user-action-column) + make_repo_on_branch "$d/wt" fm/feat-au + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-au.meta" "window=fm:fm-feat-au" "worktree=$d/wt" "kind=ship" + printf 'needs-decision: review gate\n' > "$d/state/feat-au.status" + FM_FAKE_AXI_STATUS="$(run_parked fm/feat-au)" + out=$(run_crew_state "$d" feat-au) + assert_contains "$out" "state: parked" "an ask-user row still reports parked" + assert_contains "$out" " · ask-user: authority decision" \ + "an action column of ask-user mints the human-decision component" + + # The counterexample. Nothing here is owed a human: every action column is + # auto-fix. A description enumerating the action values, and a branch named + # after the same token, must not mint the component - a crewmate that goes + # quiet before answering its own gate has to keep the wedge ladder. + reset_fakes + d=$(new_case parked-ask-user-prose-only) + make_repo_on_branch "$d/wt" fm/ask-user-authority-fix + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ap.meta" "window=fm:fm-feat-ap" "worktree=$d/wt" "kind=ship" + printf 'working: validation under way\n' > "$d/state/feat-ap.status" + FM_FAKE_AXI_STATUS="$(run_parked_crewmate_gate_with_ask_user_prose fm/ask-user-authority-fix)" + # Guard the counterexample against going vacuous: the payload this gate is read + # from must really contain the token in a position a substring or unanchored + # regex would accept, or the case below proves nothing. + assert_contains "$FM_FAKE_AXI_STATUS" ", ask-user," \ + "the counterexample payload must carry the token where a naive match accepts it" + assert_contains "$FM_FAKE_AXI_STATUS" "branch: fm/ask-user-authority-fix" \ + "the counterexample payload must also carry the token in its branch name" + out=$(run_crew_state "$d" feat-ap) + assert_contains "$out" "state: parked" "a crewmate-owed gate still reports parked" + assert_not_contains "$out" " · ask-user: authority decision" \ + "free text and a branch name must not mint the human-decision component" + + # Column order is read from the header, not assumed. + reset_fakes + d=$(new_case parked-ask-user-reordered) + make_repo_on_branch "$d/wt" fm/feat-ar + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ar.meta" "window=fm:fm-feat-ar" "worktree=$d/wt" "kind=ship" + printf 'needs-decision: review gate\n' > "$d/state/feat-ar.status" + FM_FAKE_AXI_STATUS="$(run_parked_reordered_columns fm/feat-ar)" + out=$(run_crew_state "$d" feat-ar) + assert_contains "$out" " · ask-user: authority decision" \ + "the action column is located by header index, not by fixed position" + + # A header index alone is not enough, because the row is split on raw commas. + # With `description` ahead of `action` the comma walk lands inside free text, + # so a gate whose every action is auto-fix would mint the component. The table + # is not provably safe to walk, so the derivation must refuse and the crewmate + # must keep the wedge ladder. + reset_fakes + d=$(new_case parked-free-text-before-action) + make_repo_on_branch "$d/wt" fm/feat-af + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-af.meta" "window=fm:fm-feat-af" "worktree=$d/wt" "kind=ship" + printf 'needs-decision: review gate\n' > "$d/state/feat-af.status" + FM_FAKE_AXI_STATUS="$(run_parked_free_text_before_action fm/feat-af)" + # Non-vacuity: the payload must really carry the token at the comma offset the + # `action` index resolves to, or the case below proves nothing. + assert_contains "$FM_FAKE_AXI_STATUS" "findings[1]{id,severity,file,line,description,action}:" \ + "the fixture must really place free text before the action column" + assert_contains "$FM_FAKE_AXI_STATUS" ", ask-user," \ + "the fixture description must carry the token where the comma walk would accept it" + out=$(run_crew_state "$d" feat-af) + assert_contains "$out" "state: parked" "an unsafe findings header still reports parked" + assert_not_contains "$out" " · ask-user: authority decision" \ + "a findings header that puts free text before action must not mint the human-decision component" + + # The header and the rows must come from the SAME block. An earlier unbraced + # `findings[N]:` block ahead of the live gate's braced table would otherwise + # supply the rows while the braced header supplies the count and the `action` + # index, so the walk reads the wrong rows at the right index. Here that earlier + # block carries ask-user at exactly that offset while the live gate's only row + # is auto-fix: the crewmate owes this gate its own answer and must keep the + # wedge ladder. + reset_fakes + d=$(new_case parked-unbraced-findings-precursor) + make_repo_on_branch "$d/wt" fm/feat-ub + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ub.meta" "window=fm:fm-feat-ub" "worktree=$d/wt" "kind=ship" + printf 'needs-decision: review gate\n' > "$d/state/feat-ub.status" + FM_FAKE_AXI_STATUS="$(run_parked_unbraced_findings_precursor fm/feat-ub)" + # Non-vacuity: the payload must really carry an unbraced findings block ahead + # of the braced one, with the token at the offset the walk would land on. + assert_contains "$FM_FAKE_AXI_STATUS" "findings[2]:" \ + "the fixture must really place an unbraced findings block before the gate's table" + assert_contains "$FM_FAKE_AXI_STATUS" ",ask-user," \ + "the earlier block must carry the token where the wrong-block walk would accept it" + out=$(run_crew_state "$d" feat-ub) + assert_contains "$out" "state: parked" "an unbraced findings precursor still reports parked" + assert_not_contains "$out" " · ask-user: authority decision" \ + "rows from an earlier unbraced findings block must not mint the human-decision component" + pass "the parked human-decision component is derived from the findings table's action column" +} + +test_scalar_gate_parked_not_superseded() { + reset_fakes + local d; d=$(new_case parked-scalar-gate) + make_repo_on_branch "$d/wt" fm/feat-cs + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cs.meta" "window=fm:fm-feat-cs" "worktree=$d/wt" "kind=ship" + printf 'needs-decision: review gate\n' > "$d/state/feat-cs.status" + FM_FAKE_AXI_STATUS="$(run_parked_scalar_gate_running fm/feat-cs)" + local out; out=$(run_crew_state "$d" feat-cs) + assert_contains "$out" "state: parked" "scalar gate wait -> parked" + assert_contains "$out" "source: run-step" "scalar gate wait -> run-step source" + assert_contains "$out" "parked at review" "scalar gate wait names the gate" + assert_contains "$out" "1 finding(s)" "scalar gate wait includes finding count" + assert_not_contains "$out" "superseded" "scalar gate wait not flagged stale" + pass "scalar gate parked run is not flagged superseded" +} + +test_gate_block_parked_not_superseded() { + reset_fakes + local d; d=$(new_case parked-gate-block) + make_repo_on_branch "$d/wt" fm/feat-cb + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cb.meta" "window=fm:fm-feat-cb" "worktree=$d/wt" "kind=ship" + printf 'needs-decision: review gate\n' > "$d/state/feat-cb.status" + FM_FAKE_AXI_STATUS="$(run_parked_in_gate_block fm/feat-cb)" + local out; out=$(run_crew_state "$d" feat-cb) + assert_contains "$out" "state: parked" "gate block wait -> parked" + assert_contains "$out" "source: run-step" "gate block wait -> run-step source" + assert_contains "$out" "parked at review" "gate block wait names the gate" + assert_contains "$out" "1 finding(s)" "gate block wait includes finding count" + assert_not_contains "$out" "superseded" "gate block wait not flagged stale" + pass "gate block parked run is not flagged superseded" +} + +test_ci_ready_done_log_beats_monitoring_run() { + reset_fakes + local d; d=$(new_case ci-ready) + make_repo_on_branch "$d/wt" fm/feat-ci + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ci.meta" "window=fm:fm-feat-ci" "worktree=$d/wt" "kind=ship" + printf 'done: PR https://github.com/o/r/pull/2 checks green\n' > "$d/state/feat-ci.status" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-ci)" + local out; out=$(run_crew_state "$d" feat-ci) + assert_contains "$out" "state: done" "ci-ready status log -> done" + assert_contains "$out" "source: status-log" "ci-ready state comes from the status log" + assert_contains "$out" "checks green" "ci-ready detail preserves the report" + assert_not_contains "$out" "state: working" "ci-ready is not hidden by monitoring run" + pass "ci-ready status log beats monitoring run" +} + +# Regression for the PR #252 incident: the crew's own status log never got a +# "done: ... checks green" line (log_reports_ci_ready above does not apply), +# but the ci step's log shows CI is actually green and only waiting on +# merge/close. fm-crew-state must surface this as done, not "validating +# (running)", so a green PR is never silently absorbed as still-in-progress. +test_ci_monitoring_checks_green_surfaces_done() { + reset_fakes + local d; d=$(new_case ci-green) + make_repo_on_branch "$d/wt" fm/feat-cigreen + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cigreen.meta" "window=fm:fm-feat-cigreen" "worktree=$d/wt" "kind=ship" + # No status-log line at all: the crew never reported its own checks-green line. + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cigreen)" + FM_FAKE_CI_LOGS=$(cat <<'EOF' +CI checks running, waiting for results... +all CI checks passed - still monitoring until merged or closed +EOF +) + local out; out=$(run_crew_state "$d" feat-cigreen) + assert_contains "$out" "state: done" "green ci-monitor run -> done" + assert_contains "$out" "source: run-step" "green ci-monitor -> run-step source" + assert_contains "$out" "run ci log reports CI checks passed" "the ci-monitor detail states what the run's log reported" + assert_not_contains "$out" "state: working" "green ci-monitor must not read as still validating" + pass "ci-monitoring run with checks already green surfaces done" +} + +test_top_level_ci_checks_green_surfaces_done() { + reset_fakes + local d; d=$(new_case top-level-ci-green) + make_repo_on_branch "$d/wt" fm/feat-topcigreen + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-topcigreen.meta" "window=fm:fm-feat-topcigreen" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_top_level_ci fm/feat-topcigreen)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-topcigreen) + assert_contains "$out" "state: done" "top-level ci with green log -> done" + assert_contains "$out" "source: run-step" "top-level ci green -> run-step source" + assert_contains "$out" "run ci log reports CI checks passed" "the top-level ci detail states what the run's log reported" + assert_not_contains "$out" "state: working" "top-level ci green must not stay working" + pass "top-level ci status uses ci log green marker" +} + +# CI that has not run - a fork's workflow runs held for maintainer approval +# report no checks at all - is a wait, never green. The worker's own ci-ready +# claim still answers from its status log, with the run's reading beside it. +test_ci_monitoring_no_checks_reads_waiting_not_green() { + reset_fakes + local d; d=$(new_case ci-nochecks) + make_repo_on_branch "$d/wt" fm/feat-cinochecks + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cinochecks.meta" "window=fm:fm-feat-cinochecks" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cinochecks)" + FM_FAKE_CI_LOGS="no CI checks reported - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-cinochecks) + assert_contains "$out" "state: working" "a no-checks ci monitor is still waiting: $out" + assert_contains "$out" "no CI checks reported (waiting for checks, not green)" "the wait is named, not green" + assert_not_contains "$out" "checks green" "no reported checks must not read as checks green" + + printf 'done: PR https://github.com/o/r/pull/2 checks green\n' > "$d/state/feat-cinochecks.status" + out=$(run_crew_state "$d" feat-cinochecks) + assert_contains "$out" "source: status-log" "the worker's ci-ready claim answers from its own log: $out" + assert_contains "$out" "run ci log: no CI checks reported (not green)" "the run's reading is shown beside it" + pass "a no-checks ci-monitor marker reads as waiting, not green" +} + +# The monitor logs a checks state only when it changes, and a base-branch +# advance re-arms only its idle timeout, so a green PR on a busy base ends its +# ci log with re-arm lines (the 2026-09-22 PR #5317 shape: green, then main +# advanced while it waited for merge). The green marker before them is current. +test_ci_monitoring_green_then_rearm_stays_green() { + reset_fakes + local d; d=$(new_case ci-green-then-rearm) + make_repo_on_branch "$d/wt" fm/feat-cirearm + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cirearm.meta" "window=fm:fm-feat-cirearm" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cirearm)" + FM_FAKE_CI_LOGS=$(cat <<'EOF' +all CI checks passed - still monitoring until merged or closed +base branch advanced (aaaaaaa..bbbbbbb), re-arming CI monitor timeout +base branch advanced (bbbbbbb..ccccccc), re-arming CI monitor timeout +EOF +) + local out; out=$(run_crew_state "$d" feat-cirearm) + assert_contains "$out" "state: done" "a base-advance re-arm after green keeps the PR green" + assert_contains "$out" "source: run-step" "re-armed green monitoring stays run-step sourced" + assert_contains "$out" "CI checks passed (required checks not verified" "re-armed green monitoring reads held for merge" + assert_contains "$out" "https://github.com/o/r/pull/2" "the held-for-merge reading names the run's PR" + assert_not_contains "$out" "state: working" "a re-arm line must not read as checks not ready" + pass "base-advance re-arm after green stays checks green" +} + +# The same green-then-re-arm shape, but monitored long enough that the base +# advanced past the CLI's 40-line log tail: `axi logs` without --full would +# answer with re-arm lines only, hiding the green marker entirely, and the +# green PR would read as still working for as long as main kept moving. +test_ci_monitoring_green_before_log_tail_stays_green() { + reset_fakes + local d; d=$(new_case ci-green-beyond-tail) + make_repo_on_branch "$d/wt" fm/feat-citail + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-citail.meta" "window=fm:fm-feat-citail" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-citail)" + FM_FAKE_CI_LOGS=$({ + printf 'monitoring CI for PR #2 (timeout: 4h0m0s)...\n' + printf 'all CI checks passed - still monitoring until merged or closed\n' + for i in $(seq 1 60); do + printf 'base branch advanced (%07d..%07d), re-arming CI monitor timeout\n' "$i" "$((i + 1))" + done + }) + local out; out=$(run_crew_state "$d" feat-citail) + assert_contains "$out" "state: done" "a green marker older than the log tail still reads green" + assert_contains "$out" "source: run-step" "the full-log green reading stays run-step sourced" + assert_contains "$out" "CI checks passed (required checks not verified" "the full-log reading is held for merge" + assert_contains "$out" "https://github.com/o/r/pull/2" "the full-log reading names the run's PR" + assert_not_contains "$out" "state: working" "a truncated ci log must not hide a green PR" + pass "a green marker before the ci log tail still surfaces done" +} + +test_ci_monitoring_no_checks_yet_stays_working() { + reset_fakes + local d; d=$(new_case ci-nochecks-yet) + make_repo_on_branch "$d/wt" fm/feat-cinochecksyet + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cinochecksyet.meta" "window=fm:fm-feat-cinochecksyet" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cinochecksyet)" + FM_FAKE_CI_LOGS=$(cat <<'EOF' +no CI checks reported - still monitoring until merged or closed +base branch advanced (aaaaaaa..bbbbbbb), re-arming CI monitor timeout +no CI checks reported yet, waiting for checks to register... +EOF +) + local out; out=$(run_crew_state "$d" feat-cinochecksyet) + assert_contains "$out" "state: working" "pending no-checks marker -> working" + assert_not_contains "$out" "state: done" "pending no-checks marker must not read as done" + assert_not_contains "$out" "checks green" "pending no-checks marker must not read as checks green" + pass "pending no-checks ci-monitor marker stays working" +} + +test_ci_monitoring_still_waiting_stays_working() { + reset_fakes + local d; d=$(new_case ci-waiting) + make_repo_on_branch "$d/wt" fm/feat-ciwait + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ciwait.meta" "window=fm:fm-feat-ciwait" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-ciwait)" + FM_FAKE_CI_LOGS="CI checks running, waiting for results..." + local out; out=$(run_crew_state "$d" feat-ciwait) + assert_contains "$out" "state: working" "ci step still red -> working" + assert_not_contains "$out" "checks green" "no green marker present -> no checks-green detail" + pass "ci-monitoring run with checks not yet green stays working" +} + +# A later merge-conflict auto-fix round after an earlier green reading must +# not be masked: the MOST RECENT marker in the ci log wins. +test_ci_monitoring_green_then_new_issue_stays_working() { + reset_fakes + local d; d=$(new_case ci-green-then-issue) + make_repo_on_branch "$d/wt" fm/feat-cirelapse + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cirelapse.meta" "window=fm:fm-feat-cirelapse" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cirelapse)" + FM_FAKE_CI_LOGS=$(cat <<'EOF' +all CI checks passed - still monitoring until merged or closed +base branch advanced (aaaaaaa..bbbbbbb), re-arming CI monitor timeout +issues detected: merge conflict - auto-fixing (attempt 2/10)... +EOF +) + local out; out=$(run_crew_state "$d" feat-cirelapse) + assert_contains "$out" "state: working" "a later relapse marker must win over an earlier green one" + assert_not_contains "$out" "state: done" "relapsed ci run must not read as done" + pass "a fresh issue after an earlier green reading is not masked" +} + +test_ci_ready_done_log_relapse_stays_working() { + reset_fakes + local d; d=$(new_case ci-ready-then-relapse) + make_repo_on_branch "$d/wt" fm/feat-cireadyrelapse + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cireadyrelapse.meta" "window=fm:fm-feat-cireadyrelapse" "worktree=$d/wt" "kind=ship" + printf 'done: PR https://github.com/o/r/pull/2 checks green\n' > "$d/state/feat-cireadyrelapse.status" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cireadyrelapse)" + FM_FAKE_CI_LOGS=$(cat <<'EOF' +all CI checks passed - still monitoring until merged or closed +base branch advanced (aaaaaaa..bbbbbbb), re-arming CI monitor timeout +CI checks running, waiting for results... +EOF +) + local out; out=$(run_crew_state "$d" feat-cireadyrelapse) + assert_contains "$out" "state: working" "a stale ready status must not mask a later CI relapse" + assert_contains "$out" "source: run-step" "relapsed ci run remains run-step sourced" + assert_not_contains "$out" "state: done" "relapsed ci run with stale done log must not read as done" + pass "stale checks-green status log does not mask CI relapse" +} + +test_ci_fixing_after_green_stays_working() { + reset_fakes + local d; d=$(new_case ci-fixing-after-green) + make_repo_on_branch "$d/wt" fm/feat-cifixing + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cifixing.meta" "window=fm:fm-feat-cifixing" "worktree=$d/wt" "kind=ship" + printf 'done: PR https://github.com/o/r/pull/2 checks green\n' > "$d/state/feat-cifixing.status" + FM_FAKE_AXI_STATUS="$(run_ci_fixing fm/feat-cifixing)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-cifixing) + assert_contains "$out" "state: working" "ci fixing step must stay working" + assert_contains "$out" "source: run-step" "ci fixing remains run-step sourced" + assert_not_contains "$out" "state: done" "ci fixing must not read as checks-green done" + pass "ci fixing is not overridden by an earlier green marker" +} + +test_top_level_fixing_ci_running_after_green_stays_working() { + reset_fakes + local d; d=$(new_case top-level-fixing-ci-running) + make_repo_on_branch "$d/wt" fm/feat-topfixingci + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-topfixingci.meta" "window=fm:fm-feat-topfixingci" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_fixing_ci_running fm/feat-topfixingci)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-topfixingci) + assert_contains "$out" "state: working" "top-level fixing with ci running must stay working" + assert_contains "$out" "source: run-step" "top-level fixing with ci running remains run-step sourced" + assert_contains "$out" "validating (fixing)" "top-level fixing keeps fixing detail" + assert_not_contains "$out" "state: done" "top-level fixing must not use stale green marker" + pass "top-level fixing is not overridden by a stale ci running row" +} + +test_top_level_fixing_done_log_stays_working() { + reset_fakes + local d; d=$(new_case top-level-fixing-done-log) + make_repo_on_branch "$d/wt" fm/feat-topfixing + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-topfixing.meta" "window=fm:fm-feat-topfixing" "worktree=$d/wt" "kind=ship" + printf 'done: PR https://github.com/o/r/pull/2 checks green\n' > "$d/state/feat-topfixing.status" + FM_FAKE_AXI_STATUS="$(run_fixing fm/feat-topfixing)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-topfixing) + assert_contains "$out" "state: working" "top-level fixing must stay working" + assert_contains "$out" "source: run-step" "top-level fixing remains run-step sourced" + assert_contains "$out" "validating (fixing)" "top-level fixing keeps fixing detail" + assert_not_contains "$out" "state: done" "top-level fixing must not read as stale checks-green done" + pass "top-level fixing is not overridden by a stale done log" +} + +# (d) terminal run-step is authoritative +test_terminal_passed() { + reset_fakes + local d; d=$(new_case passed) + make_repo_on_branch "$d/wt" fm/feat-d + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-d.meta" "window=fm:fm-feat-d" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" + FM_FAKE_AXI_STATUS="$(run_passed fm/feat-d)" + local out; out=$(run_crew_state "$d" feat-d) + assert_contains "$out" "state: done" "passed run -> done" + assert_contains "$out" "source: run-step" "passed -> run-step source" + assert_contains "$out" "run passed: recorded PR merged" "passed run reports merged only after the PR record says merged" + assert_not_contains "$out" "merged/closed" "passed merged PR must not keep the old ambiguous label" + pass "terminal passed run is authoritative" +} + +test_terminal_passed_with_override() { + reset_fakes + local d; d=$(new_case passed-with-override) + make_repo_on_branch "$d/wt" fm/feat-override + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-override.meta" "window=fm:fm-feat-override" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" + FM_FAKE_AXI_STATUS="$(run_passed_with_override fm/feat-override)" + local out; out=$(run_crew_state "$d" feat-override) + assert_contains "$out" "state: done" "passed-with-override run -> done, not unknown" + assert_contains "$out" "source: run-step" "passed-with-override -> run-step source" + assert_contains "$out" "run passed-with-override (approved exception, not green: live checks not all passed: Lint (fail)): recorded PR merged (forge)" \ + "passed-with-override names its exception and reports merged only after the recorded PR says merged" + assert_not_contains "$out" "state: unknown" "passed-with-override must not fall through to unknown" + assert_not_contains "$out" "outcome: passed-with-override" "passed-with-override must not surface as a raw unmapped outcome detail" + pass "terminal passed-with-override run reads done like a clean pass" +} + +test_terminal_passed_with_skips() { + reset_fakes + local d; d=$(new_case passed-with-skips) + make_repo_on_branch "$d/wt" fm/feat-skips + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-skips.meta" "window=fm:fm-feat-skips" "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" + FM_FAKE_AXI_STATUS="$(run_passed_with_skips fm/feat-skips)" + local out; out=$(run_crew_state "$d" feat-skips) + assert_contains "$out" "state: done" "passed-with-skips run -> done, not unknown" + assert_contains "$out" "source: run-step" "passed-with-skips -> run-step source" + assert_contains "$out" "run passed-with-skips: recorded PR merged" "passed-with-skips run reports merged only after the recorded PR says merged" + assert_contains "$out" "publication/CI verification skipped" "passed-with-skips keeps the skip visible, unlike a clean pass" + assert_not_contains "$out" "state: unknown" "passed-with-skips must not fall through to unknown" + assert_not_contains "$out" "outcome: passed-with-skips" "passed-with-skips must not surface as a raw unmapped outcome detail" + pass "terminal passed-with-skips run reads done with the skip kept visible" +} + +test_terminal_passed_uses_matching_retirement_receipt_without_forge() { + reset_fakes + local d url read_log out + d=$(new_case passed-receipt) + url=https://github.com/o/r/pull/1 + make_repo_on_branch "$d/wt" fm/feat-dreceipt + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dreceipt.meta" "window=fm:fm-feat-dreceipt" \ + "worktree=$d/wt" "kind=ship" "pr=$url" + seed_retired_pr_receipt "$d/state" feat-dreceipt "$url" + read_log="$d/pr-read.log" + : > "$read_log" + FM_FAKE_PR_READ_LOG=$read_log + FM_FAKE_PR_READ_FAIL=1 + FM_FAKE_AXI_STATUS="$(run_passed_no_pr fm/feat-dreceipt)" + out=$(run_crew_state "$d" feat-dreceipt) + assert_contains "$out" "state: done" "passed run with retired PR receipt -> done" + assert_contains "$out" "run passed: recorded PR merged" "matching retirement receipt is local merged evidence" + [ ! -s "$read_log" ] || fail "matching retirement receipt still attempted a forge read" + pass "terminal passed run uses matching retirement receipt without forge" +} + +test_terminal_passed_no_forge_switch_skips_read_but_keeps_receipt() { + reset_fakes + local d url read_log out + d=$(new_case passed-no-forge-switch) + url=https://github.com/o/r/pull/1 + make_repo_on_branch "$d/wt" fm/feat-dnoforge + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dnoforge.meta" "window=fm:fm-feat-dnoforge" \ + "worktree=$d/wt" "kind=ship" "pr=$url" + read_log="$d/pr-read.log" + : > "$read_log" + FM_FAKE_PR_READ_LOG=$read_log + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dnoforge "$url")" + + out=$(FM_CREW_STATE_NO_FORGE=1 run_crew_state "$d" feat-dnoforge) + assert_contains "$out" "run passed: recorded PR state unknown (forge read skipped)" "no-forge mode reports skipped read" + assert_not_contains "$out" "PR merged" "no-forge mode without a receipt must not report merged" + [ ! -s "$read_log" ] || fail "no-forge mode invoked a forge read" + + seed_retired_pr_receipt "$d/state" feat-dnoforge "$url" + out=$(FM_CREW_STATE_NO_FORGE=1 run_crew_state "$d" feat-dnoforge) + assert_contains "$out" "run passed: recorded PR merged" "no-forge mode still trusts a matching retirement receipt" + [ ! -s "$read_log" ] || fail "no-forge mode with a receipt invoked a forge read" + pass "terminal passed no-forge mode preserves local receipt evidence" +} + +test_terminal_passed_with_open_pr_does_not_claim_merged() { + reset_fakes + local d; d=$(new_case passed-open-pr) + make_repo_on_branch "$d/wt" fm/feat-dopen + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dopen.meta" "window=fm:fm-feat-dopen" \ + "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/1" + FM_FAKE_PR_STATE=OPEN + FM_FAKE_PR_MERGED=false + FM_FAKE_AXI_STATUS="$(run_passed fm/feat-dopen)" + local out; out=$(run_crew_state "$d" feat-dopen) + assert_contains "$out" "state: done" "passed run with open PR -> done" + assert_contains "$out" "run passed: recorded PR open" "open PR state is named" + assert_not_contains "$out" "merged/closed" "open PR must not get the old merged/closed label" + assert_not_contains "$out" "PR merged" "open PR must not be reported merged" + pass "terminal passed run with open PR does not claim merged" +} + +test_terminal_passed_conflicting_run_pr_claims_neither_state() { + reset_fakes + local d; d=$(new_case passed-stale-meta) + make_repo_on_branch "$d/wt" fm/feat-dstale + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dstale.meta" "window=fm:fm-feat-dstale" \ + "worktree=$d/wt" "kind=ship" "pr=https://github.com/o/r/pull/47" + FM_FAKE_PR_47_STATE=MERGED + FM_FAKE_PR_47_MERGED=true + FM_FAKE_PR_48_STATE=OPEN + FM_FAKE_PR_48_MERGED=false + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dstale https://github.com/o/r/pull/48)" + local out; out=$(run_crew_state "$d" feat-dstale) + assert_contains "$out" "state: done" "passed run with a conflicting recorded PR -> done" + assert_contains "$out" "run passed: PR state unknown (run PR https://github.com/o/r/pull/48 differs from recorded PR https://github.com/o/r/pull/47)" \ + "a run PR that differs from the recorded PR is a conflict, never either PR's state" + assert_not_contains "$out" "PR merged" "the recorded PR's merge must not be claimed for another PR's run" + pass "terminal passed run with a conflicting PR identity claims neither state" +} + +test_terminal_passed_without_readable_pr_identity_reports_unknown() { + reset_fakes + local d; d=$(new_case passed-no-pr) + make_repo_on_branch "$d/wt" fm/feat-dnopr + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dnopr.meta" "window=fm:fm-feat-dnopr" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_passed_no_pr fm/feat-dnopr)" + local out; out=$(run_crew_state "$d" feat-dnopr) + assert_contains "$out" "state: done" "passed run without PR identity -> done" + assert_contains "$out" "run passed: PR state unknown (no PR identity)" "missing PR identity is honest unknown" + assert_not_contains "$out" "merged/closed" "unknown PR state must not get the old merged/closed label" + assert_not_contains "$out" "PR merged" "unknown PR state must not be reported merged" + pass "terminal passed run without readable PR identity reports unknown" +} + +test_terminal_passed_with_open_gitlab_mr_does_not_claim_merged() { + reset_fakes + local d read_log out + d=$(new_case passed-open-gitlab-mr) + make_repo_on_branch "$d/wt" fm/feat-dgitlabopen + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dgitlabopen.meta" "window=fm:fm-feat-dgitlabopen" \ + "worktree=$d/wt" "kind=ship" "pr=https://git.example.com/group/subgroup/repo/-/merge_requests/9" + read_log="$d/glab-read.log" + : > "$read_log" + FM_FAKE_GLAB_READ_LOG=$read_log + FM_FAKE_GLAB_STATE=opened + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgitlabopen https://git.example.com/group/subgroup/repo/-/merge_requests/9)" + out=$(run_crew_state "$d" feat-dgitlabopen) + assert_contains "$out" "run passed: recorded PR open" "open GitLab MR state is named" + assert_not_contains "$out" "PR merged" "open GitLab MR must not be reported merged" + assert_grep 'git.example.com|mr view 9 -R https://git.example.com/group/subgroup/repo -F json' "$read_log" \ + "GitLab MR read uses the parsed host and project URL" + pass "terminal passed run reads open GitLab MR state" +} + +test_terminal_passed_with_merged_gitlab_mr_reports_merged() { + reset_fakes + local d out + d=$(new_case passed-merged-gitlab-mr) + make_repo_on_branch "$d/wt" fm/feat-dgitlabmerged + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dgitlabmerged.meta" "window=fm:fm-feat-dgitlabmerged" \ + "worktree=$d/wt" "kind=ship" "pr=https://gitlab.com/group/repo/-/merge_requests/10" + FM_FAKE_GLAB_STATE=merged + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgitlabmerged https://gitlab.com/group/repo/-/merge_requests/10)" + out=$(run_crew_state "$d" feat-dgitlabmerged) + assert_contains "$out" "run passed: recorded PR merged" "merged GitLab MR is reported merged" + pass "terminal passed run reads merged GitLab MR state" +} + +test_terminal_passed_with_failed_gitlab_read_reports_unknown() { + reset_fakes + local d out + d=$(new_case passed-unreadable-gitlab-mr) + make_repo_on_branch "$d/wt" fm/feat-dgitlabunknown + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dgitlabunknown.meta" "window=fm:fm-feat-dgitlabunknown" \ + "worktree=$d/wt" "kind=ship" "pr=https://gitlab.com/group/repo/-/merge_requests/11" + FM_FAKE_GLAB_READ_FAIL=1 + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgitlabunknown https://gitlab.com/group/repo/-/merge_requests/11)" + out=$(run_crew_state "$d" feat-dgitlabunknown) + assert_contains "$out" "run passed: recorded PR state unknown (unreadable)" "failed GitLab read is honest unknown" + assert_not_contains "$out" "PR merged" "failed GitLab read must not be reported merged" + pass "terminal passed run handles failed GitLab read" +} + +test_terminal_passed_with_forgejo_pull_reports_its_state() { + reset_fakes + local d read_log out spec state merged fail expected name + for spec in "open|open|false|0|run passed: recorded PR open (forge)" \ + "merged|closed|true|0|run passed: recorded PR merged (forge)" \ + "closed|closed|false|0|run passed: recorded PR closed (forge)" \ + "unreadable|open|false|1|run passed: recorded PR state unknown (unreadable)"; do + IFS='|' read -r name state merged fail expected <<<"$spec" + d=$(new_case "passed-forgejo-$name") + make_repo_on_branch "$d/wt" "fm/feat-dforgejo$name" + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dforgejo$name.meta" "window=fm:fm-feat-dforgejo$name" \ + "worktree=$d/wt" "kind=ship" "pr=https://git.example.org/team/repo/pulls/12" + read_log="$d/tea-read.log" + : > "$read_log" + FM_FAKE_TEA_READ_LOG=$read_log + FM_FAKE_TEA_STATE=$state + FM_FAKE_TEA_MERGED=$merged + FM_FAKE_TEA_READ_FAIL=$fail + FM_FAKE_AXI_STATUS="$(run_passed_with_pr "fm/feat-dforgejo$name" https://git.example.org/team/repo/pulls/12)" + out=$(run_crew_state "$d" "feat-dforgejo$name") + assert_contains "$out" "$expected" "Forgejo pull request reading $name is reported" + [ "$name" = merged ] || assert_not_contains "$out" "PR merged" "Forgejo $name pull request must not be reported merged" + assert_grep 'api --login fj --repo team/repo /repos/{owner}/{repo}/pulls/12' "$read_log" \ + "Forgejo read uses the host's tea login and the parsed project path" + done + pass "terminal passed run reads Forgejo pull request state through tea api" +} + +test_terminal_passed_with_open_gerrit_change_does_not_claim_merged() { + reset_fakes + local d url read_log out + d=$(new_case passed-open-gerrit-change) + url=https://review.internal/c/group/apps/console/+/4201 + make_repo_on_branch "$d/wt" fm/feat-dgerritopen + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dgerritopen.meta" "window=fm:fm-feat-dgerritopen" \ + "worktree=$d/wt" "kind=ship" "pr=$url" + read_log="$d/gerrit-read.log" + : > "$read_log" + FM_FAKE_GERRIT_READ_LOG=$read_log + FM_FAKE_GERRIT_STATUS=NEW + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgerritopen "$url")" + out=$(run_crew_state "$d" feat-dgerritopen) + assert_contains "$out" "run passed: recorded PR open" "open Gerrit change state is named" + assert_not_contains "$out" "PR merged" "open Gerrit change must not be reported merged" + assert_grep 'show 4201 --host review.internal --json' "$read_log" \ + "Gerrit read addresses the change by number and explicit host" + pass "terminal passed run reads open Gerrit change state" +} + +test_terminal_passed_with_merged_gerrit_change_reports_merged() { + reset_fakes + local d url out + d=$(new_case passed-merged-gerrit-change) + url=https://review.internal/c/group/apps/console/+/4200 + make_repo_on_branch "$d/wt" fm/feat-dgerritmerged + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dgerritmerged.meta" "window=fm:fm-feat-dgerritmerged" \ + "worktree=$d/wt" "kind=ship" "pr=$url" + FM_FAKE_GERRIT_STATUS=MERGED + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgerritmerged "$url")" + out=$(run_crew_state "$d" feat-dgerritmerged) + # The fixture record carries a null url, the shape a server with no + # gerrit.canonicalWebUrl returns, so the merge is reported off the change + # number the read was addressed by rather than off a URL the server may + # never compose. + assert_contains "$out" "run passed: recorded PR merged" "merged Gerrit change is reported merged" + + # An abandoned change is this report's closed, and is never merged. + FM_FAKE_GERRIT_STATUS=ABANDONED + out=$(run_crew_state "$d" feat-dgerritmerged) + assert_contains "$out" "run passed: recorded PR closed" "abandoned Gerrit change is reported closed" + assert_not_contains "$out" "PR merged" "abandoned Gerrit change must not be reported merged" + pass "terminal passed run reads merged and abandoned Gerrit change state" +} + +test_terminal_passed_with_unreadable_gerrit_change_reports_unknown() { + reset_fakes + local d url out + d=$(new_case passed-unreadable-gerrit-change) + url=https://review.internal/c/group/apps/console/+/4202 + make_repo_on_branch "$d/wt" fm/feat-dgerritunknown + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-dgerritunknown.meta" "window=fm:fm-feat-dgerritunknown" \ + "worktree=$d/wt" "kind=ship" "pr=$url" + FM_FAKE_GERRIT_READ_FAIL=1 + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgerritunknown "$url")" + out=$(run_crew_state "$d" feat-dgerritunknown) + assert_contains "$out" "run passed: recorded PR state unknown (unreadable)" "failed Gerrit read is honest unknown" + assert_not_contains "$out" "PR merged" "failed Gerrit read must not be reported merged" + + # A record naming another change can never answer for this one, however the + # server came to return it. The change number is the whole identity of the + # match, so a wrong one is an unreadable record rather than a merge. + reset_fakes + FM_FAKE_GERRIT_STATUS=MERGED + FM_FAKE_GERRIT_CHANGE=4203 + FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/feat-dgerritunknown "$url")" + out=$(run_crew_state "$d" feat-dgerritunknown) + assert_contains "$out" "run passed: recorded PR state unknown (unreadable)" "mismatched Gerrit record is honest unknown" + assert_not_contains "$out" "PR merged" "another change's merged record must not report merged" + pass "terminal passed run handles an unreadable or mismatched Gerrit read" +} + +test_terminal_failed() { + reset_fakes + local d; d=$(new_case failed) + make_repo_on_branch "$d/wt" fm/feat-e + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-e.meta" "window=fm:fm-feat-e" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_failed fm/feat-e)" + FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS/status: completed/status: failed} + local out; out=$(run_crew_state "$d" feat-e) + assert_contains "$out" "state: failed" "failed run -> failed" + assert_contains "$out" "source: run-step" "failed -> run-step source" + pass "terminal failed run is authoritative" +} + +# Recovered delivery cases, varying only the terminal route and the optional +# rebase step. The already-fixed passed-run case remains a control. +test_cancelled_delivery_and_skipped_rebase() { + local scenario failures=0 + for scenario in cancelled-outcome cancelled-status skipped-rebase cancelled-skipped-rebase passed; do + ( + reset_fakes + local d out + d=$(new_case "delivery-$scenario") + make_repo_on_branch "$d/wt" fm/delivery + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/delivery.meta" "window=fm:fm-delivery" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan fm/delivery)" + case "$scenario" in + cancelled*) FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS//failed/cancelled} ;; + esac + case "$scenario" in + *skipped-rebase) FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS/rebase,completed/rebase,skipped} ;; + cancelled-status) FM_FAKE_AXI_STATUS=$(printf '%s\n' "$FM_FAKE_AXI_STATUS" | sed '/^outcome:/d') ;; + passed) FM_FAKE_AXI_STATUS="$(run_passed_with_pr fm/delivery https://github.com/o/r/pull/203)" ;; + esac + FM_FAKE_PR_STATE=OPEN + FM_FAKE_PR_MERGED=false + FM_FAKE_PR_STATE_AXI=open + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + out=$(FM_HOME="$d" run_crew_state "$d" delivery) + assert_contains "$out" "state: done" "$scenario: delivered work remains done: $out" + assert_not_contains "$out" "PR merged" "$scenario: terminal record cannot prove a merge" + if [ "$scenario" != passed ]; then + assert_contains "$out" "https://github.com/o/r/pull/203" "$scenario: delivery identity retained" + assert_contains "$out" "ci log reported CI checks passed" "$scenario: retain positive CI evidence" + assert_contains "$out" "held for merge" "$scenario: delivery awaits merge" + assert_contains "$out" "required checks not verified" "$scenario: the ci log pass is not verified against required checks" + fi + pass "$scenario: terminal delivery reports only observed evidence" + ) || failures=$((failures + 1)) + done + [ "$failures" -eq 0 ] || fail "$failures cancelled delivery regressions" +} + +test_terminal_green_delivery_disposition() { + local route provider disposition failures=0 + for route in failed-outcome failed-status cancelled-outcome cancelled-status; do + for provider in github gitlab gerrit; do + for disposition in open merged closed unreadable skipped no-identity; do + ( + reset_fakes + local d out url expected + d=$(new_case "disposition-$route-$provider-$disposition") + make_repo_on_branch "$d/wt" fm/disposition + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/delivery.meta" "window=fm:fm-delivery" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan fm/disposition)" + case "$route" in + cancelled-*) FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS//failed/cancelled} ;; + esac + case "$route" in + *-status) FM_FAKE_AXI_STATUS=$(printf '%s\n' "$FM_FAKE_AXI_STATUS" | sed '/^outcome:/d') ;; + esac + case "$provider" in + github) url=https://github.com/o/r/pull/203 ;; + gitlab) url=https://gitlab.com/o/r/-/merge_requests/203 ;; + gerrit) url=https://review.example.com/c/r/+/203 ;; + esac + FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS//https:\/\/github.com\/o\/r\/pull\/203/$url} + FM_FAKE_PR_STATE=OPEN + FM_FAKE_PR_MERGED=false + FM_FAKE_PR_STATE_AXI=open + FM_FAKE_GLAB_STATE=opened + FM_FAKE_GERRIT_STATUS=NEW + case "$disposition" in + no-identity) FM_FAKE_AXI_STATUS=$(printf '%s\n' "$FM_FAKE_AXI_STATUS" | sed '/^[[:space:]]*pr:/d') ;; + merged) + # A merge is claimed only for the task's recorded PR identity. + printf 'pr=%s\n' "$url" >> "$d/state/delivery.meta" + FM_FAKE_PR_STATE=MERGED + FM_FAKE_PR_MERGED=true + FM_FAKE_PR_STATE_AXI=merged + FM_FAKE_GLAB_STATE=merged + FM_FAKE_GERRIT_STATUS=MERGED ;; + closed) + FM_FAKE_PR_STATE=CLOSED + FM_FAKE_PR_STATE_AXI=closed + FM_FAKE_GLAB_STATE=closed + FM_FAKE_GERRIT_STATUS=ABANDONED ;; + unreadable) + FM_FAKE_PR_READ_FAIL=1 + FM_FAKE_GLAB_READ_FAIL=1 + FM_FAKE_GERRIT_READ_FAIL=1 ;; + esac + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + if [ "$disposition" = skipped ]; then + out=$(FM_CREW_STATE_NO_FORGE=1 FM_HOME="$d" run_crew_state "$d" delivery) + else + out=$(FM_HOME="$d" run_crew_state "$d" delivery) + fi + case "$disposition" in + open|merged) + assert_contains "$out" "state: done" "$route/$provider/$disposition: delivered work: $out" + if [ "$disposition" = open ]; then + assert_contains "$out" "held for merge" "open delivery awaits merge" + else + assert_contains "$out" "PR merged" "merged delivery has current evidence" + assert_not_contains "$out" "held for merge" "merged delivery is no longer held" + fi ;; + *) + expected=failed + case "$route" in + cancelled-*) expected=unknown + assert_contains "$out" "run cancelled: no verdict" "cancellation retains no verdict" ;; + esac + assert_contains "$out" "state: $expected" "$route/$provider/$disposition: no unsupported delivery: $out" + assert_not_contains "$out" "held for merge" "unproven open delivery cannot await merge" + assert_not_contains "$out" "PR merged" "unproven merge cannot be claimed" ;; + esac + pass "$route/$provider/$disposition: terminal delivery uses current disposition" + ) || failures=$((failures + 1)) + done + done + done + [ "$failures" -eq 0 ] || fail "$failures terminal delivery disposition regressions" +} + +# Cancellation carries no verdict without the positive delivery safeguard. +# Exercise both detailed routes, selected-run attribution, and the coarse ledger. +test_cancelled_without_delivery_has_no_verdict() { + local scenario failures=0 + for scenario in outcome status selected coarse no-ci-log red-ci cancelled-test skipped-test; do + ( + reset_fakes + local d out + d=$(new_case "no-verdict-$scenario") + make_repo_on_branch "$d/wt" fm/cancelled + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/cancelled.meta" "window=fm:fm-cancelled" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_failed fm/cancelled)" + FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS//failed/cancelled} + FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS/status: completed/status: cancelled} + case "$scenario" in + status) FM_FAKE_AXI_STATUS=$(printf '%s\n' "$FM_FAKE_AXI_STATUS" | sed '/^outcome:/d') ;; + selected) + FM_FAKE_AXI_STATUS_RUN=$FM_FAKE_AXI_STATUS + FM_FAKE_AXI_HOME="count: 1 of 1 total +runs[1]{id,branch,status,head,pr}: + 01RUN,fm/cancelled,cancelled,$FM_FAKE_RUN_HEAD,\"\"" + ;; + coarse) + FM_FAKE_AXI_STATUS="$(run_running fm/another)" + FM_FAKE_RUNS_LIST=" cancelled fm/cancelled $FM_FAKE_RUN_HEAD 2026-09-26 17:00" + ;; + no-ci-log|red-ci|cancelled-test|skipped-test) + FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan fm/cancelled)" + FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS//failed/cancelled} + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + case "$scenario" in + no-ci-log) FM_FAKE_CI_LOGS= ;; + red-ci) FM_FAKE_CI_LOGS="$FM_FAKE_CI_LOGS +checks failed: 1 of 2 checks red" ;; + cancelled-test) FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS/test,completed/test,cancelled} ;; + skipped-test) FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS/test,completed/test,skipped} ;; + esac + ;; + esac + out=$(FM_HOME="$d" run_crew_state "$d" cancelled) + assert_contains "$out" "state: unknown" "$scenario: cancellation alone has no verdict: $out" + assert_contains "$out" "run cancelled: no verdict" "$scenario: explicit reason" + assert_contains "$out" "source: run-step" "$scenario: keep attribution" + assert_not_contains "$out" "held for merge" "$scenario: no unsupported delivery claim" + pass "$scenario: cancellation without delivery carries no verdict" + ) || failures=$((failures + 1)) + done + [ "$failures" -eq 0 ] || fail "$failures cancellation verdict regressions" +} + +# The real inventory consumer must not confuse a cancellation with a failed +# child contradicting an In flight row. Unknown remains explicitly partial. +test_cancelled_fleet_inventory_is_unverified_not_contradictory() { + reset_fakes + local d out summary backlog_before status_before scenario=${1:-synthetic} + d=$(new_case "cancelled-inventory-$scenario") + make_repo_on_branch "$d/wt" fm/cancelled + make_fakebin "$d" >/dev/null + mkdir -p "$d/data" "$d/config" "$d/projects" + fm_write_meta "$d/state/cancelled.meta" "window=fm:fm-cancelled" "worktree=$d/wt" \ + "project=sample" "harness=claude" "kind=ship" "mode=no-mistakes" + cat > "$d/data/backlog.md" <<'EOF' +## In flight +- [ ] cancelled - Validation in progress (repo: sample) (kind: ship) (since 2026-09-26) + +## Queued + +## Done +EOF + printf 'failed: historical cancellation projection\n' > "$d/state/cancelled.status" + backlog_before=$(cat "$d/data/backlog.md") + status_before=$(cat "$d/state/cancelled.status") + FM_FAKE_AXI_STATUS="$(run_running fm/cancelled)" + out=$(FM_HOME="$d" run_crew_state "$d" cancelled) + assert_contains "$out" 'state: working' 'fixture begins with active validation' + # Deliberately transition the external instrument fixture to cancelled. + # This executes Firstmate end to end; it does not cancel a real daemon run. + FM_FAKE_AXI_STATUS="$(run_failed fm/cancelled)" + FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS//failed/cancelled} + FM_FAKE_AXI_STATUS=${FM_FAKE_AXI_STATUS/status: completed/status: cancelled} + if [ "$scenario" = captured ]; then + # Real record supplied read-only from axi status --run + # 01M2SXM5NDEWK2KY5TG8DDYJMV; only branch/head are rebound for attribution. + # Skipped rebase and cancelled CI monitoring remain synthetic cases above. + FM_FAKE_AXI_STATUS="$(cat </dev/null || fail "cancellation must not report a terminal/backlog contradiction: $summary" + assert_equals "$backlog_before" "$(cat "$d/data/backlog.md")" 'correct backlog is unchanged' + assert_equals "$status_before" "$(cat "$d/state/cancelled.status")" 'historical event is unchanged' + pass "$scenario cancelled run leaves fleet inventory unverified without a failure contradiction" +} + +# Replay the recorded producer output through both public consumers, without +# starting or aborting a daemon run or claiming live cancellation evidence. +test_captured_cancelled_review_has_no_verdict() { + test_cancelled_fleet_inventory_is_unverified_not_contradictory captured +} + +test_terminal_failed_ci_orphan_after_green_reads_done() { + reset_fakes + local d; d=$(new_case failed-ci-orphan) + make_repo_on_branch "$d/wt" fm/feat-ci-orphan + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ci-orphan.meta" "window=fm:fm-feat-ci-orphan" "worktree=$d/wt" "kind=ship" \ + "pr=https://github.com/o/r/pull/203" + FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan fm/feat-ci-orphan)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed +daemon shutting down" + local out; out=$(run_crew_state "$d" feat-ci-orphan) + assert_contains "$out" "state: done" "orphaned ci monitor after green must read done, not failed" + assert_contains "$out" "source: run-step" "reclassified held run stays run-step sourced" + assert_contains "$out" "https://github.com/o/r/pull/203" "PR URL surfaced from the run" + assert_not_contains "$out" "state: failed" "monitor death must not read as a failed run" + pass "orphaned ci monitor after green reads as held-for-merge done" +} + +test_terminal_failed_ci_orphan_status_only_reads_done() { + reset_fakes + local d; d=$(new_case failed-ci-orphan-status-only) + make_repo_on_branch "$d/wt" fm/feat-ci-orphan2 + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ci-orphan2.meta" "window=fm:fm-feat-ci-orphan2" "worktree=$d/wt" "kind=ship" \ + "pr=https://github.com/o/r/pull/203" + FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan_status_only fm/feat-ci-orphan2)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed +daemon shutting down" + local out; out=$(run_crew_state "$d" feat-ci-orphan2) + assert_contains "$out" "state: done" "status-only failed orphaned monitor after green reads done" + assert_contains "$out" "https://github.com/o/r/pull/203" "PR URL surfaced from the run" + pass "status-only failed orphaned ci monitor after green reads done" +} + +test_terminal_failed_ci_genuine_red_stays_failed() { + reset_fakes + local d; d=$(new_case failed-ci-genuine-red) + make_repo_on_branch "$d/wt" fm/feat-ci-red + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ci-red.meta" "window=fm:fm-feat-ci-red" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan fm/feat-ci-red)" + FM_FAKE_CI_LOGS="CI checks running +checks failed: 1 of 2 checks red +daemon shutting down" + local out; out=$(run_crew_state "$d" feat-ci-red) + assert_contains "$out" "state: failed" "a genuinely red check keeps the run failed" + assert_not_contains "$out" "state: done" "genuine CI failure must not reclassify to done" + pass "genuinely failing CI keeps the failed verdict" +} + +test_terminal_failed_ci_orphan_second_failed_step_stays_failed() { + reset_fakes + local d; d=$(new_case failed-ci-second-failure) + make_repo_on_branch "$d/wt" fm/feat-ci-2fail + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ci-2fail.meta" "window=fm:fm-feat-ci-2fail" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_failed_ci_orphan_second_failure fm/feat-ci-2fail)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed +daemon shutting down" + local out; out=$(run_crew_state "$d" feat-ci-2fail) + assert_contains "$out" "state: failed" "a second failed step keeps the run failed" + assert_not_contains "$out" "state: done" "a second failed step must not reclassify to done" + pass "a second failed step disqualifies the orphaned-monitor reclassification" +} + +# (e) cross-branch attribution: `axi status` returns ANOTHER branch's run (the +# routine case once more than one crew validates the same underlying repo +# concurrently - they share ONE no-mistakes repo registration), so the helper +# falls back to the real top-level `no-mistakes runs` listing to learn whether +# THIS branch has an active run of its own. Regression coverage for the +# 2026-07-02 herdr incident: the old fallback shelled out to `no-mistakes axi` +# (bare) expecting a `runs[N]{...}:` TOON table that the real CLI never emits +# (verified against the installed v1.32.2 - the `axi` surface has no +# runs-listing subcommand at all), so attribution silently failed every time +# the repo-wide answer was not this crew's own branch. +test_cross_branch_attribution_via_runs_list() { + reset_fakes + local d short; d=$(new_case crossbranch) + make_repo_on_branch "$d/wt" fm/feat-f + short=$(git -C "$d/wt" rev-parse --short=7 HEAD) + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-f.meta" "window=fm:fm-feat-f" "worktree=$d/wt" "kind=ship" + # The repo-wide active/most-recent run belongs to a different crew's branch. + FM_FAKE_AXI_STATUS="$(run_running fm/other-crew)" + # Real `no-mistakes runs` shape: plain text, newest-first, no run id, no + # quoting - " []". + FM_FAKE_RUNS_LIST="$(cat < run-step source" + pass "cross-branch run is attributed via the real runs list" +} + +# The runs list is newest-first; a branch with an OLDER completed run must not +# shadow its own newer active one - the first (topmost) matching row wins. +test_coarse_socket_refusal_reports_blocked() { + reset_fakes + local d short; d=$(new_case coarse-socket-refused) + make_repo_on_branch "$d/wt" fm/feat-coarse-down + short=$(git -C "$d/wt" rev-parse --short=7 HEAD) + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-coarse-down.meta" "window=fm:fm-feat-coarse-down" "worktree=$d/wt" "kind=ship" + printf 'blocked: no-mistakes daemon connection refused\n' > "$d/state/feat-coarse-down.status" + FM_FAKE_AXI_STATUS="$(run_running fm/other-crew)" + FM_FAKE_RUNS_LIST="$(cat </dev/null + fm_write_meta "$d/state/feat-coarsedown.meta" "window=fm:fm-feat-coarsedown" "worktree=$d/wt" "kind=ship" + # The primary `axi status` call answers (another crew's run - the shared + # daemon serves the whole repo), so attribution falls to the coarse runs + # ledger, whose newest row for this branch is terminal failed at this + # worktree's own head. + FM_FAKE_AXI_STATUS="$(run_running fm/other-crew)" + FM_FAKE_RUNS_LIST=" failed fm/feat-coarsedown ${short} 2026-09-05 21:00" + FM_FAKE_DAEMON_DOWN=1 + local out; out=$(run_crew_state "$d" feat-coarsedown) + assert_contains "$out" "state: unknown" "daemon down + failed ledger record -> unknown" + assert_contains "$out" "no-mistakes daemon unreachable; last ledger record failed - unverified" \ + "the unverified detail names the dead instrument" + assert_not_contains "$out" "state: failed" "an instrument failure never reads as work failure" + assert_contains "$out" "source: run-step" "the ledger row is still this branch's attributed run" + + # Daemon provably up again: the same row stays a failure. + FM_FAKE_DAEMON_DOWN=0 + out=$(run_crew_state "$d" feat-coarsedown) + assert_contains "$out" "state: failed" "daemon up keeps the failed verdict over the failed record" + assert_not_contains "$out" "unverified" "no unverified qualifier while the daemon answers" + pass "failed ledger record reads unknown only when the daemon is provably down" +} + +test_cross_branch_attribution_picks_most_recent_row() { + reset_fakes + local d short; d=$(new_case crossbranch-mostrecent) + make_repo_on_branch "$d/wt" fm/feat-fq + short=$(git -C "$d/wt" rev-parse --short=7 HEAD) + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-fq.meta" "window=fm:fm-feat-fq" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_running fm/other-crew)" + FM_FAKE_RUNS_LIST="$(cat < run-step source" + pass "cross-branch attribution picks the branch's most recent row" +} + +# The plain ledger is ordered by creation time, not the time a status changed. +# A newer failure must not be hidden by an older live run, even when both heads +# bind to the worktree. These legacy CLI cases lack the AXI identity table. +test_terminal_run_keeps_newer_failure_over_live_sibling() { + reset_fakes + local d base_head live_head short_base short_live out + d=$(new_case live-beats-corpse) + make_repo_on_branch "$d/wt" fm/feat-corpse + base_head=$(git -C "$d/wt" rev-parse HEAD) + git -C "$d/wt" commit -q --allow-empty -m 'live run advanced the tip' + live_head=$(git -C "$d/wt" rev-parse HEAD) + # Worktree stays at the commit the dead run recorded; the live run is ahead. + git -C "$d/wt" reset -q --hard "$base_head" + short_base=$(git -C "$d/wt" rev-parse --short=7 "$base_head") + short_live=$(git -C "$d/wt" rev-parse --short=7 "$live_head") + [ "$short_base" != "$short_live" ] || fail "live run head did not advance past the worktree" + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/corpse.meta" "window=fm:fm-corpse" "worktree=$d/wt" "kind=ship" + # The newest run failed at this worktree's own commit. + FM_FAKE_RUN_HEAD="$base_head" + FM_FAKE_AXI_STATUS="$(run_failed fm/feat-corpse)" + # The older live run may have advanced its tip, but it did not replace this run. + FM_FAKE_RUNS_LIST="$(cat <