diff --git a/.agents/skills/bootstrap-diagnostics/SKILL.md b/.agents/skills/bootstrap-diagnostics/SKILL.md index 183aa5da2d9..8620c3280b8 100644 --- a/.agents/skills/bootstrap-diagnostics/SKILL.md +++ b/.agents/skills/bootstrap-diagnostics/SKILL.md @@ -50,6 +50,9 @@ When any diagnostic needs captain attention, report the plain consequence and re A count that grows during a session means something is still recording against unresolvable sequences, which is a bug to escalate rather than a backlog of old damage. - `WAKE_LEDGER: the wake ledger could not be read ...` - the file exists but could not be opened, so the count above is unavailable rather than zero. Repair its permissions or path before quoting any supervision-cost figure; an unreadable ledger is reported precisely so it cannot pass as a clean one. +- `WAKE_LEDGER: task(s) declared failure with no terminal record ...` - only a lock-holding session records those, so a read-only session names them and leaves the recording to the session that holds the lock. + Take no action on the count itself; the next locked session records it, and the tasks themselves are ordinary work whose state is read the usual way. + Terminal outcome counts stay diagnostic while any of them is unrecorded, so never quote a success rate from the ledger. - `FLEET_SYNC: : skipped: ` - a benign one-off skip (offline, no origin, local-only); bootstrap continued, investigate only if it blocks work. A skip can also report the bounded fleet-refresh timeout (`FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT`, or a fleet-size-aware default with a 20 second floor); a timeout never blocks startup. - `FLEET_SYNC: : recovered: ` - the clone had drifted onto a clean detached HEAD holding no unique commits and the sync self-healed it (re-attached the default branch and fast-forwarded); no action needed, it is reported only so the self-heal is visible. diff --git a/AGENTS.md b/AGENTS.md index 225eec9db2c..413e594f386 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,6 +95,7 @@ state/ volatile runtime signals; gitignored .status appended by crewmates: ": " wake-event lines, not current-state truth .turn-ended touched by turn-end hooks .childcpu identity-bound sample of the CPU consumed by the task agent's descendants, so supervision can see work happening in a child process; written by the watcher, removed by teardown + .terminal-recorded receipt proving the ledger already holds a terminal record for a task that declared failure and was never torn down, so the recording sweep never repeats it; written only by bin/fm-wake-ledger.sh, removed by teardown .grok-turnend-token firstmate-owned grok hook registry token for the task; removed by teardown .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; a ship or scout also records the task's two base references as slot_base=, contribution_target=, and base_state= (bin/fm-task-base-lib.sh); a task dispatch also records the agent-justification fields reasoning_required=, reason_code=, capability_floor=, escalation_policy=, plus tooling_gap_item= for a TOOLING_GAP dispatch (bin/fm-reasoning-lib.sh, section 7), while a secondmate provisioning spawn records none of them and an absent field reads as unknown rather than as justified reasoning; an optional traceparent= only when trace context is enabled (docs/configuration.md "Trace context propagation"); kind=secondmate also records home= and projects=, plus remote_host=/remote_root=/remote_backend=/remote_herdr_session=/remote_target= for a remote route; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-pr-merge records merge_verification= plus merge_verified_head= for the head it re-verified, or merge_verification=override for an explicitly unverified merge; fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) @@ -152,7 +153,7 @@ A lock-refused session must not spawn, steer, merge, drain the wake queue, repai 1. **Lock** - acquires the per-home session lock first, before anything mutates shared state. 2. **Bootstrap** - detect-only checks (tool/version problems, GitHub auth, the worktree-tangle check, harness override, dispatch-profile validation, backlog-backend status, and shared validation-daemon liveness) always run, but routine confirmations stay silent by default. When the lock could not be acquired, the worktree-tangle check uses read-only advisory wording without a checkout repair command. - Home-local stale Herdr projection cleanup and the six bootstrap MUTATING sweeps - non-executing legacy PR-check migration, fleet sync, secondmate convergence, secondmate liveness, pending remote handoff retry, and X-mode artifact writes - run only when this session actually holds the lock from step 1. + Home-local stale Herdr projection cleanup and bootstrap's MUTATING sweeps - enumerated only in `bin/fm-bootstrap.sh`'s header, their single owner - run only when this session actually holds the lock from step 1. The secondmate liveness sweep deterministically accounts for every registered secondmate: it relaunches only from the recovery-grade `dead` or `missing` states, preserves ambiguous, unreadable, or unreachable remote targets, and reports skipped or failed guarantees as `SECONDMATE_LIVENESS:` lines (`bin/fm-bootstrap.sh`; `bin/fm-backend.sh`'s `fm_backend_agent_state`; `docs/remote-secondmates.md`). 3. **Wake queue** - when locked, drains the durable wake queue and prints the raw records prominently as this turn's first work queue; a bounded, clearly labeled historical status-event annotation may follow a valid `signal` record but never replaces it or current-state reconciliation, and a lapsed watcher chain still surfaces here via the same guard alarm. Every locked drain also prints a bounded fleet-wide `OPEN DECISIONS` section when durable decision records remain open, including when the queue itself is empty; reconcile those entries before continuing. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 98183e12850..aebceab29cf 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -105,11 +105,14 @@ # refresh relays any completed fm-fleet-sync.sh output before the # aggregate timeout skip line with timeout and elapsed seconds. # Set FM_FLEET_PRUNE=0 to skip branch pruning during that refresh. -# Set FM_BOOTSTRAP_DETECT_ONLY=1 to skip the six MUTATING sweeps +# Set FM_BOOTSTRAP_DETECT_ONLY=1 to skip the MUTATING sweeps # (PR-check migration, secondmate_sync, secondmate_liveness_sweep, # secondmate_handoff_resume, x_mode_setup, fleet_sync) while still -# printing every read-only detect line -# above; the TANGLE line switches to advisory-only wording with no +# printing every read-only detect line above. +# wake_ledger_terminal_sweep mutates too and honours the same flag by +# switching to its dry run, so a read-only session still REPORTS the +# declared failures it declined to record. +# The TANGLE line switches to advisory-only wording with no # checkout command. Used by # fm-session-start.sh's read-only path when another live session holds # the fleet lock, so a second concurrent session never race-mutates @@ -1004,6 +1007,34 @@ wake_ledger_reconcile() { echo "WAKE_LEDGER: $unjoined outcome record(s) join no wake record - supervision-cost figures drawn from this ledger overcount until they are purged (bin/fm-wake-ledger.sh reconcile)" } +# Terminal records for tasks that declared failure and were never torn down. +# Only teardown writes a terminal line, so a failure nobody released is SILENT +# in the ledger - and silence there is indistinguishable from a task that never +# failed. A MUTATING sweep: it appends terminal records and writes a per-task +# receipt, so it runs only when this session holds the fleet lock; a read-only +# session reports the same tasks without recording them. +# bin/fm-wake-ledger.sh owns the vocabulary, the sweep, and its idempotence. +wake_ledger_terminal_sweep() { + local out n + [ -x "$SCRIPT_DIR/fm-wake-ledger.sh" ] || return 0 + if [ "${FM_BOOTSTRAP_DETECT_ONLY:-0}" = 1 ]; then + out=$("$SCRIPT_DIR/fm-wake-ledger.sh" sweep --dry-run 2>/dev/null) || return 0 + n=$(printf '%s\n' "$out" | grep -c '^unreleased failure: ' || true) + case "$n" in + ''|*[!0-9]*|0) return 0 ;; + esac + echo "WAKE_LEDGER: $n task(s) declared failure with no terminal record - read-only session left the recording to the session holding the fleet lock (bin/fm-wake-ledger.sh sweep)" + return 0 + fi + out=$("$SCRIPT_DIR/fm-wake-ledger.sh" sweep 2>/dev/null) || return 0 + n=$(printf '%s\n' "$out" \ + | sed -n 's/^recorded \([0-9][0-9]*\) unreleased failure(s) as terminal records$/\1/p') + case "$n" in + ''|*[!0-9]*|0) return 0 ;; + esac + echo "BOOTSTRAP_INFO: recorded $n declared task failure(s) that no teardown would have recorded" +} + # The entitlement probe half of the observation floor. A MUTATING sweep: it makes # live requests and writes state/model-health.json, so it runs only when this # session actually holds the fleet lock, alongside the other mutating sweeps. @@ -1215,6 +1246,7 @@ crew_dispatch_validate model_registry_validate admission_control_validate wake_ledger_reconcile +wake_ledger_terminal_sweep if [ "${FM_BOOTSTRAP_VERBOSE_FACTS:-0}" = 1 ] \ && ! fm_backlog_backend_manual "$CONFIG" && fm_tasks_axi_compatible; then echo "BOOTSTRAP_INFO: tasks-axi available" diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index a409aad7fb1..b4fb0f9989e 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -18,7 +18,7 @@ # standalone with unchanged default behavior - other flows (fm-bootstrap.sh # install after consent, /updatefirstmate, the afk daemon, existing # tests) still call them directly. The one seam this script needed - -# bootstrap running its detect-only diagnostics without its six mutating +# bootstrap running its detect-only diagnostics without its mutating # sweeps - is an opt-in FM_BOOTSTRAP_DETECT_ONLY=1 flag on fm-bootstrap.sh # itself (default unset/0 = unchanged behavior), not a fork. # @@ -29,11 +29,9 @@ # mutating step runs. # 2. bootstrap - home-local stale Herdr projection cleanup runs only # when this session actually holds the lock. Detect-only -# diagnostics always run. Bootstrap's six MUTATING sweeps -# (legacy PR-check migration, secondmate convergence, -# secondmate liveness, pending remote handoff retry, -# X-mode artifact writes, fleet sync) also run only when -# locked. +# diagnostics always run. Bootstrap's MUTATING sweeps +# also run only when locked; bin/fm-bootstrap.sh's own +# header owns that list. # 3. wake-drain - mutates the durable wake queue, so it also only runs # when locked. # 4. context digest - data/projects.md, data/secondmates.md, data/captain.md, @@ -70,7 +68,7 @@ # tasks-axi and quota-axi tool checks, and tasks-axi availability - none of # which mutate shared state and all of which are safe to compute without # verified lock ownership. -# Only projection cleanup, the five bootstrap mutating sweeps, the wake-queue +# Only projection cleanup, the bootstrap mutating sweeps, the wake-queue # drain, and the ruling-index rebuild are skipped. # The context digest below is always read-only, and so is every part of the # fleet-state digest except its RULING_RECONCILE step: publishing the derived diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index dbce2362389..e0c202d30a7 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -348,7 +348,8 @@ remote_secondmate_teardown() { tmp="$SECONDMATE_REG.tmp.$$" grep -vE "^- $ID( |$)" "$SECONDMATE_REG" > "$tmp" || true mv -f -- "$tmp" "$SECONDMATE_REG" - rm -f -- "$STATE/$ID.status" "$STATE/$ID.meta" "$STATE/$ID.turn-ended" "$STATE/$ID.childcpu" + rm -f -- "$STATE/$ID.status" "$STATE/$ID.meta" "$STATE/$ID.turn-ended" "$STATE/$ID.childcpu" \ + "$STATE/$ID.terminal-recorded" printf 'teardown %s complete (remote %s:%s)\n' "$ID" "$remote_host" "$remote_home" return 0 } @@ -2393,9 +2394,25 @@ retire_busy_state "$STATE" "$ID" "$BUSY_GEN" || exit 1 # harness/model/effort join for this task is available anywhere. # bin/fm-wake-ledger.sh owns the record format. Best effort by design - a # telemetry write must never stand between the fleet and cleanup. -LEDGER_OUTCOME=landed +# The outcome comes from what the task DECLARED, not from a constant: a task +# that reported failed: must not leave a record that says it landed. The +# mapping and its evidence vocabulary belong to fm-wake-ledger.sh; this reads +# the status file before the removal below deletes it, and falls back to the +# ledger's own unevidenced default if that read fails. +LEDGER_DERIVED=$(FM_WAKE_LEDGER="${FM_WAKE_LEDGER:-$DATA/wake-ledger.tsv}" \ + "$SCRIPT_DIR/fm-wake-ledger.sh" derive "$STATE/$ID.status" 2>/dev/null) \ + || LEDGER_DERIVED="" +case "$LEDGER_DERIVED" in + 'landed '*|'failed '*) ;; + *) LEDGER_DERIVED="landed assumed" ;; +esac +LEDGER_OUTCOME=${LEDGER_DERIVED%% *} +LEDGER_SOURCE=${LEDGER_DERIVED##* } +# A discard is the operator's own act and outranks the worker's last word: the +# work was thrown away whatever the task believed about itself. if [ "$FORCE" = "--force" ]; then LEDGER_OUTCOME=abandoned + LEDGER_SOURCE=discarded fi LEDGER_ESCALATED=$(meta_value "$META" escalated) case "$LEDGER_ESCALATED" in @@ -2417,6 +2434,7 @@ fi FM_WAKE_LEDGER="$LEDGER_PATH" \ "$SCRIPT_DIR/fm-wake-ledger.sh" task "$ID" \ --outcome "$LEDGER_OUTCOME" \ + --source "$LEDGER_SOURCE" \ --harness "$(meta_value "$META" harness)" \ --model "$(meta_value "$META" model)" \ --effort "$(meta_value "$META" effort)" \ @@ -2430,7 +2448,8 @@ FM_WAKE_LEDGER="$LEDGER_PATH" \ || echo "warning: wake ledger terminal line not recorded for $ID" >&2 rm -f "$STATE/$ID.status" "$STATE/$ID.turn-ended" "$STATE/$ID.meta" \ "$STATE/$ID.pi-ext.ts" "$STATE/$ID.grok-turnend-token" \ - "$STATE/$ID.kimi-turnend-token" "$STATE/$ID.childcpu" + "$STATE/$ID.kimi-turnend-token" "$STATE/$ID.childcpu" \ + "$STATE/$ID.terminal-recorded" if [ "$KIND" != scout ] && [ "$KIND" != secondmate ] && [ "$MODE" != local-only ]; then "$FM_ROOT/bin/fm-fleet-sync.sh" "$PROJ" || true fi diff --git a/bin/fm-wake-ledger.sh b/bin/fm-wake-ledger.sh index 3be5a05e39b..bbcfc7f0cb1 100755 --- a/bin/fm-wake-ledger.sh +++ b/bin/fm-wake-ledger.sh @@ -43,9 +43,52 @@ # # task one terminal line per task, written by bin/fm-teardown.sh # immediately before the task metadata is deleted - the last moment -# the harness/model/effort join exists. Fields: task, harness, -# model, effort, mode, kind, project, backend, outcome, route, -# escalated, findings, pr. +# the harness/model/effort join exists - and by this script's +# `sweep` for a task that declared failure and was never torn down. +# Fields: task, harness, model, effort, mode, kind, project, +# backend, outcome, outcome_source, route, escalated, findings, pr. +# A task id may carry more than one terminal line: `sweep` records a +# declared failure at declaration time and a later teardown records +# the same task's release. The LAST terminal line for a task id is +# its outcome; every reader here already resolves it that way. +# +# THE TERMINAL OUTCOME DEFINITION. This script owns it. +# +# landed the task's change reached its delivery target. +# failed the task ended without producing its change. +# abandoned the task's unlanded work was deliberately discarded. +# +# The enum is deliberately three members and is pinned to the v1 line schema. +# Widening it belongs to the unified terminal vocabulary that will arrive under +# a new schema token, not to a fourth member bolted onto v1. +# +# An outcome is worth nothing without its evidence, so every terminal line also +# carries WHERE ITS OUTCOME CAME FROM, in outcome_source: +# +# declared the task's own status log declared it (`done:` or `failed:`). +# discarded an operator tore the task down with --force. +# unreleased `sweep` found a declared failure with no terminal line and no +# teardown; the task was still holding state when this was +# written. +# assumed nothing corroborated the outcome. This is the historical +# default: teardown set outcome=landed as a constant, so a line +# with no outcome_source field reads as assumed rather than as +# evidence, and the pre-existing records stay honest without a +# rewrite. +# +# assumed exists because the constant is the actual defect. Before it, a fleet +# record reading "40 terminal (landed 40)" was not a success rate: nothing +# produced `failed`, so the numerator could not move. Naming the unsupported +# records is what makes the supported ones countable. +# +# DIAGNOSTIC ONLY - NO RATE. `report` prints terminal outcome COUNTS and never +# a success rate, and it says so in its own output. Two authoritative records +# still disagree about the same changes: this ledger counts tasks the fleet +# released, while the no-mistakes pipeline counts validation runs, and a task +# can hold many runs or none. Until that divergence is reconciled, a ratio +# computed here would be a number about neither. The divergence is a KNOWN +# NAMED GAP, not an oversight, and `report` names it every time it prints +# terminal outcomes. # # THE COORDINATOR NAMES THE COST, NOT THE SEQUENCE. `outcome ` with no # sequence resolves the most recent wake record no outcome record joins, and @@ -107,12 +150,33 @@ # --allow-unjoined says the wake records are genuinely gone. task and # after are resolved from the matching wake record when --task is absent. # fm-wake-ledger.sh task [--outcome landed|failed|abandoned] +# [--source declared|discarded|unreleased|assumed] # [--harness H] [--model M] [--effort E] [--mode M] # [--kind K] [--project P] [--backend B] [--pr URL] # [--route R] [--escalated yes|no] [--findings N] # Append one terminal task record. Absent facts record as unknown rather -# than being guessed. Called by teardown, which supplies them from the -# task metadata it is about to delete. +# than being guessed, and an absent --source records assumed rather than +# implying evidence nobody produced. Called by teardown, which supplies +# them from the task metadata it is about to delete. +# fm-wake-ledger.sh derive +# Print " " for a task's status log: the LAST +# `done:` or `failed:` line decides, and a log with neither prints +# "landed assumed". This is the one implementation of the mapping; +# teardown calls it rather than restating it. +# fm-wake-ledger.sh sweep [--dry-run] +# Record the failures teardown will never see. For every task in state/ +# whose status log declares `failed:`, append one terminal record with +# outcome=failed outcome_source=unreleased and leave a +# state/.terminal-recorded receipt so a rerun records nothing twice. +# A task that fails and is never torn down is otherwise SILENT in this +# ledger, and that silence is indistinguishable from a task that never +# failed. A task holding state with no declaration at all is a different +# thing and is not recorded here: it is already visible as an ordinary +# unfinished task in the fleet state a session start reads, and guessing +# a terminal outcome for it would put back the constant this replaced. +# --dry-run prints the same lines and writes nothing. Teardown +# removes the receipt with the rest of the task's state and writes its +# own release record, which supersedes this one. # fm-wake-ledger.sh reconcile [--count] # Count the outcome records that join no wake record. --count prints that # number alone, for a caller that formats its own line. Session-start @@ -495,12 +559,13 @@ cmd_outcome() { } cmd_task() { - local id='' outcome=landed route=unknown escalated=unknown findings=unknown + local id='' outcome=landed osource=assumed route=unknown escalated=unknown findings=unknown local harness=unknown model=unknown effort=unknown mode=unknown kind=unknown local project=unknown backend=unknown pr='' now while [ "$#" -gt 0 ]; do case "$1" in --outcome) [ "$#" -ge 2 ] || die "--outcome needs a value"; outcome=$2; shift 2 ;; + --source) [ "$#" -ge 2 ] || die "--source needs a value"; osource=$2; shift 2 ;; --harness) [ "$#" -ge 2 ] || die "--harness needs a value"; harness=$2; shift 2 ;; --model) [ "$#" -ge 2 ] || die "--model needs a value"; model=$2; shift 2 ;; --effort) [ "$#" -ge 2 ] || die "--effort needs a value"; effort=$2; shift 2 ;; @@ -524,6 +589,12 @@ cmd_task() { landed|failed|abandoned) ;; *) die "unknown terminal outcome: $outcome (landed failed abandoned)" ;; esac + # An unrecognized provenance is refused rather than downgraded to assumed: a + # caller that meant to record evidence must not have it silently erased. + case "$osource" in + declared|discarded|unreleased|assumed) ;; + *) die "unknown outcome source: $osource (declared discarded unreleased assumed)" ;; + esac case "$escalated" in yes|no|unknown) ;; *) die "--escalated must be yes, no, or unknown: $escalated" ;; @@ -547,6 +618,7 @@ cmd_task() { "project=$(ledger_sanitize "${project:-unknown}" "$LEDGER_SHORT_MAX")" \ "backend=$(ledger_sanitize "${backend:-unknown}" "$LEDGER_SHORT_MAX")" \ "outcome=$outcome" \ + "outcome_source=$osource" \ "route=$(ledger_sanitize "${route:-unknown}" "$LEDGER_SHORT_MAX")" \ "escalated=$escalated" \ "findings=$findings" @@ -558,6 +630,116 @@ cmd_task() { return 0 } +# --- terminal outcome derivation -------------------------------------------- + +# The one mapping from a task's own status log to its terminal outcome. The +# LAST done: or failed: line decides, so a task that failed, was recovered, and +# then reported done records landed rather than its worst moment. blocked: and +# needs-decision: are open states, not outcomes, and never decide one. A log +# with no terminal declaration yields the historical constant, explicitly +# marked assumed so it can never be counted as evidence. +# fm-classify-lib.sh owns status_line_verb, the one parser for a status line's +# leading verb; the derivation below must read the same verbs the watcher and +# the daemon read rather than carry a second parser. Loaded on first use, not +# at source time: `drain-record` runs on the wake path and never derives an +# outcome, and that path must stay as cheap as it was. +_ledger_need_status_parser() { + command -v status_line_verb >/dev/null 2>&1 && return 0 + # shellcheck source=bin/fm-classify-lib.sh + . "$SCRIPT_DIR/fm-classify-lib.sh" 2>/dev/null || return 1 + command -v status_line_verb >/dev/null 2>&1 +} + +ledger_derive_outcome() { # -> " " + local file=${1-} line verb outcome='' + _ledger_need_status_parser || return 1 + if [ -n "$file" ] && [ -f "$file" ]; then + while IFS= read -r line || [ -n "$line" ]; do + verb=$(status_line_verb "$line") + case "$verb" in + done) outcome=landed ;; + failed) outcome=failed ;; + esac + done < "$file" + fi + [ -n "$outcome" ] || { printf 'landed assumed'; return 0; } + printf '%s declared' "$outcome" +} + +# One key's value from a task metadata file, last assignment winning. Absent +# key and absent file both yield the empty string; every caller substitutes +# unknown rather than guessing. +ledger_meta_value() { # + [ -f "$1" ] || return 0 + LC_ALL=C awk -F= -v k="$2" '$1 == k { sub(/^[^=]*=/, ""); v = $0 } END { print v }' "$1" +} + +cmd_derive() { + local file='' derived + while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) usage; exit 0 ;; + -*) die "unknown flag for derive: $1" ;; + *) [ -z "$file" ] || die "derive takes one status file"; file=$1; shift ;; + esac + done + [ -n "$file" ] || die "derive needs a status file" + # A refusal is louder than a wrong outcome: teardown falls back to the + # unevidenced default when this exits nonzero, which is the old behavior. + derived=$(ledger_derive_outcome "$file") || die "the status-line parser is unavailable" + printf '%s\n' "$derived" +} + +cmd_sweep() { + local dry='' meta id derived outcome receipt recorded=0 + while [ "$#" -gt 0 ]; do + case "$1" in + --dry-run) dry=1; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown flag for sweep: $1" ;; + esac + done + + [ -d "$STATE" ] || return 0 + for meta in "$STATE"/*.meta; do + [ -f "$meta" ] || continue + id=$(basename "$meta" .meta) + fm_task_id_path_safe "$id" || continue + receipt="$STATE/$id.terminal-recorded" + # The receipt is the whole idempotence story: this sweep runs at every + # locked session start, and without it a task holding a declared failure + # would append a fresh terminal record on every one of them. + if [ -e "$receipt" ] || [ -L "$receipt" ]; then continue; fi + derived=$(ledger_derive_outcome "$STATE/$id.status") \ + || die "the status-line parser is unavailable" + outcome=${derived%% *} + # Only a declared failure is recorded here. A declared done: or an + # undeclared task is teardown's to record: teardown is still coming for it, + # and its record carries facts this sweep cannot see. + [ "$outcome" = failed ] || continue + printf 'unreleased failure: %s\n' "$id" + [ -z "$dry" ] || continue + cmd_task "$id" \ + --outcome failed \ + --source unreleased \ + --harness "$(ledger_meta_value "$meta" harness)" \ + --model "$(ledger_meta_value "$meta" model)" \ + --effort "$(ledger_meta_value "$meta" effort)" \ + --mode "$(ledger_meta_value "$meta" mode)" \ + --kind "$(ledger_meta_value "$meta" kind)" \ + --project "$(ledger_meta_value "$meta" project)" \ + --backend "$(ledger_meta_value "$meta" backend)" \ + --route "$(ledger_meta_value "$meta" route)" || continue + # Written only after the record is durable. A receipt ahead of the append + # would suppress the retry that a failed append needs. + printf 'failed\n' > "$receipt" 2>/dev/null || true + recorded=$((recorded + 1)) + done + [ -n "$dry" ] || [ "$recorded" -eq 0 ] \ + || printf 'recorded %s unreleased failure(s) as terminal records\n' "$recorded" + return 0 +} + cmd_reconcile() { local count_only='' counts unjoined total while [ "$#" -gt 0 ]; do @@ -682,6 +864,10 @@ cmd_report() { # The most recent terminal line wins if a task id was ever reused. profile[t] = f["harness"] "/" f["model"] "/" f["effort"] terminal[t] = f["outcome"] + # A record written before outcome_source existed carries no evidence, and + # that is exactly what assumed means, so the absent field maps onto it + # rather than needing the file rewritten. + terminal_source[t] = (f["outcome_source"] == "" ? "assumed" : f["outcome_source"]) next } END { @@ -711,7 +897,11 @@ cmd_report() { } tasks = 0 - for (t in terminal) { tasks++; term_count[terminal[t]]++ } + for (t in terminal) { + tasks++ + term_count[terminal[t]]++ + src_count[terminal_source[t]]++ + } printf "\ntasks: %d terminal", tasks if (tasks > 0) { printf " (" @@ -724,6 +914,19 @@ cmd_report() { printf ")" } printf "\n" + if (tasks > 0) { + printf " by evidence:" + split("declared discarded unreleased assumed", sv, " ") + for (i = 1; i <= 4; i++) if (sv[i] in src_count) printf " %s %d", sv[i], src_count[sv[i]] + printf "\n" + # Counts only, never a rate. Two records still disagree about the same + # changes, so any ratio printed here would describe neither of them. + printf " counts are DIAGNOSTIC ONLY - not a success rate:" + printf " assumed outcomes carry no evidence," + printf " and this ledger counts released tasks while the no-mistakes" + printf " pipeline counts validation runs.\n" + printf " that fleet/pipeline divergence is a known, named, unreconciled gap.\n" + } if (tasks > 0) { printf "\nper profile (harness/model/effort):\n" @@ -794,8 +997,10 @@ case "$SUBCOMMAND" in drain-record) cmd_drain_record "$@" ;; outcome) cmd_outcome "$@" ;; task) cmd_task "$@" ;; + derive) cmd_derive "$@" ;; + sweep) cmd_sweep "$@" ;; reconcile) cmd_reconcile "$@" ;; report) cmd_report "$@" ;; -h|--help|help) usage ;; - *) die "unknown subcommand: $SUBCOMMAND (drain-record outcome task reconcile report)" ;; + *) die "unknown subcommand: $SUBCOMMAND (drain-record outcome task derive sweep reconcile report)" ;; esac diff --git a/docs/architecture.md b/docs/architecture.md index 2c8c75c957d..d2cfabbff99 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -64,9 +64,13 @@ The script header owns the exact JSON schema. `data/wake-ledger.tsv` is the durable record of what supervision actually costs, so coordinator attention is measured rather than estimated. `bin/fm-wake-ledger.sh` is its single owner: record format, the closed outcome vocabulary, and append semantics all live in that script's header and `--help`. -It carries three record kinds - one `wake` record per drained wake, one `outcome` record per handled wake joined to it on the (seq, queued) pair, and one terminal `task` record per finished task. +It carries three record kinds - one `wake` record per drained wake, one `outcome` record per handled wake joined to it on the (seq, queued) pair, and terminal `task` records naming how each task ended. It lives under `data/` rather than `state/` because teardown clears `state/.*` and this evidence must outlive the tasks it describes. +A terminal outcome is derived from what the task itself declared, and every terminal record names the evidence behind it, so an outcome nothing corroborated cannot pass as an observed one. +Teardown is not the only producer, because a task that fails and is never released would otherwise leave the ledger silent, and silence there is indistinguishable from a task that never failed. +Terminal counts remain diagnostic rather than a success rate while this ledger counts released tasks and the no-mistakes pipeline counts validation runs; that divergence is an unreconciled gap the report names on every run. + The wake half is written deterministically by `bin/fm-wake-drain.sh` and the outcome half by the first mate. That split is the point rather than an accident: a single coordinator-written line would make measured attention cost fall whenever the recording step was skipped, so the metric would move without the underlying quantity moving. Splitting it gives a denominator that is stable under no change and turns missing coverage into a reported number instead of a silent undercount. diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index efabc83646d..d392b843181 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -1212,6 +1212,84 @@ test_validation_daemon_unused_root_is_silent() { pass "bootstrap stays silent when no daemon root exists to check" } +# The terminal sweep's session-start wiring. A locked session records declared +# failures and reports how many records it actually wrote; a read-only session +# reports the same tasks without recording them; a home with no declared +# failure stays silent on both paths. The unwritable-ledger control pins the +# count's provenance: a sweep that discovers a failure but cannot append must +# not report records as written. +test_wake_ledger_terminal_sweep_reports_only_durable_records() { + local case_dir fakebin home file out + case_dir="$TMP_ROOT/wake-ledger-terminal-sweep" + home="$case_dir/home" + mkdir -p "$home/config" "$home/data" "$home/state" + printf '%s\n' manual > "$home/config/backlog-backend" + fakebin=$(make_fake_toolchain "$case_dir") + file="$home/data/wake-ledger.tsv" + + bootstrap_out() { + PATH="$fakebin:$BASE_PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$home" \ + FM_BOOTSTRAP_DETECT_ONLY="${1:-0}" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh" + } + + # Negative control first: a task with no declared failure must keep both + # branches silent, so the positive cases below cannot be the sweep firing + # indiscriminately over every task in state/. + printf 'window=fm-alpha\n' > "$home/state/alpha.meta" + printf 'working: running\n' > "$home/state/alpha.status" + out=$(bootstrap_out) + assert_not_contains "$out" "BOOTSTRAP_INFO: recorded" \ + "a home with no declared failure must not report recorded failures" + out=$(bootstrap_out 1) + assert_not_contains "$out" "WAKE_LEDGER" \ + "a read-only session with no declared failure must stay silent" + [ ! -f "$file" ] || fail "terminal sweep: a home with no declared failure grew a ledger" + + printf 'harness=pi\nmodel=sol\nbackend=tmux\n' > "$home/state/beta.meta" + printf 'working: running\nfailed: the approach does not work\n' > "$home/state/beta.status" + + # The read-only branch reports the failure it declined to record and must + # write neither a record nor a receipt. + out=$(bootstrap_out 1) + assert_contains "$out" "WAKE_LEDGER: 1 task(s) declared failure with no terminal record" \ + "a read-only session must report the failure it declined to record" + assert_not_contains "$out" "BOOTSTRAP_INFO: recorded" \ + "a read-only session must not claim records were written" + [ ! -f "$file" ] || fail "terminal sweep: a read-only session wrote a record" + [ ! -e "$home/state/beta.terminal-recorded" ] \ + || fail "terminal sweep: a read-only session left a receipt" + + # A discovered failure whose append cannot land must not read as success. + chmod 500 "$home/data" + out=$(bootstrap_out) + chmod 700 "$home/data" + assert_not_contains "$out" "BOOTSTRAP_INFO: recorded" \ + "a failed append must not be reported as a written record" + [ ! -e "$home/state/beta.terminal-recorded" ] \ + || fail "terminal sweep: a failed append left a receipt" + + # With the ledger writable again the same locked session start records the + # failure, so the failed append above genuinely retried. + out=$(bootstrap_out) + assert_contains "$out" "BOOTSTRAP_INFO: recorded 1 declared task failure(s) that no teardown would have recorded" \ + "a locked session must record and report the declared failure" + grep -q 'task=beta' "$file" || fail "terminal sweep: the failed task was not recorded" + grep -q 'outcome=failed' "$file" || fail "terminal sweep: the outcome was not recorded as failed" + grep -q 'outcome_source=unreleased' "$file" \ + || fail "terminal sweep: the record did not say it came from an unreleased task" + [ -e "$home/state/beta.terminal-recorded" ] || fail "terminal sweep: no receipt was written" + + # Receipt idempotence through bootstrap: a rerun must not record or report + # the same failure again. + out=$(bootstrap_out) + assert_not_contains "$out" "BOOTSTRAP_INFO: recorded" \ + "a rerun re-reported an already recorded failure" + [ "$(grep -c 'task=beta' "$file")" -eq 1 ] \ + || fail "terminal sweep: a rerun appended a second record for the same task" + pass "session start records declared failures when locked, reports them read-only, and counts only durable appends" +} + test_bootstrap_reporting test_no_mistakes_min_version test_gh_axi_min_version @@ -1243,3 +1321,4 @@ test_validation_daemon_down_without_a_log_omits_the_outage_length test_validation_daemon_missing_pid_file_is_down test_validation_daemon_malformed_pid_file_is_unknown_not_down test_validation_daemon_unused_root_is_silent +test_wake_ledger_terminal_sweep_reports_only_durable_records diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 1d5eb9e6d6b..6661ad7dfaa 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -6,7 +6,7 @@ # Coverage: # - absent-file markers vs empty-but-present files in the context digest # - the lock-refusal read-only path: banner leads, every mutating step is -# skipped (including bootstrap's five mutating sweeps, verified by their +# skipped (including bootstrap's mutating sweeps, verified by their # ABSENCE), the digest still completes # - output section ordering: diagnostics/banners lead, bulk file dumps follow # - context-aware next-step guidance for read-only, AFK, X mode, and normal diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index ae5d386c7bb..0de3f85532d 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -2351,6 +2351,87 @@ test_teardown_force_records_abandoned() { pass "a --force teardown records the task as abandoned rather than landed" } +test_teardown_records_a_declared_failure_as_failed() { + local case_dir ledger + case_dir=$(make_case ledger-failed) + write_profiled_meta "$case_dir" local-only + # The task said it failed. Its work still landed on a remote, so this is an + # ordinary release, not a --force discard: without the derivation the record + # would read landed and be indistinguishable from a success. + printf 'working: started\nfailed: the approach does not work\n' \ + > "$case_dir/state/task-x1.status" + wt_commit "$case_dir" "abandoned approach" + add_fork_with_pushed_branch "$case_dir" + + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" \ + || fail "ledger-failed: teardown should succeed" + + ledger="$case_dir/data/wake-ledger.tsv" + grep -q "outcome=failed" "$ledger" \ + || fail "ledger-failed: a declared failure recorded no failed outcome:"$'\n'"$(cat "$ledger")" + grep -q "outcome_source=declared" "$ledger" \ + || fail "ledger-failed: the outcome did not record the task's own declaration as its evidence" + if grep -q "outcome=landed" "$ledger"; then + fail "ledger-failed: a failed task must not also record landed" + fi + pass "a task that declares failure and is torn down records failed, not landed" +} + +test_teardown_marks_an_uncorroborated_outcome_as_assumed() { + local case_dir ledger + case_dir=$(make_case ledger-assumed) + write_profiled_meta "$case_dir" local-only + # No terminal declaration anywhere in the log. landed is still recorded - it + # is the only thing teardown can say - but it must be marked as the + # unevidenced constant it is, or it counts as a success nobody observed. + printf 'working: still going\n' > "$case_dir/state/task-x1.status" + wt_commit "$case_dir" "fix the thing" + add_fork_with_pushed_branch "$case_dir" + + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" \ + || fail "ledger-assumed: teardown should succeed" + + ledger="$case_dir/data/wake-ledger.tsv" + grep -q "outcome=landed" "$ledger" || fail "ledger-assumed: expected the landed default" + grep -q "outcome_source=assumed" "$ledger" \ + || fail "ledger-assumed: an uncorroborated landed outcome must be marked assumed" + pass "a landed outcome no declaration corroborates records as assumed" +} + +test_teardown_supersedes_and_clears_a_sweep_receipt() { + local case_dir ledger terminal_lines + case_dir=$(make_case ledger-supersede) + write_profiled_meta "$case_dir" local-only + printf 'failed: the approach does not work\n' > "$case_dir/state/task-x1.status" + wt_commit "$case_dir" "abandoned approach" + add_fork_with_pushed_branch "$case_dir" + + # The sweep records the declared failure first, while the task still holds + # state; teardown then releases the same task. + FM_ROOT_OVERRIDE="$ROOT" FM_STATE_OVERRIDE="$case_dir/state" \ + FM_DATA_OVERRIDE="$case_dir/data" \ + "$ROOT/bin/fm-wake-ledger.sh" sweep >/dev/null \ + || fail "ledger-supersede: sweep should succeed" + [ -f "$case_dir/state/task-x1.terminal-recorded" ] \ + || fail "ledger-supersede: sweep left no receipt" + + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" \ + || fail "ledger-supersede: teardown should succeed" + + ledger="$case_dir/data/wake-ledger.tsv" + terminal_lines=$(grep -c 'task=task-x1' "$ledger" || true) + [ "$terminal_lines" -eq 2 ] \ + || fail "ledger-supersede: expected the sweep record and teardown's release record, got $terminal_lines" + grep -q "outcome_source=unreleased" "$ledger" \ + || fail "ledger-supersede: the sweep's record is missing" + # The last word is teardown's, and it is what a reader resolves. + [ "$(grep 'task=task-x1' "$ledger" | tail -1 | grep -c 'outcome_source=declared')" -eq 1 ] \ + || fail "ledger-supersede: teardown's release record did not supersede the sweep's" + [ ! -e "$case_dir/state/task-x1.terminal-recorded" ] \ + || fail "ledger-supersede: the receipt outlived the task's state" + pass "teardown supersedes a sweep record and clears its receipt with the rest of the state" +} + test_unwritable_ledger_never_fails_teardown() { local case_dir rc case_dir=$(make_case ledger-unwritable) @@ -3053,6 +3134,9 @@ test_persistent_index_lock_exhausts_retries_and_refuses_loudly test_empty_retry_wait_uses_default_without_aborting test_fractional_legacy_retry_wait_refuses_without_arithmetic_error test_teardown_records_the_terminal_ledger_line +test_teardown_records_a_declared_failure_as_failed +test_teardown_marks_an_uncorroborated_outcome_as_assumed +test_teardown_supersedes_and_clears_a_sweep_receipt test_teardown_force_records_abandoned test_unwritable_ledger_never_fails_teardown test_parked_own_run_is_aborted_before_teardown diff --git a/tests/fm-wake-ledger.test.sh b/tests/fm-wake-ledger.test.sh index 7311032f5ff..3dfe52b08a9 100755 --- a/tests/fm-wake-ledger.test.sh +++ b/tests/fm-wake-ledger.test.sh @@ -8,7 +8,14 @@ # an unwritable ledger leaves the drain's raw rows and exit status untouched, # and a deliberately slowed ledger phase never blocks a concurrent wake append. # -# Teardown's terminal-record write is covered where teardown's own fixture +# It also covers the terminal-outcome definition: the derivation from a task's +# own declaration, the closed evidence vocabulary, and the sweep that records a +# failure no teardown will ever see. That last one is the case that matters - +# a task that fails and is never torn down was silent here, and silence is +# indistinguishable from a task that never failed, so covering only the +# torn-down path would miss the actual defect. +# +# Teardown's own terminal-record write is covered where teardown's fixture # lives, in tests/fm-teardown.test.sh. set -u @@ -494,6 +501,165 @@ test_an_explicit_joinable_sequence_records_unchanged() { pass "an explicit joinable sequence, including several at once, records exactly as before" } +test_the_terminal_outcome_derivation_reads_the_task_declaration() { + local home status + home=$(make_home derive) + status="$home/state/alpha.status" + + [ "$(ledger "$home" derive "$home/state/absent.status")" = "landed assumed" ] \ + || fail "derive: a task with no status log must yield the unevidenced default" + + printf 'working: started\n' > "$status" + [ "$(ledger "$home" derive "$status")" = "landed assumed" ] \ + || fail "derive: progress alone is not evidence of an outcome" + + # An open state is not an outcome. A blocked or parked task that is later + # released normally landed its work; treating either as failure would invent + # failures out of ordinary supervision traffic. + printf 'blocked: needs a credential\nneeds-decision: which base\n' >> "$status" + [ "$(ledger "$home" derive "$status")" = "landed assumed" ] \ + || fail "derive: blocked and needs-decision are open states, not outcomes" + + printf 'failed: the approach does not work\n' >> "$status" + [ "$(ledger "$home" derive "$status")" = "failed declared" ] \ + || fail "derive: a declared failure must derive failed, with the declaration as its evidence" + + # Last declaration wins: a task that failed, was recovered and then shipped + # is a landed task, not a permanent failure. + printf 'done: PR merged\n' >> "$status" + [ "$(ledger "$home" derive "$status")" = "landed declared" ] \ + || fail "derive: a later done: must supersede an earlier failed:" + + pass "the terminal outcome derives from the task's own last declaration" +} + +test_terminal_outcome_source_is_closed_and_defaults_to_assumed() { + local home file src + home=$(make_home outcome-source) + file=$(ledger_file "$home") + + ledger "$home" task alpha --outcome failed --source declared \ + || fail "outcome-source: a declared failure should be accepted" + [ "$(last_field_of "$file" task outcome_source)" = declared ] \ + || fail "outcome-source: the evidence was not recorded" + + # An absent --source must never imply evidence nobody produced. + ledger "$home" task beta --outcome landed || fail "outcome-source: default source failed" + [ "$(last_field_of "$file" task outcome_source)" = assumed ] \ + || fail "outcome-source: an unstated evidence must record assumed" + + for src in declared discarded unreleased assumed; do + ledger "$home" task "src-$src" --outcome landed --source "$src" >/dev/null \ + || fail "outcome-source: $src should be accepted" + done + for src in observed guessed '' DECLARED declared-ish; do + if ledger "$home" task rejected --outcome landed --source "$src" >/dev/null 2>&1; then + fail "outcome-source: '$src' should have been refused" + fi + done + if grep -q 'task=rejected' "$file"; then + fail "outcome-source: a refused evidence token still wrote a record" + fi + pass "the outcome-source vocabulary is closed and an unstated evidence records assumed" +} + +# The defect this whole increment exists for: only teardown wrote a terminal +# line, so a task that failed and was NEVER torn down left the ledger silent, +# and that silence is indistinguishable from a task that never failed. A test +# that covers only the torn-down case misses exactly this. +test_a_failure_that_is_never_torn_down_is_recorded_not_silent() { + local home file out before + home=$(make_home sweep) + file=$(ledger_file "$home") + + printf 'window=fm:alpha\n' > "$home/state/alpha.meta" + printf 'working: running\n' > "$home/state/alpha.status" + printf 'harness=pi\nmodel=sol\neffort=high\nkind=ship\nmode=no-mistakes\nbackend=tmux\n' \ + > "$home/state/beta.meta" + printf 'working: running\nfailed: the approach does not work\n' > "$home/state/beta.status" + + # Negative control first: with no failure declared anywhere, the sweep must + # record nothing, so the positive result below cannot be the sweep firing + # indiscriminately over every task in state/. + mv "$home/state/beta.status" "$home/state/beta.status.held" + out=$(ledger "$home" sweep) || fail "sweep: control run failed" + [ -z "$out" ] || fail "sweep: recorded something with no declared failure:"$'\n'"$out" + [ ! -f "$file" ] || fail "sweep: wrote a record with no declared failure" + mv "$home/state/beta.status.held" "$home/state/beta.status" + + out=$(ledger "$home" sweep --dry-run) || fail "sweep: dry run failed" + printf '%s\n' "$out" | grep -q 'unreleased failure: beta' \ + || fail "sweep: dry run did not name the unreleased failure:"$'\n'"$out" + [ ! -f "$file" ] || fail "sweep: a dry run wrote a record" + [ ! -e "$home/state/beta.terminal-recorded" ] || fail "sweep: a dry run left a receipt" + + ledger "$home" sweep >/dev/null || fail "sweep: recording run failed" + [ "$(last_field_of "$file" task task)" = beta ] || fail "sweep: the failed task was not recorded" + [ "$(last_field_of "$file" task outcome)" = failed ] || fail "sweep: outcome not recorded as failed" + [ "$(last_field_of "$file" task outcome_source)" = unreleased ] \ + || fail "sweep: the record did not say it came from an unreleased task" + # The profile join is the reason the record is worth writing now rather than + # inferring it later: the metadata that carries it is still on disk. + [ "$(last_field_of "$file" task harness)" = pi ] || fail "sweep: harness not captured" + [ "$(last_field_of "$file" task model)" = sol ] || fail "sweep: model not captured" + if grep -q 'task=alpha' "$file"; then + fail "sweep: a task with no declared failure was recorded" + fi + [ -e "$home/state/beta.terminal-recorded" ] || fail "sweep: no receipt was written" + + # Idempotence, with its own negative control: removing the receipt must make + # the sweep record again, so the quiet rerun below proves the receipt works + # rather than proving the sweep stopped finding anything. + before=$(wc -l < "$file" | tr -d ' ') + ledger "$home" sweep >/dev/null || fail "sweep: rerun failed" + [ "$(wc -l < "$file" | tr -d ' ')" -eq "$before" ] \ + || fail "sweep: a rerun appended a second record for the same task" + rm -f "$home/state/beta.terminal-recorded" + ledger "$home" sweep >/dev/null || fail "sweep: post-control run failed" + [ "$(wc -l < "$file" | tr -d ' ')" -eq $((before + 1)) ] \ + || fail "sweep: removing the receipt did not make the sweep record again" + pass "a declared failure that is never torn down is recorded once, not silent" +} + +test_the_report_names_evidence_and_refuses_a_rate() { + local home file out + home=$(make_home terminal-report) + file=$(ledger_file "$home") + + ledger "$home" task alpha --outcome landed --source declared || fail "report: task failed" + ledger "$home" task beta --outcome failed --source declared || fail "report: task failed" + # A record written before this field existed. It must read as assumed rather + # than needing an append-only file rewritten. + printf 'v1\ttask\t%s\ttask=gamma\toutcome=landed\tharness=pi\tmodel=sol\teffort=low\n' \ + "$(date +%s)" >> "$file" + + out=$(ledger "$home" report) || fail "report: failed" + printf '%s\n' "$out" | grep -q 'tasks: 3 terminal (landed 2 failed 1)' \ + || fail "report: terminal counts wrong:"$'\n'"$out" + printf '%s\n' "$out" | grep -q 'by evidence:.*declared 2' \ + || fail "report: declared evidence not counted:"$'\n'"$out" + printf '%s\n' "$out" | grep -q 'by evidence:.*assumed 1' \ + || fail "report: a record predating outcome_source must count as assumed:"$'\n'"$out" + printf '%s\n' "$out" | grep -q 'DIAGNOSTIC ONLY - not a success rate' \ + || fail "report: terminal counts printed without their diagnostic-only qualifier:"$'\n'"$out" + printf '%s\n' "$out" | grep -q 'known, named, unreconciled gap' \ + || fail "report: the fleet/pipeline divergence was not named:"$'\n'"$out" + # The certification is the absence of a ratio, so assert it directly: the + # only permitted mention of a success rate is the refusal to print one, and + # no percentage may appear anywhere in the terminal-outcome section. + if printf '%s\n' "$out" | grep -o 'success rate' | grep -qv '^success rate$'; then + fail "report: unexpected success-rate text:"$'\n'"$out" + fi + if [ "$(printf '%s\n' "$out" | grep -c 'not a success rate')" \ + -ne "$(printf '%s\n' "$out" | grep -c 'success rate')" ]; then + fail "report: printed a success rate over unreconciled counts:"$'\n'"$out" + fi + if printf '%s\n' "$out" | sed -n '/^tasks:/,/^$/p' | grep -q '%'; then + fail "report: printed a ratio in the terminal-outcome section:"$'\n'"$out" + fi + pass "the report breaks outcomes down by evidence and refuses to print a rate" +} + test_reconcile_counts_outcomes_that_join_no_wake_record() { local home out now home=$(make_home reconcile) @@ -546,3 +712,7 @@ test_a_bare_outcome_records_against_the_newest_unrecorded_wake test_an_unjoinable_sequence_is_refused_without_the_override test_an_explicit_joinable_sequence_records_unchanged test_reconcile_counts_outcomes_that_join_no_wake_record +test_the_terminal_outcome_derivation_reads_the_task_declaration +test_terminal_outcome_source_is_closed_and_defaults_to_assumed +test_a_failure_that_is_never_torn_down_is_recorded_not_silent +test_the_report_names_evidence_and_refuses_a_rate