From c880c0b3a6186525068614069f5338810f8605ca Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 07:52:46 -0500 Subject: [PATCH 1/5] fix: scope open-work ledger rows to what each home owns --- bin/fm_open_loops.py | 108 ++++++++++++++++++++++++---- docs/configuration.md | 20 ++++-- tests/fm-open-loops.test.sh | 114 +++++++++++++++++++++++++++++- tests/fm-watch-open-loops.test.sh | 8 +-- 4 files changed, 223 insertions(+), 27 deletions(-) diff --git a/bin/fm_open_loops.py b/bin/fm_open_loops.py index 1772e9d2599..ded1ae07f18 100755 --- a/bin/fm_open_loops.py +++ b/bin/fm_open_loops.py @@ -33,6 +33,10 @@ r"|ssh://(?:git@)?github[.]com/|git@github[.]com:)" r"([^/]+/[^/]+?)(?:[.]git)?") PIPELINE_ENDED = {"completed", "failed", "cancelled", "aborted"} +# A worker that recorded its own stop is not missing: done or paused in its status log, or a +# done, parked, or paused current state. +STOP_EVENTS = {"done", "paused"} +STOP_STATES = {"done", "parked", "paused"} SIGNATURE = re.compile(r"usage.?limit|rate.?limit|quota|auth|unauthorized|login|network|connection" r"|timed? ?out|ECONN|ENOTFOUND|\b(?:error|failure|failed|exception|fatal)\b" r"|\b(?-i:E[A-Z][A-Z0-9_]{2,})\b", re.I) @@ -83,6 +87,22 @@ def __init__(self, home, now): self.backlog = [] self.prs = {} self.state_files = [] + self.owned_urls = set() + self.owned_branches = set() + self.unowned = {} + self.lane = self.is_secondmate_home() + + def is_secondmate_home(self): + """A genuine secondmate-home marker: a regular file holding a valid id (fm-primary-scope-lib.sh).""" + marker = self.home / ".fm-secondmate-home" + try: + if marker.is_symlink() or not marker.is_file(): + return False + first = marker.read_text().splitlines()[:1] + except (OSError, UnicodeDecodeError): + return False + ident = re.sub(r"\s", "", first[0]) if first else "" + return bool(ident) and re.fullmatch(r"[A-Za-z0-9._-]+", ident) is not None def load_config(self): path = self.config / "open-loops.json" @@ -194,13 +214,17 @@ def git(self, repo, *args): def bash(self, script, *args): return self.run(["bash", "-c", script, "open-loops", *args]) - def add(self, category, subject, action, since=None, owner="firstmate", evidence=""): + def add(self, category, subject, action, since=None, owner="firstmate", evidence="", informational=False): + """An informational row is visible context, never an obligation: it is not overdue at any age.""" identity = hashlib.sha256(f"{self.home}\0{category}\0{subject}".encode()).hexdigest()[:24] age = None if since is None or since > self.now else self.now - since limit = self.ages[category] - self.rows[identity] = dict(id=identity, category=category, subject=str(subject), owner=owner, - next_action=action, age_seconds=age, limit_seconds=limit, - overdue=age is None or age >= limit, evidence=str(evidence)[:400]) + row = dict(id=identity, category=category, subject=str(subject), owner=owner, + next_action=action, age_seconds=age, limit_seconds=limit, + overdue=not informational and (age is None or age >= limit), evidence=str(evidence)[:400]) + if informational: + row["informational"] = True + self.rows[identity] = row def source(self, name, reader, *args): """Run one reader; its failure marks the ledger partly blind and never aborts the others.""" @@ -227,20 +251,35 @@ def read_snapshot(self): if not record.get("structured") and record.get("state") != "done": self.degraded.append("unstructured backlog row: " + str(record.get("raw", ""))[:100]) + def recorded_stop(self, task): + """True when the worker's own durable record says it stopped on purpose, so its endpoint + being gone or its live state being unreadable is expected rather than a loss.""" + if task["current_state"].get("state") == "failed": + return False + last = ((task.get("paths") or {}).get("status_log") or {}).get("last_event") or {} + return task["current_state"].get("state") in STOP_STATES or last.get("state") in STOP_EVENTS + def backlog_rows(self): by_id = {t["id"]: t for t in self.tasks} + dispatched = {p.name.rsplit(".", 1)[0] for p in self.state_files if p.name.endswith((".meta", ".status"))} today = dt.datetime.fromtimestamp(self.now, dt.timezone.utc).strftime("%Y-%m-%d") for record in self.backlog: since = epoch(record.get("since")) task = by_id.get(record["id"]) if record["state"] == "in_flight" and record.get("requires_child_metadata"): - if not task or task["endpoint"].get("exists") is False: + if not task or (task["endpoint"].get("exists") is False and not self.recorded_stop(task)): self.add("missing_worker", record["id"], "recover the assigned worker without discarding work", since) elif (record["state"] == "queued" and not record.get("hold_kind") and not record.get("unresolved_blocker_ids") and (not record.get("hold_until") or record["hold_until"] <= today)): - self.add("ready_not_started", record["id"], "dispatch the dependency-cleared work", since) + # Only work whose dispatch began (a task record or status log exists) is an obligation; + # the rest of the queue is the captain's backlog, shown but never overdue. + selected = record["id"] in by_id or record["id"] in dispatched + self.add("ready_not_started", record["id"], + "dispatch the dependency-cleared work" if selected + else "informational: queued and not selected for dispatch", + since, informational=not selected) def liveness(self, task): alive = task["endpoint"].get("agent_alive") @@ -257,13 +296,20 @@ def liveness(self, task): def worker_rows(self): for task in self.tasks: - alive = self.source("worker liveness " + task["id"], self.liveness, task) - if (task["current_state"].get("state") == "unknown" + stopped = self.recorded_stop(task) + if stopped: # a recorded stop explains an unreadable endpoint; it never degrades coverage + try: + alive = self.liveness(task) + except SOURCE_ERRORS: + alive = None + else: + alive = self.source("worker liveness " + task["id"], self.liveness, task) + if (task["current_state"].get("state") == "unknown" and not stopped and not (alive in ("dead", "missing") and task["current_state"].get("detail", "").startswith("backend target gone:"))): self.degraded.append("worker current state " + task["id"] + ": " + (task["current_state"].get("detail") or "observation unavailable")) - if task["endpoint"].get("exists") is False or alive in ("dead", "missing"): + if not stopped and (task["endpoint"].get("exists") is False or alive in ("dead", "missing")): self.add("missing_worker", task["id"], "recover the assigned worker without discarding work", mtime(self.state / (task["id"] + ".status")) or mtime(self.state / (task["id"] + ".meta"))) if task["current_state"].get("state") == "failed": @@ -442,14 +488,31 @@ def pr_state(self, url): self.prs[url] = pr return pr + def load_ownership(self): + """PRs this home owns: recorded on its tasks or backlog, or opened from a task's branch.""" + self.owned_urls = {t["pr"]["url"] for t in self.tasks if t["pr"].get("url")} + self.owned_branches = {t["branch"] for t in self.tasks if t.get("branch")} + for record in self.backlog: + for link in [record.get("pr_url"), *(record.get("links") or [])]: + if isinstance(link, str) and GITHUB_PR.fullmatch(link): + self.owned_urls.add(link) + + def owns_pr(self, pr): + return pr["html_url"] in self.owned_urls or (pr.get("head") or {}).get("ref") in self.owned_branches + def repo_slugs(self): paths = {t["project"] for t in self.tasks if t.get("project")} - if (self.home / ".git").exists(): - paths.add(str(self.home)) - if self.projects_dir.is_dir(): - projects = self.source("projects inventory", lambda: list(self.projects_dir.iterdir())) or [] - paths |= {str(p) for p in projects if (p / ".git").exists()} + if not self.lane: # a lane's shared clones are not its obligations; only the main home surveys them + if (self.home / ".git").exists(): + paths.add(str(self.home)) + if self.projects_dir.is_dir(): + projects = self.source("projects inventory", lambda: list(self.projects_dir.iterdir())) or [] + paths |= {str(p) for p in projects if (p / ".git").exists()} slugs = set() + for url in self.owned_urls: + match = GITHUB_PR.fullmatch(url) + if match: + slugs.add(match[1]) for task in self.tasks: url = task["pr"].get("url") if not url: @@ -523,7 +586,20 @@ def pr_rows(self, slug): self.pr_state(url) continue self.prs[url] = pr - self.source(f"PR {slug}#{pr.get('number', '?')}", self.pr_row, slug, pr) + if self.owns_pr(pr): + self.source(f"PR {slug}#{pr.get('number', '?')}", self.pr_row, slug, pr) + else: + self.unowned.setdefault(slug, []).append(pr.get("number", "?")) + + def unowned_row(self): + """Project PRs with no fleet owner: one informational row in the main home, nothing in a lane.""" + if self.lane or not self.unowned: + return + total = sum(len(numbers) for numbers in self.unowned.values()) + listing = "; ".join(f"{slug}: " + " ".join("#" + str(n) for n in sorted(numbers, key=str)) + for slug, numbers in sorted(self.unowned.items())) + self.add("open_pr", "unowned project PRs", "informational: no fleet owner; route one only on the captain's word", + None, "none", f"{total} open PR(s) with no task or backlog owner - {listing}", informational=True) def question_rows(self): for record in self.backlog: @@ -553,9 +629,11 @@ def collect(self): self.source("state inventory", self.read_state_inventory) self.source("fleet snapshot", self.read_snapshot) self.source("backlog", self.backlog_rows) + self.source("ownership", self.load_ownership) slugs = self.source("project origins", self.repo_slugs) or [] for slug in slugs: self.source("open PRs " + slug, self.pr_rows, slug) + self.unowned_row() self.source("workers", self.worker_rows) self.source("questions", self.question_rows) if self.degraded: diff --git a/docs/configuration.md b/docs/configuration.md index be55a0081b1..72242ea5633 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -579,23 +579,33 @@ An unknown age stays `null` and counts as overdue, and an age equal to its limit | Category | A row exists when | Default limit | | --- | --- | --- | -| `missing_worker` | an In flight item requiring child metadata lacks a task record or endpoint, or an ordinary task has a confirmed missing or dead worker | 600 s | -| `ready_not_started` | a Queued item has no hold kind or unresolved blocker, and its optional hold date is absent or due | 1800 s | +| `missing_worker` | an In flight item requiring child metadata lacks a task record or endpoint, or an ordinary task has a confirmed missing or dead worker that did not record a stop | 600 s | +| `ready_not_started` | a Queued item has no hold kind or unresolved blocker, and its optional hold date is absent or due; it is overdue only once dispatch has begun | 1800 s | | `unanswered_question` | a `needs-decision` or `blocked` status key is still open, or a current captain-held backlog row is in the `live` or `aged` hold bucket | 1800 s | | `failed_task` | a task record's current state is `failed` and its deliverable is neither landed nor recorded as dropped | 1800 s | | `stalled_worker` | a live worker reads `working` but has no recent commit, status line, or pipeline progress within its limit; the row carries the last error-looking line from a bounded pane-tail sample when present, otherwise a no-progress explanation | 3600 s | | `unlanded_commit` | a ship task's copy holds nonmerge commits absent from the default branch and not represented by its open or merged PR's actual head | 86400 s | -| `open_pr` | an open PR discovered from this home's project origins or recorded task PR URLs waits on checks, a reviewer, or firstmate's review routing | 3600 s | -| `red_check` | the latest run of a check on an open PR failed; the next action is always `diagnose: code or test`, never a waiver | 0 s | +| `open_pr` | an open PR this home owns waits on checks, a reviewer, or firstmate's review routing; the main home also carries one informational group row for project PRs nobody owns | 3600 s | +| `red_check` | the latest run of a check on a PR this home owns failed; the next action is always `diagnose: code or test`, never a waiver | 0 s | | `coverage` | a source is unreadable or its forge is unsupported, so coverage is incomplete | 0 s | +Each home reports only what it owns. +A PR is owned when one of this home's task records or backlog items names its URL, or when its head branch is a branch of one of this home's tasks. +Every other open PR of the discovered repositories is unowned: it gets no `open_pr` or `red_check` row of its own, and its checks are not fetched. +The main home lists its unowned PRs once, as the single `open_pr` row named `unowned project PRs`, with owner `none` and the PR numbers per repository in its evidence. +A secondmate home, marked by a valid `.fm-secondmate-home` file, emits no such row and discovers repositories only from its own tasks and owned PR URLs, never from its shared `projects/` clones or its own checkout. +An informational row carries `informational: true` and is never overdue at any age, so it neither wakes the watcher nor appears in Bearings. +A queued item is selected for dispatch once it has a task record or a status log in this home; unselected ready items stay visible as informational rows, so the unprioritized backlog is not an overdue obligation. +A worker whose current state is `done`, `parked`, or `paused`, or whose status log ends on a `done` or `paused` event, has recorded its own stop: a gone endpoint or an unreadable live state then adds neither a `missing_worker` row nor degraded coverage. +A recorded stop never hides a `failed` task or its unlanded commits. + A source that cannot be read adds the single `coverage` row named `ledger degraded` and sets `complete: false`; it is never read as an empty fleet, and the other sources still report. The shared task/status state directory is explicitly enumerated before reading either source; an unreadable inventory remains degraded even when shell globs would otherwise yield no task or question rows. Status questions age from their stamped opening. Captain-held backlog questions use subject `:captain-hold`, owner `captain`, the existing hold reason, and the hold-set timestamp (falling back to `since`); they remain visible without status or task metadata. Blocked, dated, and Done holds are excluded, and no historical audit or new persistence is required. For local-only projects, the ledger uses the qualified local default branch advanced by `fm-merge-local` as delivery proof; other project modes retain their normal remote-default proof, with nonmerge commit patch equivalence against the actual PR head. -GitHub repositories are discovered from recorded local task PR URLs as well as project origins, including fork-only delivery; their open PRs are classified before PR-head coverage can suppress unlanded commits. +GitHub repositories are discovered from recorded local task and backlog PR URLs as well as project origins, including fork-only delivery; their open PRs are classified before PR-head coverage can suppress unlanded commits. Only exact `github.com` hosts are admitted or projected, and unsupported origins or PR URLs disclose degraded coverage; GitLab and Gerrit merge proofs are outside this ledger's current scope. An existing worker endpoint with reconciled `working` state counts as live when its backend's recovery verdict is `unverified`; other inconclusive liveness adds degraded coverage. Unreadable or unknown current task state adds degraded coverage independently of liveness, and only nonfuture progress timestamps count as work evidence. diff --git a/tests/fm-open-loops.test.sh b/tests/fm-open-loops.test.sh index b0d91943716..4fb1f1ea9f5 100755 --- a/tests/fm-open-loops.test.sh +++ b/tests/fm-open-loops.test.sh @@ -127,6 +127,8 @@ backlog = [ dict(id='waiting', structured=True, state='queued', since=iso(hours(9)), unresolved_blocker_ids=['orphan']), dict(id='held', structured=True, state='queued', since=iso(hours(9)), hold_kind='captain'), dict(id='deferred', structured=True, state='queued', since=iso(hours(9)), hold_until='2999-01-01'), + dict(id='review-8', structured=True, state='done', links=[PR_URL + '8'], pr_url=PR_URL + '8'), + dict(id='route-9', structured=True, state='done', links=[PR_URL + '9'], pr_url=PR_URL + '9'), ] (home / 'snapshot.json').write_text(json.dumps(dict(schema='fm-fleet-home-input.v1', tasks=tasks, backlog=dict(present=True, records=backlog)))) @@ -185,7 +187,8 @@ assert open_prs[PR_URL + '9']['age_seconds'] == 3600 and open_prs[PR_URL + '9'][ # Every row names an owner, a next action and an age (or an unknown age that stays overdue). for row in report['rows']: assert row['owner'] and row['next_action'] and 'age_seconds' in row and row['id'], row - assert row['overdue'] == (row['age_seconds'] is None or row['age_seconds'] >= row['limit_seconds']), row + assert row['overdue'] == (not row.get('informational') + and (row['age_seconds'] is None or row['age_seconds'] >= row['limit_seconds'])), row # Age limits live in config; the boundary is inclusive. (home / 'config/open-loops.json').write_text(json.dumps(dict(age_limits_seconds=dict(open_pr=3600, ready_not_started=9 * 3600 + 1)))) @@ -630,6 +633,108 @@ finally: assert ledger()['complete'], 'restored readable inventory must recover coverage' print('PASS: unreadable shared state inventory cannot publish complete empty coverage', flush=True) +# Ownership scoping: a home is accountable only for work its own records name. +def greens(sha, status='completed', conclusion='success'): + return [dict(id=1, name='unit', app=dict(id=3), status=status, conclusion=conclusion, completed_at=iso(hours(2)))] +scope_saved = (double['pulls'], double['checks'], double.get('pulls_by_repo')) +scope_pr = lambda number, sha, ref=None, **extra: dict( + number=number, html_url=PR_URL + str(number), head=dict(sha=sha, ref=ref), state='open', draft=False, + updated_at=iso(hours(3)), **extra) +shas = {n: chr(ord('a') + n - 31) * 40 for n in range(31, 37)} +double['pulls'] = [scope_pr(31, shas[31]), scope_pr(32, shas[32]), scope_pr(33, shas[33], 'fm/branch-owned'), + scope_pr(34, shas[34]), scope_pr(35, shas[35], requested_teams=[dict(slug='reviewers')])] +double['checks'] = {shas[31]: greens(shas[31]), shas[32]: greens(shas[32], conclusion='failure'), + shas[33]: greens(shas[33]), shas[34]: greens(shas[34], conclusion='failure'), + shas[35]: greens(shas[35])} +other = home / 'projects/other' +out(['git', 'init', '-q', '-b', 'main', other]) +git(other, 'remote', 'add', 'origin', 'https://github.com/test/other.git') +double['pulls_by_repo'] = {'/repos/test/other': [dict(scope_pr(40, 'f' * 40), html_url='https://github.com/test/other/pull/40')]} +double['checks']['f' * 40] = greens('f' * 40, conclusion='failure') +(world / 'gh.json').write_text(json.dumps(double)) +scope_tasks = [task('owns-31', fresh, pr=PR_URL + '31'), task('owns-branch', fresh)] +scope_tasks[1]['branch'] = 'fm/branch-owned' +scope_backlog = [dict(id='owns-32', structured=True, state='done', links=[PR_URL + '32'], pr_url=None)] +fixture(scope_tasks, scope_backlog) +marker_file = home / '.fm-secondmate-home' +def scoped(lane): + marker_file.unlink(missing_ok=True) + if lane: + marker_file.write_text('lane-1\n') + (world / 'gh-requests').write_text('') + return ledger(), (world / 'gh-requests').read_text() +try: + # Main home: only owned PRs are obligations; the rest is one informational group, never overdue. + main_scope, main_requests = scoped(False) + assert set(rows(main_scope, 'red_check')) == {PR_URL + '32'}, main_scope + named = {s for s in rows(main_scope, 'open_pr') if s.startswith(PR_URL)} + assert named == {PR_URL + '31', PR_URL + '33'}, main_scope + group = rows(main_scope, 'open_pr')['unowned project PRs'] + assert group['informational'] and group['overdue'] is False and group['owner'] == 'none', group + for token in ('test/project: #34 #35', 'test/other: #40'): + assert token in group['evidence'], group + assert [r['subject'] for r in main_scope['rows'] if r.get('informational') and r['category'] == 'open_pr'] \ + == ['unowned project PRs'], main_scope + assert shas[34] not in main_requests and shas[35] not in main_requests, 'unowned PR checks are not fetched' + # A lane home shares the origin but lists no unowned PR, no group, and never reads the other repository. + lane_scope, lane_requests = scoped(True) + assert set(rows(lane_scope, 'red_check')) == {PR_URL + '32'}, lane_scope + assert {s for s in rows(lane_scope, 'open_pr')} == {PR_URL + '31', PR_URL + '33'}, lane_scope + assert not any(r.get('informational') for r in lane_scope['rows'] if r['category'] == 'open_pr'), lane_scope + assert '/repos/test/other' not in lane_requests, 'a lane does not survey repositories it has no task in' + marker_file.write_text('not a valid id!\n') + assert 'unowned project PRs' in rows(ledger(), 'open_pr'), 'a malformed marker is not a lane home' + marker_file.unlink() +finally: + marker_file.unlink(missing_ok=True) + shutil.rmtree(other) + double['pulls'], double['checks'] = scope_saved[0], scope_saved[1] + double.pop('pulls_by_repo') + if scope_saved[2] is not None: + double['pulls_by_repo'] = scope_saved[2] + (world / 'gh.json').write_text(json.dumps(double)) +print('PASS: PR and red-check rows follow ownership; unowned PRs are one informational main-home group', flush=True) + +# A worker that recorded its own stop is not missing, and an unreadable live state does not degrade the ledger. +def stopped_task(name, last_event, state='unknown', detail='backend target gone: gone', exists=False, alive='missing'): + item = task(name, world / 'nowhere', state=state, exists=exists, alive=alive) + item['current_state']['detail'] = detail + item['paths']['status_log'] = dict(last_event=dict(state=last_event) if last_event else None) + return item +recorded = [stopped_task('stopped-done', 'done'), + stopped_task('stopped-paused-gone', 'paused', alive='dead'), + stopped_task('stopped-paused-live', 'paused', detail='unrecognized run status', exists=True, alive='alive'), + stopped_task('stopped-parked', None, state='parked', exists=False, alive='dead'), + stopped_task('lost-silently', 'working', alive='dead'), + stopped_task('failed-after-done', 'done', state='failed', alive='dead')] +fixture(recorded, [dict(id=t['id'], structured=True, state='in_flight', requires_child_metadata=True, + since=iso(hours(9))) for t in recorded]) +stopped_report = ledger() +assert set(rows(stopped_report, 'missing_worker')) == {'lost-silently', 'failed-after-done'}, stopped_report +assert 'failed-after-done' in rows(stopped_report, 'failed_task'), 'a recorded stop never hides a failed deliverable' +degraded_text = ' '.join(r['evidence'] for r in rows(stopped_report, 'coverage').values()) +assert not any(name in degraded_text for name in ('stopped-done', 'stopped-paused', 'stopped-parked')), stopped_report +fixture(tasks, backlog) +print('PASS: recorded stopped or parked owners are neither missing nor unknown', flush=True) + +# Unselected ready items are visible backlog, not overdue obligations; dispatched ones still are. +(home / 'state/started-ready.status').write_text(f'working [at={hours(1)}]: spawned\n') +selected_task = task('selected-ready', fresh) +fixture([selected_task], [ + dict(id='idle-ready', structured=True, state='queued', since=iso(hours(9))), + dict(id='selected-ready', structured=True, state='queued', since=iso(hours(9))), + dict(id='started-ready', structured=True, state='queued', since=iso(hours(9)))]) +ready_report = ledger() +ready_rows = rows(ready_report, 'ready_not_started') +assert set(ready_rows) == {'idle-ready', 'selected-ready', 'started-ready'}, ready_report +assert ready_rows['idle-ready']['overdue'] is False and ready_rows['idle-ready']['informational'], ready_rows +assert ready_rows['idle-ready']['next_action'].startswith('informational'), ready_rows +for name in ('selected-ready', 'started-ready'): + assert ready_rows[name]['overdue'] is True and 'informational' not in ready_rows[name], ready_rows +(home / 'state/started-ready.status').unlink() +fixture(tasks, backlog) +print('PASS: only dispatched ready work is overdue', flush=True) + # An unreadable source is one degraded row and complete:false; healthy sources still report. (home / 'snapshot-fails').write_text('') double['fail'] = True @@ -773,7 +878,8 @@ for address in (address_home, address_code): report_task['project'] = None (address / 'snapshot.json').write_text(json.dumps(dict(schema='fm-fleet-home-input.v1', tasks=[report_task], backlog=dict(present=True, records=[ - dict(id='address-ready', structured=True, state='queued')])))) + dict(id='address-ready', structured=True, state='queued'), + dict(id='address-pr', structured=True, state='done', links=[address_url])])))) empty_keys = ('FM_STATE_OVERRIDE', 'FM_DATA_OVERRIDE', 'FM_CONFIG_OVERRIDE', 'FM_PROJECTS_OVERRIDE') address_cases = [(dict(FM_HOME=str(address_home), **{key: ''}), address_home) for key in empty_keys] address_cases += [ @@ -889,7 +995,9 @@ for reader in ('origins', 'pr-checks', 'pr-state', 'questions'): for i in range(24): (deadline_home / 'state' / (str(i) + '.status')).write_text('needs-decision [key=q]: question\n') (deadline_home / 'snapshot.json').write_text(json.dumps(dict(schema='fm-fleet-home-input.v1', - tasks=deadline_tasks, backlog=dict(present=True, records=[])))) + tasks=deadline_tasks, backlog=dict(present=True, records=( + [dict(id='owns-prs', structured=True, state='done', links=[PR_URL + str(i) for i in range(24)])] + if reader == 'pr-checks' else []))))) # owned PRs are the ones whose checks are fetched script(deadline_code / 'bin/fm-fleet-snapshot.sh', '#!/usr/bin/env python3\n' 'import os\nfrom pathlib import Path\nhome = Path(os.environ["FM_HOME"])\n' '(home / "collector-pid").write_text(str(os.getppid()))\nprint((home / "snapshot.json").read_text())\n') diff --git a/tests/fm-watch-open-loops.test.sh b/tests/fm-watch-open-loops.test.sh index d19769035d7..92242d81cc6 100755 --- a/tests/fm-watch-open-loops.test.sh +++ b/tests/fm-watch-open-loops.test.sh @@ -433,7 +433,7 @@ while [ ! -e "$FM_HOME/.collector-release" ]; do [ -d "$FM_HOME" ] && [ "$SECONDS" -lt "$deadline" ] || exit 1 sleep 0.1 done -printf '%s\n' '{"schema":"fm-fleet-home-input.v1","tasks":[],"backlog":{"present":true,"records":[{"id":"retained-obligation","structured":true,"state":"queued","since":"2000-01-01T00:00:00Z"}]}}' +printf '%s\n' '{"schema":"fm-fleet-home-input.v1","tasks":[],"backlog":{"present":true,"records":[{"id":"retained-obligation","structured":true,"state":"in_flight","requires_child_metadata":true,"since":"2000-01-01T00:00:00Z"}]}}' SH chmod +x "$dir/collector-bin/fm-fleet-snapshot.sh" publish_ledger "$state" lost-launcher @@ -467,7 +467,7 @@ SH [ "$scans" -eq 1 ] || fail "watcher restart overlapped $scans collectors before publication" [ "$(wc -l < "$dir/.collector-pids" | tr -d '[:space:]')" -eq 1 ] \ || fail "a queued collector scanned after the retained publication" - jq -e '.complete == true and any(.rows[]; .subject == "retained-obligation" and .category == "ready_not_started" and .overdue)' \ + jq -e '.complete == true and any(.rows[]; .subject == "retained-obligation" and .category == "missing_worker" and .overdue)' \ "$state/open-loops.json" >/dev/null || fail "the retained scan did not publish its actual owned obligation" grep -q 'open-loop-ledger' "$state/.wake-queue" || fail "the retained scan publication was not surfaced durably" pass "a retained collector survives watcher exit, rejects overlapping restart scans, and publishes without starvation" @@ -611,7 +611,7 @@ test_real_publication_rearms_stale_incidents_without_resetting_overdue() { cat > "$dir/collector-bin/fm-fleet-snapshot.sh" <<'SH' #!/usr/bin/env bash [ "${1:-}" = --home-input ] || exit 2 -printf '%s\n' '{"schema":"fm-fleet-home-input.v1","tasks":[],"backlog":{"present":true,"records":[{"id":"recovery-obligation","structured":true,"state":"queued","since":"2000-01-01T00:00:00Z"}]}}' +printf '%s\n' '{"schema":"fm-fleet-home-input.v1","tasks":[],"backlog":{"present":true,"records":[{"id":"recovery-obligation","structured":true,"state":"in_flight","requires_child_metadata":true,"since":"2000-01-01T00:00:00Z"}]}}' SH chmod +x "$dir/collector-bin/fm-fleet-snapshot.sh" install_failing_helper "$fakebin" @@ -675,7 +675,7 @@ test_fresh_observation_rearms_recovery_during_blocked_stale_append() { cat > "$dir/collector-bin/fm-fleet-snapshot.sh" <<'SH' #!/usr/bin/env bash [ "${1:-}" = --home-input ] || exit 2 -printf '%s\n' '{"schema":"fm-fleet-home-input.v1","tasks":[],"backlog":{"present":true,"records":[{"id":"blocked-recovery-obligation","structured":true,"state":"queued","since":"2000-01-01T00:00:00Z"}]}}' +printf '%s\n' '{"schema":"fm-fleet-home-input.v1","tasks":[],"backlog":{"present":true,"records":[{"id":"blocked-recovery-obligation","structured":true,"state":"in_flight","requires_child_metadata":true,"since":"2000-01-01T00:00:00Z"}]}}' SH chmod +x "$dir/collector-bin/fm-fleet-snapshot.sh" install_failing_helper "$fakebin" From 543401034f54cd2cb2deab1a54838f9ea6a36ac6 Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 08:16:00 -0500 Subject: [PATCH 2/5] no-mistakes(review): Fix repository-scoped ownership and authoritative worker stop detection --- bin/fm_open_loops.py | 39 ++++++----------- docs/configuration.md | 14 +++--- tests/fm-open-loops.test.sh | 87 +++++++++++++++++++++++++++---------- 3 files changed, 85 insertions(+), 55 deletions(-) diff --git a/bin/fm_open_loops.py b/bin/fm_open_loops.py index ded1ae07f18..0bf3a3d8b2f 100755 --- a/bin/fm_open_loops.py +++ b/bin/fm_open_loops.py @@ -33,9 +33,6 @@ r"|ssh://(?:git@)?github[.]com/|git@github[.]com:)" r"([^/]+/[^/]+?)(?:[.]git)?") PIPELINE_ENDED = {"completed", "failed", "cancelled", "aborted"} -# A worker that recorded its own stop is not missing: done or paused in its status log, or a -# done, parked, or paused current state. -STOP_EVENTS = {"done", "paused"} STOP_STATES = {"done", "parked", "paused"} SIGNATURE = re.compile(r"usage.?limit|rate.?limit|quota|auth|unauthorized|login|network|connection" r"|timed? ?out|ECONN|ENOTFOUND|\b(?:error|failure|failed|exception|fatal)\b" @@ -89,7 +86,6 @@ def __init__(self, home, now): self.state_files = [] self.owned_urls = set() self.owned_branches = set() - self.unowned = {} self.lane = self.is_secondmate_home() def is_secondmate_home(self): @@ -254,10 +250,15 @@ def read_snapshot(self): def recorded_stop(self, task): """True when the worker's own durable record says it stopped on purpose, so its endpoint being gone or its live state being unreadable is expected rather than a loss.""" - if task["current_state"].get("state") == "failed": + state = task["current_state"].get("state") + if state in ("working", "blocked", "failed"): return False - last = ((task.get("paths") or {}).get("status_log") or {}).get("last_event") or {} - return task["current_state"].get("state") in STOP_STATES or last.get("state") in STOP_EVENTS + if state in STOP_STATES: + return True + return self.bash('. "$1"; last=$(last_status_line "$2"); ' + 'if [ "$(status_line_verb "$last")" = done ] || status_is_paused "$last"; ' + 'then printf stopped; fi', + BIN / "fm-status-event-lib.sh", self.state / (task["id"] + ".status")).strip() == "stopped" def backlog_rows(self): by_id = {t["id"]: t for t in self.tasks} @@ -296,7 +297,7 @@ def liveness(self, task): def worker_rows(self): for task in self.tasks: - stopped = self.recorded_stop(task) + stopped = self.source("worker stop " + task["id"], self.recorded_stop, task) if stopped: # a recorded stop explains an unreadable endpoint; it never degrades coverage try: alive = self.liveness(task) @@ -491,14 +492,13 @@ def pr_state(self, url): def load_ownership(self): """PRs this home owns: recorded on its tasks or backlog, or opened from a task's branch.""" self.owned_urls = {t["pr"]["url"] for t in self.tasks if t["pr"].get("url")} - self.owned_branches = {t["branch"] for t in self.tasks if t.get("branch")} for record in self.backlog: for link in [record.get("pr_url"), *(record.get("links") or [])]: if isinstance(link, str) and GITHUB_PR.fullmatch(link): self.owned_urls.add(link) - def owns_pr(self, pr): - return pr["html_url"] in self.owned_urls or (pr.get("head") or {}).get("ref") in self.owned_branches + def owns_pr(self, slug, pr): + return pr["html_url"] in self.owned_urls or (slug, (pr.get("head") or {}).get("ref")) in self.owned_branches def repo_slugs(self): paths = {t["project"] for t in self.tasks if t.get("project")} @@ -529,6 +529,8 @@ def repo_slugs(self): match = GITHUB_ORIGIN.fullmatch(origin) if match: slugs.add(match[1]) + self.owned_branches.update((match[1], t["branch"]) for t in self.tasks + if t.get("project") == path and t.get("branch")) return sorted(slugs) def repo_origin(self, path): @@ -586,20 +588,8 @@ def pr_rows(self, slug): self.pr_state(url) continue self.prs[url] = pr - if self.owns_pr(pr): + if self.owns_pr(slug, pr): self.source(f"PR {slug}#{pr.get('number', '?')}", self.pr_row, slug, pr) - else: - self.unowned.setdefault(slug, []).append(pr.get("number", "?")) - - def unowned_row(self): - """Project PRs with no fleet owner: one informational row in the main home, nothing in a lane.""" - if self.lane or not self.unowned: - return - total = sum(len(numbers) for numbers in self.unowned.values()) - listing = "; ".join(f"{slug}: " + " ".join("#" + str(n) for n in sorted(numbers, key=str)) - for slug, numbers in sorted(self.unowned.items())) - self.add("open_pr", "unowned project PRs", "informational: no fleet owner; route one only on the captain's word", - None, "none", f"{total} open PR(s) with no task or backlog owner - {listing}", informational=True) def question_rows(self): for record in self.backlog: @@ -633,7 +623,6 @@ def collect(self): slugs = self.source("project origins", self.repo_slugs) or [] for slug in slugs: self.source("open PRs " + slug, self.pr_rows, slug) - self.unowned_row() self.source("workers", self.worker_rows) self.source("questions", self.question_rows) if self.degraded: diff --git a/docs/configuration.md b/docs/configuration.md index 72242ea5633..9448aa4902c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -585,19 +585,19 @@ An unknown age stays `null` and counts as overdue, and an age equal to its limit | `failed_task` | a task record's current state is `failed` and its deliverable is neither landed nor recorded as dropped | 1800 s | | `stalled_worker` | a live worker reads `working` but has no recent commit, status line, or pipeline progress within its limit; the row carries the last error-looking line from a bounded pane-tail sample when present, otherwise a no-progress explanation | 3600 s | | `unlanded_commit` | a ship task's copy holds nonmerge commits absent from the default branch and not represented by its open or merged PR's actual head | 86400 s | -| `open_pr` | an open PR this home owns waits on checks, a reviewer, or firstmate's review routing; the main home also carries one informational group row for project PRs nobody owns | 3600 s | +| `open_pr` | an open PR this home owns waits on checks, a reviewer, or firstmate's review routing | 3600 s | | `red_check` | the latest run of a check on a PR this home owns failed; the next action is always `diagnose: code or test`, never a waiver | 0 s | | `coverage` | a source is unreadable or its forge is unsupported, so coverage is incomplete | 0 s | Each home reports only what it owns. -A PR is owned when one of this home's task records or backlog items names its URL, or when its head branch is a branch of one of this home's tasks. -Every other open PR of the discovered repositories is unowned: it gets no `open_pr` or `red_check` row of its own, and its checks are not fetched. -The main home lists its unowned PRs once, as the single `open_pr` row named `unowned project PRs`, with owner `none` and the PR numbers per repository in its evidence. -A secondmate home, marked by a valid `.fm-secondmate-home` file, emits no such row and discovers repositories only from its own tasks and owned PR URLs, never from its shared `projects/` clones or its own checkout. +A PR is owned when one of this home's task records or backlog items names its exact URL, or when its repository and head branch match the origin repository and branch of one of this home's tasks. +Every other open PR of the discovered repositories is unowned: it gets no ledger rows, and its checks are not fetched. Unowned PR triage is separate from this ledger. +A secondmate home, marked by a valid `.fm-secondmate-home` file, discovers repositories only from its own tasks and owned PR URLs, never from its shared `projects/` clones or its own checkout. An informational row carries `informational: true` and is never overdue at any age, so it neither wakes the watcher nor appears in Bearings. A queued item is selected for dispatch once it has a task record or a status log in this home; unselected ready items stay visible as informational rows, so the unprioritized backlog is not an overdue obligation. -A worker whose current state is `done`, `parked`, or `paused`, or whose status log ends on a `done` or `paused` event, has recorded its own stop: a gone endpoint or an unreadable live state then adds neither a `missing_worker` row nor degraded coverage. -A recorded stop never hides a `failed` task or its unlanded commits. +A worker whose current state is `done`, `parked`, or `paused` has recorded its own stop: a gone endpoint or an unreadable live state then adds neither a `missing_worker` row nor degraded coverage. +When current state is unknown, the latest logical status event can establish a recorded stop: `done` or the configured pause verb (`FM_CLASSIFY_PAUSED_VERB`, default `paused`). The shared status-event parser ignores continuation prose. +Known `working`, `blocked`, and `failed` current states override historical stop events. A recorded stop never hides unlanded commits. A source that cannot be read adds the single `coverage` row named `ledger degraded` and sets `complete: false`; it is never read as an empty fleet, and the other sources still report. The shared task/status state directory is explicitly enumerated before reading either source; an unreadable inventory remains degraded even when shell globs would otherwise yield no task or question rows. diff --git a/tests/fm-open-loops.test.sh b/tests/fm-open-loops.test.sh index 4fb1f1ea9f5..20668a5c2c6 100755 --- a/tests/fm-open-loops.test.sh +++ b/tests/fm-open-loops.test.sh @@ -642,15 +642,22 @@ scope_pr = lambda number, sha, ref=None, **extra: dict( updated_at=iso(hours(3)), **extra) shas = {n: chr(ord('a') + n - 31) * 40 for n in range(31, 37)} double['pulls'] = [scope_pr(31, shas[31]), scope_pr(32, shas[32]), scope_pr(33, shas[33], 'fm/branch-owned'), - scope_pr(34, shas[34]), scope_pr(35, shas[35], requested_teams=[dict(slug='reviewers')])] + scope_pr(34, shas[34]), scope_pr(35, shas[35], requested_teams=[dict(slug='reviewers')]), + scope_pr(36, shas[36], 'fm/other')] double['checks'] = {shas[31]: greens(shas[31]), shas[32]: greens(shas[32], conclusion='failure'), shas[33]: greens(shas[33]), shas[34]: greens(shas[34], conclusion='failure'), - shas[35]: greens(shas[35])} + shas[35]: greens(shas[35]), shas[36]: greens(shas[36], conclusion='failure')} other = home / 'projects/other' out(['git', 'init', '-q', '-b', 'main', other]) git(other, 'remote', 'add', 'origin', 'https://github.com/test/other.git') -double['pulls_by_repo'] = {'/repos/test/other': [dict(scope_pr(40, 'f' * 40), html_url='https://github.com/test/other/pull/40')]} +other_url = 'https://github.com/test/other/pull/' +double['pulls_by_repo'] = {'/repos/test/other': [ + dict(scope_pr(40, 'f' * 40, 'fm/branch-owned'), html_url=other_url + '40'), + dict(scope_pr(41, '1' * 40, 'fm/other'), html_url=other_url + '41'), + dict(scope_pr(42, '2' * 40, 'fm/unrelated'), html_url=other_url + '42')]} double['checks']['f' * 40] = greens('f' * 40, conclusion='failure') +double['checks']['1' * 40] = greens('1' * 40) +double['checks']['2' * 40] = greens('2' * 40, conclusion='failure') (world / 'gh.json').write_text(json.dumps(double)) scope_tasks = [task('owns-31', fresh, pr=PR_URL + '31'), task('owns-branch', fresh)] scope_tasks[1]['branch'] = 'fm/branch-owned' @@ -664,27 +671,31 @@ def scoped(lane): (world / 'gh-requests').write_text('') return ledger(), (world / 'gh-requests').read_text() try: - # Main home: only owned PRs are obligations; the rest is one informational group, never overdue. main_scope, main_requests = scoped(False) assert set(rows(main_scope, 'red_check')) == {PR_URL + '32'}, main_scope - named = {s for s in rows(main_scope, 'open_pr') if s.startswith(PR_URL)} - assert named == {PR_URL + '31', PR_URL + '33'}, main_scope - group = rows(main_scope, 'open_pr')['unowned project PRs'] - assert group['informational'] and group['overdue'] is False and group['owner'] == 'none', group - for token in ('test/project: #34 #35', 'test/other: #40'): - assert token in group['evidence'], group - assert [r['subject'] for r in main_scope['rows'] if r.get('informational') and r['category'] == 'open_pr'] \ - == ['unowned project PRs'], main_scope - assert shas[34] not in main_requests and shas[35] not in main_requests, 'unowned PR checks are not fetched' - # A lane home shares the origin but lists no unowned PR, no group, and never reads the other repository. + assert set(rows(main_scope, 'open_pr')) == {PR_URL + '31', PR_URL + '33'}, main_scope + assert all(sha not in main_requests for sha in (shas[34], shas[35], shas[36], 'f' * 40, '1' * 40, '2' * 40)), main_requests lane_scope, lane_requests = scoped(True) assert set(rows(lane_scope, 'red_check')) == {PR_URL + '32'}, lane_scope assert {s for s in rows(lane_scope, 'open_pr')} == {PR_URL + '31', PR_URL + '33'}, lane_scope assert not any(r.get('informational') for r in lane_scope['rows'] if r['category'] == 'open_pr'), lane_scope assert '/repos/test/other' not in lane_requests, 'a lane does not survey repositories it has no task in' marker_file.write_text('not a valid id!\n') - assert 'unowned project PRs' in rows(ledger(), 'open_pr'), 'a malformed marker is not a lane home' - marker_file.unlink() + (world / 'gh-requests').write_text('') + malformed_scope = ledger() + assert set(rows(malformed_scope, 'open_pr')) == {PR_URL + '31', PR_URL + '33'}, malformed_scope + assert '/repos/test/other/pulls?' in (world / 'gh-requests').read_text(), 'a malformed marker is not a lane home' + other_task = task('owns-other-branch', fresh) + other_task['project'] = str(other) + other_task['branch'] = 'fm/other' + fixture([*scope_tasks, other_task], + [*scope_backlog, dict(id='owns-other-url', structured=True, state='done', pr_url=other_url + '42')]) + for lane in (False, True): + cross_repo, requests = scoped(lane) + assert set(rows(cross_repo, 'open_pr')) == {PR_URL + '31', PR_URL + '33', other_url + '41'}, cross_repo + assert set(rows(cross_repo, 'red_check')) == {PR_URL + '32', other_url + '42'}, cross_repo + assert shas[36] not in requests and 'f' * 40 not in requests, requests + assert '1' * 40 in requests and '2' * 40 in requests, requests finally: marker_file.unlink(missing_ok=True) shutil.rmtree(other) @@ -693,29 +704,59 @@ finally: if scope_saved[2] is not None: double['pulls_by_repo'] = scope_saved[2] (world / 'gh.json').write_text(json.dumps(double)) -print('PASS: PR and red-check rows follow ownership; unowned PRs are one informational main-home group', flush=True) +print('PASS: PR ownership is repository-scoped; unowned PRs emit no rows or check requests', flush=True) # A worker that recorded its own stop is not missing, and an unreadable live state does not degrade the ledger. -def stopped_task(name, last_event, state='unknown', detail='backend target gone: gone', exists=False, alive='missing'): +def stopped_task(name, last_event, state='unknown', detail='backend target gone: gone', exists=False, alive='missing', + status_text=None): item = task(name, world / 'nowhere', state=state, exists=exists, alive=alive) item['current_state']['detail'] = detail item['paths']['status_log'] = dict(last_event=dict(state=last_event) if last_event else None) + if status_text is None: + status_text = f'{last_event}: stopped\n' if last_event else '' + (home / 'state' / (name + '.status')).write_text(status_text) return item recorded = [stopped_task('stopped-done', 'done'), stopped_task('stopped-paused-gone', 'paused', alive='dead'), stopped_task('stopped-paused-live', 'paused', detail='unrecognized run status', exists=True, alive='alive'), stopped_task('stopped-parked', None, state='parked', exists=False, alive='dead'), stopped_task('lost-silently', 'working', alive='dead'), - stopped_task('failed-after-done', 'done', state='failed', alive='dead')] + stopped_task('stopped-multiline', None, status_text='paused: waiting for CI\nThe check is still running.\n'), + stopped_task('resumed-after-pause', 'working', status_text='paused: waiting\nworking: resumed\n'), + stopped_task('lost-prose', None, status_text='The worker was paused while waiting.\n')] +for state in ('working', 'blocked', 'failed'): + for event in ('done', 'paused'): + recorded.append(stopped_task(state + '-after-' + event, event, state=state, alive='dead')) +for state in ('working', 'blocked'): + item = stopped_task(state + '-unverified-after-pause', 'paused', state=state, exists=True, alive='unverified') + item['backend'] = None + recorded.append(item) fixture(recorded, [dict(id=t['id'], structured=True, state='in_flight', requires_child_metadata=True, since=iso(hours(9))) for t in recorded]) stopped_report = ledger() -assert set(rows(stopped_report, 'missing_worker')) == {'lost-silently', 'failed-after-done'}, stopped_report -assert 'failed-after-done' in rows(stopped_report, 'failed_task'), 'a recorded stop never hides a failed deliverable' +expected_missing = {'lost-silently', 'failed-after-done', 'resumed-after-pause', 'lost-prose', + *(state + '-after-' + event for state in ('working', 'blocked', 'failed') for event in ('done', 'paused'))} +assert set(rows(stopped_report, 'missing_worker')) == expected_missing, stopped_report +assert {'failed-after-done', 'failed-after-paused'} <= set(rows(stopped_report, 'failed_task')), stopped_report degraded_text = ' '.join(r['evidence'] for r in rows(stopped_report, 'coverage').values()) -assert not any(name in degraded_text for name in ('stopped-done', 'stopped-paused', 'stopped-parked')), stopped_report +assert not any(name in degraded_text for name in ('stopped-done', 'stopped-paused', 'stopped-parked', 'stopped-multiline')), stopped_report +assert all(state + '-unverified-after-pause' in degraded_text for state in ('working', 'blocked')), stopped_report +for item in recorded: + (home / 'state' / (item['id'] + '.status')).unlink() +custom_paused = stopped_task('custom-paused', None, status_text='awaiting: waiting for CI\nThe check is still running.\n') +literal_paused = stopped_task('literal-paused', 'paused') +fixture([custom_paused, literal_paused], + [dict(id=t['id'], structured=True, state='in_flight', requires_child_metadata=True) for t in (custom_paused, literal_paused)]) +try: + custom_report = json.loads(out([code / 'bin/fm-open-loops.sh', '--json'], + env=dict(env, FM_CLASSIFY_PAUSED_VERB='awaiting'))) + assert set(rows(custom_report, 'missing_worker')) == {'literal-paused'}, custom_report + assert custom_report['complete'], custom_report +finally: + for item in (custom_paused, literal_paused): + (home / 'state' / (item['id'] + '.status')).unlink() fixture(tasks, backlog) -print('PASS: recorded stopped or parked owners are neither missing nor unknown', flush=True) +print('PASS: authoritative current state overrides historical stops; configured logical stop events are honored', flush=True) # Unselected ready items are visible backlog, not overdue obligations; dispatched ones still are. (home / 'state/started-ready.status').write_text(f'working [at={hours(1)}]: spawned\n') From 24dcd01d707be9f823f92a5fbad289023e571e69 Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 08:29:39 -0500 Subject: [PATCH 3/5] no-mistakes(review): Fix fork PR ownership and preserve standing pauses --- bin/fm_open_loops.py | 12 ++++--- docs/configuration.md | 4 +-- tests/fm-open-loops.test.sh | 71 +++++++++++++++++++++++++++---------- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/bin/fm_open_loops.py b/bin/fm_open_loops.py index 0bf3a3d8b2f..0066fdd58f8 100755 --- a/bin/fm_open_loops.py +++ b/bin/fm_open_loops.py @@ -256,8 +256,8 @@ def recorded_stop(self, task): if state in STOP_STATES: return True return self.bash('. "$1"; last=$(last_status_line "$2"); ' - 'if [ "$(status_line_verb "$last")" = done ] || status_is_paused "$last"; ' - 'then printf stopped; fi', + 'if [ "$(status_line_verb "$last")" = done ] || ' + 'status_is_paused "$(status_declared_wait_line "$2")"; then printf stopped; fi', BIN / "fm-status-event-lib.sh", self.state / (task["id"] + ".status")).strip() == "stopped" def backlog_rows(self): @@ -497,8 +497,10 @@ def load_ownership(self): if isinstance(link, str) and GITHUB_PR.fullmatch(link): self.owned_urls.add(link) - def owns_pr(self, slug, pr): - return pr["html_url"] in self.owned_urls or (slug, (pr.get("head") or {}).get("ref")) in self.owned_branches + def owns_pr(self, pr): + head = pr.get("head") or {} + return pr["html_url"] in self.owned_urls or \ + ((head.get("repo") or {}).get("full_name"), head.get("ref")) in self.owned_branches def repo_slugs(self): paths = {t["project"] for t in self.tasks if t.get("project")} @@ -588,7 +590,7 @@ def pr_rows(self, slug): self.pr_state(url) continue self.prs[url] = pr - if self.owns_pr(slug, pr): + if self.owns_pr(pr): self.source(f"PR {slug}#{pr.get('number', '?')}", self.pr_row, slug, pr) def question_rows(self): diff --git a/docs/configuration.md b/docs/configuration.md index 9448aa4902c..b401761c514 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -590,13 +590,13 @@ An unknown age stays `null` and counts as overdue, and an age equal to its limit | `coverage` | a source is unreadable or its forge is unsupported, so coverage is incomplete | 0 s | Each home reports only what it owns. -A PR is owned when one of this home's task records or backlog items names its exact URL, or when its repository and head branch match the origin repository and branch of one of this home's tasks. +A PR is owned when one of this home's task records or backlog items names its exact URL, or when its head repository (`head.repo.full_name`) and head branch match the origin repository and branch of one of this home's tasks. Matching only the base repository does not establish ownership of a fork's branch. Every other open PR of the discovered repositories is unowned: it gets no ledger rows, and its checks are not fetched. Unowned PR triage is separate from this ledger. A secondmate home, marked by a valid `.fm-secondmate-home` file, discovers repositories only from its own tasks and owned PR URLs, never from its shared `projects/` clones or its own checkout. An informational row carries `informational: true` and is never overdue at any age, so it neither wakes the watcher nor appears in Bearings. A queued item is selected for dispatch once it has a task record or a status log in this home; unselected ready items stay visible as informational rows, so the unprioritized backlog is not an overdue obligation. A worker whose current state is `done`, `parked`, or `paused` has recorded its own stop: a gone endpoint or an unreadable live state then adds neither a `missing_worker` row nor degraded coverage. -When current state is unknown, the latest logical status event can establish a recorded stop: `done` or the configured pause verb (`FM_CLASSIFY_PAUSED_VERB`, default `paused`). The shared status-event parser ignores continuation prose. +When current state is unknown, the latest logical `done` event or a standing declaration of the configured pause verb (`FM_CLASSIFY_PAUSED_VERB`, default `paused`) can establish a recorded stop. The shared status-event parser ignores continuation prose and preserves a pause behind unrelated `resolved` answers; a resolution for the pause's own phase key or any later non-resolution event retracts that pause. Known `working`, `blocked`, and `failed` current states override historical stop events. A recorded stop never hides unlanded commits. A source that cannot be read adds the single `coverage` row named `ledger degraded` and sets `complete: false`; it is never read as an empty fleet, and the other sources still report. diff --git a/tests/fm-open-loops.test.sh b/tests/fm-open-loops.test.sh index 20668a5c2c6..cc07b323b86 100755 --- a/tests/fm-open-loops.test.sh +++ b/tests/fm-open-loops.test.sh @@ -637,24 +637,32 @@ print('PASS: unreadable shared state inventory cannot publish complete empty cov def greens(sha, status='completed', conclusion='success'): return [dict(id=1, name='unit', app=dict(id=3), status=status, conclusion=conclusion, completed_at=iso(hours(2)))] scope_saved = (double['pulls'], double['checks'], double.get('pulls_by_repo')) -scope_pr = lambda number, sha, ref=None, **extra: dict( - number=number, html_url=PR_URL + str(number), head=dict(sha=sha, ref=ref), state='open', draft=False, +scope_pr = lambda number, sha, ref=None, head_repo='test/project', **extra: dict( + number=number, html_url=PR_URL + str(number), + head=dict(sha=sha, ref=ref, repo=dict(full_name=head_repo) if head_repo else None), state='open', draft=False, updated_at=iso(hours(3)), **extra) -shas = {n: chr(ord('a') + n - 31) * 40 for n in range(31, 37)} -double['pulls'] = [scope_pr(31, shas[31]), scope_pr(32, shas[32]), scope_pr(33, shas[33], 'fm/branch-owned'), +shas = {n: chr(ord('a') + n - 31) * 40 for n in range(31, 40)} +double['pulls'] = [scope_pr(31, shas[31], 'fm/branch-owned', head_repo='contributor/project'), + scope_pr(32, shas[32], 'fm/branch-owned', head_repo='contributor/project'), + scope_pr(33, shas[33], 'fm/branch-owned'), scope_pr(34, shas[34]), scope_pr(35, shas[35], requested_teams=[dict(slug='reviewers')]), - scope_pr(36, shas[36], 'fm/other')] + scope_pr(36, shas[36], 'fm/other'), + scope_pr(37, shas[37], 'fm/branch-owned', head_repo='contributor/project'), + scope_pr(38, shas[38], 'fm/branch-owned', head_repo=None), + scope_pr(39, shas[39], 'fm/branch-owned')] +double['pulls'][-1]['head'].pop('repo') double['checks'] = {shas[31]: greens(shas[31]), shas[32]: greens(shas[32], conclusion='failure'), shas[33]: greens(shas[33]), shas[34]: greens(shas[34], conclusion='failure'), shas[35]: greens(shas[35]), shas[36]: greens(shas[36], conclusion='failure')} +double['checks'].update({shas[n]: greens(shas[n], conclusion='failure') for n in (37, 38, 39)}) other = home / 'projects/other' out(['git', 'init', '-q', '-b', 'main', other]) git(other, 'remote', 'add', 'origin', 'https://github.com/test/other.git') other_url = 'https://github.com/test/other/pull/' double['pulls_by_repo'] = {'/repos/test/other': [ - dict(scope_pr(40, 'f' * 40, 'fm/branch-owned'), html_url=other_url + '40'), - dict(scope_pr(41, '1' * 40, 'fm/other'), html_url=other_url + '41'), - dict(scope_pr(42, '2' * 40, 'fm/unrelated'), html_url=other_url + '42')]} + dict(scope_pr(40, 'f' * 40, 'fm/branch-owned', head_repo='test/other'), html_url=other_url + '40'), + dict(scope_pr(41, '1' * 40, 'fm/other', head_repo='test/other'), html_url=other_url + '41'), + dict(scope_pr(42, '2' * 40, 'fm/unrelated', head_repo='test/other'), html_url=other_url + '42')]} double['checks']['f' * 40] = greens('f' * 40, conclusion='failure') double['checks']['1' * 40] = greens('1' * 40) double['checks']['2' * 40] = greens('2' * 40, conclusion='failure') @@ -674,12 +682,13 @@ try: main_scope, main_requests = scoped(False) assert set(rows(main_scope, 'red_check')) == {PR_URL + '32'}, main_scope assert set(rows(main_scope, 'open_pr')) == {PR_URL + '31', PR_URL + '33'}, main_scope - assert all(sha not in main_requests for sha in (shas[34], shas[35], shas[36], 'f' * 40, '1' * 40, '2' * 40)), main_requests + assert all(sha not in main_requests for sha in (*[shas[n] for n in range(34, 40)], 'f' * 40, '1' * 40, '2' * 40)), main_requests lane_scope, lane_requests = scoped(True) assert set(rows(lane_scope, 'red_check')) == {PR_URL + '32'}, lane_scope assert {s for s in rows(lane_scope, 'open_pr')} == {PR_URL + '31', PR_URL + '33'}, lane_scope assert not any(r.get('informational') for r in lane_scope['rows'] if r['category'] == 'open_pr'), lane_scope assert '/repos/test/other' not in lane_requests, 'a lane does not survey repositories it has no task in' + assert all(shas[n] not in lane_requests for n in range(34, 40)), lane_requests marker_file.write_text('not a valid id!\n') (world / 'gh-requests').write_text('') malformed_scope = ledger() @@ -694,8 +703,17 @@ try: cross_repo, requests = scoped(lane) assert set(rows(cross_repo, 'open_pr')) == {PR_URL + '31', PR_URL + '33', other_url + '41'}, cross_repo assert set(rows(cross_repo, 'red_check')) == {PR_URL + '32', other_url + '42'}, cross_repo - assert shas[36] not in requests and 'f' * 40 not in requests, requests + assert all(shas[n] not in requests for n in range(36, 40)) and 'f' * 40 not in requests, requests assert '1' * 40 in requests and '2' * 40 in requests, requests + double['pulls_by_repo']['/repos/test/other'].append( + dict(scope_pr(43, '3' * 40, 'fm/branch-owned'), html_url=other_url + '43')) + double['checks']['3' * 40] = greens('3' * 40) + (world / 'gh.json').write_text(json.dumps(double)) + for lane in (False, True): + fork_owned, requests = scoped(lane) + assert set(rows(fork_owned, 'open_pr')) == {PR_URL + '31', PR_URL + '33', other_url + '41', other_url + '43'}, fork_owned + assert set(rows(fork_owned, 'red_check')) == {PR_URL + '32', other_url + '42'}, fork_owned + assert '3' * 40 in requests, requests finally: marker_file.unlink(missing_ok=True) shutil.rmtree(other) @@ -704,7 +722,7 @@ finally: if scope_saved[2] is not None: double['pulls_by_repo'] = scope_saved[2] (world / 'gh.json').write_text(json.dumps(double)) -print('PASS: PR ownership is repository-scoped; unowned PRs emit no rows or check requests', flush=True) +print('PASS: PR ownership matches the head repository and branch or an exact recorded URL; unowned checks are not fetched', flush=True) # A worker that recorded its own stop is not missing, and an unreadable live state does not degrade the ledger. def stopped_task(name, last_event, state='unknown', detail='backend target gone: gone', exists=False, alive='missing', @@ -724,9 +742,23 @@ recorded = [stopped_task('stopped-done', 'done'), stopped_task('stopped-multiline', None, status_text='paused: waiting for CI\nThe check is still running.\n'), stopped_task('resumed-after-pause', 'working', status_text='paused: waiting\nworking: resumed\n'), stopped_task('lost-prose', None, status_text='The worker was paused while waiting.\n')] +recorded.extend([ + stopped_task('stopped-unrelated-answer', 'resolved', detail='observation unavailable', + status_text='needs-decision [key=choice]: choose\npaused: waiting for CI\n' + 'resolved [key=choice]: answered\n'), + stopped_task('stopped-keyed-unrelated-answer', 'resolved', exists=True, alive='unverified', + detail='unrecognized run status', + status_text='paused [key=ci]: waiting\nresolved [key=choice]: answered\n'), + stopped_task('resumed-default-phase', 'resolved', status_text='paused: waiting\nresolved: resumed\n'), + stopped_task('resumed-keyed-phase', 'resolved', + status_text='paused [key=ci]: waiting\nresolved [key=ci]: resumed\n'), + stopped_task('blocked-after-pause', 'blocked', status_text='paused: waiting\nblocked: needs help\n'), + stopped_task('stopped-done-after-pause', 'done', status_text='paused: waiting\ndone: delivered\n'), +]) for state in ('working', 'blocked', 'failed'): - for event in ('done', 'paused'): - recorded.append(stopped_task(state + '-after-' + event, event, state=state, alive='dead')) + for event in ('done', 'paused', 'resolved'): + text = 'paused: waiting\nresolved [key=choice]: answered\n' if event == 'resolved' else None + recorded.append(stopped_task(state + '-after-' + event, event, state=state, alive='dead', status_text=text)) for state in ('working', 'blocked'): item = stopped_task(state + '-unverified-after-pause', 'paused', state=state, exists=True, alive='unverified') item['backend'] = None @@ -734,16 +766,19 @@ for state in ('working', 'blocked'): fixture(recorded, [dict(id=t['id'], structured=True, state='in_flight', requires_child_metadata=True, since=iso(hours(9))) for t in recorded]) stopped_report = ledger() -expected_missing = {'lost-silently', 'failed-after-done', 'resumed-after-pause', 'lost-prose', - *(state + '-after-' + event for state in ('working', 'blocked', 'failed') for event in ('done', 'paused'))} +expected_missing = {'lost-silently', 'resumed-after-pause', 'lost-prose', 'resumed-default-phase', + 'resumed-keyed-phase', 'blocked-after-pause', + *(state + '-after-' + event for state in ('working', 'blocked', 'failed') for event in ('done', 'paused', 'resolved'))} assert set(rows(stopped_report, 'missing_worker')) == expected_missing, stopped_report assert {'failed-after-done', 'failed-after-paused'} <= set(rows(stopped_report, 'failed_task')), stopped_report degraded_text = ' '.join(r['evidence'] for r in rows(stopped_report, 'coverage').values()) -assert not any(name in degraded_text for name in ('stopped-done', 'stopped-paused', 'stopped-parked', 'stopped-multiline')), stopped_report +assert not any(t['id'] in degraded_text for t in recorded if t['id'].startswith('stopped-')), stopped_report assert all(state + '-unverified-after-pause' in degraded_text for state in ('working', 'blocked')), stopped_report for item in recorded: (home / 'state' / (item['id'] + '.status')).unlink() -custom_paused = stopped_task('custom-paused', None, status_text='awaiting: waiting for CI\nThe check is still running.\n') +custom_paused = stopped_task('custom-paused', 'resolved', + status_text='awaiting: waiting for CI\nThe check is still running.\n' + 'resolved [key=choice]: answered\n') literal_paused = stopped_task('literal-paused', 'paused') fixture([custom_paused, literal_paused], [dict(id=t['id'], structured=True, state='in_flight', requires_child_metadata=True) for t in (custom_paused, literal_paused)]) @@ -756,7 +791,7 @@ finally: for item in (custom_paused, literal_paused): (home / 'state' / (item['id'] + '.status')).unlink() fixture(tasks, backlog) -print('PASS: authoritative current state overrides historical stops; configured logical stop events are honored', flush=True) +print('PASS: standing pauses survive unrelated answers; resumption and authoritative current states override them', flush=True) # Unselected ready items are visible backlog, not overdue obligations; dispatched ones still are. (home / 'state/started-ready.status').write_text(f'working [at={hours(1)}]: spawned\n') From 610c42c09d7dbed595c5a61e49e262e3485e7f0e Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 08:41:30 -0500 Subject: [PATCH 4/5] no-mistakes(review): Normalize PR ownership and preserve generation-invalidated coverage --- bin/fm_open_loops.py | 6 +++-- docs/configuration.md | 3 ++- tests/fm-open-loops.test.sh | 45 +++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/bin/fm_open_loops.py b/bin/fm_open_loops.py index 0066fdd58f8..8030259ce7c 100755 --- a/bin/fm_open_loops.py +++ b/bin/fm_open_loops.py @@ -250,6 +250,8 @@ def read_snapshot(self): def recorded_stop(self, task): """True when the worker's own durable record says it stopped on purpose, so its endpoint being gone or its live state being unreadable is expected rather than a loss.""" + if task["current_state"].get("detail") == "task generation changed during snapshot": + return False state = task["current_state"].get("state") if state in ("working", "blocked", "failed"): return False @@ -500,7 +502,7 @@ def load_ownership(self): def owns_pr(self, pr): head = pr.get("head") or {} return pr["html_url"] in self.owned_urls or \ - ((head.get("repo") or {}).get("full_name"), head.get("ref")) in self.owned_branches + (((head.get("repo") or {}).get("full_name") or "").casefold(), head.get("ref")) in self.owned_branches def repo_slugs(self): paths = {t["project"] for t in self.tasks if t.get("project")} @@ -531,7 +533,7 @@ def repo_slugs(self): match = GITHUB_ORIGIN.fullmatch(origin) if match: slugs.add(match[1]) - self.owned_branches.update((match[1], t["branch"]) for t in self.tasks + self.owned_branches.update((match[1].casefold(), t["branch"]) for t in self.tasks if t.get("project") == path and t.get("branch")) return sorted(slugs) diff --git a/docs/configuration.md b/docs/configuration.md index b401761c514..313e0a2fa89 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -590,7 +590,7 @@ An unknown age stays `null` and counts as overdue, and an age equal to its limit | `coverage` | a source is unreadable or its forge is unsupported, so coverage is incomplete | 0 s | Each home reports only what it owns. -A PR is owned when one of this home's task records or backlog items names its exact URL, or when its head repository (`head.repo.full_name`) and head branch match the origin repository and branch of one of this home's tasks. Matching only the base repository does not establish ownership of a fork's branch. +A PR is owned when one of this home's task records or backlog items names its exact URL, or when its head repository (`head.repo.full_name`) and head branch match the origin repository and branch of one of this home's tasks. Repository identities are matched case-insensitively; branch names remain case-sensitive, and recorded URLs must match exactly. Matching only the base repository does not establish ownership of a fork's branch. Every other open PR of the discovered repositories is unowned: it gets no ledger rows, and its checks are not fetched. Unowned PR triage is separate from this ledger. A secondmate home, marked by a valid `.fm-secondmate-home` file, discovers repositories only from its own tasks and owned PR URLs, never from its shared `projects/` clones or its own checkout. An informational row carries `informational: true` and is never overdue at any age, so it neither wakes the watcher nor appears in Bearings. @@ -598,6 +598,7 @@ A queued item is selected for dispatch once it has a task record or a status log A worker whose current state is `done`, `parked`, or `paused` has recorded its own stop: a gone endpoint or an unreadable live state then adds neither a `missing_worker` row nor degraded coverage. When current state is unknown, the latest logical `done` event or a standing declaration of the configured pause verb (`FM_CLASSIFY_PAUSED_VERB`, default `paused`) can establish a recorded stop. The shared status-event parser ignores continuation prose and preserves a pause behind unrelated `resolved` answers; a resolution for the pause's own phase key or any later non-resolution event retracts that pause. Known `working`, `blocked`, and `failed` current states override historical stop events. A recorded stop never hides unlanded commits. +A snapshot invalidated with `current_state.detail` equal to `task generation changed during snapshot` cannot establish a recorded stop, even if the live status log declares one; its inconclusive liveness and current-state diagnostic retain degraded coverage. A source that cannot be read adds the single `coverage` row named `ledger degraded` and sets `complete: false`; it is never read as an empty fleet, and the other sources still report. The shared task/status state directory is explicitly enumerated before reading either source; an unreadable inventory remains degraded even when shell globs would otherwise yield no task or question rows. diff --git a/tests/fm-open-loops.test.sh b/tests/fm-open-loops.test.sh index cc07b323b86..52173d6d1b3 100755 --- a/tests/fm-open-loops.test.sh +++ b/tests/fm-open-loops.test.sh @@ -714,7 +714,25 @@ try: assert set(rows(fork_owned, 'open_pr')) == {PR_URL + '31', PR_URL + '33', other_url + '41', other_url + '43'}, fork_owned assert set(rows(fork_owned, 'red_check')) == {PR_URL + '32', other_url + '42'}, fork_owned assert '3' * 40 in requests, requests + double['pulls'][2]['head']['repo']['full_name'] = 'Test/Project' + double['pulls_by_repo']['/repos/test/other'][1]['head']['repo']['full_name'] = 'Test/Other' + double['pulls'].extend([ + scope_pr(44, '4' * 40, 'FM/branch-owned'), + dict(scope_pr(31, '5' * 40, 'fm/unrelated', head_repo='contributor/project'), + html_url='https://github.com/Test/Project/pull/31'), + ]) + double['checks'].update({sha: greens(sha, conclusion='failure') for sha in ('4' * 40, '5' * 40)}) + (world / 'gh.json').write_text(json.dumps(double)) + for origin_slug in ('test/project', 'TEST/PROJECT'): + git(home / 'projects/project', 'remote', 'set-url', 'origin', f'https://github.com/{origin_slug}.git') + for lane in (False, True): + case_scope, requests = scoped(lane) + assert set(rows(case_scope, 'open_pr')) == {PR_URL + '31', PR_URL + '33', other_url + '41', other_url + '43'}, case_scope + assert set(rows(case_scope, 'red_check')) == {PR_URL + '32', other_url + '42'}, case_scope + assert all(sha in requests for sha in (shas[33], '1' * 40, '3' * 40)), requests + assert all(sha not in requests for sha in ('4' * 40, '5' * 40)), requests finally: + git(home / 'projects/project', 'remote', 'set-url', 'origin', 'https://github.com/test/project.git') marker_file.unlink(missing_ok=True) shutil.rmtree(other) double['pulls'], double['checks'] = scope_saved[0], scope_saved[1] @@ -793,6 +811,33 @@ finally: fixture(tasks, backlog) print('PASS: standing pauses survive unrelated answers; resumption and authoritative current states override them', flush=True) +for event, status_text in ( + ('done', 'done: delivered\n'), + ('paused', 'paused: waiting for CI\n'), + ('resolved', 'paused [key=ci]: waiting\nresolved [key=choice]: answered\n')): + item = stopped_task('generation-' + event, event, exists=None, alive='unknown', + detail='task generation changed during snapshot', status_text=status_text) + item['current_state']['source'] = 'none' + records = [dict(id=item['id'], structured=True, state='in_flight', requires_child_metadata=True)] + try: + fixture([item], records) + invalidated = ledger('--heartbeat') + assert not invalidated['complete'], invalidated + evidence = rows(invalidated, 'coverage')['ledger degraded']['evidence'] + assert 'worker liveness ' + item['id'] in evidence, invalidated + assert 'worker current state ' + item['id'] + ': task generation changed during snapshot' in evidence, invalidated + assert not rows(invalidated, 'missing_worker'), invalidated + assert json.loads((home / 'state/open-loops.json').read_text()) == invalidated + item['current_state']['detail'] = 'backend target gone: gone' + item['endpoint'].update(exists=False, agent_alive='missing') + fixture([item], records) + coherent_stop = ledger() + assert coherent_stop['complete'] and not rows(coherent_stop, 'missing_worker'), coherent_stop + finally: + (home / 'state' / (item['id'] + '.status')).unlink() +fixture(tasks, backlog) +print('PASS: generation-invalidated snapshots retain coverage failures despite live stop events', flush=True) + # Unselected ready items are visible backlog, not overdue obligations; dispatched ones still are. (home / 'state/started-ready.status').write_text(f'working [at={hours(1)}]: spawned\n') selected_task = task('selected-ready', fresh) From 63095983e8acdcd0b79141f3e542ebfa0f7d98ca Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 09:03:10 -0500 Subject: [PATCH 5/5] no-mistakes(document): Correct lane-scoped ledger documentation and reconciliation guidance --- docs/architecture.md | 3 ++- docs/configuration.md | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index dc3bcdeeba8..d86ddf4e329 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -472,7 +472,8 @@ For manual project initialization, [`bin/fm-ensure-agents-md.sh`](../bin/fm-ensu The destination for each kind of knowledge, including project-intrinsic knowledge, is owned by [`AGENTS.md` section 6](../AGENTS.md#6-project-and-knowledge-management). Memory writes use inspect-then-update rather than blind append; the internal [`stow` skill](../.agents/skills/stow/SKILL.md) owns tier markers, decay, cold archival, and offload. The same pass also persists open-work record state the session is holding - filing a thread that was never recorded and correcting one the session knows went stale - bounded to the open work that session is actually holding. -It is deliberately not a reconciliation of durable records against repository or PR reality: its input is the volatile context, so it can only preserve what the session still knows, and no reconciliation that outlives a session exists today. +It is deliberately not a reconciliation of durable records against repository or PR reality: its input is the volatile context, so it can only preserve what the session still knows. +The [open-work ledger](configuration.md#open-work-ledger-configopen-loopsjson) owns reconciliation against live worker and delivery evidence. Task-scoped notes use `bin/fm-tasks-axi.sh show --full` followed by `bin/fm-tasks-axi.sh update --body-file `, adding `--archive-body` when the prior body should remain recoverable. The stow pass never writes a skill, but a separately executed, captain-approved migration may move conditional knowledge into a user-owned local skill excluded from the Firstmate clone; changes to Firstmate's tracked skills remain deliberate repository work through the normal PR pipeline. Invoked in a primary home, `/stow` then cascades the same sweep to every registered secondmate, enumerated through `bin/fm-stow-cascade.sh`: each home is accounted and curated against its own startup-memory allowance, a live secondmate sweeps its own session, and a slow or unreachable home is reported as an exception rather than blocking the primary. diff --git a/docs/configuration.md b/docs/configuration.md index 313e0a2fa89..59ddc1bbf9a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -575,29 +575,33 @@ See [`fleet-ledger.md`](fleet-ledger.md) for the opt-in setup, record contract, `bin/fm-open-loops.sh --json` is a fresh, read-only reading of this home; it never changes a worker, a PR, or the backlog, suppresses optional Git locks, and isolates temporary merge-tree objects from the inspected repository. Home selection and the state, data, config, and projects overrides follow shell defaults: unset or empty values use `FM_HOME`, then `FM_ROOT_OVERRIDE`, then the code root for the home, and the selected home's corresponding directory for each source. Every JSON row carries its category, subject, owner, next action, age in seconds, age limit, and overdue verdict. -An unknown age stays `null` and counts as overdue, and an age equal to its limit is overdue. +For a non-informational row, an unknown age stays `null` and counts as overdue, and an age equal to its limit is overdue. | Category | A row exists when | Default limit | | --- | --- | --- | -| `missing_worker` | an In flight item requiring child metadata lacks a task record or endpoint, or an ordinary task has a confirmed missing or dead worker that did not record a stop | 600 s | -| `ready_not_started` | a Queued item has no hold kind or unresolved blocker, and its optional hold date is absent or due; it is overdue only once dispatch has begun | 1800 s | +| `missing_worker` | an In flight item requiring child metadata lacks a task record, or its worker or an ordinary task's worker has a confirmed missing endpoint or dead agent without a recorded stop | 600 s | +| `ready_not_started` | a Queued item has no hold kind or unresolved blocker, and its optional hold date is absent or due; unselected items are informational | 1800 s | | `unanswered_question` | a `needs-decision` or `blocked` status key is still open, or a current captain-held backlog row is in the `live` or `aged` hold bucket | 1800 s | | `failed_task` | a task record's current state is `failed` and its deliverable is neither landed nor recorded as dropped | 1800 s | | `stalled_worker` | a live worker reads `working` but has no recent commit, status line, or pipeline progress within its limit; the row carries the last error-looking line from a bounded pane-tail sample when present, otherwise a no-progress explanation | 3600 s | | `unlanded_commit` | a ship task's copy holds nonmerge commits absent from the default branch and not represented by its open or merged PR's actual head | 86400 s | | `open_pr` | an open PR this home owns waits on checks, a reviewer, or firstmate's review routing | 3600 s | -| `red_check` | the latest run of a check on a PR this home owns failed; the next action is always `diagnose: code or test`, never a waiver | 0 s | +| `red_check` | the latest run of a check on an open PR this home owns failed; the next action is always `diagnose: code or test`, never a waiver | 0 s | | `coverage` | a source is unreadable or its forge is unsupported, so coverage is incomplete | 0 s | Each home reports only what it owns. -A PR is owned when one of this home's task records or backlog items names its exact URL, or when its head repository (`head.repo.full_name`) and head branch match the origin repository and branch of one of this home's tasks. Repository identities are matched case-insensitively; branch names remain case-sensitive, and recorded URLs must match exactly. Matching only the base repository does not establish ownership of a fork's branch. -Every other open PR of the discovered repositories is unowned: it gets no ledger rows, and its checks are not fetched. Unowned PR triage is separate from this ledger. +A PR is owned when one of this home's task records or backlog items names its exact URL, or when its head repository (`head.repo.full_name`) and head branch match the origin repository and branch of one of this home's tasks. +Repository identities are matched case-insensitively; branch names remain case-sensitive, and recorded URLs must match exactly. +Matching only the base repository does not establish ownership of a fork's branch. +Every other open PR of the discovered repositories is unowned: it gets no ledger rows, and its checks are not fetched. +Unowned PR triage is separate from this ledger. A secondmate home, marked by a valid `.fm-secondmate-home` file, discovers repositories only from its own tasks and owned PR URLs, never from its shared `projects/` clones or its own checkout. An informational row carries `informational: true` and is never overdue at any age, so it neither wakes the watcher nor appears in Bearings. A queued item is selected for dispatch once it has a task record or a status log in this home; unselected ready items stay visible as informational rows, so the unprioritized backlog is not an overdue obligation. A worker whose current state is `done`, `parked`, or `paused` has recorded its own stop: a gone endpoint or an unreadable live state then adds neither a `missing_worker` row nor degraded coverage. -When current state is unknown, the latest logical `done` event or a standing declaration of the configured pause verb (`FM_CLASSIFY_PAUSED_VERB`, default `paused`) can establish a recorded stop. The shared status-event parser ignores continuation prose and preserves a pause behind unrelated `resolved` answers; a resolution for the pause's own phase key or any later non-resolution event retracts that pause. -Known `working`, `blocked`, and `failed` current states override historical stop events. A recorded stop never hides unlanded commits. +When current state is unknown, the latest logical `done` event or a standing declaration of the configured pause verb can establish a recorded stop, using the [shared status-event parser](../bin/fm-status-event-lib.sh) for logical events, pause vocabulary, and declared-wait semantics. +Known `working`, `blocked`, and `failed` current states override historical stop events. +A recorded stop never hides unlanded commits. A snapshot invalidated with `current_state.detail` equal to `task generation changed during snapshot` cannot establish a recorded stop, even if the live status log declares one; its inconclusive liveness and current-state diagnostic retain degraded coverage. A source that cannot be read adds the single `coverage` row named `ledger degraded` and sets `complete: false`; it is never read as an empty fleet, and the other sources still report. @@ -608,8 +612,9 @@ Blocked, dated, and Done holds are excluded, and no historical audit or new pers For local-only projects, the ledger uses the qualified local default branch advanced by `fm-merge-local` as delivery proof; other project modes retain their normal remote-default proof, with nonmerge commit patch equivalence against the actual PR head. GitHub repositories are discovered from recorded local task and backlog PR URLs as well as project origins, including fork-only delivery; their open PRs are classified before PR-head coverage can suppress unlanded commits. Only exact `github.com` hosts are admitted or projected, and unsupported origins or PR URLs disclose degraded coverage; GitLab and Gerrit merge proofs are outside this ledger's current scope. -An existing worker endpoint with reconciled `working` state counts as live when its backend's recovery verdict is `unverified`; other inconclusive liveness adds degraded coverage. -Unreadable or unknown current task state adds degraded coverage independently of liveness, and only nonfuture progress timestamps count as work evidence. +An existing worker endpoint with reconciled `working` state counts as live when its backend's recovery verdict is `unverified`; other inconclusive liveness adds degraded coverage unless the worker recorded a stop. +Unknown current task state adds degraded coverage independently of liveness, except when a recorded stop or a confirmed dead or missing agent with `backend target gone:` detail explains it. +Only nonfuture progress timestamps count as work evidence. Pipeline progress uses each task's worktree to resolve a relative `NM_HOME`; unset or empty values select `$HOME/.no-mistakes`. Progress remains scoped to that resolved store and the task's project and branch. The ledger covers this home's current backlog, ordinary task records, status questions, and discovered GitHub PRs; each secondmate home runs its own watcher and reports through its own parent channel.