From b7f07306642058b089cb3e0cc7845a9c7944a3ca Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 08:12:13 -0700 Subject: [PATCH 1/3] Stamp audit runs and flag stale driftNotes (#341) Both halves of #341 are about the same failure: a point-in-time snapshot presented as current state, so an agent picking it up "fixes" what is already fixed. 1. Freshness. spec/audit.py prints a run stamp (audit run | hub ) and, per repo, the exact commit it read (@ @), and closes with a re-run reminder. AUDIT.md section 8 requires anything derived from a run - a report, and especially an onboarding issue - to quote the stamp, and requires the agent picking it up to re-run the audit and act on the live result rather than the pasted findings. 2. driftNotes. A note records a *current* deviation; once resolved it is deleted, not left describing finished work. audit.py now flags a note asserting outstanding work (pending / not yet / missing / behind / ...) on a repo that otherwise audits clean - the exact pattern reconciled reactively on three repos (#329, #335, #348). The marker list is deliberately narrow so a note recording a permanent deviation ("no get-version-task; relies on validate-task") never trips it. Codified as setup.driftnotes.current. The new check immediately earned its keep: it caught a stale note I wrote myself for HomeAutomation-Config ("OWNER TASK OPEN: ... secrets not yet set") that the owner had since resolved and I never reconciled. Verified the secrets are present in both stores and removed the note - conformance being complete, it recorded no deviation at all. Fleet-wide the heuristic flagged exactly that one note, so it is low-noise. Co-Authored-By: Claude Opus 4.8 (1M context) --- AUDIT.md | 4 ++++ registry/repos.json | 2 +- spec/audit.py | 39 +++++++++++++++++++++++++++++++++++---- spec/project-types.json | 1 + 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/AUDIT.md b/AUDIT.md index 9bb0363b..9fa88390 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -115,6 +115,10 @@ A repo is **operational** only if every applicable check passes. A single applic Write `reports//audit.md` from [`reports/_template.md`][template]: a dimension x {letter, intent, verdict, evidence} table with `file:line` citations (WORKFLOW.md 5A style), a drift section, and a list of proposed registry/spec updates (e.g. a resolved `classificationPending`). Rank findings most severe first. +**Findings are a point-in-time snapshot - stamp them and re-verify before acting.** [`spec/audit.py`][audit-runner] prints a run stamp (`audit run | hub `) and, per repo, the exact commit it read (`@ @`). Anything derived from a run - a report, and especially an **onboarding or conformance issue** - quotes that stamp, so a reader can tell whether it still applies. An agent picking up such an issue **re-runs the audit first and acts on the live result, not the pasted findings**: a repo moves between filing and pickup, so a stale block leads an agent to "fix" what is already fixed (re-requesting secrets that exist, attempting a no-op forward-sync). State the findings as evidence for *why* the issue was filed, never as the current state. + +**Reconcile `driftNotes` in the same pass.** A registry `driftNote` records a *current* deviation from the baseline. Once the deviation is resolved the note is deleted, not left describing finished work - hand-maintained prose drifts silently otherwise. `spec/audit.py` flags this: when a repo audits clean but a note still asserts outstanding work ("pending", "not yet", "missing", "behind", ...), it raises a drift finding naming the note. + ## 9. Escalate Surface spec questions rather than resolving them silently - e.g. the Python config-placement canonicalization, or a new construct no type covers. A repeated letter miss that many repos share is a signal the spec (not each repo) needs adjusting; raise it. diff --git a/registry/repos.json b/registry/repos.json index c804a3f6..ccd9e732 100644 --- a/registry/repos.json +++ b/registry/repos.json @@ -171,7 +171,7 @@ "requiredSecrets": [], "consumerModel": "pull", "releaseTrigger": "dispatch-only", - "driftNotes": ["Maintainer config/ops repo (docker-compose stacks, lifecycle scripts, Firewalla configs); Linux-consumed on the Proxmox host, so lineEndings lf.", "Renamed from HomeAutomation for fleet naming consistency (config repos are *-Config). The Vantage controller config is split out to its own Windows/CRLF Vantage-Config repo, not carried here; the legacy Vantage/ subtree is stripped.", "Private; README self-flags previously-committed secrets - secrets-hygiene concern.", "Operational conformance completed 2026-07-16 (HomeAutomation-Config #21): lint CI + dispatch-only publisher were already live; repo-config operational carry (rulesets/settings verified in sync), adapted self-audit (AUDIT.md + spec/secrets.json), and Dependabot merge-bot landed on develop. OWNER TASK OPEN: CODEGEN_APP_CLIENT_ID + CODEGEN_APP_PRIVATE_KEY not yet set in either secret store - the merge-bot is inert until then."] + "driftNotes": ["Maintainer config/ops repo (docker-compose stacks, lifecycle scripts, Firewalla configs); Linux-consumed on the Proxmox host, so lineEndings lf.", "Renamed from HomeAutomation for fleet naming consistency (config repos are *-Config). The Vantage controller config is split out to its own Windows/CRLF Vantage-Config repo, not carried here; the legacy Vantage/ subtree is stripped.", "Private; README self-flags previously-committed secrets - secrets-hygiene concern."] }, { "name": "KiCadLibrary", diff --git a/spec/audit.py b/spec/audit.py index 3e544369..7f673118 100644 --- a/spec/audit.py +++ b/spec/audit.py @@ -22,6 +22,7 @@ import re import subprocess import sys +from datetime import datetime, timezone ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -30,6 +31,9 @@ "allow_rebase_merge", "allow_auto_merge", "allow_update_branch", "delete_branch_on_merge", ] RULESET_SUBSET = ["name", "target", "enforcement", "bypass_actors", "conditions", "rules"] +# Phrases in a registry driftNote that assert work still outstanding. Deliberately specific: a note +# recording a permanent deviation ("no get-version-task; relies on validate-task") must not match. +PENDING_MARKERS = ["pending", "not yet", "owed", "todo", "still ", "behind", "missing", "absent"] def load(rel): @@ -216,7 +220,23 @@ def audit_repo(entry, spec): if gh(f"repos/{slug}/contents/{path}?ref={ground}", ok404=True) is None: findings.append(("LETTER", f"file: {path} absent on {ground} (verify intent per AUDIT.md section 7)")) - return findings + # --- Registry driftNotes freshness --- + # driftNotes are hand-maintained prose and nothing checks them against reality, so a note describing + # work that has since completed lingers (reconciled reactively on three repos before this check). + # Heuristic, deliberately narrow to stay low-noise: a note asserting *outstanding* work is suspect + # only when every other check above passed - a repo that audits clean has no outstanding work for + # such a note to describe. A note recording a permanent deviation ("relies on validate-task") uses + # no pending-marker and never trips this. + if not findings: + for note in entry.get("driftNotes", []): + marker = next((w for w in PENDING_MARKERS if w in note.lower()), None) + if marker: + findings.append(("DRIFT", f"registry: driftNote says '{marker}' but the audit is clean - verify and reconcile: \"{note[:70]}{'...' if len(note) > 70 else ''}\"")) + + audited_sha = (branch_main or branch_dev or {}).get("commit", {}).get("sha", "") + if ground == "develop" and branch_dev: + audited_sha = branch_dev["commit"]["sha"] + return findings, audited_sha def main(): @@ -235,14 +255,23 @@ def main(): print(f"Not cataloged: {', '.join(sorted(missing))}", file=sys.stderr) return 2 + # Findings are a point-in-time snapshot. Stamp the run so anything derived from it (an onboarding + # issue, a report) carries its own freshness signal and a reader can tell whether it still applies. + run_utc = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + hub = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, cwd=ROOT) + hub_sha = hub.stdout.strip() if hub.returncode == 0 else "unknown" + print(f"audit run {run_utc} | hub {hub_sha}\n") + hard = 0 for entry in repos: model = entry.get("workflowModel") or spec["registry"].get("defaults", {}).get("workflowModel") or "release" - print(f"== {entry['name']} ({', '.join(entry.get('types', []))}; {model}) ==") + ground = entry.get("groundTruthBranch", "main") try: - findings = audit_repo(entry, spec) + findings, audited_sha = audit_repo(entry, spec) except Exception as e: # a gh/JSON failure mid-audit must not abort the sweep - findings = [("ERROR", str(e))] + findings, audited_sha = [("ERROR", str(e))], "" + stamp = f" @ {ground}@{audited_sha[:7]}" if audited_sha else "" + print(f"== {entry['name']} ({', '.join(entry.get('types', []))}; {model}){stamp} ==") if not findings: print(" clean (deterministic checks; the full operational verdict is AUDIT.md's)") for kind, text in findings: @@ -250,6 +279,8 @@ def main(): if kind in ("DEFECT", "LETTER", "ERROR"): hard += 1 print(f"\n{len(repos)} repo(s) audited; {hard} defect/letter/error finding(s).") + print("Findings are a point-in-time snapshot: re-run this audit before acting on them, and quote the") + print("run stamp above in any issue derived from it (AUDIT.md section 8).") return 1 if hard else 0 diff --git a/spec/project-types.json b/spec/project-types.json index f1f50679..6e3966bd 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -124,6 +124,7 @@ "checks": [ { "id": "setup.secrets.present", "verdict": "letter", "assert": "Every requiredSecret for the repo's publish mechanisms is configured (per spec/secrets.json).", "intentRef": "repo-config/README.md" }, { "id": "setup.secrets.noforbidden", "verdict": "letter", "assert": "No forbidden secret is present (e.g. a static NUGET_API_KEY on an OIDC repo).", "intentRef": "spec/secrets.json" }, + { "id": "setup.driftnotes.current", "verdict": "intent", "assert": "A registry driftNote records a current deviation from the baseline; once resolved the note is deleted, not left describing finished work. spec/audit.py flags a note asserting outstanding work (pending / not yet / missing / behind / ...) on a repo that otherwise audits clean. Findings derived from an audit run carry its stamp and are re-verified at pickup, never trusted as current state.", "intentRef": "AUDIT.md" }, { "id": "setup.dependabot.ecosystems", "verdict": "intent", "assert": "For each ecosystem the repo's tree implies, .github/dependabot.yml declares it (dual-target main+develop per the fleet norm): github-actions when .github/workflows/ is present (its workflows reference actions, else their versions go stale and a stood-up merge-bot has no PRs to auto-merge), devcontainers when a .devcontainer is present. A missing implied ecosystem is a drift finding. Language ecosystems (nuget/uv/npm) are directory-scoped, audited by inspection.", "intentRef": "AGENTS.md#branching-model" } ] }, From d66fa1141f88260a2aa38f746e605e12a064f9ad Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 08:16:48 -0700 Subject: [PATCH 2/3] Fix the tuple-return, stamp-attribution, and marker-boundary bugs (Copilot #352) Three fixes, the first two real bugs introduced by this PR: 1. audit_repo's early gh-failure path still returned a bare findings list while the normal path returned (findings, sha), so unpacking it raised ValueError - which main's except Exception then reported INSTEAD of the original gh error, masking it. Returns a tuple now. Regression-tested by forcing the first gh call to fail: the 403 is preserved. 2. audited_sha fell back to (branch_main or branch_dev), so a repo whose ground truth is develop but whose develop is missing would stamp "develop@" - misattributing every finding. It now reads the branch matching groundTruthBranch and stamps nothing when that branch is absent. 3. The driftNote marker match used plain substrings, missing "still."/"missing," at punctuation and risking hits inside longer words. Now a word-boundary regex; the "still " trailing-space hack is gone. Table-tested: 4 true positives and 3 negatives, including "missingness metric" and the permanent-deviation note. Co-Authored-By: Claude Opus 4.8 (1M context) --- spec/audit.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/spec/audit.py b/spec/audit.py index 7f673118..342cbde4 100644 --- a/spec/audit.py +++ b/spec/audit.py @@ -33,7 +33,7 @@ RULESET_SUBSET = ["name", "target", "enforcement", "bypass_actors", "conditions", "rules"] # Phrases in a registry driftNote that assert work still outstanding. Deliberately specific: a note # recording a permanent deviation ("no get-version-task; relies on validate-task") must not match. -PENDING_MARKERS = ["pending", "not yet", "owed", "todo", "still ", "behind", "missing", "absent"] +PENDING_MARKERS = ["pending", "not yet", "owed", "todo", "still", "behind", "missing", "absent"] def load(rel): @@ -78,7 +78,7 @@ def audit_repo(entry, spec): try: live = gh(f"repos/{slug}") except RuntimeError as e: - return [("ERROR", str(e))] + return [("ERROR", str(e))], "" # --- Branch facts --- branch_main = gh(f"repos/{slug}/branches/main", ok404=True) @@ -229,13 +229,14 @@ def audit_repo(entry, spec): # no pending-marker and never trips this. if not findings: for note in entry.get("driftNotes", []): - marker = next((w for w in PENDING_MARKERS if w in note.lower()), None) + marker = next((w for w in PENDING_MARKERS if re.search(rf"\b{re.escape(w)}\b", note, re.I)), None) if marker: findings.append(("DRIFT", f"registry: driftNote says '{marker}' but the audit is clean - verify and reconcile: \"{note[:70]}{'...' if len(note) > 70 else ''}\"")) - audited_sha = (branch_main or branch_dev or {}).get("commit", {}).get("sha", "") - if ground == "develop" and branch_dev: - audited_sha = branch_dev["commit"]["sha"] + # Stamp the commit actually read for the ground-truth branch. Never fall back to the other branch: + # a stamp naming develop while carrying main's sha would misattribute every finding. + ground_branch = branch_dev if ground == "develop" else branch_main + audited_sha = (ground_branch or {}).get("commit", {}).get("sha", "") return findings, audited_sha From 690b1bf35ae52dbebda28876e7dc926c55051c71 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 08:20:20 -0700 Subject: [PATCH 3/3] Trim the heuristic comment; disambiguate the clean-verdict wording (Copilot #352) The driftNotes comment ran six lines against the repo's comment-hygiene rule (one line default, a second only to carry a constraint); trimmed to two that keep the gating rationale and the false-positive guard. The clean-run message said "the full operational verdict", which collides with workflowModel: operational and misreads on a release repo. It now says "the full letter+intent verdict", naming the tiers AUDIT.md actually evaluates. Co-Authored-By: Claude Opus 4.8 (1M context) --- spec/audit.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/spec/audit.py b/spec/audit.py index 342cbde4..e01f8861 100644 --- a/spec/audit.py +++ b/spec/audit.py @@ -221,12 +221,8 @@ def audit_repo(entry, spec): findings.append(("LETTER", f"file: {path} absent on {ground} (verify intent per AUDIT.md section 7)")) # --- Registry driftNotes freshness --- - # driftNotes are hand-maintained prose and nothing checks them against reality, so a note describing - # work that has since completed lingers (reconciled reactively on three repos before this check). - # Heuristic, deliberately narrow to stay low-noise: a note asserting *outstanding* work is suspect - # only when every other check above passed - a repo that audits clean has no outstanding work for - # such a note to describe. A note recording a permanent deviation ("relies on validate-task") uses - # no pending-marker and never trips this. + # Gated on everything else passing: a clean repo has no outstanding work for a pending-marker note to + # describe. Narrow markers keep a permanent-deviation note ("relies on validate-task") from tripping. if not findings: for note in entry.get("driftNotes", []): marker = next((w for w in PENDING_MARKERS if re.search(rf"\b{re.escape(w)}\b", note, re.I)), None) @@ -274,7 +270,7 @@ def main(): stamp = f" @ {ground}@{audited_sha[:7]}" if audited_sha else "" print(f"== {entry['name']} ({', '.join(entry.get('types', []))}; {model}){stamp} ==") if not findings: - print(" clean (deterministic checks; the full operational verdict is AUDIT.md's)") + print(" clean (deterministic checks; the full letter+intent verdict is AUDIT.md's)") for kind, text in findings: print(f" {kind:6} {text}") if kind in ("DEFECT", "LETTER", "ERROR"):