diff --git a/.gitignore b/.gitignore index 29a8119..5a4b3ae 100644 --- a/.gitignore +++ b/.gitignore @@ -59,8 +59,18 @@ venv/ # instance's failure history, spend calibration and availability stay local. # Sanitized public twins of the load-bearing docs are committed under their # canonical names; these are the full versions. +# +# IMPROVEMENT_BACKLOG.md IS NO LONGER LISTED, ON PURPOSE. Ignoring it made it +# invisible: a gitignored file does not exist in a git worktree, and agents work +# in worktrees, so CLAUDE.md's two mandatory rules about it were unfollowable by +# the workers they bind. The log now lives OUTSIDE the tree at +# $ORCH_LOCAL_RUNTIME, reached via improvement_log.py, and the tracked +# IMPROVEMENT_BACKLOG.md here is a short POINTER to it. Keeping it ignored as +# well would let the 481 KB evidence file reappear at that path silently, which +# is the failure this whole change undoes; tracked, it announces itself in +# `git status` instead. test_improvement_log.py fails if the pointer grows into +# a log. LOCAL_POLICY.md -IMPROVEMENT_BACKLOG.md CAPABILITY_USEFULNESS.md *.local.md diff --git a/.verify-floor.json b/.verify-floor.json index eb70c9b..0a8f493 100644 --- a/.verify-floor.json +++ b/.verify-floor.json @@ -1,8 +1,8 @@ { - "collected": 387, - "passed": 387, + "collected": 391, + "passed": 391, "skipped_max": 26, "selftest_skipped_max": 7, "gate_skipped_max": 2, - "note": "Recorded by verify.py --update-floor, except the *_max ceilings, which are edited BY HAND and never re-measured. `collected` catches tests that stopped being collected; `passed` is compared against passed+skipped, so a check may move between passing and consciously-skipped but the two together may never shrink. The *_max ceilings bound the skipped side: 24/7/2 is exactly what a machine with none of this instance's local prerequisites skips (a GitHub runner: no agent CLIs, no ~/.codex/skills, no /Applications/ChatGPT.app, no populated capability ledger), measured 2026-08-21. On the owner's machine all prerequisites exist and nothing skips at all. Raising a ceiling is a deliberate act: it means agreeing that one more thing is allowed to go unchecked, so say which and why in the commit. LOWERED 26 -> 24 on 2026-08-22, reverting the raise made earlier the same day. The two kill-switch exemption tests no longer need to skip on a bare runner: their declarations moved out of the running instance's ledger and into capabilities.KNOWN_DECLARATIONS, so they assert code-derived truth and run everywhere. Moving a test back below the ceiling is the preferred way to lower it -- fix what made it machine-dependent, rather than agreeing to check less. FLOOR 345 -> 353 on 2026-08-22: 345 was measured on a branch cut before #13 (research panels/rounds/domain studies) merged, so the recorded floor sat 8 tests BELOW what main actually collects. A floor below reality is the permissive direction -- those 8 could have silently stopped being collected and still cleared the check, which is exactly the hole this file exists to close. Measure the floor on the merge result, not on the branch. Raised again on 2026-08-22 by the producer-identity-scope branch, which adds tests on top of the 353 recorded by #15; re-measured after rebasing rather than assumed. NOTE: `verify.py --update-floor` REPLACES this note with a generic one, so it must be restored by hand after every use \u2014 the ceiling rationale is the only record of which prerequisite justifies each skip. FLOOR 365 -> 366 on 2026-08-22 (heartbeat-ordering work, PR #18): exactly one new test, test_capabilities.test_no_tick_producer_runs_above_the_heartbeat_export. No ceiling moved and nothing new is skipped -- it reads source files rather than a populated ledger, so it runs on any machine. The branch recorded 354 because it was cut before #16 merged; re-measured on the MERGE RESULT per the rule above, which is exactly the mistake that put the floor 8 below reality last time. FLOOR 366 -> 368 on 2026-08-23: main collected 368 while this file recorded 366, drift left by #34 (evidence-acquisition landed, +1) and #37 (tick capability evidence, +1) whose authors each measured against a branch cut before the other merged. A floor BELOW reality is the permissive direction this file exists to close -- those two could have silently stopped being collected and still cleared the check. Measured on the merge result per the rule above: 368 passed, 0 failed, 0 skipped, 83/83 selftests, 43/43 can-fire, 5/5 gates. CEILING 24 -> 26 and FLOOR 368 -> 387 on 2026-08-23 (profiles/provenance branch, PR #42). This file CONFLICTED with #50, which raised the floor 366 -> 368 on main while this branch raised it to 387; resolved as the UNION rather than by taking a side -- #50's rationale is retained above and the count was RE-MEASURED on the new merge result instead of keeping either number. 368 (main) + 19 (this branch's net new tests) = 387; #50 corrected recorded drift rather than adding coverage, which is why 387 is unchanged from the pre-conflict measurement. Measured in a runner sandbox reproducing CI exactly (361 passed, 26 skipped, 387 collected) AND on the owner's machine (387 passed, 0 skipped, 5/5 gates). The two new skips are drift detectors against a REAL installed agent runtime, so neither can be moved below the ceiling -- the preferred way to lower one: (1) agy advertised-models cache absent, since comparing declared model ids against the catalogue agy actually advertises needs that catalogue, and a fixture would exercise the comparison while detecting no real drift; (2) vibe config absent (~/.vibe/config.toml), since active_model cannot be read to check for drift when there is no config to read. Both name their missing prerequisite, so a green run still states what it did not check. A third candidate skip was REFUSED: dispatcher's per-run agy-log assertion failed on a bare runner because adapters.advertised_models shells out to `agy models` when its disk cache is cold, and that probe landed inside a monkeypatched subprocess.run and overwrote the captured command. That is a stub leak, so it was fixed by ISOLATING the double rather than by skipping -- which makes CI run MORE." + "note": "Recorded by verify.py --update-floor, except the *_max ceilings, which are edited BY HAND and never re-measured. `collected` catches tests that stopped being collected; `passed` is compared against passed+skipped, so a check may move between passing and consciously-skipped but the two together may never shrink. The *_max ceilings bound the skipped side: 24/7/2 is exactly what a machine with none of this instance's local prerequisites skips (a GitHub runner: no agent CLIs, no ~/.codex/skills, no /Applications/ChatGPT.app, no populated capability ledger), measured 2026-08-21. On the owner's machine all prerequisites exist and nothing skips at all. Raising a ceiling is a deliberate act: it means agreeing that one more thing is allowed to go unchecked, so say which and why in the commit. LOWERED 26 -> 24 on 2026-08-22, reverting the raise made earlier the same day. The two kill-switch exemption tests no longer need to skip on a bare runner: their declarations moved out of the running instance's ledger and into capabilities.KNOWN_DECLARATIONS, so they assert code-derived truth and run everywhere. Moving a test back below the ceiling is the preferred way to lower it -- fix what made it machine-dependent, rather than agreeing to check less. FLOOR 345 -> 353 on 2026-08-22: 345 was measured on a branch cut before #13 (research panels/rounds/domain studies) merged, so the recorded floor sat 8 tests BELOW what main actually collects. A floor below reality is the permissive direction -- those 8 could have silently stopped being collected and still cleared the check, which is exactly the hole this file exists to close. Measure the floor on the merge result, not on the branch. Raised again on 2026-08-22 by the producer-identity-scope branch, which adds tests on top of the 353 recorded by #15; re-measured after rebasing rather than assumed. NOTE: `verify.py --update-floor` REPLACES this note with a generic one, so it must be restored by hand after every use \u2014 the ceiling rationale is the only record of which prerequisite justifies each skip. FLOOR 365 -> 366 on 2026-08-22 (heartbeat-ordering work, PR #18): exactly one new test, test_capabilities.test_no_tick_producer_runs_above_the_heartbeat_export. No ceiling moved and nothing new is skipped -- it reads source files rather than a populated ledger, so it runs on any machine. The branch recorded 354 because it was cut before #16 merged; re-measured on the MERGE RESULT per the rule above, which is exactly the mistake that put the floor 8 below reality last time. FLOOR 366 -> 368 on 2026-08-23: main collected 368 while this file recorded 366, drift left by #34 (evidence-acquisition landed, +1) and #37 (tick capability evidence, +1) whose authors each measured against a branch cut before the other merged. A floor BELOW reality is the permissive direction this file exists to close -- those two could have silently stopped being collected and still cleared the check. Measured on the merge result per the rule above: 368 passed, 0 failed, 0 skipped, 83/83 selftests, 43/43 can-fire, 5/5 gates. CEILING 24 -> 26 and FLOOR 368 -> 387 on 2026-08-23 (profiles/provenance branch, PR #42). This file CONFLICTED with #50, which raised the floor 366 -> 368 on main while this branch raised it to 387; resolved as the UNION rather than by taking a side -- #50's rationale is retained above and the count was RE-MEASURED on the new merge result instead of keeping either number. 368 (main) + 19 (this branch's net new tests) = 387; #50 corrected recorded drift rather than adding coverage, which is why 387 is unchanged from the pre-conflict measurement. Measured in a runner sandbox reproducing CI exactly (361 passed, 26 skipped, 387 collected) AND on the owner's machine (387 passed, 0 skipped, 5/5 gates). The two new skips are drift detectors against a REAL installed agent runtime, so neither can be moved below the ceiling -- the preferred way to lower one: (1) agy advertised-models cache absent, since comparing declared model ids against the catalogue agy actually advertises needs that catalogue, and a fixture would exercise the comparison while detecting no real drift; (2) vibe config absent (~/.vibe/config.toml), since active_model cannot be read to check for drift when there is no config to read. Both name their missing prerequisite, so a green run still states what it did not check. A third candidate skip was REFUSED: dispatcher's per-run agy-log assertion failed on a bare runner because adapters.advertised_models shells out to `agy models` when its disk cache is cold, and that probe landed inside a monkeypatched subprocess.run and overwrote the captured command. That is a stub leak, so it was fixed by ISOLATING the double rather than by skipping -- which makes CI run MORE. FLOOR 387 -> 391 on 2026-08-23 (improvement-log accessor, PR #59): exactly four new tests, all in test_improvement_log.py -- three read tracked files in the tree (the pointer's size and content, and that CLAUDE.md 0 step 3 and 5 name the accessor rather than a bare path) and one runs the accessor as a subprocess against a path that cannot exist. None reads a populated ledger, an agent CLI or ~/.codex, so all four RUN on a bare runner and NO ceiling moved: nothing new is skipped. Measured on the MERGE RESULT after rebasing onto origin/main af6654d, which collected 387 -- not on the branch base, per the rule above." } diff --git a/ADDING_CAPABILITIES.md b/ADDING_CAPABILITIES.md index 4cf57b1..dc5313e 100644 --- a/ADDING_CAPABILITIES.md +++ b/ADDING_CAPABILITIES.md @@ -24,7 +24,8 @@ in a table nothing wrote for it. The dated evidence for all nine failure modes is deliberately **not** committed — it names this instance's repositories, PRs, spend and working constraints. It lives in -`ADDING_CAPABILITIES.local.md` alongside `IMPROVEMENT_BACKLOG.md` (see `LOCAL_POLICY.md`). +`ADDING_CAPABILITIES.local.md` (see `LOCAL_POLICY.md`), and the item-by-item status history lives in +the machine-local improvement log, reached with `python3 improvement_log.py search `. The governing lesson survives the split, and it is the reason this file has a test file rather than only prose: **a rule that lives only in a document does not survive the next session.** `CLAUDE.md` diff --git a/CLAUDE.md b/CLAUDE.md index f0871e3..efec381 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,11 +2,19 @@ > **Local vs public.** This repository is the TOOL: generic capabilities, gates and tests. This > instance's EVIDENCE is not committed — the failure history, spend calibration and attention-budget -> figures live in `LOCAL_POLICY.md`, `IMPROVEMENT_BACKLOG.md`, `CAPABILITY_USEFULNESS.md`, -> `*.local.md` and `Code/Audits/Orchestrator/`, all gitignored. Where a rule below cites a figure, -> read it from `LOCAL_POLICY.md`. Runtime state (Brain, ledger, stamps, worktrees) lives outside the -> tree at `$ORCH_STATE_DIR`, default `~/.codex/orchestrator`, so a second instance needs a different -> `ORCH_STATE_DIR` and no code change. +> figures live in `LOCAL_POLICY.md`, `CAPABILITY_USEFULNESS.md`, `*.local.md` and +> `Code/Audits/Orchestrator/`, all gitignored. Where a rule below cites a figure, read it from +> `LOCAL_POLICY.md`. Runtime state and durable evidence live OUTSIDE the tree, behind two variables: +> `$ORCH_LOCAL_RUNTIME` (the Brain, the capability ledger, and the **improvement log** — reached with +> `improvement_log.py`, never by path) and `$ORCH_STATE_DIR` (the audit cache, cadence stamps, +> monitors, worktrees). Both default to `~/.codex/orchestrator`, so a second instance needs different +> values and no code change. +> +> **Gitignored is not the same as machine-local, and confusing the two hid a rule from its own +> workers.** A gitignored file does not exist in a git WORKTREE, and agents work in worktrees — so +> the improvement log, which §0 and §5 both require, was unreachable by every worker those rules +> bind. Evidence that agents must READ belongs outside the tree behind an accessor, not gitignored +> inside it. When you add such a thing, leave a tracked pointer naming the accessor. Read `README.md` first for what this project is and its important functionality. This file is the rules for anyone (human or agent) *changing* it. @@ -143,7 +151,12 @@ back. Adding a "new" feature that already exists is the easy mistake here. 2. Read the historical dormancy inventory: `Code/Audits/Orchestrator/2026-07-08-dormancy-rescan.md`, then generate current activation truth with `python3 capabilities.py inventory`. Feature maturity is not activation evidence. -3. Check `IMPROVEMENT_BACKLOG.md` — items carry status notes; many "ideas" are already DONE. +3. Search the improvement log — `python3 improvement_log.py search `. Items carry status + notes and many "ideas" are already DONE. The log is machine-local evidence living outside the + tree, so **use the accessor, never a path**: it resolves `$ORCH_LOCAL_RUNTIME` for you, prints + each hit under the item that owns it, and — when the log is not on this machine — names what is + missing and exits 2 instead of looking like "no matches". `IMPROVEMENT_BACKLOG.md` in the tree is + a pointer, not the log. 4. If the capability EXISTS: the task is to **wire/activate/extend or un-gate it** (and say so), not rebuild it. If it exists but is deliberately gated, treat flipping the gate as the change and justify it. Only build new if the concept genuinely isn't present. @@ -223,11 +236,16 @@ Do not create a second event log, model registry, or capability inventory. `capability_activation_audit.entrypoint_presence` / `absent_entrypoint_note`, which name the sibling checkout the code was found in — but read the message rather than the missing-parts list. - **The split is TOOL vs EVIDENCE.** Generic capabilities, gates and tests are committed. This - instance's evidence is not: `IMPROVEMENT_BACKLOG.md`, `CAPABILITY_USEFULNESS.md`, - `LOCAL_POLICY.md`, `*.local.md`, `experiments/`, `ux_reviews/`, `data/`, `Audits/`. When adding a + instance's evidence is not: `CAPABILITY_USEFULNESS.md`, `LOCAL_POLICY.md`, `*.local.md`, + `experiments/`, `ux_reviews/`, `data/`, `Audits/` — gitignored in the tree — plus the ledger, the + Brain and the **improvement log**, which live outside it under `$ORCH_LOCAL_RUNTIME`. When adding a personal figure — spend, availability, a habit — it goes in `LOCAL_POLICY.md`; the code refers to it. The boundary is file location, not recall. Published VENDOR list prices are fine and are kept deliberately: they are public and they are the model-tier rationale. + **Evidence an agent is REQUIRED to read must go outside the tree behind an accessor, not + gitignored inside it** — a gitignored path is absent from every worktree, which is how §0 step 3 + became unfollowable. `improvement_log.py` is the pattern: a tracked pointer of the same name, an + accessor that resolves the path, and a named absence when the file is not on this machine. - A live fleet tick runs hourly and writes only to worktrees/state, never to this canonical tree — but if another interactive/headless session is editing here too, coordinate (the fleet-checkout hazard is real; see the user's memory). Look before overwriting a file you didn't create. @@ -282,5 +300,9 @@ and update the gated-features list in README.md + the dormancy inventory. When you activate a dormant feature, un-gate a flag, or add a subsystem: update its lifecycle record, regenerate the capability inventory, update README.md's functionality section if the -topology changed, and append a status note to the relevant IMPROVEMENT_BACKLOG.md item. Do not -duplicate lifecycle verdicts in prose; stale parallel inventories are how features get forgotten. +topology changed, and record a status note on the relevant improvement-log item with +`python3 improvement_log.py append ""`. Use the accessor rather than editing a +file: the log is machine-local (outside the tree), the accessor finds the item and places the dated +note inside it, and it REFUSES on an ambiguous or unknown ref rather than guessing — a note filed +against the wrong item corrupts the record it exists to improve. Do not duplicate lifecycle verdicts +in prose; stale parallel inventories are how features get forgotten. diff --git a/IMPROVEMENT_BACKLOG.md b/IMPROVEMENT_BACKLOG.md new file mode 100644 index 0000000..a976791 --- /dev/null +++ b/IMPROVEMENT_BACKLOG.md @@ -0,0 +1,27 @@ +# Improvement backlog — POINTER. The log itself is machine-local. + +**Do not append to this file.** It is a tracked pointer, a few lines long, and `test_improvement_log.py` +fails if it grows into a log. The real improvement log — the numbered items and their status notes — +is this instance's EVIDENCE, not the tool, so it lives outside the tree with the ledger and the Brain +(`$ORCH_LOCAL_RUNTIME/IMPROVEMENT_BACKLOG.md`, default `~/.codex/orchestrator/`). Reach it through the +accessor, which resolves the path for you: + +```bash +python3 improvement_log.py search # CLAUDE.md §0 step 3 — is this already DONE? +python3 improvement_log.py append "" # CLAUDE.md §5 — record a status note +python3 improvement_log.py path # where it resolved to, and whether it is here +``` + +`search` prints each hit under the item heading that owns it, plus the number of lines and sections it +read, so "no matching items" is a statement about a file that was actually read. When the log is not on +this machine at all — a fresh clone, a CI runner, a second instance — every command says what is +missing and where it would be, and exits 2 rather than returning empty. + +## Why this pointer exists + +The rules above are mandatory and, until this file, unfollowable by the workers they bind. The log was +gitignored, and **a gitignored file does not exist in a git worktree** — agents work in worktrees, so +the project's own countermeasure against its #1 defect (building something that already exists) was +invisible to every worker required to consult it. Nothing in a worktree even hinted the log existed. +Moving it to machine-local state is what `CLAUDE.md` §1 already prescribes for runtime state and +instance evidence; this pointer is what makes it findable from anywhere. diff --git a/README.md b/README.md index 5b7bd7c..d65f784 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,8 @@ safety switch, not dead code. ## Governing docs `ORCHESTRATOR.md` (role/philosophy) · `ARCHITECTURE.md` (data flow) · `FEEDBACK_LOOP.md` (learning loop) · `EVAL_AND_TESTING.md` (selftest/gate regime) · `PLANNING.md` (roadmap) · -`IMPROVEMENT_BACKLOG.md` (numbered items + status log) · `CLAUDE.md` (agent rules) · +`improvement_log.py` (the machine-local numbered items + status log; `IMPROVEMENT_BACKLOG.md` in the +tree is a pointer to it) · `CLAUDE.md` (agent rules) · **`ADDING_CAPABILITIES.md`** (the enforced procedure for adding or reviving a capability, and the nine failure modes it exists to stop). Durable audit history: `Code/Audits/Orchestrator/`. diff --git a/capability_admission.py b/capability_admission.py index dc60147..bb0f373 100644 --- a/capability_admission.py +++ b/capability_admission.py @@ -391,10 +391,15 @@ def preflight(spec: dict) -> dict: r"([A-Z][A-Z_]*(?:UNTIL|DEADLINE|EXPIRES?)[A-Z_]*)[^\n]{0,60}?" r"(20\d\d-\d\d-\d\d)" ) SCAN_SUFFIXES = (".py", ".sh", ".md") -# The backlog narrates history and cites closed records; this module necessarily NAMES the records -# that were never written, because documenting them is the whole point. Both would otherwise report -# themselves forever, and a check that cries wolf about itself gets muted. -SKIP_NAMES = {"IMPROVEMENT_BACKLOG.md", "capability_admission.py", "CAPABILITY_USEFULNESS.md"} +# This module necessarily NAMES the records that were never written, because documenting them is the +# whole point; the usefulness log likewise narrates history and cites closed records. Both would +# otherwise report themselves forever, and a check that cries wolf about itself gets muted. +# +# `IMPROVEMENT_BACKLOG.md` was the third entry and is deliberately GONE from this set: the file that +# narrated history now lives outside the tree (`improvement_log.py`), and what remains at that path +# is a short pointer with nothing to exempt. A stale allowlist entry is an incident record for a +# condition that no longer exists, and it would silence the scan over a file this gate should read. +SKIP_NAMES = {"capability_admission.py", "CAPABILITY_USEFULNESS.md"} def _now() -> int: diff --git a/improvement_log.py b/improvement_log.py new file mode 100644 index 0000000..df037c3 --- /dev/null +++ b/improvement_log.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 +"""improvement_log.py — search and append THIS instance's improvement log, from ANY worktree. + +WHY THIS EXISTS. `CLAUDE.md` depends on the improvement log in two places, and until this module +both were unfollowable by exactly the workers they constrain: + + * §0 step 3, part of the MANDATORY dedup-before-develop check — "check the improvement log; items + carry status notes; many 'ideas' are already DONE"; + * §5 — "append a status note to the relevant improvement-log item". + +Both named a bare path, `IMPROVEMENT_BACKLOG.md`, which was gitignored. **A gitignored file does not +exist in a git worktree, and agents work in worktrees** — it was verified absent from every worktree +on this machine. Three agents in one day were structurally unable to do step 3; two said so and fell +back to ledger notes and committed docstrings. The project's stated #1-failure-mode countermeasure — +its accumulated record of what is already DONE — was invisible to every worker that the rule binds. + +So the log moved OUT OF THE TREE, to machine-local state, and this module is how it is reached. No +agent needs to know the path. The tree keeps a small COMMITTED pointer of the same name +(`IMPROVEMENT_BACKLOG.md`), so the thirteen places that cite that filename still resolve and a +worktree still hints that the log exists. + +WHICH STATE VARIABLE, AND WHY `ORCH_LOCAL_RUNTIME`. `CLAUDE.md` §1 partitions the two: `ORCH_STATE_DIR` +holds the audit cache, the firing monitor and redirect-sweep state — derived artifacts that regenerate +themselves — while `ORCH_LOCAL_RUNTIME` holds the capability LEDGER and the Brain: durable, irreplaceable +instance evidence. The improvement log is the second kind. It cannot be regenerated from anything, and +its §0 role is the prose twin of the ledger's `notes` field (both answer "is this already DONE?"), so it +belongs beside the ledger rather than beside the caches. + +A NAMED ABSENCE, NEVER A SILENT ONE. On a fresh clone, a CI runner, or a second instance with its own +`ORCH_LOCAL_RUNTIME`, there is no log. Every command then says what is missing and where it would be, +and exits 2 — never an empty result. A reason-less empty answer is indistinguishable from "no matching +items", which is this repository's founding defect wearing a different hat. The two are also separate +exit codes so a script can tell them apart: 1 is an honest empty, 2 is an absent log. + +DEDUP FINDING (CLAUDE.md §0), recorded before a line was written. Grepped by concept, not name: +`git grep IMPROVEMENT_BACKLOG` returns thirteen references and every one is prose or a comment — no +reader, no writer, no accessor of any kind. `improvement_backlog|improvement-backlog|backlog_notes` +over `*.py` returns nothing. The only machinery that touches the filename is +`capability_admission.SKIP_NAMES`, an exclusion rather than an accessor. `backlog.py` is a different +subject entirely (this tool's own fleet work-discovery lane, writing `backlog.json`) and is not +extended here. The concept is genuinely absent, so this is new. + +NOT A CAPABILITY, DELIBERATELY. This is documentation access: it has no dispatch path, no outcome and +no learning sink, so the admission gate does not bind on it and it gets no ledger row — the same +reasoning `env_prereq.py` records for itself. Registering a row would also be actively harmful right +now: the ledger is shared per MACHINE while code is branch-isolated per WORKTREE, so a row added here +would turn every sibling worktree's `verify.py` red for a module they cannot see (CLAUDE.md §1). + +CLI + python3 improvement_log.py path + python3 improvement_log.py search [--limit N] [--context N] + python3 improvement_log.py append + python3 improvement_log.py --selftest +""" +from __future__ import annotations + +import argparse +import datetime +import os +import re +import shutil +import sys +import tempfile +from pathlib import Path + +# The two env knobs, in the order they are consulted. Read through `log_path()` rather than captured +# at import: the selftest and the tests point a subprocess at a synthetic file, and a constant frozen +# at import time would make the module untestable without touching the owner's real 481 KB log. +ENV_DIRECT = "ORCH_IMPROVEMENT_LOG" +ENV_RUNTIME = "ORCH_LOCAL_RUNTIME" +LOG_NAME = "IMPROVEMENT_BACKLOG.md" + +# Exit codes, so a caller can distinguish the two kinds of nothing. +EXIT_OK = 0 +EXIT_NO_MATCH = 1 # ran, read the whole log, found nothing — an HONEST empty +EXIT_ABSENT = 2 # the log itself is not on this machine — a NAMED absence +EXIT_REFUSED = 3 # the caller's item-ref matched zero or many items; nothing was written + +HEADING_RE = re.compile(r"^(#{2,3})\s+(.*?)\s*$") +# `## 🟢 8. GitHub API rate-limit awareness` -> item number 8. The status emoji and any other leading +# punctuation are skipped, because the marker changes over an item's life and the number does not. +ITEM_NUM_RE = re.compile(r"^[^0-9A-Za-z]*(\d+)\s*[.)]") + + +def log_path() -> Path: + """Where the improvement log lives, resolved the way the rest of this tree resolves state.""" + direct = os.environ.get(ENV_DIRECT) + if direct: + return Path(direct).expanduser() + runtime = Path(os.environ.get(ENV_RUNTIME, Path.home() / ".codex" / "orchestrator")) + return runtime.expanduser() / LOG_NAME + + +def absence_note(path: Path) -> str: + """What is missing, where it would be, and what to do — never an empty result.""" + return ( + f"PROBLEM: this instance's improvement log is NOT on this machine.\n" + f" looked for: {path}\n" + f" controlled by: ${ENV_DIRECT} (a full path), else ${ENV_RUNTIME}/{LOG_NAME}\n" + f" what it is: the numbered improvement items plus their status log — machine-local\n" + f" instance EVIDENCE, deliberately never committed (CLAUDE.md §1, tool vs evidence).\n" + f" so: a fresh clone, a CI runner and a second instance all legitimately lack it. This is\n" + f" NOT an empty search result. Do NOT create one inside the tree: the tracked\n" + f" {LOG_NAME} is a pointer, and committing the evidence is what the split forbids." + ) + + +# -------------------------------------------------------------------------------------------- +# Parsing: sections, so a hit can be reported under the item that owns it. +# -------------------------------------------------------------------------------------------- + +def parse_sections(text: str) -> list[dict]: + """Every `##`/`###` heading with its line span and item number, in file order. + + One pass, and the span END is exclusive, so `append` and `search` agree on where an item stops. + """ + lines = text.splitlines() + heads: list[dict] = [] + for idx, line in enumerate(lines): + m = HEADING_RE.match(line) + if not m: + continue + level, title = len(m.group(1)), m.group(2) + num_m = ITEM_NUM_RE.match(title) + heads.append({"level": level, "title": title, "line": idx + 1, + "item": num_m.group(1) if num_m and level == 2 else None, + "end": len(lines)}) + # A section ends where the next heading of the same-or-shallower level begins. + for i, head in enumerate(heads): + for nxt in heads[i + 1:]: + if nxt["level"] <= head["level"]: + head["end"] = nxt["line"] - 1 + break + return heads + + +def _owning(heads: list[dict], line_no: int) -> dict | None: + """The DEEPEST section containing this line — the `###` if there is one, else the `##`.""" + best = None + for head in heads: + if head["line"] <= line_no <= head["end"]: + if best is None or head["level"] >= best["level"]: + best = head + return best + + +# -------------------------------------------------------------------------------------------- +# search — CLAUDE.md §0 step 3, in one command. +# -------------------------------------------------------------------------------------------- + +def search(term: str, *, path: Path | None = None, limit: int = 40, + context: int = 0) -> dict: + """Case-insensitive hits, each reported under the item heading that owns it. + + Returns the DENOMINATOR too (lines and sections searched), because "no matches" only means + something when the reader can see what was read. + """ + path = path or log_path() + if not path.is_file(): + return {"present": False, "path": str(path), "term": term, + "absent_reason": absence_note(path), "matches": [], + "lines": 0, "sections": 0} + text = path.read_text(encoding="utf-8", errors="ignore") + lines = text.splitlines() + heads = parse_sections(text) + needle = term.lower() + matches = [] + for idx, line in enumerate(lines): + if needle not in line.lower(): + continue + own = _owning(heads, idx + 1) + row = {"line": idx + 1, "text": line.strip(), + "section": own["title"] if own else "(before the first heading)", + "section_line": own["line"] if own else 0, + "item": (own or {}).get("item")} + if context: + lo, hi = max(0, idx - context), min(len(lines), idx + context + 1) + row["context"] = [ln.rstrip() for ln in lines[lo:hi]] + matches.append(row) + truncated = len(matches) > limit + return {"present": True, "path": str(path), "term": term, "absent_reason": None, + "matches": matches[:limit], "total_matches": len(matches), + "truncated": truncated, "lines": len(lines), "sections": len(heads)} + + +def render_search(rep: dict) -> str: + if not rep["present"]: + return rep["absent_reason"] + head = (f"improvement log: {rep['path']}\n" + f"searched {rep['lines']} lines / {rep['sections']} sections for {rep['term']!r}") + if not rep["matches"]: + # An honest empty, and it says so in words as well as in the exit code. + return (f"{head}\nNO MATCHING ITEMS. The log was read in full and nothing mentions " + f"{rep['term']!r} — this is a real absence of matches, not a missing file.") + out = [head, f"{rep['total_matches']} match(es)" + + (f", showing the first {len(rep['matches'])}" if rep["truncated"] else "")] + last = None + for m in rep["matches"]: + if m["section"] != last: + out.append("") + out.append(f"## {m['section']} (line {m['section_line']}" + + (f", item {m['item']}" if m["item"] else "") + ")") + last = m["section"] + out.append(f" {m['line']}: {m['text']}") + for ctx in m.get("context") or []: + out.append(f" | {ctx}") + return "\n".join(out) + + +# -------------------------------------------------------------------------------------------- +# append — CLAUDE.md §5, in one command. +# -------------------------------------------------------------------------------------------- + +def find_section(ref: str, heads: list[dict]) -> dict: + """Resolve an item-ref to exactly ONE section, or refuse and say which candidates it saw. + + A bare number means the numbered item (`8`, `#8` -> `## 🟢 8. GitHub API rate-limit ...`); + anything else is a case-insensitive substring of the heading. Zero or many matches REFUSE: + guessing which item a note belongs to would corrupt the record it is meant to improve. + """ + ref = ref.strip() + bare = ref.lstrip("#§").strip() + if bare.isdigit(): + hits = [h for h in heads if h["item"] == str(int(bare))] + kind = f"item number {int(bare)}" + else: + needle = ref.lower() + hits = [h for h in heads if needle in h["title"].lower()] + kind = f"heading containing {ref!r}" + if len(hits) == 1: + return {"ok": True, "section": hits[0]} + return {"ok": False, "section": None, "candidates": hits, "kind": kind, + "reason": ("no section matches" if not hits + else f"{len(hits)} sections match — the ref is ambiguous")} + + +def append_note(ref: str, note: str, *, path: Path | None = None, + today: str | None = None) -> dict: + """Append one dated status note at the END of the matched section. Atomic, with one backup.""" + path = path or log_path() + if not path.is_file(): + return {"ok": False, "absent": True, "path": str(path), + "reason": absence_note(path)} + if not note.strip(): + return {"ok": False, "absent": False, "path": str(path), + "reason": "refusing to append an empty note"} + text = path.read_text(encoding="utf-8") + heads = parse_sections(text) + found = find_section(ref, heads) + if not found["ok"]: + return {"ok": False, "absent": False, "path": str(path), "ref": ref, + "candidates": [{"title": h["title"], "line": h["line"], "item": h["item"]} + for h in found["candidates"]], + "reason": f"{found['reason']} for {found['kind']}"} + sec = found["section"] + lines = text.splitlines() + stamp = today or datetime.datetime.now(datetime.timezone.utc).date().isoformat() + entry = f"- **STATUS {stamp}:** {note.strip()}" + # Insert at the section's end, after trailing blank lines are stepped back over, so the note + # lands inside the item it belongs to rather than under the next heading. + at = min(sec["end"], len(lines)) + while at > sec["line"] and not lines[at - 1].strip(): + at -= 1 + new = lines[:at] + ["", entry] + lines[at:] + body = "\n".join(new) + "\n" + backup = path.with_suffix(path.suffix + ".prev") + shutil.copy2(path, backup) # ONE rolling backup: this history is unversioned. + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=".improvement-log-", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(body) + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + return {"ok": True, "absent": False, "path": str(path), "section": sec["title"], + "section_line": sec["line"], "inserted_at": at + 2, "entry": entry, + "backup": str(backup)} + + +def render_append(rep: dict) -> str: + if rep.get("absent"): + return rep["reason"] + if not rep["ok"]: + out = [f"REFUSED — nothing was written: {rep['reason']}."] + cands = rep.get("candidates") or [] + if cands: + out.append("candidates (give a longer, unambiguous ref):") + out += [f" line {c['line']}: {c['title']}" for c in cands[:10]] + else: + out.append(f"run `improvement_log.py search ` against {rep['path']} to find the " + f"item, then use its number or a distinctive phrase from its heading.") + return "\n".join(out) + return (f"appended to {rep['path']}\n section: {rep['section']} (line {rep['section_line']})\n" + f" line {rep['inserted_at']}: {rep['entry']}\n backup: {rep['backup']}") + + +# -------------------------------------------------------------------------------------------- +# selftest — the CALLER's contract, exercised through the CLI on a synthetic log in a tempdir. +# -------------------------------------------------------------------------------------------- + +SYNTHETIC = """# Synthetic improvement log + +## 🟢 4. Durable repo-knowledge memory + +Something about caching a pattern. + +## ✅ 7. Telemetry integrity + +### 7a. Cost-aware scoring + +Thompson sampling was wired here and is DONE. + +### 7b. Thompson sampling flag + +Held OFF behind a switch. + +## ⚪ 9. Thompson routing follow-up + +Still open. +""" + + +def _selftest() -> int: + import json + import subprocess + + failures: list[str] = [] + + def check(name: str, cond: bool, detail: str = "") -> None: + if cond: + print(f" ok {name}") + else: + failures.append(f"{name}: {detail}") + print(f" FAIL {name} — {detail}") + + def run(argv: list[str], log: str) -> subprocess.CompletedProcess: + # A SUBPROCESS on purpose: the assertions below are about what a CALLER receives from the + # CLI — text and exit code — not about what an internal helper returns. + env = dict(os.environ, **{ENV_DIRECT: log}) + return subprocess.run([sys.executable, str(Path(__file__).resolve()), *argv], + capture_output=True, text=True, env=env) + + with tempfile.TemporaryDirectory() as td: + log = Path(td) / LOG_NAME + log.write_text(SYNTHETIC, encoding="utf-8") + missing = Path(td) / "nowhere" / LOG_NAME + + print("improvement_log selftest") + + # 1. search finds a term and reports the ITEM that owns it. + p = run(["search", "thompson"], str(log)) + check("search exits 0 on a hit", p.returncode == EXIT_OK, f"rc={p.returncode}") + check("search names the owning section", "7a. Cost-aware scoring" in p.stdout, + p.stdout[-200:]) + check("search reports the denominator", "sections for 'thompson'" in p.stdout, + p.stdout[:200]) + check("search finds every occurrence", "3 match(es)" in p.stdout, p.stdout[:200]) + + # 2. An HONEST empty: different exit code, and it says the file WAS read. + p = run(["search", "zzz-nothing-here"], str(log)) + check("no match exits 1", p.returncode == EXIT_NO_MATCH, f"rc={p.returncode}") + check("no match says so in words", "NO MATCHING ITEMS" in p.stdout, p.stdout[:200]) + check("no match is not reported as absence", "PROBLEM:" not in p.stdout, p.stdout[:200]) + + # 3. The NAMED ABSENCE: what is missing, where, and which env vars control it. + p = run(["search", "thompson"], str(missing)) + check("absent log exits 2", p.returncode == EXIT_ABSENT, f"rc={p.returncode}") + check("absence names the path", str(missing) in p.stdout, p.stdout[:300]) + check("absence names both env vars", + ENV_DIRECT in p.stdout and ENV_RUNTIME in p.stdout, p.stdout[:300]) + check("absence is not an empty result", p.stdout.strip() != "", "empty stdout") + + # 4. append lands the note INSIDE the referenced item, dated. The target is item 4, which + # is deliberately NOT the last section: appending to the last one makes "end of section" + # and "end of file" the same position, so the placement assertion could not fail. It was + # written that way first and a break->revert proved it discriminated nothing. + p = run(["append", "4", "wired and verified"], str(log)) + check("append exits 0", p.returncode == EXIT_OK, f"rc={p.returncode}\n{p.stderr[-300:]}") + after = log.read_text(encoding="utf-8") + check("append wrote a dated note", "**STATUS " in after and "wired and verified" in after, + after[-200:]) + head, _, rest = after.partition("## ✅ 7. Telemetry integrity") + check("note is INSIDE item 4, above the next heading", "wired and verified" in head, + f"landed after item 4: {rest[-160:]!r}") + check("note did not land at end of file", "wired and verified" not in rest, rest[-160:]) + check("append did not touch other items", + after.count("Thompson sampling was wired here and is DONE.") == 1, "duplicated") + check("append left one backup", (Path(td) / f"{LOG_NAME}.prev").is_file(), "no .prev") + + # 5. A ref that matches many REFUSES and changes nothing. + before = log.read_text(encoding="utf-8") + p = run(["append", "Thompson", "ambiguous"], str(log)) + check("ambiguous ref refuses", p.returncode == EXIT_REFUSED, f"rc={p.returncode}") + check("refusal lists candidates", "candidates" in p.stdout, p.stdout[:300]) + check("refusal wrote nothing", log.read_text(encoding="utf-8") == before, "file changed") + + # 6. A ref that matches nothing REFUSES too, and points at `search`. + p = run(["append", "99", "no such item"], str(log)) + check("unknown ref refuses", p.returncode == EXIT_REFUSED, f"rc={p.returncode}") + check("unknown ref suggests search", "search" in p.stdout, p.stdout[:300]) + check("unknown ref wrote nothing", log.read_text(encoding="utf-8") == before, + "file changed") + + # 7. append to an absent log names the absence and does NOT create the file. + p = run(["append", "9", "note"], str(missing)) + check("append to absent log exits 2", p.returncode == EXIT_ABSENT, f"rc={p.returncode}") + check("append to absent log creates nothing", not missing.exists(), "file created") + + # 8. `path` answers where the log is even when it is missing, for both env knobs. + p = run(["path", "--json"], str(missing)) + check("path exits 2 when absent", p.returncode == EXIT_ABSENT, f"rc={p.returncode}") + try: + doc = json.loads(p.stdout) + except Exception: # noqa: BLE001 + doc = {} + check("path reports resolved + present", doc.get("path") == str(missing) + and doc.get("present") is False, p.stdout[:200]) + + print(f"improvement_log selftest: {len(failures)} failure(s)") + for f in failures: + print(f" - {f}") + return 1 if failures else 0 + + +# -------------------------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------------------------- + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description="Search and append this instance's improvement log (machine-local evidence).") + ap.add_argument("--selftest", action="store_true") + sub = ap.add_subparsers(dest="cmd") + + s = sub.add_parser("search", help="CLAUDE.md §0 step 3 — is this already DONE?") + s.add_argument("term") + s.add_argument("--limit", type=int, default=40) + s.add_argument("--context", type=int, default=0, help="lines of context around each hit") + s.add_argument("--json", action="store_true") + + a = sub.add_parser("append", help="CLAUDE.md §5 — record a status note on an item") + a.add_argument("ref", help="item number (e.g. 9) or a distinctive phrase from its heading") + a.add_argument("note") + a.add_argument("--json", action="store_true") + + p = sub.add_parser("path", help="where the log resolves to, and whether it is here") + p.add_argument("--json", action="store_true") + + args = ap.parse_args(argv) + if args.selftest: + return _selftest() + if not args.cmd: + ap.print_help() + return EXIT_OK + + import json + + if args.cmd == "path": + target = log_path() + present = target.is_file() + doc = {"path": str(target), "present": present, + "env": {ENV_DIRECT: os.environ.get(ENV_DIRECT), + ENV_RUNTIME: os.environ.get(ENV_RUNTIME)}} + if args.json: + print(json.dumps(doc, indent=2)) + else: + print(f"improvement log: {target}\npresent: {present}") + if not present: + print(absence_note(target)) + return EXIT_OK if present else EXIT_ABSENT + + if args.cmd == "search": + rep = search(args.term, limit=args.limit, context=args.context) + print(json.dumps(rep, indent=2) if args.json else render_search(rep)) + if not rep["present"]: + return EXIT_ABSENT + return EXIT_OK if rep["matches"] else EXIT_NO_MATCH + + rep = append_note(args.ref, args.note) + print(json.dumps(rep, indent=2) if args.json else render_append(rep)) + if rep.get("absent"): + return EXIT_ABSENT + return EXIT_OK if rep["ok"] else EXIT_REFUSED + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test_improvement_log.py b/test_improvement_log.py new file mode 100644 index 0000000..ba7b2bf --- /dev/null +++ b/test_improvement_log.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""test_improvement_log.py — the TREE invariants behind the improvement-log accessor. + +`improvement_log.py --selftest` covers the module's own behaviour (search, append, the named +absence) against a synthetic log in a tempdir. It deliberately never touches this instance's real +log, which is machine-local and absent on any other machine. + +What it CANNOT cover is the three facts about the repository that made the defect possible, and each +one rots silently: + + 1. the tracked `IMPROVEMENT_BACKLOG.md` is a POINTER, and stays one. The whole change is undone the + moment someone appends evidence to it: the 481 KB machine-local log starts becoming committed, + which is exactly what the tool-vs-evidence split forbids. Prose asking nicely is what failed + before — `ADDING_CAPABILITIES.md` opens with that lesson — so the size limit is a test. + 2. the pointer NAMES the accessor. A pointer that does not say how to reach the log leaves a + worktree exactly as blind as a gitignored file did. + 3. `CLAUDE.md` §0 step 3 and §5 name the ACCESSOR, not a bare path. Those two rules are the reason + the log exists; re-writing either back into a path re-breaks it for every worker in a worktree. + +These are cheap, and they are the only thing that survives the next session. +""" +from __future__ import annotations + +import pathlib +import re +import subprocess +import sys + +HERE = pathlib.Path(__file__).resolve().parent +POINTER = HERE / "IMPROVEMENT_BACKLOG.md" +CLAUDE_MD = HERE / "CLAUDE.md" +ACCESSOR = "improvement_log.py" + +# A pointer is a paragraph and three commands. The real log is ~480 KB / 6,000 lines, so anything in +# between is someone having started to use this file as the log. The gap between the two is three +# orders of magnitude wide: this bound cannot be tripped by ordinary prose edits. +POINTER_MAX_BYTES = 8_192 +POINTER_MAX_SECTIONS = 4 + + +def test_tracked_pointer_stays_a_pointer_and_never_becomes_the_log(): + """The 481 KB of machine-local evidence must never arrive at this tracked path.""" + assert POINTER.is_file(), f"{POINTER.name} must be tracked in the tree — it is what makes the " \ + f"machine-local log discoverable from a worktree" + size = POINTER.stat().st_size + assert size <= POINTER_MAX_BYTES, ( + f"{POINTER.name} is {size} bytes, over the {POINTER_MAX_BYTES}-byte pointer limit. This file " + f"is a POINTER; the log itself is machine-local evidence and must not be committed. Append " + f"with `python3 {ACCESSOR} append \"\"` instead.") + sections = len(re.findall(r"(?m)^##\s", POINTER.read_text(encoding="utf-8"))) + assert sections <= POINTER_MAX_SECTIONS, ( + f"{POINTER.name} has {sections} `##` sections — it is turning into the log it points at.") + + +def test_tracked_pointer_names_the_accessor_and_both_rules(): + """A pointer that does not name the accessor leaves a worktree as blind as before.""" + text = POINTER.read_text(encoding="utf-8") + assert ACCESSOR in text, f"the pointer must name {ACCESSOR} — it is the only way to reach the log" + for cmd in ("search", "append"): + assert f"{ACCESSOR} {cmd}" in text, f"the pointer must show `{ACCESSOR} {cmd}`" + assert "ORCH_LOCAL_RUNTIME" in text, "the pointer must say WHERE the log lives" + + +def test_claude_md_rules_name_the_accessor_not_a_bare_path(): + """§0 step 3 and §5 are the two rules the accessor exists to make followable.""" + text = CLAUDE_MD.read_text(encoding="utf-8") + # §0 step 3 — the dedup check. + step3 = [ln for ln in text.splitlines() if ln.lstrip().startswith("3. ") + and "improvement log" in ln.lower()] + assert step3, "CLAUDE.md §0 step 3 must tell the reader to search the improvement log" + dedup = text.split("## 0. Dedup-before-develop", 1)[-1].split("## 1. Editing", 1)[0] + assert f"{ACCESSOR} search" in dedup, ( + f"CLAUDE.md §0 must name `{ACCESSOR} search` — a bare path is unreadable from a worktree, " + f"which is what made this mandatory step unfollowable") + # §5 — the status note. + keep_true = text.split("## 5. Keep the docs true", 1)[-1] + assert f"{ACCESSOR} append" in keep_true, ( + f"CLAUDE.md §5 must name `{ACCESSOR} append` rather than telling the reader to edit a file") + + +def test_accessor_reports_a_named_absence_to_a_caller(): + """The caller-facing contract, asserted on what a CALLER receives: text and exit code. + + Duplicated from the selftest on purpose and kept minimal: this is the assertion that the + founding defect — an empty result indistinguishable from a missing file — stays fixed, and it + must hold under `pytest` (which is what CI actually runs) and not only under `--selftest`. + Points at a path that cannot exist, so it never reads this instance's real log. + """ + missing = HERE / "no-such-dir-for-tests" / "IMPROVEMENT_BACKLOG.md" + assert not missing.exists() + proc = subprocess.run([sys.executable, str(HERE / ACCESSOR), "search", "anything"], + capture_output=True, text=True, cwd=str(HERE), + env={"PATH": "/usr/bin:/bin", "HOME": str(HERE), + "ORCH_IMPROVEMENT_LOG": str(missing)}) + out = proc.stdout + proc.stderr + assert proc.returncode == 2, f"an absent log must exit 2, not {proc.returncode}: {out[:300]}" + assert str(missing) in out, "the absence must name the path it looked for" + assert "ORCH_LOCAL_RUNTIME" in out, "the absence must name the env var that controls the path" + assert out.strip(), "an absent log must never produce an empty result" + + +def main() -> int: + failures = [] + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + print(f" ok {name}") + except AssertionError as exc: + failures.append(name) + print(f" FAIL {name} — {exc}") + print(f"improvement-log tree invariants: {len(failures)} failure(s)") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())