diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e14fb8..3ae9243 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,18 @@ name: CI # for weeks without anyone knowing: 9 pytest failures plus a module selftest broken since # 2026-06-18. A clean-machine CI run is the single strongest reason this code is in a repo at all. # -# The suite is STATE-INDEPENDENT: verified 2026-08-21 by running it with ORCH_STATE_DIR pointed at -# an empty directory — 330 passed. So CI needs no Brain database, no capability ledger and no -# seeded state, which is what keeps the owner's rule (no evaluation databases stored remotely) -# compatible with having CI at all. +# CORRECTED 2026-08-21 after the first run came back red. The claim here was that the suite is +# STATE-INDEPENDENT, "verified by running it with ORCH_STATE_DIR pointed at an empty directory — +# 330 passed". That verification moved the WRONG KNOB. There are two: ORCH_STATE_DIR (the audit +# cache, firing monitor, redirect sweep) and ORCH_LOCAL_RUNTIME (the capability LEDGER and the +# Brain). Only the second one holds the state those checks read, so the experiment left the +# owner's populated 40-row ledger in place and proved nothing about a fresh machine. On a real +# runner the ledger bootstraps to the 14 rows the code declares, and 21 checks that read the +# other 26 failed. +# +# Both knobs are now set, so the run really is state-free, and the checks that need this +# instance's registration history SKIP with the missing row named instead of failing. verify.py +# bounds that skipping with a ceiling — see .verify-floor.json. on: push: @@ -42,14 +50,20 @@ jobs: - name: Verify env: - # State is machine-local by design. A fresh directory proves the suite does not depend on - # this developer's Brain, ledger or cadence stamps. + # State is machine-local by design, and it lives behind TWO variables. Setting only one + # is what made the "state-independent" claim above wrong for two months of one day. ORCH_STATE_DIR: ${{ runner.temp }}/orch-state + ORCH_LOCAL_RUNTIME: ${{ runner.temp }}/orch-runtime run: | - mkdir -p "$ORCH_STATE_DIR" + mkdir -p "$ORCH_STATE_DIR" "$ORCH_LOCAL_RUNTIME" python3 verify.py # verify.py already fails on: any pytest failure, ZERO tests collected, a collection count - # below the recorded floor, a module selftest that exits 0 without saying anything, and any - # of the five capability gates. Deliberately no `|| true` anywhere — an exit code that cannot - # fail is the defect this repo exists to stop repeating. + # below the recorded floor, passed+skipped dropping below the floor, MORE SKIPS THAN THE + # AGREED CEILING, a module selftest that exits 0 without saying anything, a selftest or gate + # that exits 0 having skipped without naming what is missing, and any of the five capability + # gates. Deliberately no `|| true` anywhere — an exit code that cannot fail is the defect + # this repo exists to stop repeating. + # + # It also prints every skip and its reason, so a green run here always states what it did + # not check. If that list grows, the ceiling turns it into a red rather than a footnote. diff --git a/.verify-floor.json b/.verify-floor.json index 4ea129e..8b04820 100644 --- a/.verify-floor.json +++ b/.verify-floor.json @@ -1,5 +1,8 @@ { "collected": 330, "passed": 330, - "note": "floor recorded by verify.py --update-floor; a later run collecting fewer tests FAILS, because silently running fewer tests looks exactly like passing" + "skipped_max": 24, + "selftest_skipped_max": 7, + "gate_skipped_max": 2, + "note": "Recorded by verify.py --update-floor, except the *_max ceilings, which are edited BY HAND and never re-measured. `collected` catches tests that stopped being collected; `passed` is compared against passed+skipped, so a check may move between passing and consciously-skipped but the two together may never shrink. The *_max ceilings bound the skipped side: 24/7/2 is exactly what a machine with none of this instance's local prerequisites skips (a GitHub runner: no agent CLIs, no ~/.codex/skills, no /Applications/ChatGPT.app, no populated capability ledger), measured 2026-08-21. On the owner's machine all prerequisites exist and nothing skips at all. Raising a ceiling is a deliberate act: it means agreeing that one more thing is allowed to go unchecked, so say which and why in the commit." } diff --git a/CLAUDE.md b/CLAUDE.md index 926114c..3712457 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,6 +75,24 @@ Do not create a second event log, model registry, or capability inventory. real pytest, reads the COUNTS rather than the exit status, enforces a collection floor so tests silently ceasing to run cannot look like tests passing, treats a silent zero-exit selftest as a failure, and runs the five capability gates. CI runs the same command on a clean machine. +- **A check whose PREREQUISITE is absent skips with the missing thing NAMED, and skipping is + bounded.** Some checks need what only a running instance has: the populated capability ledger, + an installed agent CLI, `~/.codex/skills`, the version-capable Codex binary. Those gates live in + `env_prereq.py` — detect the prerequisite, never `$CI`, so the same code is right on any machine. + Three rules, and they are enforced, not advisory: every skip carries a reason naming what is + missing; `verify.py` prints all of them so a green run always states what it did not check; and + `.verify-floor.json` caps the number of skipped tests, selftests and gates, so skipping one more + thing than agreed is a RED, not a footnote. Raising a ceiling means agreeing that one more thing + goes unchecked — do it deliberately and say why. When a check fails only because a stub leaked + (a monkeypatched `Popen` catching a model-catalog probe, say), the fix is isolation, not a skip: + that makes CI run MORE. **Never turn a real failure into a skip**, and never add a skip without + a reason string — a reason-less skip is indistinguishable from a pass, which is this repo's + founding defect wearing a different hat. +- **State lives behind TWO variables and they are not the same.** `ORCH_STATE_DIR` holds the audit + cache, firing-monitor and redirect-sweep state; `ORCH_LOCAL_RUNTIME` holds the capability LEDGER + and the Brain. Pointing only the first at an empty directory and concluding "the suite is + state-independent" is exactly the mistake that made the first CI run red — the ledger never + moved. Set both when testing a fresh-machine claim. - **The split is TOOL vs EVIDENCE.** Generic capabilities, gates and tests are committed. This instance's evidence is not: `IMPROVEMENT_BACKLOG.md`, `CAPABILITY_USEFULNESS.md`, `LOCAL_POLICY.md`, `*.local.md`, `experiments/`, `ux_reviews/`, `data/`, `Audits/`. When adding a diff --git a/README.md b/README.md index 5e7bb29..33c0c33 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,13 @@ HANDOFF: ~/.codex/handoff/ (heartbeat orchestrator.json — legacy lan canonical tree — so canonical edits are yours alone, but always re-sync so the schedule sees them. - **Every module has a `--selftest`.** Run it after editing that module; it is the project's test suite (there is no separate pytest tree). `python3 .py --selftest`. +- **`python3 verify.py` is the whole verdict.** Real pytest plus every module selftest plus the five + capability gates, judged on the COUNTS rather than exit codes, against a recorded floor in + `.verify-floor.json`. It also bounds SKIPPING: a check needing something only a running instance + has (the populated capability ledger, an installed agent CLI, `~/.codex/skills`) skips with the + missing thing named — see `env_prereq.py` — and the floor file caps how many such skips are + allowed, so quietly checking less is a red. Every skip and its reason is printed, so a green run + always states what it did not check. On a machine with all prerequisites nothing skips at all. - **Activation is evidence-backed.** `features.py` describes reusable code maturity; `capabilities.py` is the activation authority. An `active` declaration must prove its matcher, invocation, artifact consumer, outcome sink, expiry, kill switch, and rollback. Each active tick diff --git a/adapters.py b/adapters.py index 8fda0c2..8dcd02e 100644 --- a/adapters.py +++ b/adapters.py @@ -733,7 +733,7 @@ def _selftest(): old_codex_sandbox = os.environ.pop("CODEX_SANDBOX", None) old_codex_bypass = os.environ.pop("ORCH_CODEX_BYPASS_INNER_SANDBOX", None) try: - _selftest_inner() + _selftest_inner(gaps=[]) finally: if old_codex_sandbox is not None: os.environ["CODEX_SANDBOX"] = old_codex_sandbox @@ -741,7 +741,9 @@ def _selftest(): os.environ["ORCH_CODEX_BYPASS_INNER_SANDBOX"] = old_codex_bypass -def _selftest_inner(): +def _selftest_inner(*, gaps: list[str] | None = None): + import env_prereq # imported here: env_prereq reads this module + gaps = gaps if gaps is not None else [] c = build_command("cursor", "do x") # Composer is PINNED, not implied: omitting --model selects `auto`, which routes across every # frontier model cursor sells. Owner policy is Composer only (2026-08-08). @@ -772,23 +774,30 @@ def _selftest_inner(): # Non-tier modes still pass NO --model and keep the legacy lane tag. assert "--model" not in build_command("codex", "x", mode="assess"), "assess must not pin a model" assert model_identity("codex", None) == "codex:full:default" - profile_commands = {} - for profile in execution_profiles.profiles_for_agent("codex"): - cmd = build_command("codex", "x", mode="full", profile=profile, transport="local") - assert cmd[0] == str(CODEX_PROFILE_BIN), cmd - assert cmd[cmd.index("--model") + 1] == profile["requested_model"], cmd - assert cmd[cmd.index("--sandbox") + 1] == "workspace-write", cmd - assert cmd[cmd.index("-c") + 1] == f'model_reasoning_effort="{profile["reasoning_effort"]}"', cmd - profile_commands[profile["profile_id"]] = cmd - assess = build_command( - "codex", "x", mode="assess", profile=profile, transport="offload", - permission_mode="read-only", - ) - assert assess[assess.index("--sandbox") + 1] == "read-only", assess - assert "--json" not in assess and assess[assess.index("--model") + 1] == profile["requested_model"], assess - assert { - cmd[cmd.index("--model") + 1] for cmd in profile_commands.values() - } == {"gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"} + # An EXACT profile resolves the version-capable Codex binary and `profile_codex_binary()` + # fails closed rather than falling back to PATH — deliberately, since a profile that cannot + # pin its version is not an exact profile. So this SECTION needs that binary installed; the + # default lives inside a macOS app bundle and cannot exist on a Linux runner. Everything else + # in this selftest runs anywhere. + if env_prereq.runnable(gaps, env_prereq.codex_profile_binary_absent()): + profile_commands = {} + for profile in execution_profiles.profiles_for_agent("codex"): + cmd = build_command("codex", "x", mode="full", profile=profile, transport="local") + assert cmd[0] == str(CODEX_PROFILE_BIN), cmd + assert cmd[cmd.index("--model") + 1] == profile["requested_model"], cmd + assert cmd[cmd.index("--sandbox") + 1] == "workspace-write", cmd + assert cmd[cmd.index("-c") + 1] == f'model_reasoning_effort="{profile["reasoning_effort"]}"', cmd + profile_commands[profile["profile_id"]] = cmd + assess = build_command( + "codex", "x", mode="assess", profile=profile, transport="offload", + permission_mode="read-only", + ) + assert assess[assess.index("--sandbox") + 1] == "read-only", assess + assert "--json" not in assess and assess[assess.index("--model") + 1] == profile["requested_model"], assess + assert { + cmd[cmd.index("--model") + 1] for cmd in profile_commands.values() + } == {"gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"} + env_prereq.report_gaps("adapters.py", gaps) codex_cwd = HOME / ".codex" / "orchestrator" / "worktrees" / "selftest" ccwd = build_command("codex", "x", cwd=codex_cwd) assert "--cd" in ccwd and ccwd[ccwd.index("--cd") + 1] == str(codex_cwd), ccwd diff --git a/capability_activation_audit.py b/capability_activation_audit.py index b9fa23e..64120b2 100644 --- a/capability_activation_audit.py +++ b/capability_activation_audit.py @@ -43,6 +43,7 @@ import json import os import re +import shutil import subprocess import sys import time @@ -444,9 +445,19 @@ def _fleet_label_index(*, use_cache: bool = True) -> dict: except (OSError, ValueError): pass index = {} + # A FAILED gh call already means "unknown for this repo" (unauthenticated, offline, no + # access), and the loop skips it. An ABSENT gh binary meant an uncaught FileNotFoundError + # that took the whole audit down — same information, opposite outcome. Named here, and it + # short-circuits: with no gh at all there is nothing to ask 12 times. + if not shutil.which("gh"): + return {"generated_at": time.time(), "repos": {}, + "unreadable": "gh CLI not installed; fleet label vocabulary unknown"} for full in getattr(backlog, "SUPPORTED_REPOS", []): - proc = subprocess.run(["gh", "label", "list", "--repo", full, "--limit", "300", - "--json", "name"], capture_output=True, text=True, timeout=120) + try: + proc = subprocess.run(["gh", "label", "list", "--repo", full, "--limit", "300", + "--json", "name"], capture_output=True, text=True, timeout=120) + except (OSError, subprocess.SubprocessError): + continue # unknown for this repo, exactly like a nonzero exit if proc.returncode != 0: continue try: diff --git a/capability_admission.py b/capability_admission.py index afe2059..0ed19c9 100644 --- a/capability_admission.py +++ b/capability_admission.py @@ -44,6 +44,7 @@ import sys import capabilities +import env_prereq HERE = pathlib.Path(__file__).resolve().parent def _audits_dir() -> pathlib.Path: @@ -468,9 +469,39 @@ def format_report(rep: dict) -> str: return "\n".join(out) + "\n" +def _probe_commitments(probe: pathlib.Path) -> None: + """Prove the commitment DETECTOR works, against synthetic files in `probe`. + + Split out of `_selftest` so `AUDITS` can be swapped for a synthetic empty record set around + exactly this block and restored after — the assertions are verbatim. + """ + (probe / "fake.sh").write_text("# see 2026-01-02-nonexistent-decision.md\n") + got = commitments(root=probe) + assert any(d["record"] == "2026-01-02-nonexistent-decision.md" + for d in got["dangling_citations"]), got + assert not got["clean"] + # A PLAN is not a decision record; citing one must not be flagged. + (probe / "fake.sh").write_text("# see 2026-01-02-thing-PLAN.md\n") + assert commitments(root=probe)["clean"], "a -PLAN citation must not be treated as a record" + # An overdue deadline with no record is flagged. + # The MOTIVATING SHAPE, verbatim: a shell default with the var repeated inside. The first + # pattern here could not match this, which would have made the whole check theatre. + (probe / "fake2.sh").write_text( + 'ORCH_THING_TRIAL_UNTIL="${ORCH_THING_TRIAL_UNTIL:-2020-01-01}"\n') + over = commitments(root=probe) + assert any(o["date"] == "2020-01-01" for o in over["overdue_without_record"]), over + # A future deadline is not overdue. + (probe / "fake2.sh").write_text('ORCH_X_TRIAL_UNTIL="${ORCH_X_TRIAL_UNTIL:-2099-01-01}"\n') + assert not commitments(root=probe)["overdue_without_record"] + + def _selftest() -> None: ledger = capabilities.load(capabilities.REG) assert ledger, "ledger must load" + # Sections needing this instance's registration history are gated individually and named at + # the end — gating the WHOLE selftest for one block would drop everything below it, which is + # running less to report green. + gaps: list[str] = [] # Every requirement must be able to FAIL. A predicate that always passes is decoration. ctx = {"audit_rows": {}, "fixtures": set()} @@ -501,12 +532,16 @@ def _selftest() -> None: # Legacy scoping lives on the ROW (`legacy`/`enforced`), so a pre-gate capability still reports # exactly what it is missing — it simply does not block the suite. Were the exemption pushed # into the predicates instead, the debt would read as compliance and disappear. - ledger_rows = report()["rows"] - legacy_rows = [r for r in ledger_rows if r["legacy"]] - assert legacy_rows, "expected pre-gate capabilities to be marked legacy" - assert any(r["missing"] for r in legacy_rows), \ - "legacy rows must still report their missing parts, not be silently passed" - assert all(not r["enforced"] for r in legacy_rows), "legacy rows must not block the suite" + # "Pre-gate" means registered before 2026-08-21 on the running instance, so this block can + # only be exercised where that history exists. A ledger bootstrapped from the committed tree + # has no legacy population at all. + if env_prereq.runnable(gaps, env_prereq.ledger_legacy_rows_absent()): + ledger_rows = report()["rows"] + legacy_rows = [r for r in ledger_rows if r["legacy"]] + assert legacy_rows, "expected pre-gate capabilities to be marked legacy" + assert any(r["missing"] for r in legacy_rows), \ + "legacy rows must still report their missing parts, not be silently passed" + assert all(not r["enforced"] for r in legacy_rows), "legacy rows must not block the suite" # A waiver must EXPIRE. An exception with no end date is how "temporary" became a month. for cid, waiver in WAIVERS.items(): @@ -523,26 +558,27 @@ def _selftest() -> None: import tempfile with tempfile.TemporaryDirectory(prefix="cap-adm-") as td: probe = pathlib.Path(td) - (probe / "fake.sh").write_text("# see 2026-01-02-nonexistent-decision.md\n") - got = commitments(root=probe) - assert any(d["record"] == "2026-01-02-nonexistent-decision.md" - for d in got["dangling_citations"]), got - assert not got["clean"] - # A PLAN is not a decision record; citing one must not be flagged. - (probe / "fake.sh").write_text("# see 2026-01-02-thing-PLAN.md\n") - assert commitments(root=probe)["clean"], "a -PLAN citation must not be treated as a record" - # An overdue deadline with no record is flagged. - # The MOTIVATING SHAPE, verbatim: a shell default with the var repeated inside. The first - # pattern here could not match this, which would have made the whole check theatre. - (probe / "fake2.sh").write_text( - 'ORCH_THING_TRIAL_UNTIL="${ORCH_THING_TRIAL_UNTIL:-2020-01-01}"\n') - over = commitments(root=probe) - assert any(o["date"] == "2020-01-01" for o in over["overdue_without_record"]), over - # A future deadline is not overdue. - (probe / "fake2.sh").write_text('ORCH_X_TRIAL_UNTIL="${ORCH_X_TRIAL_UNTIL:-2099-01-01}"\n') - assert not commitments(root=probe)["overdue_without_record"] + # POINT `AUDITS` AT A SYNTHETIC, EMPTY RECORD SET for the duration of the probe. Two + # reasons, and neither is a skip: + # 1. `commitments()` returns "cannot judge" when the audit ledger is absent — correctly, + # since record existence is unanswerable without it. But that made this block, whose + # whole job is to prove the DETECTOR works either way, unprovable on a machine that + # has no audit ledger: the first CI run died right here. + # 2. Even where the ledger exists, the verdicts below depended on which records happen + # to be in it. A synthetic empty set makes all four deterministic everywhere. + # This is the harness, not an assertion: every assert below is unchanged, and now runs on + # any machine instead of only on this one. + audits_probe = probe / "audits" + audits_probe.mkdir() + saved_audits = globals()["AUDITS"] + globals()["AUDITS"] = audits_probe + try: + _probe_commitments(probe) + finally: + globals()["AUDITS"] = saved_audits # preflight must report obligations rather than pretending it verified them. + pf = preflight({"capability_id": "capability:proposed-thing", "notes": "dedup: checked X; absent", "downstream_consumer": "a.py:b", "learning_sink": "feedback.outcomes", "kill_switch": "F=0", @@ -553,8 +589,10 @@ def _selftest() -> None: pf2 = preflight({"capability_id": "capability:bare"}) assert not pf2["ready_to_build"] and "kill_switch" in pf2["declarable_missing"], pf2 + env_prereq.report_gaps("capability_admission.py", gaps) print("capability_admission.py selftest: OK (every requirement can fail and can pass, " - "grandfathering visible, waivers expire, dangling + overdue commitments detected)") + "grandfathering visible, waivers expire, dangling + overdue commitments detected)" + + (f" — {len(set(gaps))} section(s) skipped, see above" if gaps else "")) def main() -> int: diff --git a/capability_advisor.py b/capability_advisor.py index 6f3dbe1..da5e8eb 100644 --- a/capability_advisor.py +++ b/capability_advisor.py @@ -36,6 +36,7 @@ import sys import capabilities +import env_prereq # Free text -> the task_type vocabulary the fleet actually records. Deterministic and inspectable; # a model call here would make the same task classify differently on different days, which would @@ -334,7 +335,15 @@ def _selftest_front_door() -> None: capability) was unreachable because TASK_SIGNALS had no entry for it AND its kind-based matcher can never match a task_type trigger; and the legacy-removal campaign — proven-valuable codemod work — classified as nothing at all. + + Every case names a capability the advisor must FIND, so the whole function needs those rows + present. They are registered by running the system, not by checking out the tree. """ + gaps: list[str] = [] + if not env_prereq.runnable(gaps, env_prereq.ledger_rows_absent( + "offload", "codemod-campaign", "testgen-lane")): + env_prereq.report_gaps("capability_advisor.py front-door", gaps) + return cases = [ ("summarise these 200 pages of docs", "offload"), ("offload this big read to a cheap agent", "offload"), diff --git a/capability_outcome_bridge.py b/capability_outcome_bridge.py index 415e7df..99f6d3c 100644 --- a/capability_outcome_bridge.py +++ b/capability_outcome_bridge.py @@ -44,6 +44,7 @@ from pathlib import Path import capabilities +import env_prereq import feedback # Terminal verdicts worth propagating. A still-pending outcome is not evidence yet. @@ -594,47 +595,58 @@ def _selftest() -> None: # This is the bug the resolver shipped with: it reads row["capability_ids"], the SELECT never # fetched them, and there is no runs.capability_ids column — so every tag written by # record_run(capability_ids=...) produced a Brain edge that never reached the ledger. - import tempfile as _tf - _old_db = feedback.DB_PATH - with _tf.TemporaryDirectory(prefix="bridge-tagged-") as _td: - feedback.DB_PATH = Path(_td) / "brain.db" - try: - feedback.record_run("tagged:run", "o/r#1", "implement", "gemini", - capability_ids=["agy-runtime-isolation"]) - feedback.record_outcome("tagged:run", adjudicated_verdict="PASS", merged=True, - durability="durable") - rows = collect() - row = [r for r in rows if r["run_id"] == "tagged:run"][0] - assert row["capability_ids"] == ["agy-runtime-isolation"], row - mapped = attribute(rows, known={"agy-runtime-isolation"}) - assert [l["capability_id"] for l in mapped["links"]] == ["agy-runtime-isolation"], mapped - assert mapped["links"][0]["resolver"] == "run_tagged", mapped - - # ---- offload backfill: the link is backend_run_id, and only backend_run_id -------- - # record_role_run now tags `offload` at record time, so a CURRENT row needs no repair. - # These two are shaped like HISTORY — recorded before that tagging existed — which is - # exactly what the backfill is for. - for rid, target, payload in ( - ("role:redirect:withoffload", "o/r#2", - {"role": "redirect", "backend_run_id": "offload:xyz"}), - ("role:redirect:replayed", "o/r#3", {"role": "redirect"}), - ): - feedback.record_run(rid, target, "role:redirect", "cursor", - role_name="redirect", decomposition=payload) - first = backfill_offload_capability_edges() - assert first["backfilled"] == 1, first - with feedback._conn() as c: - tagged = {r[0] for r in c.execute( - "SELECT target_run_id FROM influence_edges WHERE influence_type='capability' " - "AND capability_id='offload'").fetchall()} - assert tagged == {"role:redirect:withoffload"}, tagged - assert backfill_offload_capability_edges()["backfilled"] == 0, "not idempotent" - finally: - feedback.DB_PATH = _old_db - + # The Brain below is a fresh tmp DB, but the edge writer resolves `agy-runtime-isolation`'s + # and `offload`'s version lineage from the LEDGER and refuses an edge without one — so the + # tags this block asserts on can only be written where those rows are registered. Gated as a + # SECTION: everything above and below it runs on any machine. + _gaps: list[str] = [] + if env_prereq.runnable( + _gaps, + env_prereq.ledger_rows_absent("agy-runtime-isolation", "offload"), + env_prereq.ledger_version_lineage_absent("agy-runtime-isolation", "offload")): + import tempfile as _tf + _old_db = feedback.DB_PATH + with _tf.TemporaryDirectory(prefix="bridge-tagged-") as _td: + feedback.DB_PATH = Path(_td) / "brain.db" + try: + feedback.record_run("tagged:run", "o/r#1", "implement", "gemini", + capability_ids=["agy-runtime-isolation"]) + feedback.record_outcome("tagged:run", adjudicated_verdict="PASS", merged=True, + durability="durable") + rows = collect() + row = [r for r in rows if r["run_id"] == "tagged:run"][0] + assert row["capability_ids"] == ["agy-runtime-isolation"], row + mapped = attribute(rows, known={"agy-runtime-isolation"}) + assert [l["capability_id"] for l in mapped["links"]] == ["agy-runtime-isolation"], mapped + assert mapped["links"][0]["resolver"] == "run_tagged", mapped + + # ---- offload backfill: the link is backend_run_id, and only backend_run_id -------- + # record_role_run now tags `offload` at record time, so a CURRENT row needs no repair. + # These two are shaped like HISTORY — recorded before that tagging existed — which is + # exactly what the backfill is for. + for rid, target, payload in ( + ("role:redirect:withoffload", "o/r#2", + {"role": "redirect", "backend_run_id": "offload:xyz"}), + ("role:redirect:replayed", "o/r#3", {"role": "redirect"}), + ): + feedback.record_run(rid, target, "role:redirect", "cursor", + role_name="redirect", decomposition=payload) + first = backfill_offload_capability_edges() + assert first["backfilled"] == 1, first + with feedback._conn() as c: + tagged = {r[0] for r in c.execute( + "SELECT target_run_id FROM influence_edges WHERE influence_type='capability' " + "AND capability_id='offload'").fetchall()} + assert tagged == {"role:redirect:withoffload"}, tagged + assert backfill_offload_capability_edges()["backfilled"] == 0, "not idempotent" + finally: + feedback.DB_PATH = _old_db + + env_prereq.report_gaps("capability_outcome_bridge.py", _gaps) print("capability_outcome_bridge.py selftest: OK (explicit attribution, no entrypoint " "inference, unknown ids refused, idempotent, dry-run inert, collect supplies " - "recorded tags so run_tagged is live, offload backfill keyed on backend_run_id)") + "recorded tags so run_tagged is live, offload backfill keyed on backend_run_id)" + + (f" — {len(set(_gaps))} section(s) skipped, see above" if _gaps else "")) def main(argv: list[str]) -> int: diff --git a/capability_recurrence_check.py b/capability_recurrence_check.py index e608d8a..764b0f7 100644 --- a/capability_recurrence_check.py +++ b/capability_recurrence_check.py @@ -39,6 +39,7 @@ import backlog import capabilities +import env_prereq # --------------------------------------------------------------------------- fixtures # Each entry: the capability under test, the real instance, and how to decide if it would fire. @@ -804,6 +805,15 @@ def format_report(rep: dict) -> str: def _selftest() -> None: + # `docs-drift-fix-agent`'s fixture reads its matcher out of the LIVE ledger, so on a machine + # where that row was never registered the fixture errors — and an errored fixture is + # indistinguishable from a broken one, which is what the loop below exists to catch. Gate it + # by NAME and keep the guard live for every other fixture: excusing one row is bounded, and + # the reason says which. Two places need it, so it is computed once. + gaps: list[str] = [] + _docs_drift_ok = env_prereq.runnable( + gaps, env_prereq.ledger_rows_absent("docs-drift-fix-agent")) + # The must-not-fire guards are the ones that keep this honest: a check that only ever says # "yes" would pass trivially and tell us nothing. rep = replay(offline=True) @@ -831,6 +841,8 @@ def _selftest() -> None: # a real miss. Every fixture must exercise real machinery. for row in rep["rows"]: det = row.get("detail") + if not _docs_drift_ok and row["capability"] == "docs-drift-fix-agent": + continue # the ONE excused row, named in `gaps` above if isinstance(det, dict): assert "error" not in det, f"fixture for {row['capability']} ERRORED: {det['error']}" @@ -839,23 +851,24 @@ def _selftest() -> None: # very latched-state bug the fixture exists to catch (a blocked verdict whose clear path is # never re-checked). So test the mechanism in BOTH directions; a regression to a hardcoded # verdict fails here rather than quietly under-reporting forever. - import capability_activation_audit as _audit - _stub = {"repo": "R", "workflow": "w", "path": "/p"} - _real = _audit.external_caller - try: - _audit.external_caller = lambda cap: {**_stub, "exists": False} - absent = _external_caller_state("docs-drift-fix-agent") - _audit.external_caller = lambda cap: {**_stub, "exists": True} - present = _external_caller_state("docs-drift-fix-agent") - finally: - _audit.external_caller = _real - assert not absent["fires"] and absent["external_blocker"] is True, absent - assert present["fires"] and present["external_blocker"] is False, present - # ...and the LIVE row must agree with the caller actually on disk, in whichever direction. - docs = [r for r in rep["rows"] if r["capability"] == "docs-drift-fix-agent"][0] - _cap = capabilities.load(capabilities.REG)["docs-drift-fix-agent"] - _live = _audit.external_caller(_cap) - assert docs["fires"] is bool(_live and _live.get("exists")), (docs, _live) + if _docs_drift_ok: + import capability_activation_audit as _audit + _stub = {"repo": "R", "workflow": "w", "path": "/p"} + _real = _audit.external_caller + try: + _audit.external_caller = lambda cap: {**_stub, "exists": False} + absent = _external_caller_state("docs-drift-fix-agent") + _audit.external_caller = lambda cap: {**_stub, "exists": True} + present = _external_caller_state("docs-drift-fix-agent") + finally: + _audit.external_caller = _real + assert not absent["fires"] and absent["external_blocker"] is True, absent + assert present["fires"] and present["external_blocker"] is False, present + # ...and the LIVE row must agree with the caller actually on disk, in whichever direction. + docs = [r for r in rep["rows"] if r["capability"] == "docs-drift-fix-agent"][0] + _cap = capabilities.load(capabilities.REG)["docs-drift-fix-agent"] + _live = _audit.external_caller(_cap) + assert docs["fires"] is bool(_live and _live.get("exists")), (docs, _live) # tick_env must EXECUTE orchestrate.sh's conditionals, not regex-scrape its defaults. The # range-lane flag is the proof case: its naive default is 1, but past the trial-review date the @@ -1009,8 +1022,10 @@ def _no_exports() -> tuple[dict, dict]: assert not str(row.get("provenance", "")).startswith("live "), row text = format_report(rep) assert "WOULD FIRE" in text and "WOULD MISS" in text + env_prereq.report_gaps("capability_recurrence_check.py", gaps) print("capability_recurrence_check.py selftest: OK (must-not-fire guards hold, unemittable " - "task_type reads as a miss, offline stays offline)") + "task_type reads as a miss, offline stays offline)" + + (f" — {len(set(gaps))} section(s) skipped, see above" if gaps else "")) def main(argv: list[str]) -> int: diff --git a/env_prereq.py b/env_prereq.py new file mode 100644 index 0000000..df78156 --- /dev/null +++ b/env_prereq.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""env_prereq.py — "is the thing this check needs actually HERE?", answered BY NAME. + +WHY THIS EXISTS. The first CI run of the public repo (2026-08-21) was red: 21 pytest +failures, 8 module selftests and 2 capability gates, against 330 green on the owner's +machine. Not one was a defect in this tree. Each check needed something that exists only on +the machine the system runs on, and a GitHub runner has none of it: no agent CLIs, no +`~/.codex/skills`, no `/Applications/ChatGPT.app`, and — the big one — no capability ledger. +The ledger is machine-local state by design (`$ORCH_STATE_DIR`, never committed); a fresh +bootstrap holds the 14 rows the code declares, while this instance's ledger holds 40. The +other 26 are accumulated registration history and cannot be reconstructed from source. + +So the tree was fine and the instrument said RED. The opposite error — declaring green while +running less — is the failure this project is named for, so the fix could not be a blanket +skip, a try/except, or a narrower CI invocation. What it is instead: + + * A check whose prerequisite is genuinely absent SKIPS, and its skip carries a reason that + NAMES the missing thing. `MissingPrerequisite` subclasses `unittest.SkipTest`, which + pytest reports as SKIPPED with the message from a test body, a fixture, or a module-level + `skipif`; the dual-mode `main()` runners in `test_capability_admission.py` and + `test_capability_set_coverage.py` catch the same exception explicitly. One exception type, + one reason string, every harness. + * Detection is of the PREREQUISITE, never of CI. Nothing here reads `$CI` or + `$GITHUB_ACTIONS`: the question is "does this binary exist", "does this ledger row carry + version lineage" — so the same code gives the right answer on a runner, on the owner's + box, and on a second instance with a different `ORCH_STATE_DIR`. + * Skipping is BOUNDED, not open-ended. `verify.py` enforces a ceiling on the number of + skipped tests, skipped selftests and skipped gates (`.verify-floor.json`), and prints + every skip reason, so "green" always states what did not run. A future change that skips + more than the agreed set FAILS. + +Assertions are untouched. A check that RUNS still asserts exactly what it asserted before; +only its applicability gate is new. + +DEDUP FINDING (CLAUDE.md §0), recorded 2026-08-21 before writing a line. Grepped the tree for +the concept, not the name: `pytest.skip` appears in exactly one file +(`test_model_tier_resolution.py`, twice, both `shutil.which`-gated — the idiom this module +generalises); there is no `conftest.py`, no `pytest.ini`/`pyproject.toml`, and no shared +prerequisite/applicability helper of any kind. The nearest relatives are single-call-site +degradations, not reusable machinery: `capability_admission.py:335` ("cannot judge without the +ledger; never fail on absence") and `capability_activation_audit._fleet_label_index`, which +skips a repo whose `gh` call fails. Nothing to wire, activate or un-gate — this concept is +genuinely absent, so it is new. This module is test-applicability infrastructure, not an +orchestrator capability: it has no dispatch path, no outcome and no ledger row, so the +admission gate does not bind on it. +""" +from __future__ import annotations + +import os +import shutil +import unittest +from pathlib import Path + +# The single token verify.py greps for in a selftest's or gate's output to classify it as +# SKIPPED rather than passed. Defined once here and consumed by both sides, so the writer and +# the reader cannot drift apart. +PREREQ_ABSENT_MARK = "PREREQUISITE ABSENT:" + + +class MissingPrerequisite(unittest.SkipTest): + """This check cannot apply here, and the message says what is missing. + + Subclasses `unittest.SkipTest` on purpose: pytest natively reports it as a skip carrying + the reason, and a plain `except MissingPrerequisite` works in the hand-rolled runners. It + is NOT an assertion failure and must never be raised to paper over one — the prerequisite + is a fact about the machine, never about the code under test. + """ + + +# --------------------------------------------------------------------------- the ledger +# The capability ledger is machine-local accumulated history. Three distinct facets of its +# absence produce three distinct failures, so each gets its own named detector rather than one +# vague "ledger looks empty". + +def _ledger() -> dict: + # Imported lazily: `capabilities` imports `feedback`, and `feedback`'s own selftest imports + # this module — a module-level import here would close that cycle. + import capabilities + return capabilities.load_declared(capabilities.REG) + + +def ledger_rows_absent(*capability_ids: str) -> str | None: + """Reason string when any named capability has no row in the ledger at all.""" + try: + ledger = _ledger() + except Exception as exc: # noqa: BLE001 + return f"capability ledger unreadable ({type(exc).__name__}: {exc})" + missing = sorted(c for c in capability_ids if c not in ledger) + if not missing: + return None + import capabilities + return (f"capability ledger has no row for {', '.join(missing)} — the ledger is " + f"machine-local state ({capabilities.REG}); it holds {len(ledger)} row(s) here, " + f"and these are registered by running the system, not by checking out the tree") + + +def ledger_version_lineage_absent(*capability_ids: str) -> str | None: + """Reason string when a row exists but carries no `capability_version_id`. + + A freshly bootstrapped row has `capability_version_id: None`, and the influence-edge writer + refuses an edge without lineage — by design, so a capability cannot be credited with a + version it never had. That refusal is correct behaviour and not something to assert around. + """ + try: + ledger = _ledger() + except Exception as exc: # noqa: BLE001 + return f"capability ledger unreadable ({type(exc).__name__}: {exc})" + unversioned = sorted(c for c in capability_ids + if not (ledger.get(c) or {}).get("capability_version_id")) + if not unversioned: + return None + return (f"capability ledger carries no version lineage for {', '.join(unversioned)} " + f"(capability_version_id is unset) — lineage is established by real registration " + f"on the running instance, so a fresh bootstrap has none") + + +def ledger_invocation_history_absent(*capability_ids: str) -> str | None: + """Reason string when a row has never recorded an invocation on this machine.""" + try: + ledger = _ledger() + except Exception as exc: # noqa: BLE001 + return f"capability ledger unreadable ({type(exc).__name__}: {exc})" + silent = sorted(c for c in capability_ids + if not (ledger.get(c) or {}).get("last_invocation")) + if not silent: + return None + return (f"capability ledger records no invocation for {', '.join(silent)} " + f"(last_invocation is unset) — liveness classification reads that history, which " + f"only accrues on the running instance") + + +def ledger_legacy_rows_absent() -> str | None: + """Reason string when the ledger holds no capability registered before the admission gate. + + `capability_admission` scopes enforcement to capabilities registered from 2026-08-21 and + reports the earlier ones as legacy debt. A ledger bootstrapped today has no earlier ones, + so "legacy debt is reported, not forgiven" has nothing to report on. + """ + try: + import capability_admission as admission + rows = admission.report()["rows"] + except Exception as exc: # noqa: BLE001 + return f"admission report unavailable ({type(exc).__name__}: {exc})" + if any(r.get("legacy") for r in rows): + return None + return ("capability ledger holds no pre-admission-gate (legacy) capability — legacy debt " + "accrues from this instance's registration history, and a ledger bootstrapped " + "from the committed tree has none") + + +# --------------------------------------------------------------------------- local files & CLIs + +def skill_resource_absent() -> str | None: + """Reason string when the reference skill's bundled script is not installed. + + `capability_compiler.reference_skill_source()` hashes a real installed skill resource under + `~/.codex/skills`; the compiler is exercised against a genuine file on purpose, so there is + nothing to compile when the skill is not installed. + """ + import capability_compiler + try: + resource = Path(capability_compiler.reference_skill_source()["resources"][0]["source_path"]) + except (OSError, KeyError, IndexError): + # reference_skill_source() hashes the file as it builds the dict, so an absent resource + # raises here — which is the answer, not an error. + resource = (Path.home() / ".codex" / "skills" / "code-workspace-hygiene" + / "scripts" / "audit_code_root.sh") + if resource.is_file(): + return None + return (f"reference skill resource not installed: {resource} — the skill compiler is " + f"deliberately exercised against a real installed skill, not a fixture") + + +def codex_profile_binary_absent() -> str | None: + """Reason string when the version-capable Codex binary exact profiles require is absent. + + `adapters.profile_codex_binary()` fails closed rather than falling back to whatever `codex` + is on PATH — the whole point of an exact profile is that the binary can pin a version. The + default location is inside a macOS app bundle, so it cannot exist on a Linux runner. + """ + import adapters + if adapters.CODEX_PROFILE_BIN.is_file(): + return None + return (f"exact-profile Codex binary absent: {adapters.CODEX_PROFILE_BIN} " + f"(set ORCH_CODEX_PROFILE_BIN to a version-capable Codex binary) — " + f"adapters.profile_codex_binary() fails closed rather than using PATH") + + +def agent_cli_absent(*agents: str) -> str | None: + """Reason string when a seat's CLI is not installed, so its auth probe cannot run.""" + import adapters + missing = [] + for agent in agents: + probe = (adapters.AUTH_PROBES.get(agent) or {}).get("cmd") or [] + binary = probe[0] if probe else agent + if not shutil.which(str(binary)): + missing.append(f"{agent} ({binary})") + if not missing: + return None + return (f"agent CLI not installed for {', '.join(missing)} — with no CLI and no credential " + f"file there is genuinely no free signal for the seat, which is the documented " + f"UNKNOWN case, not a broken credential") + + +def credential_file_absent(*agents: str) -> str | None: + """Reason string when the credential FILE the fleet sources at dispatch time is absent.""" + import agent_auth_check + missing = [] + for agent in agents: + entry = agent_auth_check.CREDENTIAL_FILES.get(agent) + if entry is None: + missing.append(f"{agent} (no credential file registered)") + elif not entry[0].is_file(): + missing.append(f"{agent} ({entry[0]})") + if not missing: + return None + return (f"credential file absent for {', '.join(missing)} — the fleet authenticates by " + f"sourcing that file, so without it the seat's verdict is BROKEN by design") + + +def seat_has_no_free_signal() -> str | None: + """Reason string naming every seat that has neither an installed CLI nor a credential file. + + Gate for the "no seat may report UNKNOWN" check. A seat with no probe and no credential + file has, by `agent_auth_check`'s own documented rule, no free signal at all — and UNKNOWN + is explicitly never treated as a failure there. Asserting the absence of UNKNOWN on such a + machine tests the machine, not the code. + """ + import adapters + import agent_auth_check + blind = [] + for agent in agent_auth_check.AGENTS: + probe = (adapters.AUTH_PROBES.get(agent) or {}).get("cmd") or [] + binary = str(probe[0]) if probe else None + has_cli = bool(binary and shutil.which(binary)) + entry = agent_auth_check.CREDENTIAL_FILES.get(agent) + has_file = bool(entry and entry[0].is_file()) + if not has_cli and not has_file: + blind.append(agent) + if not blind: + return None + return (f"no free auth signal on this machine for {', '.join(sorted(blind))}: neither an " + f"installed CLI probe nor a credential file — agent_auth_check's documented " + f"UNKNOWN case") + + +# --------------------------------------------------------------------------- harness glue + +def require(*reasons: str | None) -> None: + """Raise `MissingPrerequisite` for the first named absence, or return. + + Call with detector results: `require(ledger_rows_absent("issue-readiness"))`. + """ + for reason in reasons: + if reason: + raise MissingPrerequisite(reason) + + +def selftest_skipped(module: str, *reasons: str | None) -> bool: + """For a module `--selftest`: print the marked reason and report that it did not run. + + Returns True when the selftest must not run. The caller exits 0 — a skip is not a failure — + but the marked line is what stops `verify.py` from counting it as a pass. verify.py already + treats a silent zero-exit as a failure; this closes the matching hole, a zero-exit that + SPOKE while executing nothing. + """ + for reason in reasons: + if reason: + print(f"{module} selftest: {PREREQ_ABSENT_MARK} {reason}") + return True + return False + + +def runnable(gaps: list[str], *reasons: str | None) -> bool: + """Should this SECTION of a selftest run here? Records the reason when it must not. + + Gating a whole `--selftest` because one block of it needs the ledger would throw away the + other few hundred assertions in the same function — running less to report green, which is + the exact trade this project refuses. So the gate goes around the smallest block that needs + the missing thing, `gaps` collects why, and `report_gaps` says so at the end. + """ + for reason in reasons: + if reason: + gaps.append(reason) + return False + return True + + +def report_gaps(module: str, gaps: list[str]) -> None: + """Print the marked line naming every section that did not run, or nothing if all did.""" + for reason in dict.fromkeys(gaps): + print(f"{module} selftest: {PREREQ_ABSENT_MARK} section skipped — {reason}") + + +def _selftest() -> None: + # The exception must be a skip to every harness that will see it. + assert issubclass(MissingPrerequisite, unittest.SkipTest) + try: + require(None, None) + except MissingPrerequisite: # pragma: no cover + raise AssertionError("require() must not raise when nothing is absent") + try: + require(None, "the named thing is missing", "a later reason") + except MissingPrerequisite as exc: + assert str(exc) == "the named thing is missing", exc + else: + raise AssertionError("require() must raise on the first named absence") + + # EVERY detector must answer with a reason that NAMES the thing, or with None. A detector + # returning a bare True/False is the failure this module exists to prevent: a skip with no + # reason is indistinguishable from a pass. + detectors = [ + ("ledger_rows_absent", lambda: ledger_rows_absent("definitely-not-a-capability")), + ("ledger_version_lineage_absent", + lambda: ledger_version_lineage_absent("definitely-not-a-capability")), + ("ledger_invocation_history_absent", + lambda: ledger_invocation_history_absent("definitely-not-a-capability")), + ("ledger_legacy_rows_absent", ledger_legacy_rows_absent), + ("skill_resource_absent", skill_resource_absent), + ("codex_profile_binary_absent", codex_profile_binary_absent), + ("agent_cli_absent", lambda: agent_cli_absent("codex")), + ("credential_file_absent", lambda: credential_file_absent("vibe")), + ("seat_has_no_free_signal", seat_has_no_free_signal), + ] + for name, fn in detectors: + got = fn() + assert got is None or (isinstance(got, str) and len(got) > 20), (name, got) + # The three that were handed a capability that cannot exist MUST report absence, or the + # detector is not detecting anything. + for name, fn in detectors[:3]: + got = fn() + assert got and "definitely-not-a-capability" in got, (name, got) + + # A skipped selftest must SPEAK, and its line must carry the shared mark verify.py greps + # for. A skip that prints nothing is a silent zero-exit by another name. + import io + import contextlib + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + assert selftest_skipped("mod", None, "thing X is absent") is True + assert selftest_skipped("mod", None, None) is False + text = buf.getvalue() + assert PREREQ_ABSENT_MARK in text and "thing X is absent" in text, text + assert text.count(PREREQ_ABSENT_MARK) == 1, text + + print("env_prereq.py selftest: OK (skip-is-a-skip, every detector names the missing thing, " + "marked selftest skip speaks)") + + +def main(argv: list[str]) -> int: + if "--selftest" in argv: + _selftest() + return 0 + # Default: report what this machine can and cannot check. Useful on a new box. + checks = { + "reference skill resource": skill_resource_absent(), + "exact-profile Codex binary": codex_profile_binary_absent(), + "seat auth signal": seat_has_no_free_signal(), + "vibe credential file": credential_file_absent("vibe"), + "ledger legacy rows": ledger_legacy_rows_absent(), + } + for name, reason in checks.items(): + print(f"{'ABSENT ' if reason else 'present'} {name}" + (f"\n {reason}" if reason else "")) + return 0 + + +if __name__ == "__main__": + import sys + raise SystemExit(main(sys.argv[1:])) diff --git a/exp_abcd.py b/exp_abcd.py index 23311de..8c70802 100644 --- a/exp_abcd.py +++ b/exp_abcd.py @@ -2337,6 +2337,19 @@ def fake_subject_lifecycle(exp_id, lifecycle, reason=None): old_handoff, old_ledger = adapters.HANDOFF, adapters.LEDGER old_build_command = adapters.build_command old_popen = subprocess.Popen + # ISOLATION, not a skip. This block replaces subprocess.Popen wholesale to fake the agent + # processes, and `_eval_command` resolves each seat's model on the way — which spawns a CLI + # catalog probe whenever the advertised-model cache is cold. The probe then lands in FakePopen, + # which is built for the evaluator's stdout contract, and dies on `stdout.write` with an int. + # It never showed up locally because the cache is always warm here; on a fresh machine (the + # first CI run, 2026-08-21) it was a hard AttributeError. + # + # ORCH_MODEL_PROBE is the module's own documented kill-switch for exactly this: catalog probes + # off, pinned models only, NO subprocess. Turning it off for the stubbed window makes the + # selftest hermetic on every machine — it now runs MORE than it did, not less, which is why + # this is a fix and not an applicability gate. + old_model_probe = os.environ.get("ORCH_MODEL_PROBE") + old_advertised_memo = dict(adapters._ADVERTISED_MEMO) class FakePopen: next_pid = 4900 @@ -2385,6 +2398,8 @@ def fake_build_command(agent, prompt, mode, cwd=None): return ["printf", "fake-agent"] adapters.build_command = fake_build_command + os.environ["ORCH_MODEL_PROBE"] = "0" + adapters._ADVERTISED_MEMO.clear() subprocess.Popen = FakePopen run_id = "e1:codex" @@ -2520,6 +2535,12 @@ def fake_build_command(agent, prompt, mode, cwd=None): adapters.LEDGER = old_ledger adapters.build_command = old_build_command subprocess.Popen = old_popen + if old_model_probe is None: + os.environ.pop("ORCH_MODEL_PROBE", None) + else: + os.environ["ORCH_MODEL_PROBE"] = old_model_probe + adapters._ADVERTISED_MEMO.clear() + adapters._ADVERTISED_MEMO.update(old_advertised_memo) shutil.rmtree(tmp, ignore_errors=True) print( "exp_abcd.py selftest: OK (branch/worktree naming, mode map, frozen implement prompt, " diff --git a/feedback.py b/feedback.py index 34656ea..f729b14 100644 --- a/feedback.py +++ b/feedback.py @@ -4638,7 +4638,10 @@ def _effective(cell_agent: str, s: dict, m: str) -> tuple[float, str]: def _selftest(): import tempfile + import env_prereq # imported here: this module is env_prereq's own dep + tmp = tempfile.mkdtemp(prefix="feedback-selftest-") + gaps: list[str] = [] global DB_PATH DB_PATH = Path(tmp) / "t.db" try: @@ -5663,18 +5666,26 @@ def _selftest(): # ---- an offload-backed role run carries the transport capability, and one without # ---- a backend run does not: the tag follows the recorded link, never the role name. - record_role_run("role:redirect:offloaded", "redirect", "owner/repo#tr", "cursor", - backend_run_id="offload:abc123") - record_role_run("role:redirect:replayed", "redirect", "owner/repo#tr", "cursor") - with _conn() as c: - offloaded = {r[0] for r in c.execute( - "SELECT capability_id FROM influence_edges WHERE target_run_id=? " - "AND influence_type='capability'", ("role:redirect:offloaded",)).fetchall()} - replayed = {r[0] for r in c.execute( - "SELECT capability_id FROM influence_edges WHERE target_run_id=? " - "AND influence_type='capability'", ("role:redirect:replayed",)).fetchall()} - assert offloaded == {"role-redirect", "offload"}, offloaded - assert replayed == {"role-redirect"}, replayed + # The DB here is a fresh tmp file, but the tags come out of the LEDGER: this writer + # refuses a capability edge with no version lineage, on purpose, so a capability is never + # credited with a version it never had. `offload`'s row exists only on an instance that + # has run it, so the SECTION is gated and named — everything else here still runs. + if env_prereq.runnable( + gaps, + env_prereq.ledger_rows_absent("role-redirect", "offload"), + env_prereq.ledger_version_lineage_absent("role-redirect", "offload")): + record_role_run("role:redirect:offloaded", "redirect", "owner/repo#tr", "cursor", + backend_run_id="offload:abc123") + record_role_run("role:redirect:replayed", "redirect", "owner/repo#tr", "cursor") + with _conn() as c: + offloaded = {r[0] for r in c.execute( + "SELECT capability_id FROM influence_edges WHERE target_run_id=? " + "AND influence_type='capability'", ("role:redirect:offloaded",)).fetchall()} + replayed = {r[0] for r in c.execute( + "SELECT capability_id FROM influence_edges WHERE target_run_id=? " + "AND influence_type='capability'", ("role:redirect:replayed",)).fetchall()} + assert offloaded == {"role-redirect", "offload"}, offloaded + assert replayed == {"role-redirect"}, replayed # ---- a REJECTED edge must never inherit the acting run's success --------------- # `_propagate_outcome_lineage_in_conn` filters on accepted=1. That filter is the whole @@ -5709,56 +5720,63 @@ def _selftest(): assert lineage_health["accepted_influence_linked"] >= 3, lineage_health assert lineage_health["orphan_edges"] == 0, lineage_health - # ---- capability attribution must reach a run that CAN resolve ------------------- - # A role run is advisory and never gets an `outcomes` row, so its own capability edge is - # permanently unresolvable. The run that ACTS on the proposal inherits the attribution, - # and outcome propagation then resolves it on the already-existing path. - record_role_run("role:triage:x:1", "triage", "o/r#7", "gemini") - capq = ("SELECT capability_id,target_run_id,durability FROM influence_edges " - "WHERE influence_type='capability'") - with _conn() as cc: - pre = cc.execute(capq + " AND target_run_id='role:triage:x:1'").fetchall() - assert pre and pre[0][2] is None, f"advisory role run should have no outcome: {pre}" - - record_run("cap-inherit-work", target="o/r#7", task_type="implement", agent="codex", - influenced_by_role_run_ids=["role:triage:x:1"]) - with _conn() as cc: - mid = cc.execute(capq + " AND target_run_id='cap-inherit-work'").fetchall() - assert mid and mid[0][0] == "role-triage", f"attribution not inherited: {mid}" - - record_outcome("cap-inherit-work", verifier_verdict="PASS", merged=1, durability="durable") - with _conn() as cc: - post = cc.execute(capq + " AND target_run_id='cap-inherit-work'").fetchall() - assert post and post[0][2] == "durable", f"outcome did not reach the edge: {post}" - - # A run declaring the capability ITSELF must not also get an inherited duplicate. - record_role_run("role:triage:x:2", "triage", "o/r#8", "gemini") - record_run("cap-nodupe", target="o/r#8", task_type="implement", agent="codex", - capability_ids=["role-triage"], - influenced_by_role_run_ids=["role:triage:x:2"]) - with _conn() as cc: - dupes = cc.execute(capq + " AND target_run_id='cap-nodupe'").fetchall() - assert len(dupes) == 1, f"double-counted the same capability: {dupes}" - - # DELIBERATE BREAK -> REVERT: stub the lookup and nothing is inherited, so the edge stays - # unresolvable — proving the inheritance is what makes measurement possible. - saved_lookup = globals()["_capability_attribution_of"] - try: - globals()["_capability_attribution_of"] = lambda *a, **k: [] - record_role_run("role:triage:x:3", "triage", "o/r#9", "gemini") - record_run("cap-broken", target="o/r#9", task_type="implement", agent="codex", - influenced_by_role_run_ids=["role:triage:x:3"]) + # Same LEDGER prerequisite as above, and the same reason it is a SECTION rather than the + # whole selftest: every edge here is a `role-triage` capability edge, and the writer needs + # that row's version lineage to create one at all. + if env_prereq.runnable( + gaps, + env_prereq.ledger_rows_absent("role-triage"), + env_prereq.ledger_version_lineage_absent("role-triage")): + # ---- capability attribution must reach a run that CAN resolve ------------------- + # A role run is advisory and never gets an `outcomes` row, so its own capability edge is + # permanently unresolvable. The run that ACTS on the proposal inherits the attribution, + # and outcome propagation then resolves it on the already-existing path. + record_role_run("role:triage:x:1", "triage", "o/r#7", "gemini") + capq = ("SELECT capability_id,target_run_id,durability FROM influence_edges " + "WHERE influence_type='capability'") with _conn() as cc: - broken = cc.execute(capq + " AND target_run_id='cap-broken'").fetchall() - assert not broken, "break did not change behaviour — test is vacuous" - finally: - globals()["_capability_attribution_of"] = saved_lookup - record_role_run("role:triage:x:4", "triage", "o/r#10", "gemini") - record_run("cap-reverted", target="o/r#10", task_type="implement", agent="codex", - influenced_by_role_run_ids=["role:triage:x:4"]) - with _conn() as cc: - rev = cc.execute(capq + " AND target_run_id='cap-reverted'").fetchall() - assert rev, "revert did not restore inheritance" + pre = cc.execute(capq + " AND target_run_id='role:triage:x:1'").fetchall() + assert pre and pre[0][2] is None, f"advisory role run should have no outcome: {pre}" + + record_run("cap-inherit-work", target="o/r#7", task_type="implement", agent="codex", + influenced_by_role_run_ids=["role:triage:x:1"]) + with _conn() as cc: + mid = cc.execute(capq + " AND target_run_id='cap-inherit-work'").fetchall() + assert mid and mid[0][0] == "role-triage", f"attribution not inherited: {mid}" + + record_outcome("cap-inherit-work", verifier_verdict="PASS", merged=1, durability="durable") + with _conn() as cc: + post = cc.execute(capq + " AND target_run_id='cap-inherit-work'").fetchall() + assert post and post[0][2] == "durable", f"outcome did not reach the edge: {post}" + + # A run declaring the capability ITSELF must not also get an inherited duplicate. + record_role_run("role:triage:x:2", "triage", "o/r#8", "gemini") + record_run("cap-nodupe", target="o/r#8", task_type="implement", agent="codex", + capability_ids=["role-triage"], + influenced_by_role_run_ids=["role:triage:x:2"]) + with _conn() as cc: + dupes = cc.execute(capq + " AND target_run_id='cap-nodupe'").fetchall() + assert len(dupes) == 1, f"double-counted the same capability: {dupes}" + + # DELIBERATE BREAK -> REVERT: stub the lookup and nothing is inherited, so the edge stays + # unresolvable — proving the inheritance is what makes measurement possible. + saved_lookup = globals()["_capability_attribution_of"] + try: + globals()["_capability_attribution_of"] = lambda *a, **k: [] + record_role_run("role:triage:x:3", "triage", "o/r#9", "gemini") + record_run("cap-broken", target="o/r#9", task_type="implement", agent="codex", + influenced_by_role_run_ids=["role:triage:x:3"]) + with _conn() as cc: + broken = cc.execute(capq + " AND target_run_id='cap-broken'").fetchall() + assert not broken, "break did not change behaviour — test is vacuous" + finally: + globals()["_capability_attribution_of"] = saved_lookup + record_role_run("role:triage:x:4", "triage", "o/r#10", "gemini") + record_run("cap-reverted", target="o/r#10", task_type="implement", agent="codex", + influenced_by_role_run_ids=["role:triage:x:4"]) + with _conn() as cc: + rev = cc.execute(capq + " AND target_run_id='cap-reverted'").fetchall() + assert rev, "revert did not restore inheritance" # JSON snapshot (the reviewable project copy of the dataset) snap = snapshot_json(Path(tmp) / "snap.json") @@ -5769,12 +5787,14 @@ def _selftest(): and snap["rows"]["execution_attempts"] >= 5 and "evidence_types" in snap["rows"] ), snap + env_prereq.report_gaps("feedback.py", gaps) print( "feedback.py selftest: OK (prior→posterior learning, durability/verifier-as-success, late updates, " "versioned weights, eval matrix, human calibration, evidence-gap growth+prune+approval, " "trace retention, test_evaluator_trace_cannot_resolve_worker_model, conservative legacy migration, " "quality-magnitude/outcome learner + effort reward, safe completion lineage + " "rejected-edge non-inheritance, json snapshot)" + + (f" — {len(set(gaps))} section(s) skipped, see above" if gaps else "") ) finally: import shutil diff --git a/range_lane_rollout.py b/range_lane_rollout.py index e420d77..6cea1b4 100644 --- a/range_lane_rollout.py +++ b/range_lane_rollout.py @@ -25,6 +25,7 @@ import backlog import capabilities +import env_prereq import capacity import claims import dispatcher @@ -478,6 +479,13 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.selftest: + # Every `build_rollout` here runs a dry dispatch preview, and a codex assignment resolves + # an EXACT profile — which needs the version-capable Codex binary and fails closed rather + # than falling back to PATH. That is the whole selftest's spine, not one section of it, so + # the gate is the selftest. The reason names the binary; verify.py counts it and bounds it. + if env_prereq.selftest_skipped("range_lane_rollout.py", + env_prereq.codex_profile_binary_absent()): + return 0 _selftest() return 0 diff --git a/test_capabilities.py b/test_capabilities.py index aed3290..501fada 100644 --- a/test_capabilities.py +++ b/test_capabilities.py @@ -5,6 +5,7 @@ import pytest import capabilities +import env_prereq @pytest.fixture @@ -284,6 +285,12 @@ def test_gate_blocks_execution_is_opt_in_and_narrow(): {**base, "gate_blocks_execution": True, "gate_reason": None}, now=100) == "invoked_without_outcomes" + # From here the check reads the LIVE ledger, which is machine-local state: `issue-readiness` + # is registered by running the system, not by checking out the tree, so on a machine that has + # never run it there is nothing to classify. Skip with the row named — never silently pass. + env_prereq.require(env_prereq.ledger_rows_absent( + "thompson-hybrid-routing", "range-lane-rollout", "issue-readiness", "role-triage")) + # `load_declared`, not `load(create=False)`: `gate_blocks_execution` is a DECLARATION-owned # field, so a raw read answers with whatever is on disk at that instant. Both of these rows had # it reconciled mid-suite on 2026-08-21 (08:15:07), which is how this file produced a red that @@ -320,6 +327,13 @@ def test_evidence_gate_kind_is_not_blanket_observer(): # `tick_phase` at 08:07:28, mid-suite (the row's own `declaration_reconciled` event records it). # A raw `create=False` read asks "which side of that write did I land on?"; this asks the # question the test actually means, and still writes nothing. + # + # `observing` is a verdict about RECORDED HISTORY: a supervisor row that has never been + # invoked on this machine classifies `deliberately_gated`, correctly. So the prerequisite is + # the invocation history, and its absence is named rather than asserted around. + env_prereq.require( + env_prereq.ledger_rows_absent("live-keepalive-supervisor", "redirect-apply-bootstrap"), + env_prereq.ledger_invocation_history_absent("live-keepalive-supervisor")) ledger = capabilities.load_declared(capabilities.REG) supervisor = ledger["live-keepalive-supervisor"] diff --git a/test_capability_admission.py b/test_capability_admission.py index c952fa5..3451a26 100644 --- a/test_capability_admission.py +++ b/test_capability_admission.py @@ -17,6 +17,7 @@ import capabilities import capability_admission as admission +import env_prereq def test_new_capabilities_carry_all_required_parts(): @@ -26,6 +27,10 @@ def test_new_capabilities_carry_all_required_parts(): a capability without them is how six subsystems went dormant, how `issue-readiness` shipped with no heartbeat, and how `reference-sync-hygiene` accrued 367 events its own gate could not read. """ + # Enforcement is scoped to capabilities registered from 2026-08-21, and registration happens + # on the running instance. A ledger with no pre-gate rows has no enforced population either, + # so there is nothing for this to be a gate ON. Name that, do not pass on an empty set. + env_prereq.require(env_prereq.ledger_legacy_rows_absent()) rep = admission.report() failing = rep["enforced_failing"] rows = {r["capability_id"]: r for r in rep["rows"]} @@ -76,6 +81,7 @@ def test_legacy_debt_is_reported_not_forgiven(): pushed into the predicates instead of the row, the debt would read as compliance — which is precisely how "all findings resolved; nothing dropped" coexisted with 13 blocked capabilities. """ + env_prereq.require(env_prereq.ledger_legacy_rows_absent()) rep = admission.report() legacy = [r for r in rep["rows"] if r["legacy"]] assert legacy, "expected pre-gate capabilities to be marked legacy" @@ -100,17 +106,28 @@ def test_waivers_are_bounded_not_just_dated(): def test_the_gate_admits_itself(): """Dogfooding, and not for style: a gate exempt from its own rule is the rule being optional.""" + # `admit()` raises ValueError on a capability the ledger has never heard of, so the gate can + # only be asked about itself where it is registered. + env_prereq.require(env_prereq.ledger_rows_absent("capability-admission-gate")) own = admission.admit("capability-admission-gate") assert own["admitted"], f"the admission gate fails its own requirements: {own['missing']}" def main() -> int: tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] - failures = [] + failures, skipped = [], [] for fn in tests: try: fn() print(f" OK {fn.__name__}") + # A skip is not a pass and not a failure. Catching it BEFORE AssertionError matters: + # MissingPrerequisite is a SkipTest, not an AssertionError, so an uncaught one would + # crash this runner — and printing it as OK would be worse, because the count would + # then claim coverage this machine cannot provide. + except env_prereq.MissingPrerequisite as exc: + skipped.append((fn.__name__, str(exc))) + print(f" SKIP {fn.__name__}") + print(f" {env_prereq.PREREQ_ABSENT_MARK} {str(exc)[:400]}") except AssertionError as exc: failures.append(fn.__name__) print(f" FAIL {fn.__name__}") @@ -118,6 +135,13 @@ def main() -> int: if failures: print(f"\n{len(failures)} of {len(tests)} admission checks FAILED") return 1 + if skipped: + # Green, and saying exactly what did not run. verify.py greps the mark and counts it + # against a ceiling, so this can never quietly become the whole file. + print(f"\n{len(tests) - len(skipped)} of {len(tests)} admission checks passed, " + f"{len(skipped)} skipped: " + + "; ".join(f"{n} ({r[:80]})" for n, r in skipped)) + return 0 rep = admission.report() print(f"\nall {len(tests)} admission checks passed — " f"{rep['enforced_total']} enforced, {len(rep['legacy_debt'])} legacy debt, " diff --git a/test_capability_lifecycle_e2e.py b/test_capability_lifecycle_e2e.py index 57898a7..d59283e 100644 --- a/test_capability_lifecycle_e2e.py +++ b/test_capability_lifecycle_e2e.py @@ -11,6 +11,7 @@ import capability_compiler as compiler import capability_lifecycle import capability_targets +import env_prereq import feedback import roles from test_evidence_contract_compiler import _plan as evidence_contract_plan @@ -166,6 +167,11 @@ def _record_target_run(run_id: str, subject: str) -> None: def test_all_target_kinds_complete_shadow_canary_lifecycle( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # The `skill` kind compiles the reference skill package, which hashes a REAL installed skill + # resource under ~/.codex/skills — deliberately, so the compiler is exercised against a + # genuine file rather than a fixture. Without it installed there is no skill to take through + # the lifecycle, and the other four kinds are covered by their own tests. + env_prereq.require(env_prereq.skill_resource_absent()) monkeypatch.setattr(feedback, "DB_PATH", tmp_path / "brain.db") now = int(time.time()) for kind in ("role", "workflow", "skill", "playbook", "gate"): diff --git a/test_capability_set_coverage.py b/test_capability_set_coverage.py index 8ec6459..9aadf99 100644 --- a/test_capability_set_coverage.py +++ b/test_capability_set_coverage.py @@ -27,6 +27,7 @@ import capabilities import capability_activation_audit as audit import capability_recurrence_check as recurrence +import env_prereq # Capabilities exempt from needing a recurrence fixture, each with a REASON. This list exists so an # exemption is a deliberate, reviewable act rather than a silent omission. Keep it empty if possible. @@ -57,6 +58,11 @@ def test_every_capability_has_a_recurrence_fixture(): def test_no_fixture_names_an_unknown_capability(): """A fixture pointing at a nonexistent capability covers nothing while looking like coverage.""" + # This check compares fixtures against the LIVE ledger, so it can only distinguish a typo + # from an unregistered capability where the whole registered set is present. On a machine + # that has never run the system the ledger holds only the rows the code declares, and every + # fixture beyond those would read as a typo. Name the absent rows instead of asserting. + env_prereq.require(env_prereq.ledger_rows_absent(*sorted(_fixture_capabilities()))) ledger = set(capabilities.load(capabilities.REG)) unknown = sorted(_fixture_capabilities() - ledger) assert not unknown, f"fixtures name capabilities absent from the ledger: {unknown}" @@ -146,11 +152,19 @@ def main() -> int: print(roster(), end="") return 0 tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] - failures = [] + failures, skipped = [], [] for fn in tests: try: fn() print(f" OK {fn.__name__}") + # MissingPrerequisite is a SkipTest, not an AssertionError — caught first so it neither + # crashes this runner nor gets counted as a pass. "5 of 6 passed, 1 skipped because X" + # is the honest line; "all 6 passed" over a set the machine cannot see is the lie this + # whole file exists to prevent. + except env_prereq.MissingPrerequisite as exc: + skipped.append((fn.__name__, str(exc))) + print(f" SKIP {fn.__name__}") + print(f" {env_prereq.PREREQ_ABSENT_MARK} {str(exc)[:400]}") except AssertionError as exc: failures.append((fn.__name__, str(exc))) print(f" FAIL {fn.__name__}") @@ -159,6 +173,11 @@ def main() -> int: print(f"\n{len(failures)} of {len(tests)} capability-set coverage checks FAILED") return 1 ledger = capabilities.load(capabilities.REG) + if skipped: + print(f"\n{len(tests) - len(skipped)} of {len(tests)} capability-set coverage checks " + f"passed over {len(ledger)} ledger capabilities, {len(skipped)} skipped: " + + "; ".join(f"{n} ({r[:80]})" for n, r in skipped)) + return 0 print(f"\nall {len(tests)} capability-set coverage checks passed " f"over ALL {len(ledger)} ledger capabilities " f"(--roster for the per-capability table)") diff --git a/test_capacity_profiles.py b/test_capacity_profiles.py index 32396f7..e2e4b5c 100644 --- a/test_capacity_profiles.py +++ b/test_capacity_profiles.py @@ -7,6 +7,7 @@ import adapters import capacity import dispatcher +import env_prereq import execution_profiles import feedback import ledger_reconcile @@ -63,6 +64,10 @@ def test_capacity_build_reads_shared_pool_burn_once(tmp_path, monkeypatch, codex def test_exact_codex_profile_commands_preserve_permission_rails(monkeypatch): + # `build_command` with an exact profile resolves the version-capable Codex binary and fails + # closed if it is absent, rather than falling back to whatever `codex` is on PATH. Nothing + # about the permission rails can be observed without a command to inspect. + env_prereq.require(env_prereq.codex_profile_binary_absent()) monkeypatch.setenv("ORCH_CODEX_BYPASS_INNER_SANDBOX", "0") models = set() for profile in execution_profiles.profiles_for_agent("codex"): @@ -82,6 +87,7 @@ def test_exact_codex_profile_commands_preserve_permission_rails(monkeypatch): def test_nested_sandbox_never_widens_read_only_profile(monkeypatch): + env_prereq.require(env_prereq.codex_profile_binary_absent()) monkeypatch.setenv("CODEX_SANDBOX", "seatbelt") monkeypatch.delenv("ORCH_CODEX_BYPASS_INNER_SANDBOX", raising=False) profile = execution_profiles.get_profile("codex-5.6-sol-high") diff --git a/test_experiment_arm_identity.py b/test_experiment_arm_identity.py index 9a26f99..777a38e 100644 --- a/test_experiment_arm_identity.py +++ b/test_experiment_arm_identity.py @@ -141,6 +141,17 @@ def test_arm_evaluate_dual_writes_exact_v2_and_parent_legacy(tmp_path, monkeypat monkeypatch.setenv("ORCH_OBJECTIVE_ANCHOR", "0") monkeypatch.setattr(exp_abcd, "_record_execution_start", lambda *a, **kw: 1) monkeypatch.setattr(exp_abcd, "_record_execution_complete", lambda *a, **kw: None) + # ISOLATION, not a skip. FakePopen below replaces subprocess.Popen for the whole call, and + # `_eval_command` resolves each seat's model on the way — which spawns a CLI catalog probe + # when the advertised-model cache is cold. That probe then hits FakePopen, whose `stdout` is + # PIPE (an int), and dies on `stdout.write`. Warm cache here, cold on a fresh machine: it is + # why this test was red on the first CI run and green locally. + # + # ORCH_MODEL_PROBE=0 is adapters' own documented kill-switch — catalog probes off, pinned + # models only, no subprocess — so the test becomes hermetic instead of skipping. It asserts + # exactly what it asserted before, and now asserts it on every machine. + monkeypatch.setenv("ORCH_MODEL_PROBE", "0") + monkeypatch.setattr(exp_abcd.adapters, "_ADVERTISED_MEMO", {}) class FakePopen: def __init__(self, *_args, stdout=None, **_kwargs): diff --git a/test_feedback_model_provenance.py b/test_feedback_model_provenance.py index 185326e..d200088 100644 --- a/test_feedback_model_provenance.py +++ b/test_feedback_model_provenance.py @@ -3,6 +3,7 @@ import pytest +import env_prereq import feedback @@ -395,6 +396,13 @@ def test_multi_capability_run_records_one_edge_each(tmp_path): one-capability case — which meant a run declaring two capabilities recorded attribution for neither. Edges are the many-to-many surface (2026-08-09). """ + # The Brain here is a fresh tmp DB, but the LEDGER is not: the edge writer resolves each + # capability's version lineage from it and refuses an edge without one (all-or-nothing, so a + # capability is never credited with a borrowed version). A ledger row with no lineage is the + # unregistered case, which the sibling test asserts produces no attribution. + env_prereq.require( + env_prereq.ledger_rows_absent("adversarial-review", "testgen-lane"), + env_prereq.ledger_version_lineage_absent("adversarial-review", "testgen-lane")) old_db = feedback.DB_PATH feedback.DB_PATH = tmp_path / "feedback.db" try: @@ -435,6 +443,10 @@ def test_role_run_creates_a_capability_tagged_edge(tmp_path): payload and never to record_run, so 81 influence edges carried 0 capability tags and reconcile_causal_lifecycle — which reads exactly those tags — saw no evidence ever. """ + # Same prerequisite as the multi-capability case: no version lineage in the ledger, no edge. + env_prereq.require( + env_prereq.ledger_rows_absent("role-triage"), + env_prereq.ledger_version_lineage_absent("role-triage")) old_db = feedback.DB_PATH feedback.DB_PATH = tmp_path / "feedback.db" try: diff --git a/test_model_tier_resolution.py b/test_model_tier_resolution.py index 68db5c6..944a7fc 100644 --- a/test_model_tier_resolution.py +++ b/test_model_tier_resolution.py @@ -23,6 +23,7 @@ import adapters import capacity +import env_prereq def _live_advertised_models() -> list[str]: @@ -500,6 +501,11 @@ def test_presence_probe_still_detects_a_missing_credential(monkeypatch): def test_every_seat_has_some_free_signal(): """No seat may report UNKNOWN: seats without a CLI probe fall back to a credential-file check. vibe is the case that forced this — `vibe -p` bills and everything else is a TUI.""" + # The fallback chain is CLI probe -> credential file -> UNKNOWN, and `agent_auth_check`'s own + # rule is that UNKNOWN is never a failure. A seat with neither an installed CLI nor a + # credential file therefore has no free signal by design; asserting the absence of UNKNOWN + # on such a machine measures the machine's installation, not this chain. + env_prereq.require(env_prereq.seat_has_no_free_signal()) import agent_auth_check for agent in agent_auth_check.AGENTS: row = agent_auth_check.check(agent) @@ -508,6 +514,9 @@ def test_every_seat_has_some_free_signal(): def test_file_only_seats_are_labelled_configured_not_ok(): """A file check must never be sold as a working credential.""" + # CONFIGURED is the verdict for "the credential file is there and holds the key". With the + # file absent the correct verdict is BROKEN, which the sibling test asserts directly. + env_prereq.require(env_prereq.credential_file_absent("vibe", "aider")) import agent_auth_check for agent in ("vibe", "aider"): assert agent not in adapters.AUTH_PROBES, agent diff --git a/test_skill_compiler.py b/test_skill_compiler.py index a0a86ff..4d151b9 100644 --- a/test_skill_compiler.py +++ b/test_skill_compiler.py @@ -11,11 +11,24 @@ import capabilities import capability_compiler as compiler +import env_prereq import feedback QUICK_VALIDATE = Path.home() / ".codex" / "skills" / ".system" / "skill-creator" / "scripts" / "quick_validate.py" +# EVERY test here builds from `compiler.reference_skill_source()`, which hashes a real installed +# skill resource under ~/.codex/skills — on purpose: the skill compiler is exercised against a +# genuine installed skill, not a synthetic fixture, because a fixture could not catch a manifest +# that fails the real validator. So the resource is this file's prerequisite in full. +# +# `skipif` rather than a module-level raise: skipif leaves all 7 items COLLECTED and skips them +# individually with the reason attached, while a raise at import time would drop the collection +# count by 7 — and a dropped collection count is exactly what verify.py's floor exists to catch. +_SKILL_RESOURCE_ABSENT = env_prereq.skill_resource_absent() +pytestmark = pytest.mark.skipif(bool(_SKILL_RESOURCE_ABSENT), + reason=_SKILL_RESOURCE_ABSENT or "") + @pytest.fixture def generated_skill(tmp_path: Path) -> dict: diff --git a/verify.py b/verify.py index b4253ab..65919c3 100644 --- a/verify.py +++ b/verify.py @@ -28,6 +28,18 @@ identical to all tests passing. * **zero collected is always a failure**, whatever the exit status. * the summary states what actually executed, never "the suite passed". + * a **SKIP CEILING** (added 2026-08-21, with the first CI run). Skipping is the other way to + run less while reading green, so it is bounded exactly like collection: the floor file + records the maximum number of skipped tests, skipped selftests and skipped gates, and + exceeding any of them FAILS. Every skip must also NAME its missing prerequisite — this + module prints all of them, so "green" always states what did not run. + + The floor is now `passed + skipped >= floor.passed`, not `passed >= floor.passed`: a check + may move between passing and consciously-skipped, but the two together may never shrink. + Turning a failure into a pass by skipping it is what the ceiling forbids; letting a machine + without the prerequisite report honestly is what the floor change allows. One constant per + ceiling, defined once in the floor file and read once here, so the measuring and draining + windows cannot drift apart. """ from __future__ import annotations @@ -41,13 +53,23 @@ HERE = pathlib.Path(__file__).resolve().parent FLOOR = HERE / ".verify-floor.json" +# The token a selftest or gate prints to say "I did not run this, and here is what is missing". +# Imported from env_prereq rather than duplicated: a shared literal in two files is a pair that +# drifts, and a mark that drifts turns a skip back into a silent pass. +try: + from env_prereq import PREREQ_ABSENT_MARK +except Exception: # noqa: BLE001 + PREREQ_ABSENT_MARK = "PREREQUISITE ABSENT:" + # pytest's terse summary line, e.g. "182 passed, 3 skipped in 41.20s" COUNT_RE = re.compile(r"(\d+) (passed|failed|error|errors|skipped|xfailed|xpassed)") def run_pytest(*, extra: list[str] | None = None) -> dict: """Execute the suite and read the COUNTS, not the exit code.""" - cmd = [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", "--no-header"] + # `-rs` makes pytest print the reason behind every skip. Without it a skip count is a number + # with no story, which is the shape a silent narrowing hides in. + cmd = [sys.executable, "-m", "pytest", "-q", "-rs", "-p", "no:cacheprovider", "--no-header"] cmd += extra or [] proc = subprocess.run(cmd, cwd=HERE, capture_output=True, text=True) tail = (proc.stdout or "") + (proc.stderr or "") @@ -60,10 +82,19 @@ def run_pytest(*, extra: list[str] | None = None) -> dict: # A usage error prints to stderr and exits 0. Absence of counts is therefore a failure, never # an empty success. usage_error = "usage: pytest" in tail or "unrecognized arguments" in tail + lines = tail.strip().splitlines() + # EVERY failure, not the last 12 lines of output. The first CI run reported 21 failures and + # the log named 7 of them, because this was `lines[-12:]` — a truncated red costs a whole + # round trip to diagnose. + failures = [ln.strip() for ln in lines + if ln.startswith(("FAILED ", "ERROR ")) or ln.lstrip().startswith(("FAILED ", "ERROR "))] + skips = [ln.strip() for ln in lines if ln.lstrip().startswith("SKIPPED ")] return {"counts": counts, "collected": collected, "passed": counts.get("passed", 0), + "skipped": counts.get("skipped", 0), "failed": counts.get("failed", 0) + counts.get("error", 0), "returncode": proc.returncode, "usage_error": usage_error, - "tail": tail.strip().splitlines()[-12:]} + "failures": failures, "skips": skips, + "tail": lines[-12:]} def selftest_modules() -> list[str]: @@ -82,19 +113,34 @@ def selftest_modules() -> list[str]: def run_selftests(modules: list[str]) -> dict: - ok, bad = [], {} + """Run each `--selftest` and sort it into ran / skipped-with-a-reason / failed. + + THREE outcomes, not two. A selftest that exits 0 having executed nothing was already caught + (the silent zero-exit rule). The matching hole is a selftest that exits 0, SPEAKS, and still + executed nothing — indistinguishable from a pass by the old two-way split. So a selftest that + cannot run here prints the shared `PREREQUISITE ABSENT:` mark with the missing thing named, + and lands in `skipped`, which is counted, printed, and ceilinged. `ok` therefore means "ran", + and the number after it is trustworthy again. + """ + ok, bad, skipped = [], {}, {} for mod in modules: proc = subprocess.run([sys.executable, f"{mod}.py", "--selftest"], cwd=HERE, capture_output=True, text=True) + out = (proc.stdout or "") + (proc.stderr or "") # A selftest must both exit 0 AND say something. A silent zero-exit is the very failure # this module exists to catch. - spoke = bool((proc.stdout or "").strip() or (proc.stderr or "").strip()) - if proc.returncode == 0 and spoke: - ok.append(mod) - else: + spoke = bool(out.strip()) + if proc.returncode != 0 or not spoke: bad[mod] = ("silent zero-exit — did it run?" if proc.returncode == 0 - else ((proc.stdout or "") + (proc.stderr or "")).strip()[-200:]) - return {"ok": ok, "failed": bad} + else out.strip()[-200:]) + continue + reasons = [ln.split(PREREQ_ABSENT_MARK, 1)[1].strip() + for ln in out.splitlines() if PREREQ_ABSENT_MARK in ln] + if reasons: + skipped[mod] = reasons + else: + ok.append(mod) + return {"ok": ok, "failed": bad, "skipped": skipped} GATES = ( @@ -112,7 +158,12 @@ def run_gates() -> dict: proc = subprocess.run([sys.executable, *argv], cwd=HERE, capture_output=True, text=True) text = (proc.stdout or "") + (proc.stderr or "") - out[name] = {"ok": proc.returncode == 0, "line": _headline(name, text)} + # Same three-way split as the selftests: a gate that could not judge here says so with + # the shared mark, and is reported as SKIP rather than folded into `ok`. + reasons = [ln.split(PREREQ_ABSENT_MARK, 1)[1].strip() + for ln in text.splitlines() if PREREQ_ABSENT_MARK in ln] + out[name] = {"ok": proc.returncode == 0, "skipped": reasons, + "line": _headline(name, text)} return out @@ -137,6 +188,56 @@ def load_floor() -> dict: return {} +# Ceiling keys, and what each bounds. Named once so the check below and `--update-floor` cannot +# disagree about which number they mean. +CEILINGS = ( + ("skipped_max", "skipped test(s)"), + ("selftest_skipped_max", "skipped selftest(s)"), + ("gate_skipped_max", "skipped gate(s)"), +) + + +def _floor_problems(floor: dict, py: dict) -> list[str]: + """Did the amount of CHECKING drop? Pure, so the selftest exercises the real rule. + + Two independent drops, both of which look like passing: + * fewer tests COLLECTED — an import error, a rename, a deletion; + * fewer tests passed-or-consciously-skipped — a test that stopped running without becoming + a named skip. `passed` alone cannot be the floor once skipping is legitimate, or the + machine missing a prerequisite fails for being honest; `passed + skipped` can be, and the + ceiling is what stops the skipped side swallowing everything. + """ + problems = [] + fc, fp = int(floor.get("collected", 0)), int(floor.get("passed", 0)) + if fc and py["collected"] < fc: + problems.append(f"collection DROPPED: {py['collected']} < floor {fc} — tests stopped " + f"running, which looks identical to tests passing") + if fp and py["passed"] + py.get("skipped", 0) < fp: + problems.append(f"executed-or-skipped count dropped: {py['passed']} passed + " + f"{py.get('skipped', 0)} skipped < floor {fp} — a test stopped being run " + f"without becoming a named skip") + return problems + + +def _ceiling_problems(floor: dict, actual: dict) -> list[str]: + """Is anything skipping MORE than the agreed maximum? Pure, for the same reason. + + An UNSET ceiling means "nothing agreed yet", not "zero" — reading a missing key as 0 would + condemn every machine that legitimately lacks a prerequisite. + """ + problems = [] + for key, label in CEILINGS: + limit = floor.get(key) + if limit is None: + continue + if actual.get(key, 0) > int(limit): + problems.append( + f"SKIP CEILING exceeded: {actual[key]} {label} > agreed maximum {limit}. " + f"Skipping is bounded on purpose — either the new skip is wrong, or raise " + f"`{key}` in .verify-floor.json deliberately and say why.") + return problems + + def verify(*, update_floor: bool = False) -> tuple[int, str]: py = run_pytest() floor = load_floor() @@ -152,11 +253,7 @@ def verify(*, update_floor: bool = False) -> tuple[int, str]: if py["failed"]: problems.append(f"{py['failed']} pytest failure(s)/error(s)") fc, fp = int(floor.get("collected", 0)), int(floor.get("passed", 0)) - if fc and py["collected"] < fc: - problems.append(f"collection DROPPED: {py['collected']} < floor {fc} — tests stopped " - f"running, which looks identical to tests passing") - if fp and py["passed"] < fp: - problems.append(f"passing count dropped: {py['passed']} < floor {fp}") + problems += _floor_problems(floor, py) if st["failed"]: problems.append(f"{len(st['failed'])} module selftest(s) failed: " f"{', '.join(sorted(st['failed']))}") @@ -164,13 +261,46 @@ def verify(*, update_floor: bool = False) -> tuple[int, str]: if not res["ok"]: problems.append(f"gate failed: {name}") + # THE SKIP CEILING. Skipping is bounded, not open-ended: exceeding an agreed maximum fails, + # so a future change cannot quietly convert a red into a skip. Each ceiling reports its own + # value against the limit in the same breath, per the house rule that a gate must always say + # both numbers — `24/24` alone reads as "fine", `24/24 (ceiling)` reads as "at the limit". + actual = {"skipped_max": py["skipped"], + "selftest_skipped_max": len(st["skipped"]), + "gate_skipped_max": sum(1 for r in gates.values() if r["skipped"])} + problems += _ceiling_problems(floor, actual) + + def _cap(key: str) -> str: + """Both numbers, always in the same place: the count AND what bounds it.""" + limit = floor.get(key) + return f"{actual[key]}" + (f"/{limit} max" if limit is not None else " (no ceiling set)") + lines = ["# verify.py", ""] lines.append(f" pytest: {py['passed']} passed, {py['failed']} failed, " - f"{py['counts'].get('skipped', 0)} skipped " + f"{_cap('skipped_max')} skipped " f"({py['collected']} collected; floor {fc or 'unset'})") - lines.append(f" selftests: {len(st['ok'])} of {len(mods)} modules exposing --selftest") + lines.append(f" selftests: {len(st['ok'])} of {len(mods)} modules ran, " + f"{_cap('selftest_skipped_max')} skipped") for name, res in gates.items(): - lines.append(f" {name:<18} {'ok ' if res['ok'] else 'FAIL'} {res['line']}") + state = "SKIP" if res["skipped"] else ("ok " if res["ok"] else "FAIL") + lines.append(f" {name:<18} {state} {res['line']}") + if actual["gate_skipped_max"]: + lines.append(f" gates: {_cap('gate_skipped_max')} skipped") + + # WHAT DID NOT RUN, always — under a green verdict as much as a red one. A number of skips + # with no reasons beside it is how "green" quietly stops meaning "checked". + if py["skips"] or st["skipped"] or actual["gate_skipped_max"]: + lines.append("") + lines.append(" SKIPPED (prerequisite absent on this machine — nothing here was checked):") + for ln in py["skips"]: + lines.append(f" pytest {ln}") + for mod, reasons in sorted(st["skipped"].items()): + for reason in reasons: + lines.append(f" selftest {mod}: {reason}") + for name, res in gates.items(): + for reason in res["skipped"]: + lines.append(f" gate {name}: {reason}") + lines.append("") if problems: lines.append(" PROBLEMS:") @@ -178,18 +308,42 @@ def verify(*, update_floor: bool = False) -> tuple[int, str]: for mod, why in sorted(st["failed"].items()): lines.append(f" selftest {mod}: {why[:120]}") if py["failed"] or py["usage_error"]: + # EVERY failure by name, then the tail for context. A truncated failure list costs a + # whole CI round trip, which is what happened on the first run. + lines += ["", f" pytest failures ({len(py['failures'])}):"] + lines += [f" {ln}" for ln in py["failures"]] lines += ["", " pytest tail:"] + [f" {ln}" for ln in py["tail"]] else: lines.append(f" VERIFIED — {py['passed']} tests actually executed and passed, " - f"{len(st['ok'])} selftests spoke, {len(gates)} gates green") + f"{len(st['ok'])} selftests spoke, " + f"{len(gates) - actual['gate_skipped_max']} of {len(gates)} gates green" + + (f"; {py['skipped']} test(s), {actual['selftest_skipped_max']} selftest(s) " + f"and {actual['gate_skipped_max']} gate(s) SKIPPED for a named missing " + f"prerequisite, listed above" + if (py["skipped"] or actual["selftest_skipped_max"] + or actual["gate_skipped_max"]) else "")) if update_floor and not problems: - FLOOR.write_text(json.dumps({"collected": py["collected"], "passed": py["passed"], - "note": "floor recorded by verify.py --update-floor; a later " - "run collecting fewer tests FAILS, because silently " - "running fewer tests looks exactly like passing"}, - indent=1) + "\n", encoding="utf-8") - lines.append(f" floor updated: collected={py['collected']} passed={py['passed']}") + # `collected` and `passed` are re-measured; the CEILINGS are NOT. A ceiling re-recorded + # from whatever the last run happened to skip is not a ceiling, it is a ratchet that + # follows the leak — and on the machine that has every prerequisite it would record 0 and + # fail every other machine. So the agreed maxima are preserved from the existing file and + # only ever changed by hand, deliberately. + # `passed + skipped`, not `passed`: the floor means "checks that ran or were named", so + # it records the same number on a machine with every prerequisite and on one without. + # Recording bare `passed` from a skipping machine would RATCHET THE FLOOR DOWN by exactly + # the amount that was skipped — the floor following the leak instead of catching it. + blob = {"collected": py["collected"], "passed": py["passed"] + py["skipped"]} + for key, _label in CEILINGS: + if floor.get(key) is not None: + blob[key] = int(floor[key]) + blob["note"] = ("floor recorded by verify.py --update-floor; a later run collecting fewer " + "tests FAILS, because silently running fewer tests looks exactly like " + "passing. `passed` is compared against passed+skipped. The *_max ceilings " + "bound skipping and are NOT re-measured here — edit them by hand.") + FLOOR.write_text(json.dumps(blob, indent=1) + "\n", encoding="utf-8") + lines.append(f" floor updated: collected={py['collected']} passed={py['passed']} " + f"(ceilings preserved)") return (1 if problems else 0), "\n".join(lines) + "\n" @@ -227,16 +381,62 @@ def _selftest() -> None: 'import sys\nif "--selftest" in sys.argv:\n sys.exit(0)\n') (pathlib.Path(td) / "loud_mod.py").write_text( 'import sys\nif "--selftest" in sys.argv:\n print("loud selftest: OK")\n') + # A LOUD ZERO-EXIT THAT EXECUTED NOTHING MUST NOT READ AS A PASS. This is the silent + # zero-exit's twin, and the reason `ok` had to stop meaning "exited 0 and spoke": a + # skipped selftest speaks. It must land in `skipped`, with its reason carried out. + (pathlib.Path(td) / "skipping_mod.py").write_text( + 'import sys\n' + 'if "--selftest" in sys.argv:\n' + f' print("skipping_mod selftest: {PREREQ_ABSENT_MARK} the widget is not installed")\n') try: globals()["HERE"] = pathlib.Path(td) - got = run_selftests(["silent_mod", "loud_mod"]) + got = run_selftests(["silent_mod", "loud_mod", "skipping_mod"]) finally: globals()["HERE"] = saved assert "silent_mod" in got["failed"], f"a silent zero-exit must FAIL: {got}" assert "did it run?" in got["failed"]["silent_mod"], got - assert got["ok"] == ["loud_mod"], got - - print("verify.py selftest: OK (count parsing, selftest discovery, silent-zero-exit is a FAILURE)") + assert got["ok"] == ["loud_mod"], f"a skipped selftest must not be counted as ok: {got}" + assert got["skipped"] == {"skipping_mod": ["the widget is not installed"]}, got + + # ---- THE SKIP CEILING, in both directions ------------------------------------------------- + # Bounding skips is the whole reason skipping was allowed at all, so the bound is tested the + # way a gate must be: it has to FAIL when exceeded and PASS when not, and the failure has to + # name the key to raise. `_ceiling_problems` is the same code path `verify()` uses. + at_limit = _ceiling_problems({"skipped_max": 24}, {"skipped_max": 24, + "selftest_skipped_max": 0, + "gate_skipped_max": 0}) + assert at_limit == [], f"exactly at the ceiling must pass: {at_limit}" + over = _ceiling_problems({"skipped_max": 24}, {"skipped_max": 25, + "selftest_skipped_max": 0, + "gate_skipped_max": 0}) + assert len(over) == 1 and "SKIP CEILING exceeded" in over[0], over + assert "skipped_max" in over[0], "the failure must name the key to raise deliberately" + # Each ceiling is independent — one slipping must not be masked by the others holding. + for key in ("selftest_skipped_max", "gate_skipped_max"): + counts = {"skipped_max": 0, "selftest_skipped_max": 0, "gate_skipped_max": 0} + counts[key] = 3 + got_c = _ceiling_problems({key: 2}, counts) + assert len(got_c) == 1 and key in got_c[0], (key, got_c) + # An UNSET ceiling is not a ceiling of zero — it means nothing has been agreed yet. Reading + # `None` as 0 would fail every machine that legitimately skips anything. + assert _ceiling_problems({}, {"skipped_max": 99, "selftest_skipped_max": 9, + "gate_skipped_max": 9}) == [] + + # ---- the floor counts what RAN OR WAS NAMED, and the ceiling stops that being a loophole -- + # 330 -> 300 passed with 30 named skips is fine; 300 passed with 0 skips is a test that + # vanished. Both directions, because only having one is how a floor becomes decoration. + assert _floor_problems({"collected": 330, "passed": 330}, + {"collected": 330, "passed": 306, "skipped": 24}) == [] + dropped = _floor_problems({"collected": 330, "passed": 330}, + {"collected": 330, "passed": 306, "skipped": 0}) + assert len(dropped) == 1 and "dropped" in dropped[0], dropped + shrank = _floor_problems({"collected": 330, "passed": 330}, + {"collected": 320, "passed": 320, "skipped": 0}) + assert any("collection DROPPED" in p for p in shrank), shrank + + print("verify.py selftest: OK (count parsing, selftest discovery, silent-zero-exit is a " + "FAILURE, a loud skip is not a pass, skip ceiling fails when exceeded and holds when " + "not, floor counts passed+skipped)") def main() -> int: