diff --git a/manifests/evidence.json b/manifests/evidence.json index 325097860..4e3a91b50 100644 --- a/manifests/evidence.json +++ b/manifests/evidence.json @@ -14164,8 +14164,8 @@ }, { "path": "tests/test_gap_wave_ledger.py", - "sha256": "1ce3ed5fb281a4bca2a316e0c5db473f29c7a0b97d3773df7fdb118c03f2a883", - "bytes": 4174 + "sha256": "186b9c6909939136f2f4a6177410c13709f3acdece31978e6e08ca488b4f2d23", + "bytes": 9020 }, { "path": "tests/test_gitleaks_config.py", @@ -14444,8 +14444,8 @@ }, { "path": "tools/sota-convergence/gap_wave_ledger.py", - "sha256": "6dc6ec19d3c8db944371cbde86f8c7b0dcced125f38b4b14f8ff9fcf455f652b", - "bytes": 8675 + "sha256": "8de1e940df6f214728b98fd2fb4e98d544de3a528e2b8805886283ba1cd03153", + "bytes": 12685 }, { "path": "tools/sota-convergence/github_freshness.py", diff --git a/tests/test_gap_wave_ledger.py b/tests/test_gap_wave_ledger.py index 2546fa5b4..3b92bf022 100644 --- a/tests/test_gap_wave_ledger.py +++ b/tests/test_gap_wave_ledger.py @@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1] LEDGER = ROOT / "catalogs/landscape/gap-wave2-20260923--gap-resolution.json" -RANK = {"not_run": 0, "not_settled": 1, "advanced": 2, "settled": 3} +RANK = {"not_run": 0, "deferred": 1, "covered_elsewhere": 2, "not_settled": 3, "advanced": 4, "settled": 5} def _tool(): @@ -57,13 +57,81 @@ def test_settle_key_rejects_unknown_values(self): with self.assertRaises(SystemExit): self.tool.settle_key("maybe") + def test_catalog_layer_dir_style_reads_prefixed_dirs_and_defaults_the_layer(self): + import tempfile + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) / "evidence/artifacts/wave-x" + (base / "foundation__native-clients").mkdir(parents=True) + (base / "native-clients").mkdir() + receipt = {"id": "r1", "source_revision": "92bb279", "gap_refs": [{"gap_index": 2}, 5], + "settles_gap": "partially", "verdict_impact": {"direction": "inconclusive"}} + (base / "foundation__native-clients/r1.json").write_text(json.dumps(receipt)) + (base / "foundation__native-clients/results.json").write_text(json.dumps({"gap_refs": "not a receipt"})) + (base / "native-clients/r2.json").write_text(json.dumps({**receipt, "id": "r2", + "gap_refs": [{"layer_id": "native-clients", "gap_index": 0}]})) + prefixed = self.tool.load_receipts(Path(tmp), "wave-x", "catalog__layer") + self.assertEqual([r["id"] for r in prefixed], ["r1"]) + self.assertEqual(prefixed[0]["gap_refs"], [["native-clients", 2], ["native-clients", 5]]) + plain = self.tool.load_receipts(Path(tmp), "wave-x", "layer") + self.assertEqual([r["id"] for r in plain], ["r2"]) + + def test_catalog_layer_accepts_outcome_only_receipts_and_rejects_non_receipts(self): + import tempfile + with tempfile.TemporaryDirectory() as tmp: + layer = Path(tmp) / "evidence/artifacts/wave-y/foundation__workers" + (layer / "raw").mkdir(parents=True) + (layer / "raw/stdout.json").write_text("{}") # subdirectories are never read + (layer / "results.json").write_text(json.dumps({"0": "advanced"})) + (layer / "_index.json").write_text("{}") + (layer / "preregistrations-fixround.json").write_text("{}") + shapes = {"a": {"gap_index": 0, "outcome": "advanced"}, + "b": {"gap_index": 3, "outcome": "deferred", "blocker": "needs the user"}, + "c": {"gap_index": 4, "outcome": "covered_elsewhere"}, + "d": {"gap_index": 5, "outcome": "settled"}} + for name, body in shapes.items(): + (layer / f"{name}.json").write_text(json.dumps({"id": name, **body})) + loaded = {r["id"]: r for r in self.tool.load_receipts(Path(tmp), "wave-y", "catalog__layer")} + self.assertEqual(sorted(loaded), ["a", "b", "c", "d"]) + self.assertEqual(loaded["a"]["gap_refs"], [["workers", 0]]) + self.assertEqual(self.tool.SETTLES[loaded["a"]["settles_key"]], "advanced") + self.assertEqual(self.tool.SETTLES[loaded["b"]["settles_key"]], "deferred") + self.assertEqual(self.tool.SETTLES[loaded["c"]["settles_key"]], "covered_elsewhere") + self.assertEqual(self.tool.SETTLES[loaded["d"]["settles_key"]], "settled") + self.assertIsNone(loaded["a"]["source_revision"]) # build() defaults and flags it + (layer / "stray.json").write_text(json.dumps({"note": "not a receipt"})) + with self.assertRaises(SystemExit) as raised: + self.tool.load_receipts(Path(tmp), "wave-y", "catalog__layer") + self.assertIn("stray.json", str(raised.exception.code)) + (layer / "stray.json").write_text(json.dumps({"gap_index": 1, "outcome": "maybe"})) + with self.assertRaises(SystemExit): + self.tool.load_receipts(Path(tmp), "wave-y", "catalog__layer") + + def test_receipt_selector_never_reads_raw_capture_subdirs(self): + import tempfile + with tempfile.TemporaryDirectory() as tmp: + layer = Path(tmp) / "evidence/artifacts/wave-z/foundation__document-retrieval" + (layer / "raw/bench-out").mkdir(parents=True) + (layer / "raw/bench-out/run-cold-1.json").write_text("") # empty capture of a failed run + (layer / "r.json").write_text(json.dumps({"gap_index": 1, "outcome": "advanced"})) + names = [p.name for p, _ in self.tool.iter_receipt_files(Path(tmp), "wave-z", "catalog__layer")] + self.assertEqual(names, ["r.json"]) + self.assertEqual(len(self.tool.load_receipts(Path(tmp), "wave-z", "catalog__layer")), 1) + + def test_rank_keeps_deferral_below_any_executed_outcome(self): + rank = self.tool.RANK + self.assertLess(rank["not_run"], rank["deferred"]) + self.assertLess(rank["deferred"], rank["not_settled"]) + self.assertLess(rank["covered_elsewhere"], rank["not_settled"]) + def test_every_receipt_names_the_crosswalk_revision(self): for receipt in self.doc["receipts"]: self.assertTrue(self.doc["source_revision"].startswith(receipt["source_revision"][:7]), receipt["path"]) def test_repo_raw_artifacts_match_their_recorded_hash(self): import hashlib - for path in sorted((ROOT / "evidence/artifacts/gap-wave2-20260923").rglob("*.json")): + files = [p for style in ("layer", "catalog__layer") + for p, _ in self.tool.iter_receipt_files(ROOT, "gap-wave2-20260923", style)] + for path in files: # the same receipt selector the ledger uses; raw captures are never parsed data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict): continue diff --git a/tools/sota-convergence/gap_wave_ledger.py b/tools/sota-convergence/gap_wave_ledger.py index ad8f26684..ce9b8a64e 100644 --- a/tools/sota-convergence/gap_wave_ledger.py +++ b/tools/sota-convergence/gap_wave_ledger.py @@ -1,17 +1,23 @@ """Ledger of a gap-evidence wave keyed to the gap crosswalk (offline, deterministic). Reads the crosswalk at catalogs/landscape/gap-crosswalk-92bb279.json and the wave's receipts -under evidence/artifacts///*.json (directories without "__"; another session's -wave uses __), and writes a ledger plus a summary page: +under evidence/artifacts///*.json (default) or /__/*.json +(--dir-style catalog__layer, where gap_refs may omit layer_id; results.json, _index.json and +preregistration*.json are skipped and any other JSON must be a receipt), and writes +a ledger plus a summary page: python3 tools/sota-convergence/gap_wave_ledger.py --wave gap-wave2-20260923 --owner gap-resolution [--check] + python3 tools/sota-convergence/gap_wave_ledger.py --wave gap-wave2-20260923 --owner agent-lab-17 --dir-style catalog__layer Status per crosswalk gap: settled / advanced / not_settled from the receipts' settles_gap (true / partially / false), or from a receipt's per_gap_settles entry (":") when it has one, best over the receipts that name the gap. A receipt naming several gaps carries one settles_gap value, so it credits each named gap at most "advanced": only a single-gap receipt can settle a gap. Gaps the owner was assigned but no receipt names are -"not_run". Nothing here changes a verdict. +"not_run". In catalog__layer mode a receipt may instead carry gap_index and an outcome +(settled / advanced / not_settled / deferred / covered_elsewhere); deferred and covered_elsewhere +give no settling credit but stay visible as their own status, and a missing source_revision +defaults to the crosswalk's (flagged source_revision_defaulted). Nothing here changes a verdict. """ import argparse import hashlib @@ -20,50 +26,105 @@ import sys CROSSWALK = "catalogs/landscape/gap-crosswalk-92bb279.json" -RANK = {"not_run": 0, "not_settled": 1, "advanced": 2, "settled": 3} -SETTLES = {"true": "settled", "partially": "advanced", "false": "not_settled"} +RANK = {"not_run": 0, "deferred": 1, "covered_elsewhere": 2, "not_settled": 3, "advanced": 4, "settled": 5} +SETTLES = {"true": "settled", "partially": "advanced", "false": "not_settled", + # outcome-only receipts (catalog__layer): no settling credit, but kept visible as their own status + "deferred": "deferred", "covered_elsewhere": "covered_elsewhere"} +OUTCOME_TO_SETTLES = {"settled": "true", "advanced": "partially", "not_settled": "false", + "deferred": "deferred", "covered_elsewhere": "covered_elsewhere"} +STATUS_ORDER = ("settled", "advanced", "not_settled", "covered_elsewhere", "deferred", "not_run") def settle_key(value): if isinstance(value, bool): return "true" if value else "false" + if value in ("deferred", "covered_elsewhere"): + return value key = str(value).strip().split(" ")[0].strip("(),").lower() if key not in SETTLES: raise SystemExit(f"unknown settles_gap value {value!r}") return key -def load_receipts(root, wave): +def _receipt_dirs(base, dir_style): + """(directory, default layer_id) pairs. "layer": / (no "__"); "catalog__layer": + __/, where gap_refs entries may omit layer_id.""" + for directory in sorted(p for p in base.iterdir() if p.is_dir()): + if dir_style == "layer" and "__" not in directory.name: + yield directory, directory.name + elif dir_style == "catalog__layer" and "__" in directory.name: + yield directory, directory.name.split("__", 1)[1] + + +def _refs(data, default_layer, path): + refs = [] + for g in data["gap_refs"]: + if isinstance(g, dict) and "gap_index" in g: + refs.append((g.get("layer_id", default_layer), int(g["gap_index"]))) + elif isinstance(g, int): + refs.append((default_layer, g)) + else: + raise SystemExit(f"{path}: unrecognised gap_refs entry {g!r}") + return refs + + +def iter_receipt_files(root, wave, dir_style="layer"): + """(path, default layer_id) for every candidate receipt file: top-level *.json in each layer + dir, minus per-layer indexes and preregistration records. Subdirectories (raw/, bench-out/...) + hold raw captures and are never read.""" base = root / "evidence/artifacts" / wave - out = [] - for directory in sorted(p for p in base.iterdir() if p.is_dir() and "__" not in p.name): + for directory, default_layer in _receipt_dirs(base, dir_style): for path in sorted(directory.glob("*.json")): - data = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(data, dict) or "gap_refs" not in data: + if path.name in ("results.json", "_index.json") or path.name.startswith("preregistration"): continue - refs = [(g["layer_id"], g["gap_index"]) for g in data["gap_refs"]] - per_gap = {k: settle_key(v) for k, v in (data.get("per_gap_settles") or {}).items()} - for key in per_gap: - if key not in {f"{l}:{i}" for l, i in refs}: - raise SystemExit(f"{path}: per_gap_settles names {key}, which is not in gap_refs") - vi = data.get("verdict_impact") or {} - calls = data.get("model_calls") or [] - out.append({ - "path": str(path.relative_to(root)), "id": data.get("id"), - "source_revision": data.get("source_revision"), "gap_refs": [list(r) for r in refs], - "settles_gap": data.get("settles_gap"), "settles_key": settle_key(data.get("settles_gap")), - "per_gap_settles": per_gap, - "direction": vi.get("direction"), "evidence_class": data.get("evidence_class"), - "model_calls": sum(int(c.get("count", 0) or 0) for c in calls if isinstance(c, dict)), - }) + yield path, default_layer + + +def load_receipts(root, wave, dir_style="layer"): + out = [] + for path, default_layer in iter_receipt_files(root, wave, dir_style): + data = json.loads(path.read_text(encoding="utf-8")) + if dir_style == "catalog__layer": + # Every top-level JSON here except results.json is a receipt: fail loudly, never skip. + if not isinstance(data, dict) or ("gap_refs" not in data and "gap_index" not in data): + raise SystemExit(f"{path}: not a receipt (needs gap_refs or gap_index)") + if "gap_refs" not in data: + gi = data["gap_index"] + data = {**data, "gap_refs": gi if isinstance(gi, list) else [gi]} + if "settles_gap" not in data: + outcome = data.get("outcome") + if outcome not in OUTCOME_TO_SETTLES: + raise SystemExit(f"{path}: needs settles_gap or an outcome in {sorted(OUTCOME_TO_SETTLES)}") + data = {**data, "settles_gap": OUTCOME_TO_SETTLES[outcome]} + elif not isinstance(data, dict) or "gap_refs" not in data: + continue + refs = _refs(data, default_layer, path) + per_gap = {k: settle_key(v) for k, v in (data.get("per_gap_settles") or {}).items()} + for key in per_gap: + if key not in {f"{l}:{i}" for l, i in refs}: + raise SystemExit(f"{path}: per_gap_settles names {key}, which is not in gap_refs") + vi = data.get("verdict_impact") or {} + calls = data.get("model_calls") or [] + out.append({ + "path": str(path.relative_to(root)), "id": data.get("id"), + "source_revision": data.get("source_revision"), "gap_refs": [list(r) for r in refs], + "outcome": data.get("outcome"), + "settles_gap": data.get("settles_gap"), "settles_key": settle_key(data.get("settles_gap")), + "per_gap_settles": per_gap, + "direction": vi.get("direction"), "evidence_class": data.get("evidence_class"), + "model_calls": sum(int(c.get("count", 0) or 0) for c in calls if isinstance(c, dict)), + }) return out -def build(root, wave, owner): +def build(root, wave, owner, dir_style="layer"): crosswalk = json.loads((root / CROSSWALK).read_text(encoding="utf-8")) - receipts = load_receipts(root, wave) + receipts = load_receipts(root, wave, dir_style) rev = crosswalk["source_revision"] for r in receipts: + if r["source_revision"] is None and dir_style == "catalog__layer": + # Outcome-only receipts carry crosswalk indexes by construction; record that it was defaulted. + r["source_revision"], r["source_revision_defaulted"] = rev[:7], True if not rev.startswith(str(r["source_revision"])[:7]): raise SystemExit(f"{r['path']}: source_revision {r['source_revision']} is not the crosswalk's {rev}") by_gap = {} @@ -99,7 +160,8 @@ def build(root, wave, owner): "credit settled/advanced/not_settled, but a receipt naming several gaps credits each at most " "advanced; not_run means no receipt names the gap. No verdict changes here."), "counts": dict(sorted(counts.items())), "layers": layers, - "receipts": sorted(({k: v for k, v in r.items() if k != "settles_key" and not (k == "per_gap_settles" and not v)} + "receipts": sorted(({k: v for k, v in r.items() if k != "settles_key" and not (k == "per_gap_settles" and not v) + and not (k == "outcome" and v is None)} for r in receipts), key=lambda r: r["path"]), } return doc @@ -112,7 +174,8 @@ def render(doc): f"assigned to `{doc['owner']}`, keyed to the rows at `{doc['source_revision'][:7]}`. " "Each receipt was reviewed by an Opus evidence reviewer and corrected in one fix round. No verdict changes here.", "", f"Rule: {doc['rule']}", "", "## Totals", "", "| Status | Gaps |", "| --- | ---: |"] - lines += [f"| {k} | {doc['counts'].get(k, 0)} |" for k in ("settled", "advanced", "not_settled", "not_run")] + lines += [f"| {k} | {doc['counts'].get(k, 0)} |" for k in STATUS_ORDER + if k in doc["counts"] or k in ("settled", "advanced", "not_settled", "not_run")] lines += ["", f"Receipts: {len(doc['receipts'])}; native model calls recorded: " f"{sum(r['model_calls'] for r in doc['receipts'])}.", "", "## Per gap", "", "| Layer | Gap | Status | Receipts |", "| --- | ---: | --- | --- |"] @@ -135,9 +198,11 @@ def main(argv=None): ap.add_argument("--wave", required=True) ap.add_argument("--owner", required=True) ap.add_argument("--check", action="store_true") + ap.add_argument("--dir-style", choices=("layer", "catalog__layer"), default="layer", + help="receipt directory layout: / (default) or __/") args = ap.parse_args(argv) root = args.root.resolve() - doc = build(root, args.wave, args.owner) + doc = build(root, args.wave, args.owner, args.dir_style) text = json.dumps(doc, indent=1, ensure_ascii=False) + "\n" page = render(doc) out, md = root / f"catalogs/landscape/{doc['id']}.json", root / f"docs/{doc['id']}.md"