From dab5395d40f04df1950b622433079d60e673dc94 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Fri, 8 May 2026 08:23:11 -0700 Subject: [PATCH 1/2] fix(bench/run): strip per_case from merged canonical output (#437) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror of the per_question strip rule — structmemeval emits a per_case list whose context strings bloat the canonical JSON to 6.4 MB on a clean 11/11 run. The detail isn't band-checked (tolerance.check_report walks summary leaves only), so it's pure diff weight. Test extended to assert both per_question and per_case are stripped from the merged output. --- benchmarks/run.py | 1 + tests/test_bench_dispatcher.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/run.py b/benchmarks/run.py index a6341e222..cafe6afdc 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -195,6 +195,7 @@ def _default_runner(cmd: list[str], out_path: Path) -> subprocess.CompletedProce # needs to read it directly. _DETAIL_FIELDS_TO_STRIP: frozenset[str] = frozenset({ "per_question", + "per_case", }) diff --git a/tests/test_bench_dispatcher.py b/tests/test_bench_dispatcher.py index 616c3932c..aa6bd2f36 100644 --- a/tests/test_bench_dispatcher.py +++ b/tests/test_bench_dispatcher.py @@ -179,13 +179,16 @@ def test_headline_cut_recorded(tmp_path): def test_per_question_detail_stripped_from_merged_output(tmp_path): - """The merged JSON drops `per_question` lists — they bloat the file - and aren't read by the band-check (#437 calibration finding 2026-05-06). + """The merged JSON drops `per_question` and `per_case` lists — they bloat + the file and aren't read by the band-check (#437 calibration finding + 2026-05-06; `per_case` added 2026-05-08 after structmemeval bloated the + canonical to 6.4 MB). """ out = tmp_path / "stripped.json" payload = { "f1": 0.5, "exact_match": 0.3, "per_question": [{"id": i, "score": 0.5} for i in range(2000)], + "per_case": [{"case_id": f"c{i}", "accuracy": 1.0} for i in range(50)], } rc = bench_run.main_all( out_path=out, canonical=False, smoke=True, @@ -199,6 +202,8 @@ def test_per_question_detail_stripped_from_merged_output(tmp_path): ama_out = data["results"]["amabench"]["_"]["output"] assert "per_question" not in mab_out assert "per_question" not in ama_out + assert "per_case" not in mab_out + assert "per_case" not in ama_out # Summary metrics retained. assert mab_out["f1"] == 0.5 assert ama_out["exact_match"] == 0.3 From 1d9b6e2d80009a9933dbe8b667b60744d8a2a8cd Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Fri, 8 May 2026 08:24:05 -0700 Subject: [PATCH 2/2] feat(bench-canonical): calibrate v2.0.0 from 3-run canonical pass (#437) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the partial first-pass canonical (6 of 11 invocations on git_commit bce8311, captured 2026-05-07) with a fully calibrated canonical from three back-to-back `aelf bench all --canonical` runs on git_commit 9e3f8de2 (after #473 fix shipped). Headline numbers (run-1, source of truth): mab/Conflict_Resolution f1=0.0065 sub_em=0.7025 N=800 mab/Test_Time_Learning f1=0.0001 sub_em=0.0929 N=700 mab/Long_Range_Understand. f1=0.1811 sub_em=0.0234 N=171 mab/Accurate_Retrieval f1=0.0106 sub_em=0.1530 N=2000 locomo overall_f1=0.0212 N=1986 longmemeval avg_latency_ms=5.55 N=500 structmemeval/location accuracy=0.9048 N=42 cases structmemeval/accounting accuracy=0.0000 N=15 cases structmemeval/recommend. accuracy=0.1540 N=84 cases structmemeval/tree accuracy=0.0000 N=22 cases amabench total_episodes=208 / qa=2496 Tolerance bands: spec called for observed range × 1.5 → metric override when required pct exceeds the default band. Across 3 runs, MAB and StructMemEval metrics are bit-deterministic (range = 0.0); LongMemEval avg_latency_ms varies 5.55–6.04 ms (well inside the default ±25% latency band). Result: metric_overrides = {} — defaults cover observed variance. Self-check: tolerance.check_report(canonical, canonical) → 36/36 pass. Cross-check: run-2 and run-3 against the canonical → 36/36 pass each. The earlier partial canonical's _calibration_notes referenced "missing /tmp/ data dirs" as the LoCoMo blocker and "_status: error" for StructMemEval × 4 (the #473 retrieve_v2 temporal_sort bug). Both are resolved at github/main HEAD. Closes the v2.0 reproducibility ship-gate. Per #437 acceptance: on a fresh clone, `uv sync --extra benchmarks && aelf bench all --canonical --out PATH` reproduces every published number within the documented bands. --- benchmarks/results/v2.0.0.json | 110 +++++++++++++++++++++++---------- tests/test_benchmarks_badge.py | 15 ++++- 2 files changed, 88 insertions(+), 37 deletions(-) diff --git a/benchmarks/results/v2.0.0.json b/benchmarks/results/v2.0.0.json index 129ed031f..ddd6b77eb 100644 --- a/benchmarks/results/v2.0.0.json +++ b/benchmarks/results/v2.0.0.json @@ -1,8 +1,8 @@ { - "_calibration_notes": "First canonical pass 2026-05-07. 6 of 11 invocations succeeded (MAB \u00d74, LongMemEval, AMA-Bench); 5 failed due to missing /tmp/ data dirs (LoCoMo + StructMemEval \u00d74). Re-run those adapters after populating /tmp/LoCoMo and /tmp/StructMemEval. Per-metric override bands (3+ runs \u00d7 1.5 spread) not yet calibrated \u2014 first-run values are placeholders for the working adapters.", - "aelfrice_version": "1.6.0", - "captured_at_utc": "2026-05-07T02:10:35Z", - "git_commit": "bce8311", + "_calibration_notes": "Calibrated 2026-05-08 from 3 full canonical runs (11/11 ok each) on git_commit 9e3f8dee. Tolerance bands per spec: observed range x 1.5; metric_overrides left empty (defaults \u2014 F1 plus or minus 7%, EM plus or minus 10%, latency plus or minus 25%, fallback plus or minus 10%, absolute floor plus or minus 0.02 \u2014 cover observed variance. Across 3 runs, MAB and StructMemEval metrics are bit-deterministic; LongMemEval avg_latency_ms varied 5.55\u20136.04 ms (well inside default plus or minus 25%). Per-run inputs held in /tmp/v2.0.0-cal/run-{1,2,3}.json during calibration session.", + "aelfrice_version": "2.0.0", + "captured_at_utc": "2026-05-08T15:00:00Z", + "git_commit": "9e3f8dee2898d21ed5250618d86f84b3235caab4", "harness_version": "1", "headline_cut": { "amabench": [ @@ -92,12 +92,12 @@ } ] }, - "label": "v2.0.0 canonical (first calibration pass \u2014 partial)", + "label": "v2.0.0 canonical", "metric_overrides": {}, "results": { "amabench": { "_": { - "_elapsed_sec": 266.921, + "_elapsed_sec": 189.73, "_status": "ok", "output": { "domain_counts": { @@ -121,51 +121,61 @@ }, "locomo": { "_": { - "_elapsed_sec": 1.041, - "_error_message": "Traceback (most recent call last):\n File \"\", line 198, in _run_module_as_main\n File \"\", line 88, in _run_code\n File \"$HOME/projects/aelfrice/benchmarks/locomo_adapter.py\", line 565, in \n main()\n ~~~~^^\n File \"$HOME/projects/aelfrice/benchmarks/locomo_adapter.py\", line 503, in main\n conversations: list[LoCoMoConversation] = load_locomo(args.data)\n ~~~~~~~~~~~^^^^^^^^^^^\n File \"/Users", - "_status": "error" + "_elapsed_sec": 34.403, + "_status": "ok", + "output": { + "category_f1": { + "1": 0.1051, + "2": 0.0074, + "3": 0.008, + "4": 0.011, + "5": 0.0 + }, + "overall_f1": 0.0212, + "total_qa": 1986 + } } }, "longmemeval": { "_": { - "_elapsed_sec": 70.552, + "_elapsed_sec": 51.822, "_status": "ok", "output": { "avg_beliefs_per_query": 49.15, - "avg_latency_ms": 7.8, + "avg_latency_ms": 5.55, "category_stats": { "knowledge-update": { "avg_beliefs": 50.45, - "avg_latency_ms": 8.0, + "avg_latency_ms": 6.05, "count": 78 }, "multi-session": { "avg_beliefs": 50.51, - "avg_latency_ms": 10.43, + "avg_latency_ms": 7.19, "count": 133 }, "single-session-assistant": { "avg_beliefs": 38.93, - "avg_latency_ms": 2.9, + "avg_latency_ms": 2.06, "count": 56 }, "single-session-preference": { "avg_beliefs": 50.23, - "avg_latency_ms": 5.4, + "avg_latency_ms": 3.77, "count": 30 }, "single-session-user": { "avg_beliefs": 49.94, - "avg_latency_ms": 4.9, + "avg_latency_ms": 3.37, "count": 70 }, "temporal-reasoning": { "avg_beliefs": 50.68, - "avg_latency_ms": 9.16, + "avg_latency_ms": 6.64, "count": 133 } }, - "total_ingest_time_s": 61.63, + "total_ingest_time_s": 44.69, "total_ingest_turns": 10960, "total_questions": 500 } @@ -173,7 +183,7 @@ }, "mab": { "Accurate_Retrieval": { - "_elapsed_sec": 541.933, + "_elapsed_sec": 381.34, "_status": "ok", "output": { "exact_match": 0.0, @@ -185,7 +195,7 @@ } }, "Conflict_Resolution": { - "_elapsed_sec": 83.214, + "_elapsed_sec": 60.041, "_status": "ok", "output": { "exact_match": 0.0, @@ -197,7 +207,7 @@ } }, "Long_Range_Understanding": { - "_elapsed_sec": 540.45, + "_elapsed_sec": 385.399, "_status": "ok", "output": { "exact_match": 0.0, @@ -209,7 +219,7 @@ } }, "Test_Time_Learning": { - "_elapsed_sec": 452.277, + "_elapsed_sec": 311.118, "_status": "ok", "output": { "exact_match": 0.0, @@ -223,24 +233,56 @@ }, "structmemeval": { "accounting": { - "_elapsed_sec": 0.266, - "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_accounting.json", - "_status": "error" + "_elapsed_sec": 0.658, + "_status": "ok", + "output": { + "accuracy": 0.0, + "bench": "big", + "perfect_cases": 0, + "task": "accounting", + "total_cases": 15, + "total_correct": 0, + "total_queries": 15 + } }, "location": { - "_elapsed_sec": 0.406, - "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_location.json", - "_status": "error" + "_elapsed_sec": 1.526, + "_status": "ok", + "output": { + "accuracy": 0.9048, + "bench": "big", + "perfect_cases": 38, + "task": "location", + "total_cases": 42, + "total_correct": 38, + "total_queries": 42 + } }, "recommendations": { - "_elapsed_sec": 0.271, - "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_recommendations.json", - "_status": "error" + "_elapsed_sec": 14.803, + "_status": "ok", + "output": { + "accuracy": 0.154, + "bench": "big", + "perfect_cases": 0, + "task": "recommendations", + "total_cases": 84, + "total_correct": 170, + "total_queries": 1104 + } }, "tree": { - "_elapsed_sec": 0.266, - "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_tree.json", - "_status": "error" + "_elapsed_sec": 2.141, + "_status": "ok", + "output": { + "accuracy": 0.0, + "bench": "big", + "perfect_cases": 0, + "task": "tree", + "total_cases": 22, + "total_correct": 0, + "total_queries": 43 + } } } }, diff --git a/tests/test_benchmarks_badge.py b/tests/test_benchmarks_badge.py index d0f3405dc..42ba4fc29 100644 --- a/tests/test_benchmarks_badge.py +++ b/tests/test_benchmarks_badge.py @@ -90,13 +90,22 @@ def test_skipped_counts_as_not_ok(tmp_path): assert text.startswith("reproducibility: ⚠️") -def test_canonical_v200_partial(tmp_path): - """Sanity: today's checked-in canonical reports 6/11.""" +def test_canonical_v200_full_pass(tmp_path): + """Sanity: today's checked-in canonical reports 11/11 ok. + + Was 6/11 during the partial first-pass calibration on 2026-05-07 + (LoCoMo data missing + StructMemEval × 4 hitting the #473 + temporal_sort kwarg bug). Calibrated to 11/11 on 2026-05-08 once + #473 shipped and /tmp/LoCoMo + /tmp/StructMemEval data dirs were + populated. Ratchets to detect regression — flip back to a partial + cut would surface here. + """ canonical = Path(__file__).parent.parent / "benchmarks" / "results" / "v2.0.0.json" if not canonical.exists(): pytest.skip("canonical baseline not present") text = badge.compute_badge_text(canonical, today="2026-05-08") - assert "6/11 ok" in text + assert "11/11 ok" in text + assert text.startswith("reproducibility: ✅") def test_zero_total_does_not_render_check(tmp_path):