From 252f8166a1dc31c4e661257fbac87e7c4d829a7b Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Fri, 8 May 2026 00:43:38 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(bench):=20badge.py=20=E2=80=94=20rende?= =?UTF-8?q?r=20reproducibility=20badge=20text=20from=20canonical=20JSON=20?= =?UTF-8?q?(#477)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure helper that reads the merged bench-canonical JSON and produces the one-line text the cron splices into the README marker block. Unit-tested so the workflow stays free of count logic. Skipped invocations (per #479) do not count as ok. --- benchmarks/badge.py | 59 +++++++++++++++++ tests/test_benchmarks_badge.py | 118 +++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 benchmarks/badge.py create mode 100644 tests/test_benchmarks_badge.py diff --git a/benchmarks/badge.py b/benchmarks/badge.py new file mode 100644 index 000000000..32401d15f --- /dev/null +++ b/benchmarks/badge.py @@ -0,0 +1,59 @@ +"""Compute the README reproducibility-badge text from a canonical bench JSON. + +The nightly `bench-canonical` cron writes a merged report under +`benchmarks/results/v2.0.0-cron-.json`. This module reads that +report and produces the one-line badge text that the workflow splices +between the `` and +`` markers in README.md. + +Issue: #477. +""" +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Mapping + + +def _count_invocations(headline_cut: Mapping[str, list]) -> int: + return sum(len(v) for v in headline_cut.values()) + + +def _count_ok(results: Mapping[str, Mapping[str, Mapping]]) -> int: + ok = 0 + for by_sub in results.values(): + for entry in by_sub.values(): + if isinstance(entry, Mapping) and entry.get("_status") == "ok": + ok += 1 + return ok + + +def compute_badge_text(report_path: Path, *, today: str | None = None) -> str: + """Render the badge line from the report at *report_path*. + + `today` defaults to UTC `YYYY-MM-DD`; the parameter exists so tests + can pin a date. + """ + data = json.loads(Path(report_path).read_text()) + total = _count_invocations(data["headline_cut"]) + ok = _count_ok(data["results"]) + if today is None: + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + icon = "✅" if ok == total and total > 0 else "⚠️" + return f"reproducibility: {icon} {ok}/{total} ok · last run {today}" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path, help="path to merged canonical JSON") + parser.add_argument("--today", default=None, help="override UTC date (YYYY-MM-DD)") + args = parser.parse_args(argv) + sys.stdout.write(compute_badge_text(args.report, today=args.today) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_benchmarks_badge.py b/tests/test_benchmarks_badge.py new file mode 100644 index 000000000..d0f3405dc --- /dev/null +++ b/tests/test_benchmarks_badge.py @@ -0,0 +1,118 @@ +"""Tests for benchmarks.badge — README reproducibility-badge text formatter. + +Issue: #477. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from benchmarks import badge + + +def _write_report( + tmp_path: Path, + *, + headline_cut: dict, + results: dict, +) -> Path: + p = tmp_path / "report.json" + p.write_text( + json.dumps( + { + "label": "test", + "headline_cut": headline_cut, + "metric_overrides": {}, + "results": results, + } + ) + ) + return p + + +def test_all_ok_renders_check_icon(tmp_path): + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]}, + results={ + "a": {"_": {"_status": "ok"}}, + "b": {"_": {"_status": "ok"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert text == "reproducibility: ✅ 2/2 ok · last run 2026-05-08" + + +def test_partial_renders_warn_icon(tmp_path): + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]}, + results={ + "a": {"_": {"_status": "ok"}}, + "b": {"_": {"_status": "error"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert text == "reproducibility: ⚠️ 1/2 ok · last run 2026-05-08" + + +def test_total_counts_subkeys_not_adapters(tmp_path): + """An adapter with N parametrised invocations contributes N to total.""" + report = _write_report( + tmp_path, + headline_cut={ + "mab": [{"sub_key": "x"}, {"sub_key": "y"}, {"sub_key": "z"}], + "amabench": [{"sub_key": None}], + }, + results={ + "mab": {"x": {"_status": "ok"}, "y": {"_status": "ok"}, "z": {"_status": "ok"}}, + "amabench": {"_": {"_status": "ok"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert "4/4" in text + + +def test_skipped_counts_as_not_ok(tmp_path): + """Per #479, skipped_data_missing is distinct from ok; it does not count.""" + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]}, + results={ + "a": {"_": {"_status": "ok"}}, + "b": {"_": {"_status": "skipped_data_missing"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert "1/2 ok" in text + assert text.startswith("reproducibility: ⚠️") + + +def test_canonical_v200_partial(tmp_path): + """Sanity: today's checked-in canonical reports 6/11.""" + canonical = Path(__file__).parent.parent / "benchmarks" / "results" / "v2.0.0.json" + if not canonical.exists(): + pytest.skip("canonical baseline not present") + text = badge.compute_badge_text(canonical, today="2026-05-08") + assert "6/11 ok" in text + + +def test_zero_total_does_not_render_check(tmp_path): + """Empty headline_cut shouldn't produce the all-green icon.""" + report = _write_report(tmp_path, headline_cut={}, results={}) + text = badge.compute_badge_text(report, today="2026-05-08") + assert text.startswith("reproducibility: ⚠️ 0/0 ok") + + +def test_today_defaults_to_utc(tmp_path): + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}]}, + results={"a": {"_": {"_status": "ok"}}}, + ) + text = badge.compute_badge_text(report) + # YYYY-MM-DD shape; don't pin the actual day. + suffix = text.rsplit("last run ", 1)[1] + assert len(suffix) == 10 and suffix[4] == "-" and suffix[7] == "-" From 1f850c4d1e0611ab5f881d74cf5c19390a0b4f1d Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Fri, 8 May 2026 00:43:44 -0700 Subject: [PATCH 2/2] ci(bench-canonical): wire README badge in-place rewrite on cron (#477) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two steps before the existing commit+push: 1. Compute badge text via `python -m benchmarks.badge` against the merged JSON (gated on bench succeeding so an errored run doesn't rewrite the badge to a stale 0/N line). 2. Sync the current main README into the bench-canonical-results worktree and awk-rewrite the marker block. Same commit as the JSON entry; one cron commit per run. main is not touched — operator cherry-picks if they want main current. --- .github/workflows/bench-canonical.yml | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/bench-canonical.yml b/.github/workflows/bench-canonical.yml index 5cc33d5eb..40c0d4311 100644 --- a/.github/workflows/bench-canonical.yml +++ b/.github/workflows/bench-canonical.yml @@ -114,6 +114,39 @@ jobs: sys.exit(0 if overall.value == 'pass' else (1 if overall.value == 'fail' else 0)) " + - name: Compute badge text + # Renders the one-line badge from the merged JSON. Uses + # `benchmarks.badge` (unit-tested) so the workflow stays free + # of count logic. Output is stashed in $GITHUB_OUTPUT for the + # next step. Skips when bench errored — leaves the previous + # badge in place rather than rewriting to a stale 0/N line. + id: badge + if: steps.bench.outputs.out != '' + run: | + set -euo pipefail + text=$(uv run python -m benchmarks.badge "${{ steps.bench.outputs.out }}") + echo "text=${text}" >> "$GITHUB_OUTPUT" + echo "computed: ${text}" + + - name: Sync README into bench-canonical-results worktree + rewrite badge + # Issue #477: badge in README on the bench-canonical-results + # branch is auto-rewritten on each cron run. main is not + # touched (operator cherry-picks if they want main current). + # README is copied from the main checkout each run so the + # bench-canonical-results branch never carries a stale README. + if: steps.badge.outputs.text != '' + run: | + set -euo pipefail + cp README.md .bench-results-branch/README.md + new="${{ steps.badge.outputs.text }}" awk ' + BEGIN { in_block=0 } + // { print; print ENVIRON["new"]; in_block=1; next } + // { print; in_block=0; next } + in_block { next } + { print } + ' .bench-results-branch/README.md > .bench-results-branch/README.md.new + mv .bench-results-branch/README.md.new .bench-results-branch/README.md + - name: Commit + push to bench-canonical-results env: GIT_AUTHOR_NAME: aelfrice-bench-bot