diff --git a/.github/workflows/bench-canonical.yml b/.github/workflows/bench-canonical.yml index 5cc33d5eb..40c0d4311 100644 --- a/.github/workflows/bench-canonical.yml +++ b/.github/workflows/bench-canonical.yml @@ -114,6 +114,39 @@ jobs: sys.exit(0 if overall.value == 'pass' else (1 if overall.value == 'fail' else 0)) " + - name: Compute badge text + # Renders the one-line badge from the merged JSON. Uses + # `benchmarks.badge` (unit-tested) so the workflow stays free + # of count logic. Output is stashed in $GITHUB_OUTPUT for the + # next step. Skips when bench errored — leaves the previous + # badge in place rather than rewriting to a stale 0/N line. + id: badge + if: steps.bench.outputs.out != '' + run: | + set -euo pipefail + text=$(uv run python -m benchmarks.badge "${{ steps.bench.outputs.out }}") + echo "text=${text}" >> "$GITHUB_OUTPUT" + echo "computed: ${text}" + + - name: Sync README into bench-canonical-results worktree + rewrite badge + # Issue #477: badge in README on the bench-canonical-results + # branch is auto-rewritten on each cron run. main is not + # touched (operator cherry-picks if they want main current). + # README is copied from the main checkout each run so the + # bench-canonical-results branch never carries a stale README. + if: steps.badge.outputs.text != '' + run: | + set -euo pipefail + cp README.md .bench-results-branch/README.md + new="${{ steps.badge.outputs.text }}" awk ' + BEGIN { in_block=0 } + // { print; print ENVIRON["new"]; in_block=1; next } + // { print; in_block=0; next } + in_block { next } + { print } + ' .bench-results-branch/README.md > .bench-results-branch/README.md.new + mv .bench-results-branch/README.md.new .bench-results-branch/README.md + - name: Commit + push to bench-canonical-results env: GIT_AUTHOR_NAME: aelfrice-bench-bot diff --git a/benchmarks/badge.py b/benchmarks/badge.py new file mode 100644 index 000000000..32401d15f --- /dev/null +++ b/benchmarks/badge.py @@ -0,0 +1,59 @@ +"""Compute the README reproducibility-badge text from a canonical bench JSON. + +The nightly `bench-canonical` cron writes a merged report under +`benchmarks/results/v2.0.0-cron-.json`. This module reads that +report and produces the one-line badge text that the workflow splices +between the `` and +`` markers in README.md. + +Issue: #477. +""" +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Mapping + + +def _count_invocations(headline_cut: Mapping[str, list]) -> int: + return sum(len(v) for v in headline_cut.values()) + + +def _count_ok(results: Mapping[str, Mapping[str, Mapping]]) -> int: + ok = 0 + for by_sub in results.values(): + for entry in by_sub.values(): + if isinstance(entry, Mapping) and entry.get("_status") == "ok": + ok += 1 + return ok + + +def compute_badge_text(report_path: Path, *, today: str | None = None) -> str: + """Render the badge line from the report at *report_path*. + + `today` defaults to UTC `YYYY-MM-DD`; the parameter exists so tests + can pin a date. + """ + data = json.loads(Path(report_path).read_text()) + total = _count_invocations(data["headline_cut"]) + ok = _count_ok(data["results"]) + if today is None: + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + icon = "✅" if ok == total and total > 0 else "⚠️" + return f"reproducibility: {icon} {ok}/{total} ok · last run {today}" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path, help="path to merged canonical JSON") + parser.add_argument("--today", default=None, help="override UTC date (YYYY-MM-DD)") + args = parser.parse_args(argv) + sys.stdout.write(compute_badge_text(args.report, today=args.today) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_benchmarks_badge.py b/tests/test_benchmarks_badge.py new file mode 100644 index 000000000..d0f3405dc --- /dev/null +++ b/tests/test_benchmarks_badge.py @@ -0,0 +1,118 @@ +"""Tests for benchmarks.badge — README reproducibility-badge text formatter. + +Issue: #477. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from benchmarks import badge + + +def _write_report( + tmp_path: Path, + *, + headline_cut: dict, + results: dict, +) -> Path: + p = tmp_path / "report.json" + p.write_text( + json.dumps( + { + "label": "test", + "headline_cut": headline_cut, + "metric_overrides": {}, + "results": results, + } + ) + ) + return p + + +def test_all_ok_renders_check_icon(tmp_path): + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]}, + results={ + "a": {"_": {"_status": "ok"}}, + "b": {"_": {"_status": "ok"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert text == "reproducibility: ✅ 2/2 ok · last run 2026-05-08" + + +def test_partial_renders_warn_icon(tmp_path): + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]}, + results={ + "a": {"_": {"_status": "ok"}}, + "b": {"_": {"_status": "error"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert text == "reproducibility: ⚠️ 1/2 ok · last run 2026-05-08" + + +def test_total_counts_subkeys_not_adapters(tmp_path): + """An adapter with N parametrised invocations contributes N to total.""" + report = _write_report( + tmp_path, + headline_cut={ + "mab": [{"sub_key": "x"}, {"sub_key": "y"}, {"sub_key": "z"}], + "amabench": [{"sub_key": None}], + }, + results={ + "mab": {"x": {"_status": "ok"}, "y": {"_status": "ok"}, "z": {"_status": "ok"}}, + "amabench": {"_": {"_status": "ok"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert "4/4" in text + + +def test_skipped_counts_as_not_ok(tmp_path): + """Per #479, skipped_data_missing is distinct from ok; it does not count.""" + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]}, + results={ + "a": {"_": {"_status": "ok"}}, + "b": {"_": {"_status": "skipped_data_missing"}}, + }, + ) + text = badge.compute_badge_text(report, today="2026-05-08") + assert "1/2 ok" in text + assert text.startswith("reproducibility: ⚠️") + + +def test_canonical_v200_partial(tmp_path): + """Sanity: today's checked-in canonical reports 6/11.""" + canonical = Path(__file__).parent.parent / "benchmarks" / "results" / "v2.0.0.json" + if not canonical.exists(): + pytest.skip("canonical baseline not present") + text = badge.compute_badge_text(canonical, today="2026-05-08") + assert "6/11 ok" in text + + +def test_zero_total_does_not_render_check(tmp_path): + """Empty headline_cut shouldn't produce the all-green icon.""" + report = _write_report(tmp_path, headline_cut={}, results={}) + text = badge.compute_badge_text(report, today="2026-05-08") + assert text.startswith("reproducibility: ⚠️ 0/0 ok") + + +def test_today_defaults_to_utc(tmp_path): + report = _write_report( + tmp_path, + headline_cut={"a": [{"sub_key": None}]}, + results={"a": {"_": {"_status": "ok"}}}, + ) + text = badge.compute_badge_text(report) + # YYYY-MM-DD shape; don't pin the actual day. + suffix = text.rsplit("last run ", 1)[1] + assert len(suffix) == 10 and suffix[4] == "-" and suffix[7] == "-"