ci(bench-canonical): in-place README badge rewrite from cron (#477)

.github/workflows/bench-canonical.yml

@@ -114,6 +114,39 @@
           sys.exit(0 if overall.value == 'pass' else (1 if overall.value == 'fail' else 0))
           "
 
+      - name: Compute badge text
+        # Renders the one-line badge from the merged JSON. Uses
+        # `benchmarks.badge` (unit-tested) so the workflow stays free
+        # of count logic. Output is stashed in $GITHUB_OUTPUT for the
+        # next step. Skips when bench errored — leaves the previous
+        # badge in place rather than rewriting to a stale 0/N line.
+        id: badge
+        if: steps.bench.outputs.out != ''
+        run: |
+          set -euo pipefail
+          text=$(uv run python -m benchmarks.badge "${{ steps.bench.outputs.out }}")
+          echo "text=${text}" >> "$GITHUB_OUTPUT"
+          echo "computed: ${text}"
+
+      - name: Sync README into bench-canonical-results worktree + rewrite badge
+        # Issue #477: badge in README on the bench-canonical-results
+        # branch is auto-rewritten on each cron run. main is not
+        # touched (operator cherry-picks if they want main current).
+        # README is copied from the main checkout each run so the
+        # bench-canonical-results branch never carries a stale README.
+        if: steps.badge.outputs.text != ''
+        run: |
+          set -euo pipefail
+          cp README.md .bench-results-branch/README.md
+          new="${{ steps.badge.outputs.text }}" awk '
+            BEGIN { in_block=0 }
+            /<!-- bench-canonical-badge:start -->/ { print; print ENVIRON["new"]; in_block=1; next }
+            /<!-- bench-canonical-badge:end -->/   { print; in_block=0; next }
+            in_block { next }
+            { print }
+          ' .bench-results-branch/README.md > .bench-results-branch/README.md.new
+          mv .bench-results-branch/README.md.new .bench-results-branch/README.md
+
       - name: Commit + push to bench-canonical-results
         env:
           GIT_AUTHOR_NAME: aelfrice-bench-bot

benchmarks/badge.py

@@ -0,0 +1,59 @@
+"""Compute the README reproducibility-badge text from a canonical bench JSON.
+
+The nightly `bench-canonical` cron writes a merged report under
+`benchmarks/results/v2.0.0-cron-<date>.json`. This module reads that
+report and produces the one-line badge text that the workflow splices
+between the `<!-- bench-canonical-badge:start -->` and
+`<!-- bench-canonical-badge:end -->` markers in README.md.
+
+Issue: #477.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Mapping
+
+
+def _count_invocations(headline_cut: Mapping[str, list]) -> int:
+    return sum(len(v) for v in headline_cut.values())
+
+
+def _count_ok(results: Mapping[str, Mapping[str, Mapping]]) -> int:
+    ok = 0
+    for by_sub in results.values():
+        for entry in by_sub.values():
+            if isinstance(entry, Mapping) and entry.get("_status") == "ok":
+                ok += 1
+    return ok
+
+
+def compute_badge_text(report_path: Path, *, today: str | None = None) -> str:
+    """Render the badge line from the report at *report_path*.
+
+    `today` defaults to UTC `YYYY-MM-DD`; the parameter exists so tests
+    can pin a date.
+    """
+    data = json.loads(Path(report_path).read_text())
+    total = _count_invocations(data["headline_cut"])
+    ok = _count_ok(data["results"])
+    if today is None:
+        today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    icon = "✅" if ok == total and total > 0 else "⚠️"
+    return f"reproducibility: {icon} {ok}/{total} ok · last run {today}"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("report", type=Path, help="path to merged canonical JSON")
+    parser.add_argument("--today", default=None, help="override UTC date (YYYY-MM-DD)")
+    args = parser.parse_args(argv)
+    sys.stdout.write(compute_badge_text(args.report, today=args.today) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_benchmarks_badge.py

@@ -0,0 +1,118 @@
+"""Tests for benchmarks.badge — README reproducibility-badge text formatter.
+
+Issue: #477.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from benchmarks import badge
+
+
+def _write_report(
+    tmp_path: Path,
+    *,
+    headline_cut: dict,
+    results: dict,
+) -> Path:
+    p = tmp_path / "report.json"
+    p.write_text(
+        json.dumps(
+            {
+                "label": "test",
+                "headline_cut": headline_cut,
+                "metric_overrides": {},
+                "results": results,
+            }
+        )
+    )
+    return p
+
+
+def test_all_ok_renders_check_icon(tmp_path):
+    report = _write_report(
+        tmp_path,
+        headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]},
+        results={
+            "a": {"_": {"_status": "ok"}},
+            "b": {"_": {"_status": "ok"}},
+        },
+    )
+    text = badge.compute_badge_text(report, today="2026-05-08")
+    assert text == "reproducibility: ✅ 2/2 ok · last run 2026-05-08"
+
+
+def test_partial_renders_warn_icon(tmp_path):
+    report = _write_report(
+        tmp_path,
+        headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]},
+        results={
+            "a": {"_": {"_status": "ok"}},
+            "b": {"_": {"_status": "error"}},
+        },
+    )
+    text = badge.compute_badge_text(report, today="2026-05-08")
+    assert text == "reproducibility: ⚠️ 1/2 ok · last run 2026-05-08"
+
+
+def test_total_counts_subkeys_not_adapters(tmp_path):
+    """An adapter with N parametrised invocations contributes N to total."""
+    report = _write_report(
+        tmp_path,
+        headline_cut={
+            "mab": [{"sub_key": "x"}, {"sub_key": "y"}, {"sub_key": "z"}],
+            "amabench": [{"sub_key": None}],
+        },
+        results={
+            "mab": {"x": {"_status": "ok"}, "y": {"_status": "ok"}, "z": {"_status": "ok"}},
+            "amabench": {"_": {"_status": "ok"}},
+        },
+    )
+    text = badge.compute_badge_text(report, today="2026-05-08")
+    assert "4/4" in text
+
+
+def test_skipped_counts_as_not_ok(tmp_path):
+    """Per #479, skipped_data_missing is distinct from ok; it does not count."""
+    report = _write_report(
+        tmp_path,
+        headline_cut={"a": [{"sub_key": None}], "b": [{"sub_key": None}]},
+        results={
+            "a": {"_": {"_status": "ok"}},
+            "b": {"_": {"_status": "skipped_data_missing"}},
+        },
+    )
+    text = badge.compute_badge_text(report, today="2026-05-08")
+    assert "1/2 ok" in text
+    assert text.startswith("reproducibility: ⚠️")
+
+
+def test_canonical_v200_partial(tmp_path):
+    """Sanity: today's checked-in canonical reports 6/11."""
+    canonical = Path(__file__).parent.parent / "benchmarks" / "results" / "v2.0.0.json"
+    if not canonical.exists():
+        pytest.skip("canonical baseline not present")
+    text = badge.compute_badge_text(canonical, today="2026-05-08")
+    assert "6/11 ok" in text
+
+
+def test_zero_total_does_not_render_check(tmp_path):
+    """Empty headline_cut shouldn't produce the all-green icon."""
+    report = _write_report(tmp_path, headline_cut={}, results={})
+    text = badge.compute_badge_text(report, today="2026-05-08")
+    assert text.startswith("reproducibility: ⚠️ 0/0 ok")
+
+
+def test_today_defaults_to_utc(tmp_path):
+    report = _write_report(
+        tmp_path,
+        headline_cut={"a": [{"sub_key": None}]},
+        results={"a": {"_": {"_status": "ok"}}},
+    )
+    text = badge.compute_badge_text(report)
+    # YYYY-MM-DD shape; don't pin the actual day.
+    suffix = text.rsplit("last run ", 1)[1]
+    assert len(suffix) == 10 and suffix[4] == "-" and suffix[7] == "-"