fix(rescore): handle uncategorised results without dividing by zero

benchmarks/locomo_rescore_streaming.py

@@ -145,11 +145,29 @@ def write_checkpoint(path: Path, data: dict) -> None:
     tmp.replace(path)
 
 
+def _row_stats(rs: list[dict]) -> dict:
+    """Compute per-bucket metrics from a list of rescored results."""
+    f1 = statistics.mean(r["f1"] for r in rs)
+    orig = statistics.mean(r["judge"] for r in rs)
+    tol = statistics.mean(r.get("judge_tolerant", r["judge"]) for r in rs)
+    rv = [r["judge_rejudged"] for r in rs if r.get("judge_rejudged") is not None]
+    rj = statistics.mean(rv) if rv else float("nan")
+    delta = (rj - orig) if rv else 0.0
+    return {"f1": f1, "orig": orig, "tol": tol, "rj": rj, "rv": rv, "delta": delta}
+
+
 def print_scorecard(path: Path, data: dict, judge_model: str) -> None:
+    """Print a per-category and overall scorecard.
+
+    LoCoMo results carry integer category labels (1-4); LongMemEval-KU
+    results have no category field at all. Falls back to a single
+    "Overall" bucket when no LoCoMo categories match so the script
+    stays useful for both benchmarks instead of dividing by zero.
+    """
     results = data["results"]
     print()
     print("=" * 88)
-    print(f"LoCoMo Rescore (streaming) — {path.name}  [judge: {judge_model}]")
+    print(f"Rescore (streaming) — {path.name}  [judge: {judge_model}]")
     print("=" * 88)
     print(f"{'Category':<16} {'Count':>6} {'F1':>8} {'Orig':>8} {'Tol':>8} {'Rejudge':>10} {'Delta':>8}  Covered")
     print("-" * 88)
@@ -158,22 +176,50 @@ def print_scorecard(path: Path, data: dict, judge_model: str) -> None:
         rs = [r for r in results if r.get("category") == c]
         if not rs:
             continue
-        f1 = statistics.mean(r["f1"] for r in rs)
-        orig = statistics.mean(r["judge"] for r in rs)
-        tol = statistics.mean(r.get("judge_tolerant", r["judge"]) for r in rs)
-        rv = [r["judge_rejudged"] for r in rs if r.get("judge_rejudged") is not None]
-        rj = statistics.mean(rv) if rv else float("nan")
-        delta = (rj - orig) if rv else 0.0
-        cov = f"{len(rv)}/{len(rs)} ({100*len(rv)/len(rs):.0f}%)"
-        print(f"{CAT_NAMES[c]} ({c})   {len(rs):>6} {f1:>8.3f} {orig:>8.2f} {tol:>8.2f} {rj:>10.2f} {delta:>+8.2f}  {cov}")
+        s = _row_stats(rs)
+        cov = f"{len(s['rv'])}/{len(rs)} ({100*len(s['rv'])/len(rs):.0f}%)"
+        print(
+            f"{CAT_NAMES[c]} ({c})   {len(rs):>6} {s['f1']:>8.3f} {s['orig']:>8.2f} "
+            f"{s['tol']:>8.2f} {s['rj']:>10.2f} {s['delta']:>+8.2f}  {cov}"
+        )
         tot["count"] += len(rs)
-        tot["f1"] += f1 * len(rs)
-        tot["orig"] += orig * len(rs)
-        tot["tol"] += tol * len(rs)
-        tot["rj_sum"] += sum(rv)
-        tot["rj_n"] += len(rv)
+        tot["f1"] += s["f1"] * len(rs)
+        tot["orig"] += s["orig"] * len(rs)
+        tot["tol"] += s["tol"] * len(rs)
+        tot["rj_sum"] += sum(s["rv"])
+        tot["rj_n"] += len(s["rv"])
     print("-" * 88)
+    # Fold any uncategorised rows into Overall so a malformed/mixed dataset
+    # (some rows with category in {1,2,3,4}, some without) is fully counted
+    # rather than silently dropping the leftovers.
+    leftovers = [r for r in results if r.get("category") not in {1, 2, 3, 4}]
+    if tot["count"] > 0 and leftovers:
+        s = _row_stats(leftovers)
+        m = len(leftovers)
+        tot["count"] += m
+        tot["f1"] += s["f1"] * m
+        tot["orig"] += s["orig"] * m
+        tot["tol"] += s["tol"] * m
+        tot["rj_sum"] += sum(s["rv"])
+        tot["rj_n"] += len(s["rv"])
     n = tot["count"]
+    if n == 0:
+        # No LoCoMo categories matched and no rows at all — print a
+        # placeholder so benchmarks without integer category labels
+        # (e.g. LongMemEval-KU) still get a usable Overall row.
+        if not results:
+            print(f"{'Overall':<21} {0:>6}  (no results)")
+            print("=" * 88)
+            return
+        s = _row_stats(results)
+        n = len(results)
+        cov = f"{len(s['rv'])}/{n} ({100*len(s['rv'])/n:.0f}%)"
+        print(
+            f"{'Overall':<21} {n:>6} {s['f1']:>8.3f} {s['orig']:>8.2f} "
+            f"{s['tol']:>8.2f} {s['rj']:>10.2f} {s['delta']:>+8.2f}  {cov}"
+        )
+        print("=" * 88)
+        return
     all_f1 = tot["f1"] / n
     all_orig = tot["orig"] / n
     all_tol = tot["tol"] / n

tests/test_locomo_rescore_streaming.py

@@ -0,0 +1,117 @@
+"""Regression tests for benchmarks/locomo_rescore_streaming.py print_scorecard.
+
+Covers:
+- LoCoMo-shaped results (integer categories 1-4) — print the per-category breakdown
+- LongMemEval-KU-shaped results (no category) — fall back to one Overall bucket
+- Empty results — print "no results" instead of dividing by zero
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import sys
+from contextlib import redirect_stdout
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+RESCORE_PATH = REPO_ROOT / "benchmarks" / "locomo_rescore_streaming.py"
+
+
+def _load_rescore_module():
+    """Load the rescore script as a module without running its CLI."""
+    spec = importlib.util.spec_from_file_location("locomo_rescore_streaming", RESCORE_PATH)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _capture(fn):
+    buf = io.StringIO()
+    with redirect_stdout(buf):
+        fn()
+    return buf.getvalue()
+
+
+def test_print_scorecard_locomo_categories():
+    """LoCoMo-shaped results: per-category lines plus Overall."""
+    rescore = _load_rescore_module()
+    data = {
+        "results": [
+            {"category": 1, "f1": 0.5, "judge": 0.6, "judge_rejudged": 0.7},
+            {"category": 1, "f1": 0.4, "judge": 0.5, "judge_rejudged": 0.6},
+            {"category": 4, "f1": 0.8, "judge": 0.9, "judge_rejudged": 0.85},
+        ],
+    }
+    out = _capture(lambda: rescore.print_scorecard(Path("test.json"), data, "qwen3:4b"))
+    assert "Single-hop" in out  # category 1
+    assert "Open-dom" in out  # category 4
+    assert "Overall" in out
+    # Spot-check the Overall count is 3 and the table prints
+    assert " 3 " in out
+
+
+def test_print_scorecard_uncategorised_results_lmeku_path():
+    """Regression: LongMemEval-KU results have no category — must not crash."""
+    rescore = _load_rescore_module()
+    data = {
+        "results": [
+            {"f1": 0.2, "judge": 0.0, "judge_rejudged": 0.5},
+            {"f1": 0.4, "judge": 0.0, "judge_rejudged": 0.6},
+            {"f1": 0.3, "judge": 0.0, "judge_rejudged": 0.7},
+            {"category": None, "f1": 0.5, "judge": 0.0, "judge_rejudged": 0.4},
+        ],
+    }
+    out = _capture(lambda: rescore.print_scorecard(Path("lmeku.json"), data, "qwen3:4b"))
+    # No per-category line should print (none match c in [1,2,3,4])
+    assert "Single-hop" not in out
+    # Falls back to one Overall bucket covering all 4 results
+    assert "Overall" in out
+    assert " 4 " in out  # count column
+    # Must not raise — that was the original ZeroDivisionError bug
+
+
+def test_print_scorecard_empty_results():
+    """Empty results print a 'no results' line instead of crashing."""
+    rescore = _load_rescore_module()
+    out = _capture(lambda: rescore.print_scorecard(Path("empty.json"), {"results": []}, "qwen3:4b"))
+    assert "no results" in out
+
+
+def test_print_scorecard_mixed_categories_includes_uncategorised_in_overall():
+    """Mixed datasets (some category=1-4, some category=None) must fold
+    uncategorised rows into Overall instead of silently dropping them."""
+    rescore = _load_rescore_module()
+    data = {
+        "results": [
+            # Two LoCoMo-shaped rows
+            {"category": 1, "f1": 0.5, "judge": 0.6, "judge_rejudged": 0.7},
+            {"category": 1, "f1": 0.4, "judge": 0.5, "judge_rejudged": 0.6},
+            # Two rows missing the category field
+            {"f1": 0.3, "judge": 0.4, "judge_rejudged": 0.5},
+            {"category": None, "f1": 0.2, "judge": 0.3, "judge_rejudged": 0.4},
+        ],
+    }
+    out = _capture(lambda: rescore.print_scorecard(Path("mixed.json"), data, "qwen3:4b"))
+    assert "Single-hop" in out  # the categorised rows still show
+    # Overall count must be 4 (all rows), not 2 (only the categorised ones)
+    overall_line = next(line for line in out.splitlines() if line.startswith("Overall"))
+    assert " 4 " in overall_line, f"Overall should count all 4 rows, got: {overall_line!r}"
+
+
+def test_print_scorecard_locomo_with_partial_rescore():
+    """Partial rescore coverage (some judge_rejudged is None) still prints."""
+    rescore = _load_rescore_module()
+    data = {
+        "results": [
+            {"category": 1, "f1": 0.5, "judge": 0.6, "judge_rejudged": 0.7},
+            {"category": 1, "f1": 0.4, "judge": 0.5, "judge_rejudged": None},
+            {"category": 2, "f1": 0.8, "judge": 0.9, "judge_rejudged": 0.85},
+        ],
+    }
+    out = _capture(lambda: rescore.print_scorecard(Path("partial.json"), data, "qwen3:4b"))
+    assert "Single-hop" in out
+    assert "Temporal" in out
+    # Coverage on category 1 should reflect 1/2 rescored
+    assert "1/2" in out