feat: posterior-ranking eval harness — MRR uplift + ECE — slice 1 of #151

benchmarks/posterior_ranking/__init__.py

@@ -0,0 +1 @@
+# posterior_ranking benchmark package

benchmarks/posterior_ranking/ece.py

@@ -0,0 +1,152 @@
+"""Expected Calibration Error (ECE) scorer for the posterior-ranking eval harness.
+
+Implements the ECE contract from docs/v2_posterior_ranking_residual.md § Slice 1.
+
+For each (query, retrieved_belief, rank) triple in the eval set, treat
+posterior_mean(b) as the predicted probability that the user will rate b
+positive.
+
+Bucketing: 10 equal-width buckets [0.0, 0.1), [0.1, 0.2), ..., [0.9, 1.0].
+Per bucket: (mean_predicted, mean_actual) where mean_actual is the empirical
+positive-feedback rate from the synthetic feedback stream replayed in the eval.
+
+ECE = sum_b (|bucket_b| / N) * |mean_predicted_b - mean_actual_b|
+
+Pass criterion: ECE <= 0.10
+"""
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+
+from aelfrice.scoring import posterior_mean as _posterior_mean
+
+N_BUCKETS: int = 10
+DEFAULT_ECE_THRESHOLD: float = 0.10
+
+
+@dataclass
+class BucketStat:
+    """Statistics for one equal-width probability bucket."""
+
+    bucket_idx: int
+    bucket_lo: float
+    bucket_hi: float
+    count: int
+    mean_predicted: float
+    mean_actual: float
+    weight: float  # count / N_total
+
+
+@dataclass
+class ECEResult:
+    """ECE calibration result."""
+
+    ece: float
+    buckets: list[BucketStat]
+    n_total: int
+    pass_threshold: float
+    passed: bool
+
+
+def _bucket_index(predicted: float) -> int:
+    """Map a predicted probability in [0, 1] to bucket index in [0, N_BUCKETS-1]."""
+    idx = int(predicted * N_BUCKETS)
+    # Clamp 1.0 exactly into the last bucket.
+    return min(idx, N_BUCKETS - 1)
+
+
+def compute_ece(
+    triples: list[tuple[float, float, float]],
+    threshold: float = DEFAULT_ECE_THRESHOLD,
+) -> ECEResult:
+    """Compute ECE from a list of (alpha, beta, actual_positive_rate) triples.
+
+    Each triple represents one (query, retrieved_belief, rank) observation:
+      - alpha, beta: the belief's current posterior parameters
+      - actual_positive_rate: 1.0 if this belief received positive feedback
+        in the synthetic stream, 0.0 otherwise
+
+    Returns an ECEResult with per-bucket statistics and overall ECE.
+    """
+    n_total = len(triples)
+    if n_total == 0:
+        # Degenerate: no observations; ECE is 0, pass trivially.
+        buckets = [
+            BucketStat(
+                bucket_idx=i,
+                bucket_lo=i / N_BUCKETS,
+                bucket_hi=(i + 1) / N_BUCKETS,
+                count=0,
+                mean_predicted=0.0,
+                mean_actual=0.0,
+                weight=0.0,
+            )
+            for i in range(N_BUCKETS)
+        ]
+        return ECEResult(ece=0.0, buckets=buckets, n_total=0, pass_threshold=threshold, passed=True)
+
+    # Accumulate per bucket.
+    bucket_predicted: list[list[float]] = [[] for _ in range(N_BUCKETS)]
+    bucket_actual: list[list[float]] = [[] for _ in range(N_BUCKETS)]
+
+    for alpha, beta, actual in triples:
+        pred = _posterior_mean(alpha, beta)
+        idx = _bucket_index(pred)
+        bucket_predicted[idx].append(pred)
+        bucket_actual[idx].append(actual)
+
+    buckets: list[BucketStat] = []
+    ece = 0.0
+
+    for i in range(N_BUCKETS):
+        preds = bucket_predicted[i]
+        acts = bucket_actual[i]
+        count = len(preds)
+        weight = count / n_total
+
+        if count > 0:
+            mean_pred = sum(preds) / count
+            mean_act = sum(acts) / count
+        else:
+            mean_pred = (i + 0.5) / N_BUCKETS  # midpoint for empty bucket
+            mean_act = 0.0
+
+        ece += weight * abs(mean_pred - mean_act)
+
+        buckets.append(BucketStat(
+            bucket_idx=i,
+            bucket_lo=i / N_BUCKETS,
+            bucket_hi=(i + 1) / N_BUCKETS,
+            count=count,
+            mean_predicted=mean_pred,
+            mean_actual=mean_act,
+            weight=weight,
+        ))
+
+    passed = ece <= threshold
+    return ECEResult(ece=ece, buckets=buckets, n_total=n_total, pass_threshold=threshold, passed=passed)
+
+
+def compute_ece_from_stores(
+    fixture_observations: list[dict[str, object]],
+    threshold: float = DEFAULT_ECE_THRESHOLD,
+) -> ECEResult:
+    """Compute ECE from a list of observation dicts.
+
+    Each dict must have keys:
+      "alpha": float
+      "beta": float
+      "received_positive": bool  (True if this observation received positive feedback)
+
+    This is the interface used by run.py after replaying the synthetic feedback stream.
+    """
+    triples: list[tuple[float, float, float]] = [
+        (
+            float(obs["alpha"]),
+            float(obs["beta"]),
+            1.0 if obs["received_positive"] else 0.0,
+        )
+        for obs in fixture_observations
+    ]
+    return compute_ece(triples, threshold=threshold)

benchmarks/posterior_ranking/fixtures/default.jsonl

@@ -0,0 +1,7 @@
+{"id": "q1", "query": "python asyncio event loop", "known_belief_content": "asyncio event loop runs coroutines and callbacks via a single-threaded scheduler", "noise_belief_contents": ["threading module provides OS-level threads for parallel execution", "multiprocessing spawns separate processes for CPU-bound tasks", "the GIL prevents true thread parallelism for CPU-bound code in CPython", "subprocess module launches child processes for shell commands"]}
+{"id": "q2", "query": "SQLite WAL journal mode", "known_belief_content": "SQLite WAL mode allows concurrent readers and one writer without blocking reads", "noise_belief_contents": ["PostgreSQL MVCC uses row-level versioning for concurrent transactions", "MySQL InnoDB uses a clustered index for primary key lookups", "Redis persistence uses RDB snapshots and AOF append-only logs", "database indexes trade write overhead for faster read performance"]}
+{"id": "q3", "query": "Beta Bernoulli posterior update", "known_belief_content": "Beta-Bernoulli model updates alpha on positive observation and beta on negative observation", "noise_belief_contents": ["Gaussian processes model function distributions over continuous input spaces", "Dirichlet-Categorical model extends Beta-Bernoulli to multinomial outcomes", "Thompson sampling draws from posterior to balance exploration and exploitation", "MCMC methods approximate intractable posterior distributions by sampling"]}
+{"id": "q4", "query": "BM25 term frequency saturation", "known_belief_content": "BM25 term frequency component saturates at high counts via k1 parameter preventing dominant terms", "noise_belief_contents": ["TF-IDF weights terms by frequency divided by log of document count", "cosine similarity normalizes document length before comparing term vectors", "PageRank assigns authority scores via iterative link-following random walk", "dense retrieval encodes queries and documents into shared embedding space"]}
+{"id": "q5", "query": "exponential decay half-life", "known_belief_content": "exponential decay with half-life h reduces a quantity by factor 0.5 every h time units", "noise_belief_contents": ["Poisson process models events arriving at constant average rate over time", "Markov chain memoryless property means future state depends only on present state", "geometric distribution models number of trials until first success in Bernoulli process", "log-normal distribution arises when logarithm of variable is normally distributed"]}
+{"id": "q6", "query": "FTS5 full text search porter stemmer", "known_belief_content": "SQLite FTS5 with porter tokenizer applies English stemming to normalize query and document terms", "noise_belief_contents": ["Elasticsearch uses inverted index with configurable analyzers for full-text search", "Lucene query parser supports boolean operators AND OR NOT and phrase queries", "n-gram tokenization splits text into overlapping character sequences for fuzzy matching", "stop words removal eliminates high-frequency function words before indexing"]}
+{"id": "q7", "query": "Jeffreys prior Beta distribution", "known_belief_content": "Jeffreys prior for Bernoulli likelihood is Beta(0.5, 0.5) which is invariant under reparameterization", "noise_belief_contents": ["Laplace prior Beta(1,1) is uniform and assigns equal probability to all values", "conjugate prior for binomial likelihood is Beta distribution updated by successes and failures", "maximum likelihood estimation finds parameters maximizing probability of observed data", "Bayesian credible interval contains true parameter with specified posterior probability"]}