From e51e4059e4b81fa30505304e3c9c24fe3d9cb2db Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sun, 3 May 2026 19:52:56 -0700 Subject: [PATCH 1/3] feat(wonder): three phantom-generation strategies (#228) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RW (random walk), TC (triangle closure), STS (span-topic sampling) implementing the candidates from docs/v2_wonder_consolidation.md "The three candidate strategies". Each is a pure read over a MemoryStore returning a list of Phantom dataclasses; output sorted by composition for the bake-off determinism story. No write-path activation here — this is the research surface only; ship-decision PR per spec § "Adoption criteria" wires the chosen strategy into production retrieval. --- src/aelfrice/wonder/__init__.py | 22 +++ src/aelfrice/wonder/models.py | 45 ++++++ src/aelfrice/wonder/strategies.py | 231 ++++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 src/aelfrice/wonder/__init__.py create mode 100644 src/aelfrice/wonder/models.py create mode 100644 src/aelfrice/wonder/strategies.py diff --git a/src/aelfrice/wonder/__init__.py b/src/aelfrice/wonder/__init__.py new file mode 100644 index 00000000..b8db131b --- /dev/null +++ b/src/aelfrice/wonder/__init__.py @@ -0,0 +1,22 @@ +"""Wonder-consolidation bake-off harness (#228). + +Three offline phantom-generation strategies plus a deterministic +synthetic corpus + feedback simulator + evaluator + runner that +together close the v2.0 ship-decision per +``docs/v2_wonder_consolidation.md``. + +The harness is a research surface: nothing here writes to a live +``Store`` outside the bake-off. The chosen-strategy production +wiring is a follow-up issue per the spec. +""" +from __future__ import annotations + +from .models import Phantom +from .strategies import random_walk, span_topic_sampling, triangle_closure + +__all__ = [ + "Phantom", + "random_walk", + "span_topic_sampling", + "triangle_closure", +] diff --git a/src/aelfrice/wonder/models.py b/src/aelfrice/wonder/models.py new file mode 100644 index 00000000..7525ab78 --- /dev/null +++ b/src/aelfrice/wonder/models.py @@ -0,0 +1,45 @@ +"""Wonder-package dataclasses (#228). + +Kept out of top-level ``aelfrice.models`` because ``Phantom`` is a +v2.0 research-only concept and carries strategy-specific fields +(``construction_cost``) that don't belong alongside the +load-bearing ``Belief`` / ``Edge`` dataclasses. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +STRATEGY_RW: Final[str] = "RW" +STRATEGY_TC: Final[str] = "TC" +STRATEGY_STS: Final[str] = "STS" + +STRATEGIES: Final[frozenset[str]] = frozenset({ + STRATEGY_RW, + STRATEGY_TC, + STRATEGY_STS, +}) + + +@dataclass(frozen=True) +class Phantom: + """A speculative composition produced by a wonder generation strategy. + + ``composition`` is the sorted tuple of belief ids the strategy + bundled together. Sorted so two strategies that produce the + same belief set hash identically — required for the Jaccard + metric to detect overlap. + + ``construction_cost`` is in ``atoms-touched`` units (Decision E + in the planning memo) — every belief the strategy read while + producing this phantom counts as one. Reproducible across + machines and substrate-neutral. + + ``seed_id`` records the random-walk start atom; ``None`` for TC + and STS which have no single-seed anchor. + """ + + composition: tuple[str, ...] + strategy: str + construction_cost: float + seed_id: str | None = None diff --git a/src/aelfrice/wonder/strategies.py b/src/aelfrice/wonder/strategies.py new file mode 100644 index 00000000..a022d76c --- /dev/null +++ b/src/aelfrice/wonder/strategies.py @@ -0,0 +1,231 @@ +"""Three phantom-generation strategies for the #228 bake-off. + +Per ``docs/v2_wonder_consolidation.md`` §"The three candidate strategies": + +* **RW (random walk)** — start from a high-uncertainty atom, walk N + hops on the typed-edge graph, bundle visited atoms. +* **TC (triangle closure)** — find pairs (A, B) where A→C and B→C + exist with edge type ∈ {SUPPORTS, CITES, RELATES_TO}; propose + (A, B) keyed on shared target C. +* **STS (span-topic sampling)** — sample compositions whose + constituents span sessions (max session-id diversity in the + no-embedding form). + +Each strategy is a pure function over a ``MemoryStore``. None +mutate the store. Output ordering is sorted by composition for +determinism — the Jaccard evaluator and seed-sweep depend on it. +""" +from __future__ import annotations + +import random +from typing import TYPE_CHECKING + +from aelfrice.models import ( + EDGE_CITES, + EDGE_RELATES_TO, + EDGE_SUPPORTS, +) +from aelfrice.scoring import uncertainty_score + +from .models import ( + STRATEGY_RW, + STRATEGY_STS, + STRATEGY_TC, + Phantom, +) + +if TYPE_CHECKING: + from aelfrice.store import MemoryStore + +# Edge types eligible for TC's shared-target shape. Mirrors the +# spec § "The three candidate strategies" gloss for TC. CONTRADICTS +# and SUPERSEDES are excluded because two beliefs sharing a +# CONTRADICTS target are not candidates for composition — they're +# candidates for resolution, which is a different surface. +TC_EDGE_TYPES: frozenset[str] = frozenset({ + EDGE_SUPPORTS, + EDGE_CITES, + EDGE_RELATES_TO, +}) + +# Default RW seed-selection floor. Beta(1,1) has differential entropy +# 0; beliefs with α≈β≈1 (uninformative prior, the synthetic-corpus +# default) sit just above. Tighter beliefs slip below 0; flatter ones +# climb above. The ratification's "0.7" referred to a normalized +# scale not used in v1.x; the floor here is differential-entropy +# units. Tunable per-bake-off via the ``uncertainty_floor`` arg. +DEFAULT_RW_UNCERTAINTY_FLOOR: float = -0.5 + + +def _all_belief_ids(store: "MemoryStore") -> list[str]: + return store.list_belief_ids() + + +def random_walk( + store: "MemoryStore", + *, + rng: random.Random, + n_walks: int = 50, + depth: int = 2, + uncertainty_floor: float = DEFAULT_RW_UNCERTAINTY_FLOOR, +) -> list[Phantom]: + """RW: start from high-uncertainty atoms; walk ``depth`` hops. + + Seed selection: any belief whose ``uncertainty_score(α, β)`` + exceeds ``uncertainty_floor`` is eligible. RNG samples + ``n_walks`` seeds *with replacement* — empirically observed + sample variance is what the spec wants to falsify, so resampling + is allowed. + + Walk: at each step, pick a uniformly random outgoing edge and + follow it. Self-revisits are dropped from the bundle but counted + in cost (atoms touched). A walk that hits a dead-end before + completing ``depth`` hops still produces a phantom from the + truncated bundle, as long as it has ≥2 atoms. + """ + eligible: list[str] = [] + for bid in _all_belief_ids(store): + b = store.get_belief(bid) + if b is None: + continue + if uncertainty_score(b.alpha, b.beta) >= uncertainty_floor: + eligible.append(bid) + if not eligible: + return [] + + phantoms: list[Phantom] = [] + for _ in range(n_walks): + seed = rng.choice(eligible) + visited: list[str] = [seed] + seen: set[str] = {seed} + cost = 1.0 # the seed itself + cursor = seed + for _ in range(depth): + outgoing = store.edges_from(cursor) + if not outgoing: + break + edge = rng.choice(outgoing) + cost += 1.0 + cursor = edge.dst + if cursor not in seen: + seen.add(cursor) + visited.append(cursor) + if len(visited) < 2: + continue + phantoms.append( + Phantom( + composition=tuple(sorted(visited)), + strategy=STRATEGY_RW, + construction_cost=cost, + seed_id=seed, + ) + ) + # Deduplicate by composition; first occurrence wins on cost. + seen_comp: dict[tuple[str, ...], Phantom] = {} + for p in phantoms: + if p.composition not in seen_comp: + seen_comp[p.composition] = p + return sorted(seen_comp.values(), key=lambda p: p.composition) + + +def triangle_closure( + store: "MemoryStore", + *, + edge_types: frozenset[str] = TC_EDGE_TYPES, +) -> list[Phantom]: + """TC: propose (A, B) for every pair with a shared edge target C. + + Deterministic — no RNG. Cost per phantom is 3 (A + B + C touched). + Skips self-pairs and dedup'd unordered pairs (the resulting + composition tuple is sorted, so duplicates collapse naturally). + """ + # target_id -> set of source ids that point at it via an eligible + # edge type. + incoming: dict[str, set[str]] = {} + for edge in store.iter_all_edges(): + if edge.type not in edge_types: + continue + if edge.src == edge.dst: + continue + incoming.setdefault(edge.dst, set()).add(edge.src) + + phantoms: dict[tuple[str, ...], Phantom] = {} + for target, sources in incoming.items(): + if len(sources) < 2: + continue + ordered = sorted(sources) + for i, a in enumerate(ordered): + for b in ordered[i + 1:]: + comp = (a, b) if a < b else (b, a) + if comp in phantoms: + continue + phantoms[comp] = Phantom( + composition=comp, + strategy=STRATEGY_TC, + construction_cost=3.0, + seed_id=None, + ) + return sorted(phantoms.values(), key=lambda p: p.composition) + + +def span_topic_sampling( + store: "MemoryStore", + *, + rng: random.Random, + n_samples: int = 50, + composition_size: int = 2, +) -> list[Phantom]: + """STS: sample compositions that span the most distinct sessions. + + No-embedding form per the spec: session-id diversity stands in + for "topic" diversity. Beliefs without a ``session_id`` are + bucketed under a synthetic ``"__none__"`` session — that bucket + contributes no diversity, so phantoms drawn entirely from it are + discarded. + + Sampling: pick ``composition_size`` distinct sessions uniformly + at random, then one belief uniformly from each. Repeat + ``n_samples`` times. Cost per phantom is ``composition_size`` + (one atom touched per slot). + """ + if composition_size < 2: + raise ValueError("composition_size must be >= 2") + by_session: dict[str, list[str]] = {} + for bid in _all_belief_ids(store): + b = store.get_belief(bid) + if b is None: + continue + key = b.session_id if b.session_id else "__none__" + by_session.setdefault(key, []).append(bid) + real_sessions = [k for k in by_session if k != "__none__"] + if len(real_sessions) < composition_size: + return [] + real_sessions.sort() # deterministic seeding into the rng + + phantoms: dict[tuple[str, ...], Phantom] = {} + for _ in range(n_samples): + chosen_sessions = rng.sample(real_sessions, composition_size) + picks: list[str] = [] + for sess in chosen_sessions: + picks.append(rng.choice(sorted(by_session[sess]))) + if len(set(picks)) < composition_size: + continue + comp = tuple(sorted(picks)) + if comp in phantoms: + continue + phantoms[comp] = Phantom( + composition=comp, + strategy=STRATEGY_STS, + construction_cost=float(composition_size), + seed_id=None, + ) + return sorted(phantoms.values(), key=lambda p: p.composition) + + +__all__ = [ + "DEFAULT_RW_UNCERTAINTY_FLOOR", + "TC_EDGE_TYPES", + "random_walk", + "span_topic_sampling", + "triangle_closure", +] From 97af9ca99f1975b61516f0094c8767a90d1ca976 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sun, 3 May 2026 19:56:55 -0700 Subject: [PATCH 2/3] feat(wonder): synthetic corpus + evaluator + bake-off runner (#228) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit simulator.py: deterministic corpus generator (n_topics × n_atoms_per_topic) + feedback simulator (single-topic agreement = confirm) + populate_store helper. Synthetic from seed; no lab-fixture dependency. evaluator.py: four metrics from spec § Adoption criteria — confirmation_rate, retrieval_freq_per_cost, pairwise_jaccard, junk_rate — plus the verdict function applying the four ship rules (single / ensemble / defer / drop). runner.py: orchestrates a multi-seed sweep (default N=10 per planning memo Decision D), aggregates mean metrics across seeds, emits result JSON. CLI entry point at `python -m aelfrice.wonder.runner`. wonder_consolidation.py: thin shim keeping the existing bench-gate stub honest (Decision A). Token-overlap relatedness in [0,1]; not the bake-off. Smoke run on 2-seed × 20-walk preview produces sane verdict distribution (defer at default densities — corpus tuning is the running session's job for R, not part of this harness PR). --- src/aelfrice/wonder/evaluator.py | 222 ++++++++++++++++++++++++ src/aelfrice/wonder/runner.py | 229 +++++++++++++++++++++++++ src/aelfrice/wonder/simulator.py | 247 +++++++++++++++++++++++++++ src/aelfrice/wonder_consolidation.py | 74 ++++++++ 4 files changed, 772 insertions(+) create mode 100644 src/aelfrice/wonder/evaluator.py create mode 100644 src/aelfrice/wonder/runner.py create mode 100644 src/aelfrice/wonder/simulator.py create mode 100644 src/aelfrice/wonder_consolidation.py diff --git a/src/aelfrice/wonder/evaluator.py b/src/aelfrice/wonder/evaluator.py new file mode 100644 index 00000000..a071ce1f --- /dev/null +++ b/src/aelfrice/wonder/evaluator.py @@ -0,0 +1,222 @@ +"""Bake-off evaluator metrics for #228. + +Direct port of the four metrics in +``docs/v2_wonder_consolidation.md`` § "Adoption criteria for v2.0 +ship": + +* ``confirmation_rate`` — promotions / phantoms_generated under + fixed feedback budget. +* ``retrieval_freq_per_cost`` — (mean retrieval count) / (mean + construction_cost). Retrieval count is simulated as a function of + composition size: larger compositions surface more often per + the spec's "retrieval surface frequency per unit construction + cost" criterion. +* ``pairwise_jaccard[s1, s2]`` — overlap between two strategies' + composition sets. Sorted-tuple compositions hash identically + across strategies so set algebra works directly. +* ``junk_rate`` — phantoms gc'd within 14d / generated. The + simulator marks a phantom for gc when it junks repeatedly + before any confirm; this evaluator treats "junked under fixed + budget" as a proxy for "would gc within 14d" — the spec's + threshold is qualitative, this makes it computable. + +Output is a single ``BakeoffResult`` dataclass with one +``StrategyMetrics`` per strategy plus the cross-strategy Jaccard +matrix and the adoption-criterion verdict. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +from .models import STRATEGY_RW, STRATEGY_STS, STRATEGY_TC, Phantom +from .simulator import ( + SyntheticCorpus, + feedback_verdict, + simulate_promotion, +) + +Verdict = Literal["single", "ensemble", "defer", "drop"] + +# Adoption criteria thresholds from spec § "Adoption criteria for +# v2.0 ship". Constants here so the runner can echo them in the +# result JSON for audit. +H0_PLUS_PP_FLOOR: float = 0.10 +JACCARD_REDUNDANCY: float = 0.6 +JACCARD_COMPLEMENT: float = 0.3 +JUNK_RATE_DEFER: float = 0.60 +H0_NULL_RATE: float = 0.065 # midpoint of spec's 5–8% prediction + + +@dataclass(frozen=True) +class StrategyMetrics: + strategy: str + n_phantoms: int + confirmation_rate: float + retrieval_freq_per_cost: float + junk_rate: float + mean_construction_cost: float + + +@dataclass(frozen=True) +class BakeoffResult: + metrics: tuple[StrategyMetrics, ...] + pairwise_jaccard: dict[tuple[str, str], float] + verdict: Verdict + h0_floor: float = H0_NULL_RATE + H0_PLUS_PP_FLOOR + notes: tuple[str, ...] = field(default_factory=tuple) + + +def _retrieval_count(composition: tuple[str, ...]) -> int: + """Simulated retrieval surface frequency. + + Each constituent contributes one retrieval event under the + no-embedding form (the corpus has no real query log). Reuses + composition size as a stand-in — strategies that generate + larger phantoms surface more often, by design. + """ + return len(composition) + + +def evaluate_strategy( + phantoms: list[Phantom], + corpus: SyntheticCorpus, + *, + feedback_budget_per_phantom: int = 16, +) -> StrategyMetrics: + """Score a strategy's phantom set against the corpus. + + ``feedback_budget_per_phantom`` is the simulated number of + feedback events each phantom can absorb. Confirms accumulate α; + junks accumulate β. The promotion gate is from + ``ALPHA_PROMOTION_THRESHOLD``. + + A phantom counts toward ``junk_rate`` if every event in its + budget was a junk verdict (the simulator is deterministic + given corpus + composition, so all events for one phantom + share the same verdict — but the rate definition is preserved + in case the simulator gains randomness later). + """ + if not phantoms: + return StrategyMetrics( + strategy="", + n_phantoms=0, + confirmation_rate=0.0, + retrieval_freq_per_cost=0.0, + junk_rate=0.0, + mean_construction_cost=0.0, + ) + strategy = phantoms[0].strategy + promotions = 0 + junked = 0 + total_retrieval = 0 + total_cost = 0.0 + for phantom in phantoms: + verdicts = [ + feedback_verdict(phantom.composition, corpus) + for _ in range(feedback_budget_per_phantom) + ] + confirms = sum(1 for v in verdicts if v == "confirm") + junks = sum(1 for v in verdicts if v == "junk") + if simulate_promotion(confirms, junks): + promotions += 1 + if confirms == 0 and junks > 0: + junked += 1 + total_retrieval += _retrieval_count(phantom.composition) + total_cost += phantom.construction_cost + n = len(phantoms) + confirmation_rate = promotions / n + junk_rate = junked / n + mean_cost = total_cost / n + mean_retrieval = total_retrieval / n + retrieval_per_cost = mean_retrieval / mean_cost if mean_cost > 0 else 0.0 + return StrategyMetrics( + strategy=strategy, + n_phantoms=n, + confirmation_rate=confirmation_rate, + retrieval_freq_per_cost=retrieval_per_cost, + junk_rate=junk_rate, + mean_construction_cost=mean_cost, + ) + + +def pairwise_jaccard( + a: list[Phantom], b: list[Phantom] +) -> float: + """Jaccard over composition tuples between two phantom sets.""" + set_a = {p.composition for p in a} + set_b = {p.composition for p in b} + if not set_a and not set_b: + return 1.0 + inter = len(set_a & set_b) + union = len(set_a | set_b) + return inter / union if union else 0.0 + + +def adoption_verdict( + metrics: tuple[StrategyMetrics, ...], + jaccard: dict[tuple[str, str], float], + *, + h0_floor: float | None = None, +) -> Verdict: + """Apply the four spec rules in order. + + 1. **Drop** if all strategies fall below the H0 null floor. + 2. **Defer** if no strategy clears H0+10pp, or any strategy's + junk_rate exceeds 60%. + 3. **Single-strategy ship** if exactly one strategy clears the + floor with retrieval-per-cost within 25% of best, and the + others are not complementary (any pairwise Jaccard ≥ 0.6). + 4. **Ensemble** if the top two strategies are complementary + (Jaccard < 0.3) and each clears the floor. + + Order matters: drop dominates defer, which dominates the ship + decisions. + """ + if h0_floor is None: + h0_floor = H0_NULL_RATE + H0_PLUS_PP_FLOOR + by_strategy = {m.strategy: m for m in metrics} + rates = [m.confirmation_rate for m in metrics] + if all(r < H0_NULL_RATE for r in rates): + return "drop" + if any(m.junk_rate > JUNK_RATE_DEFER for m in metrics): + return "defer" + clearing = [m for m in metrics if m.confirmation_rate >= h0_floor] + if not clearing: + return "defer" + best_retrieval = max(m.retrieval_freq_per_cost for m in clearing) + cost_window = [ + m for m in clearing + if m.retrieval_freq_per_cost >= 0.75 * best_retrieval + ] + if len(clearing) == 1: + return "single" + sorted_clearing = sorted( + clearing, key=lambda m: m.confirmation_rate, reverse=True + ) + top_two = sorted_clearing[:2] + pair = tuple(sorted(p.strategy for p in top_two)) + j = jaccard.get(pair, 0.0) + if j < JACCARD_COMPLEMENT: + return "ensemble" + if any(j >= JACCARD_REDUNDANCY for j in jaccard.values()): + # Top strategy plus any other with high overlap → single + # ship the leader. + if cost_window: + return "single" + return "single" + + +__all__ = [ + "BakeoffResult", + "H0_NULL_RATE", + "H0_PLUS_PP_FLOOR", + "JACCARD_COMPLEMENT", + "JACCARD_REDUNDANCY", + "JUNK_RATE_DEFER", + "StrategyMetrics", + "Verdict", + "adoption_verdict", + "evaluate_strategy", + "pairwise_jaccard", +] diff --git a/src/aelfrice/wonder/runner.py b/src/aelfrice/wonder/runner.py new file mode 100644 index 00000000..79e97118 --- /dev/null +++ b/src/aelfrice/wonder/runner.py @@ -0,0 +1,229 @@ +"""Bake-off runner for the #228 wonder-consolidation campaign. + +Wires the corpus, the three strategies, and the evaluator into a +single ``run_bakeoff`` entry point. Multi-seed sweeping per +Decision D in the planning memo (default N=10): metrics are +reported as mean across seeds, with per-seed detail kept in the +result dict for variance auditing. + +CLI usage:: + + uv run python -m aelfrice.wonder.runner \\ + --feedback-budget 16 --seeds 10 --output bake_off_R0.json + +Output JSON shape:: + + { + "config": {...}, + "per_seed": [{seed, strategy_metrics, jaccard, verdict}, ...], + "aggregate": { + "strategy_metrics": {strategy: {metric: mean_value}}, + "jaccard": {"RW|TC": mean, ...}, + "verdict_distribution": {"single": n, ...}, + "majority_verdict": "..." + } + } +""" +from __future__ import annotations + +import argparse +import json +import random +import sys +from collections import Counter +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from aelfrice.store import MemoryStore + +from .evaluator import ( + H0_NULL_RATE, + H0_PLUS_PP_FLOOR, + StrategyMetrics, + adoption_verdict, + evaluate_strategy, + pairwise_jaccard, +) +from .models import STRATEGY_RW, STRATEGY_STS, STRATEGY_TC, Phantom +from .simulator import build_corpus, populate_store +from .strategies import ( + random_walk, + span_topic_sampling, + triangle_closure, +) + + +def _run_one_seed( + seed: int, + *, + n_topics: int, + n_atoms_per_topic: int, + n_walks: int, + n_sts_samples: int, + feedback_budget: int, +) -> dict[str, Any]: + rng = random.Random(seed) + corpus = build_corpus( + rng=rng, + n_topics=n_topics, + n_atoms_per_topic=n_atoms_per_topic, + ) + store = MemoryStore(":memory:") + populate_store(store, corpus, rng=rng) + + rw = random_walk(store, rng=random.Random(seed + 1), n_walks=n_walks) + tc = triangle_closure(store) + sts = span_topic_sampling( + store, rng=random.Random(seed + 2), n_samples=n_sts_samples + ) + + metrics_list: list[StrategyMetrics] = [ + evaluate_strategy(rw, corpus, feedback_budget_per_phantom=feedback_budget), + evaluate_strategy(tc, corpus, feedback_budget_per_phantom=feedback_budget), + evaluate_strategy(sts, corpus, feedback_budget_per_phantom=feedback_budget), + ] + strategy_to_phantoms: dict[str, list[Phantom]] = { + STRATEGY_RW: rw, + STRATEGY_TC: tc, + STRATEGY_STS: sts, + } + j: dict[tuple[str, str], float] = {} + pairs = [ + (STRATEGY_RW, STRATEGY_TC), + (STRATEGY_RW, STRATEGY_STS), + (STRATEGY_STS, STRATEGY_TC), + ] + for s1, s2 in pairs: + key = tuple(sorted((s1, s2))) + j[key] = pairwise_jaccard( + strategy_to_phantoms[s1], strategy_to_phantoms[s2] + ) + + metrics_tuple = tuple(metrics_list) + verdict = adoption_verdict(metrics_tuple, j) + + return { + "seed": seed, + "strategy_metrics": [asdict(m) for m in metrics_tuple], + "jaccard": {f"{a}|{b}": v for (a, b), v in j.items()}, + "verdict": verdict, + } + + +def _aggregate(per_seed: list[dict[str, Any]]) -> dict[str, Any]: + strategies = [m["strategy"] for m in per_seed[0]["strategy_metrics"]] + metric_names = [ + "confirmation_rate", + "retrieval_freq_per_cost", + "junk_rate", + "mean_construction_cost", + "n_phantoms", + ] + agg: dict[str, dict[str, float]] = {s: {} for s in strategies} + for s_idx, strategy in enumerate(strategies): + for name in metric_names: + values = [ + seed["strategy_metrics"][s_idx][name] for seed in per_seed + ] + agg[strategy][name] = sum(values) / len(values) + + jaccard_keys = list(per_seed[0]["jaccard"].keys()) + jaccard_agg = { + k: sum(s["jaccard"][k] for s in per_seed) / len(per_seed) + for k in jaccard_keys + } + verdict_counter = Counter(s["verdict"] for s in per_seed) + return { + "strategy_metrics": agg, + "jaccard": jaccard_agg, + "verdict_distribution": dict(verdict_counter), + "majority_verdict": verdict_counter.most_common(1)[0][0], + } + + +def run_bakeoff( + *, + n_topics: int = 8, + n_atoms_per_topic: int = 25, + n_walks: int = 50, + n_sts_samples: int = 50, + feedback_budget: int = 16, + seeds: int = 10, +) -> dict[str, Any]: + """Run the full bake-off and return the result dict. + + ``seeds`` controls the number of random seeds swept per + Decision D in the planning memo. Defaults to 10 (variance + estimate for the public-side R run). + """ + per_seed = [ + _run_one_seed( + seed=s, + n_topics=n_topics, + n_atoms_per_topic=n_atoms_per_topic, + n_walks=n_walks, + n_sts_samples=n_sts_samples, + feedback_budget=feedback_budget, + ) + for s in range(seeds) + ] + return { + "config": { + "n_topics": n_topics, + "n_atoms_per_topic": n_atoms_per_topic, + "n_walks": n_walks, + "n_sts_samples": n_sts_samples, + "feedback_budget": feedback_budget, + "seeds": seeds, + "h0_null_rate": H0_NULL_RATE, + "h0_floor": H0_NULL_RATE + H0_PLUS_PP_FLOOR, + }, + "per_seed": per_seed, + "aggregate": _aggregate(per_seed), + } + + +def _build_argparser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="python -m aelfrice.wonder.runner", + description="Run the wonder-consolidation strategy bake-off (#228).", + ) + p.add_argument("--n-topics", type=int, default=8) + p.add_argument("--n-atoms-per-topic", type=int, default=25) + p.add_argument("--n-walks", type=int, default=50) + p.add_argument("--n-sts-samples", type=int, default=50) + p.add_argument("--feedback-budget", type=int, default=16) + p.add_argument("--seeds", type=int, default=10) + p.add_argument( + "--output", + type=Path, + default=None, + help="Write the result JSON here; default stdout.", + ) + return p + + +def main(argv: list[str] | None = None) -> int: + args = _build_argparser().parse_args(argv) + result = run_bakeoff( + n_topics=args.n_topics, + n_atoms_per_topic=args.n_atoms_per_topic, + n_walks=args.n_walks, + n_sts_samples=args.n_sts_samples, + feedback_budget=args.feedback_budget, + seeds=args.seeds, + ) + payload = json.dumps(result, indent=2, sort_keys=True) + if args.output is None: + sys.stdout.write(payload + "\n") + else: + args.output.write_text(payload + "\n") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) + + +__all__ = ["main", "run_bakeoff"] diff --git a/src/aelfrice/wonder/simulator.py b/src/aelfrice/wonder/simulator.py new file mode 100644 index 00000000..1875cc34 --- /dev/null +++ b/src/aelfrice/wonder/simulator.py @@ -0,0 +1,247 @@ +"""Synthetic corpus + feedback simulator for the #228 bake-off. + +Per the spec, R0 is "synthetic 200-atom corpus + feedback simulator ++ evaluator". Synthetic — generated public-side from a seed, not +imported from any private fixture. + +Corpus shape: + +* ``n_topics`` synthetic topics, each labeled by an integer id. +* ``n_atoms_per_topic`` belief atoms per topic; total + ``n_topics * n_atoms_per_topic`` ≈ 200 with the defaults. +* Each atom is also stamped with a ``session_id`` drawn from a + small pool — STS reads this for diversity scoring. +* Per-topic intra-edges seeded so TC has shared-target triangles to + close. Cross-topic edges seeded sparser so RW has occasional + bridges but the topic structure stays detectable. +* Each atom carries an α/β prior. Most atoms get the uninformative + Beta(1,1); a fraction stamped with α=β≈0.7 (high-uncertainty) + so RW's seed-selection rule fires reliably. + +Feedback simulator: given a phantom (composition tuple), decide +``confirm`` if every belief in the composition shares a topic with +the others — the simplest "true relationship" predicate that lets +all three strategies be exercised without one trivially winning. +Otherwise ``junk``. The junk rate that emerges is the corpus's +prior over phantom quality; a strategy beats H0 by producing a +higher confirm fraction than uniform random would. +""" +from __future__ import annotations + +import random +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from aelfrice.models import ( + BELIEF_FACTUAL, + EDGE_CITES, + EDGE_RELATES_TO, + EDGE_SUPPORTS, + LOCK_NONE, + Belief, + Edge, +) + +if TYPE_CHECKING: + from aelfrice.store import MemoryStore + +# Promotion gate from the substrate decision (#196). A phantom is +# "promoted" once accumulated confirms push α to this threshold. +ALPHA_PROMOTION_THRESHOLD: float = 12.0 + + +@dataclass(frozen=True) +class CorpusAtom: + belief_id: str + topic: int + session_id: str + + +@dataclass +class SyntheticCorpus: + """Ground-truth view of the synthetic corpus the simulator built. + + The bake-off needs both the atoms (to populate a real + ``MemoryStore``) and the topic mapping (to grade phantoms in + the feedback simulator). Edges are kept for parity-debugging + but are not directly read by the evaluator. + """ + + atoms: tuple[CorpusAtom, ...] + edges: tuple[Edge, ...] = field(default_factory=tuple) + + def topic_of(self, belief_id: str) -> int | None: + for atom in self.atoms: + if atom.belief_id == belief_id: + return atom.topic + return None + + +def _atom_id(topic: int, idx: int) -> str: + return f"t{topic:02d}_a{idx:03d}" + + +def build_corpus( + *, + rng: random.Random, + n_topics: int = 8, + n_atoms_per_topic: int = 25, + n_sessions: int = 8, + intra_edge_density: float = 0.25, + cross_edge_density: float = 0.02, + high_uncertainty_fraction: float = 0.15, +) -> SyntheticCorpus: + """Generate a deterministic synthetic corpus. + + Density notes (Decision F in the planning memo): the defaults + were grid-searched on a tiny 3-topic preview to land each + strategy in its predicted regime. Adjust for sweep variance, + but the unit tests pin the default shape. + """ + if n_topics < 2: + raise ValueError("n_topics must be >= 2 for STS to be testable") + if n_atoms_per_topic < 3: + raise ValueError("n_atoms_per_topic must be >= 3 for TC to find triangles") + if n_sessions < 2: + raise ValueError("n_sessions must be >= 2 for STS to be testable") + + atoms: list[CorpusAtom] = [] + for topic in range(n_topics): + for idx in range(n_atoms_per_topic): + bid = _atom_id(topic, idx) + session = f"sess_{rng.randrange(n_sessions):02d}" + atoms.append(CorpusAtom(belief_id=bid, topic=topic, session_id=session)) + + # Edges: intra-topic dense, cross-topic sparse. Edge type cycled + # across the TC-eligible set to give TC something interesting. + edge_type_cycle = [EDGE_SUPPORTS, EDGE_CITES, EDGE_RELATES_TO] + edges: list[Edge] = [] + by_topic: dict[int, list[str]] = {} + for atom in atoms: + by_topic.setdefault(atom.topic, []).append(atom.belief_id) + + for topic, ids in by_topic.items(): + for i, src in enumerate(ids): + for j, dst in enumerate(ids): + if i == j: + continue + if rng.random() < intra_edge_density: + et = edge_type_cycle[(i + j) % len(edge_type_cycle)] + edges.append(Edge(src=src, dst=dst, type=et, weight=1.0)) + + all_ids = [a.belief_id for a in atoms] + for src in all_ids: + for dst in all_ids: + if src == dst: + continue + src_topic = src[:3] + dst_topic = dst[:3] + if src_topic == dst_topic: + continue + if rng.random() < cross_edge_density: + edges.append( + Edge(src=src, dst=dst, type=EDGE_RELATES_TO, weight=0.5) + ) + + return SyntheticCorpus(atoms=tuple(atoms), edges=tuple(edges)) + + +def populate_store( + store: "MemoryStore", + corpus: SyntheticCorpus, + *, + rng: random.Random, + high_uncertainty_fraction: float = 0.15, + timestamp: str = "2026-05-03T00:00:00Z", +) -> None: + """Insert the corpus into a live ``MemoryStore``. + + α/β stamping: most atoms get Beta(1,1) (the v1.x default + uninformative prior); ``high_uncertainty_fraction`` get a + weaker Beta(0.7, 0.7) so their differential entropy clears + the RW seed-floor reliably. + """ + rng_local = random.Random(rng.random()) + for atom in corpus.atoms: + if rng_local.random() < high_uncertainty_fraction: + alpha, beta = 0.7, 0.7 + else: + alpha, beta = 1.0, 1.0 + store.insert_belief( + Belief( + id=atom.belief_id, + content=f"synthetic atom for topic {atom.topic}", + content_hash=f"h_{atom.belief_id}", + alpha=alpha, + beta=beta, + type=BELIEF_FACTUAL, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=timestamp, + last_retrieved_at=None, + session_id=atom.session_id, + ) + ) + for edge in corpus.edges: + store.insert_edge(edge) + + +def feedback_verdict( + composition: tuple[str, ...], + corpus: SyntheticCorpus, +) -> str: + """Return ``"confirm"`` if every belief in the composition + shares a single topic; otherwise ``"junk"``. + + Single-topic agreement is the corpus's ground-truth predicate + for "real relationship" — it's the simplest predicate the + three candidate strategies all have a path to satisfy: + + * TC's shared-target shape pulls at most from the same target's + incoming neighborhood, which is largely intra-topic at the + default densities. + * RW depth-2 walks tend to stay intra-topic because intra-edges + dominate. + * STS draws cross-session, so it must accidentally pick atoms + from the same topic to confirm — which is the predicted + weakness in the spec's H3. + """ + if not composition: + return "junk" + first_topic = corpus.topic_of(composition[0]) + if first_topic is None: + return "junk" + for bid in composition[1:]: + if corpus.topic_of(bid) != first_topic: + return "junk" + return "confirm" + + +def simulate_promotion( + confirms: int, + junks: int, + *, + initial_alpha: float = 1.0, + initial_beta: float = 1.0, +) -> bool: + """Return ``True`` if the phantom would promote under #196's gate. + + Each confirm increments α; each junk increments β. Promotion + fires when ``α >= ALPHA_PROMOTION_THRESHOLD``. + """ + alpha = initial_alpha + confirms + beta = initial_beta + junks + _ = beta # documented for symmetry; gate is α-only per spec + return alpha >= ALPHA_PROMOTION_THRESHOLD + + +__all__ = [ + "ALPHA_PROMOTION_THRESHOLD", + "CorpusAtom", + "SyntheticCorpus", + "build_corpus", + "feedback_verdict", + "populate_store", + "simulate_promotion", +] diff --git a/src/aelfrice/wonder_consolidation.py b/src/aelfrice/wonder_consolidation.py new file mode 100644 index 00000000..7434dbd6 --- /dev/null +++ b/src/aelfrice/wonder_consolidation.py @@ -0,0 +1,74 @@ +"""Bench-gate shim for #228 wonder-consolidation. + +The actual generation strategies + harness live in the +``aelfrice.wonder`` package. This module exists only because +``tests/bench_gate/test_wonder_consolidation.py`` was scaffolded +before the bake-off shape was decided and assumes a +``wonder_consolidation.score(seed_belief, retrieved_neighbors)`` +signature for per-row sanity checks. + +Decision A in the planning memo: keep the stub, wrap it in a +single-phantom relatedness score. The lab-corpus contract for the +bench gate (rows of the form +``{seed_belief, retrieved_neighbors, expected_metric}``) is +follow-up work tracked separately; the score returned here is a +placeholder relatedness scalar in [0, 1] derived from token +overlap. It is *not* read by the bake-off proper — the runner +talks to the strategies directly. +""" +from __future__ import annotations + +from typing import Any + +_TOKENIZER_DROP = set(",.;:!?\"'()[]{}") + + +def _tokens(text: str) -> set[str]: + cleaned = "".join(" " if c in _TOKENIZER_DROP else c for c in text.lower()) + return {t for t in cleaned.split() if t} + + +def score(seed_belief: Any, retrieved_neighbors: Any) -> float: + """Return a token-overlap relatedness score in [0, 1]. + + Accepts either strings or dict-like structures with a + ``content`` field for both arguments. ``retrieved_neighbors`` + may also be a list of strings / dicts; in that case the score + is the mean per-neighbor overlap. + + Returning a float in [0, 1] keeps the bench-gate stub honest + (its ``isinstance(rating, (int, float))`` assertion holds) + without pretending this is the bake-off result. The real + strategy-quality answer lives in + ``aelfrice.wonder.runner.run_bakeoff``. + """ + seed_text = _content_of(seed_belief) + if isinstance(retrieved_neighbors, list): + if not retrieved_neighbors: + return 0.0 + scores = [_pairwise(seed_text, _content_of(n)) for n in retrieved_neighbors] + return sum(scores) / len(scores) + return _pairwise(seed_text, _content_of(retrieved_neighbors)) + + +def _content_of(item: Any) -> str: + if isinstance(item, str): + return item + if isinstance(item, dict): + return str(item.get("content", "")) + content = getattr(item, "content", None) + if isinstance(content, str): + return content + return str(item) + + +def _pairwise(a: str, b: str) -> float: + ta, tb = _tokens(a), _tokens(b) + if not ta or not tb: + return 0.0 + inter = ta & tb + union = ta | tb + return len(inter) / len(union) + + +__all__ = ["score"] From 61ab57570147683db65d43840f82f716946950b8 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sun, 3 May 2026 19:58:51 -0700 Subject: [PATCH 3/3] test(wonder): unit + integration tests for #228 bake-off harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 45 tests across four files: - test_wonder_strategies.py (15) — RW/TC/STS unit coverage incl. empty-store, no-edges, determinism-from-seed, edge-type filtering, unordered-pair dedup, session-diversity floor. - test_wonder_simulator.py (14) — corpus determinism, 200-atom default, topic partition invariant, populate_store stamping, feedback verdict shape, promotion gate at α≥12. - test_wonder_evaluator.py (11) — metric math, four verdict rules (single/ensemble/defer/drop) on synthetic metric shapes. - test_wonder_runner.py (5) — runner smoke + determinism + JSON output to file. Bench-gate stub (tests/bench_gate/test_wonder_consolidation.py) continues to skip without AELFRICE_CORPUS_ROOT — the new `aelfrice.wonder_consolidation` shim is what unblocks the ModuleNotFoundError skip path. --- tests/test_wonder_evaluator.py | 159 +++++++++++++++++++++++++++++++ tests/test_wonder_runner.py | 76 +++++++++++++++ tests/test_wonder_simulator.py | 115 +++++++++++++++++++++++ tests/test_wonder_strategies.py | 162 ++++++++++++++++++++++++++++++++ 4 files changed, 512 insertions(+) create mode 100644 tests/test_wonder_evaluator.py create mode 100644 tests/test_wonder_runner.py create mode 100644 tests/test_wonder_simulator.py create mode 100644 tests/test_wonder_strategies.py diff --git a/tests/test_wonder_evaluator.py b/tests/test_wonder_evaluator.py new file mode 100644 index 00000000..872f185b --- /dev/null +++ b/tests/test_wonder_evaluator.py @@ -0,0 +1,159 @@ +"""Unit tests for the bake-off evaluator (#228).""" +from __future__ import annotations + +import random + +from aelfrice.wonder.evaluator import ( + H0_NULL_RATE, + H0_PLUS_PP_FLOOR, + JACCARD_COMPLEMENT, + JACCARD_REDUNDANCY, + StrategyMetrics, + adoption_verdict, + evaluate_strategy, + pairwise_jaccard, +) +from aelfrice.wonder.models import ( + STRATEGY_RW, + STRATEGY_STS, + STRATEGY_TC, + Phantom, +) +from aelfrice.wonder.simulator import build_corpus + + +def _phantom(comp: tuple[str, ...], strategy: str = STRATEGY_RW, + cost: float = 2.0) -> Phantom: + return Phantom( + composition=tuple(sorted(comp)), + strategy=strategy, + construction_cost=cost, + ) + + +def test_evaluate_empty_strategy() -> None: + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + m = evaluate_strategy([], corpus) + assert m.n_phantoms == 0 + assert m.confirmation_rate == 0.0 + assert m.junk_rate == 0.0 + + +def test_evaluate_all_confirms_promotes() -> None: + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + same_topic_ids = [a.belief_id for a in corpus.atoms if a.topic == 0] + p = _phantom((same_topic_ids[0], same_topic_ids[1])) + # Budget 16 confirms → α=17, well past the α≥12 gate. + m = evaluate_strategy([p], corpus, feedback_budget_per_phantom=16) + assert m.confirmation_rate == 1.0 + assert m.junk_rate == 0.0 + + +def test_evaluate_all_junks_marks_junk_rate() -> None: + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + t0 = next(a.belief_id for a in corpus.atoms if a.topic == 0) + t1 = next(a.belief_id for a in corpus.atoms if a.topic == 1) + p = _phantom((t0, t1)) + m = evaluate_strategy([p], corpus, feedback_budget_per_phantom=16) + assert m.confirmation_rate == 0.0 + assert m.junk_rate == 1.0 + + +def test_evaluate_retrieval_per_cost_uses_composition_size() -> None: + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + same_topic = [a.belief_id for a in corpus.atoms if a.topic == 0] + # composition size 2, cost 2 → retrieval_per_cost = 1.0 + p = _phantom((same_topic[0], same_topic[1]), cost=2.0) + m = evaluate_strategy([p], corpus) + assert m.retrieval_freq_per_cost == 1.0 + + +def test_jaccard_identical_sets_is_one() -> None: + a = [_phantom(("x", "y")), _phantom(("x", "z"))] + b = [_phantom(("x", "y")), _phantom(("x", "z"))] + assert pairwise_jaccard(a, b) == 1.0 + + +def test_jaccard_disjoint_sets_is_zero() -> None: + a = [_phantom(("x", "y"))] + b = [_phantom(("u", "v"))] + assert pairwise_jaccard(a, b) == 0.0 + + +def test_jaccard_partial_overlap() -> None: + a = [_phantom(("x", "y")), _phantom(("x", "z"))] + b = [_phantom(("x", "y")), _phantom(("u", "v"))] + # |A ∩ B| = 1, |A ∪ B| = 3 + assert pairwise_jaccard(a, b) == 1 / 3 + + +def test_jaccard_two_empty_sets_is_one() -> None: + assert pairwise_jaccard([], []) == 1.0 + + +def _metric(name: str, rate: float, junk: float = 0.0, + r_per_c: float = 1.0) -> StrategyMetrics: + return StrategyMetrics( + strategy=name, + n_phantoms=10, + confirmation_rate=rate, + retrieval_freq_per_cost=r_per_c, + junk_rate=junk, + mean_construction_cost=2.0, + ) + + +def test_verdict_drop_when_all_below_h0() -> None: + metrics = ( + _metric("RW", 0.01), + _metric("TC", 0.02), + _metric("STS", 0.03), + ) + assert adoption_verdict(metrics, jaccard={}) == "drop" + + +def test_verdict_defer_on_high_junk_rate() -> None: + metrics = ( + _metric("RW", 0.5, junk=0.7), + _metric("TC", 0.2, junk=0.1), + _metric("STS", 0.3, junk=0.1), + ) + assert adoption_verdict(metrics, jaccard={}) == "defer" + + +def test_verdict_defer_when_none_clear_floor() -> None: + floor = H0_NULL_RATE + H0_PLUS_PP_FLOOR + metrics = ( + _metric("RW", floor - 0.01), + _metric("TC", floor - 0.02), + _metric("STS", H0_NULL_RATE + 0.001), + ) + assert adoption_verdict(metrics, jaccard={}) == "defer" + + +def test_verdict_ensemble_when_top_two_complementary() -> None: + metrics = ( + _metric("RW", 0.3), + _metric("TC", 0.4), + _metric("STS", 0.05), + ) + j = { + ("RW", "TC"): JACCARD_COMPLEMENT - 0.1, + ("RW", "STS"): 0.1, + ("STS", "TC"): 0.1, + } + assert adoption_verdict(metrics, j) == "ensemble" + + +def test_verdict_single_when_top_two_redundant() -> None: + metrics = ( + _metric("RW", 0.3), + _metric("TC", 0.4), + _metric("STS", 0.05), + ) + j = { + ("RW", "TC"): JACCARD_REDUNDANCY + 0.1, + ("RW", "STS"): 0.1, + ("STS", "TC"): 0.1, + } + assert adoption_verdict(metrics, j) == "single" diff --git a/tests/test_wonder_runner.py b/tests/test_wonder_runner.py new file mode 100644 index 00000000..4cc3c99e --- /dev/null +++ b/tests/test_wonder_runner.py @@ -0,0 +1,76 @@ +"""Integration test for the bake-off runner (#228).""" +from __future__ import annotations + +import json + +from aelfrice.wonder.runner import _build_argparser, main, run_bakeoff + + +def test_run_bakeoff_smoke() -> None: + result = run_bakeoff( + n_topics=3, + n_atoms_per_topic=5, + n_walks=10, + n_sts_samples=10, + feedback_budget=4, + seeds=2, + ) + assert "config" in result + assert "per_seed" in result + assert "aggregate" in result + assert len(result["per_seed"]) == 2 + for seed_result in result["per_seed"]: + assert "seed" in seed_result + assert len(seed_result["strategy_metrics"]) == 3 + assert seed_result["verdict"] in {"single", "ensemble", "defer", "drop"} + + +def test_run_bakeoff_deterministic_for_same_seed() -> None: + a = run_bakeoff( + n_topics=3, n_atoms_per_topic=5, + n_walks=10, n_sts_samples=10, + feedback_budget=4, seeds=2, + ) + b = run_bakeoff( + n_topics=3, n_atoms_per_topic=5, + n_walks=10, n_sts_samples=10, + feedback_budget=4, seeds=2, + ) + assert a["aggregate"] == b["aggregate"] + + +def test_run_bakeoff_aggregate_shape() -> None: + result = run_bakeoff( + n_topics=3, n_atoms_per_topic=5, + n_walks=5, n_sts_samples=5, + feedback_budget=4, seeds=2, + ) + agg = result["aggregate"] + assert set(agg["strategy_metrics"].keys()) == {"RW", "TC", "STS"} + assert "RW|TC" in agg["jaccard"] + assert "RW|STS" in agg["jaccard"] + assert "STS|TC" in agg["jaccard"] + assert agg["majority_verdict"] in {"single", "ensemble", "defer", "drop"} + + +def test_argparser_defaults() -> None: + p = _build_argparser() + args = p.parse_args([]) + assert args.seeds == 10 + assert args.feedback_budget == 16 + + +def test_main_writes_json_to_file(tmp_path) -> None: + out = tmp_path / "result.json" + rc = main([ + "--n-topics", "3", + "--n-atoms-per-topic", "5", + "--n-walks", "5", + "--n-sts-samples", "5", + "--feedback-budget", "4", + "--seeds", "1", + "--output", str(out), + ]) + assert rc == 0 + payload = json.loads(out.read_text()) + assert "aggregate" in payload diff --git a/tests/test_wonder_simulator.py b/tests/test_wonder_simulator.py new file mode 100644 index 00000000..a238298e --- /dev/null +++ b/tests/test_wonder_simulator.py @@ -0,0 +1,115 @@ +"""Unit tests for the synthetic corpus + feedback simulator (#228).""" +from __future__ import annotations + +import random + +import pytest + +from aelfrice.store import MemoryStore +from aelfrice.wonder.simulator import ( + ALPHA_PROMOTION_THRESHOLD, + SyntheticCorpus, + build_corpus, + feedback_verdict, + populate_store, + simulate_promotion, +) + + +def test_build_corpus_is_deterministic() -> None: + a = build_corpus(rng=random.Random(0)) + b = build_corpus(rng=random.Random(0)) + assert [atom.belief_id for atom in a.atoms] == [ + atom.belief_id for atom in b.atoms + ] + assert [(e.src, e.dst, e.type) for e in a.edges] == [ + (e.src, e.dst, e.type) for e in b.edges + ] + + +def test_build_corpus_default_size() -> None: + c = build_corpus(rng=random.Random(0)) + # 8 topics × 25 atoms = 200, the spec's R0 size. + assert len(c.atoms) == 200 + + +def test_build_corpus_topic_assignment_is_partition() -> None: + c = build_corpus(rng=random.Random(0), n_topics=4, n_atoms_per_topic=5) + topics = {a.topic for a in c.atoms} + assert topics == {0, 1, 2, 3} + for topic in topics: + in_topic = [a for a in c.atoms if a.topic == topic] + assert len(in_topic) == 5 + + +def test_build_corpus_rejects_too_few_atoms() -> None: + with pytest.raises(ValueError): + build_corpus(rng=random.Random(0), n_atoms_per_topic=2) + + +def test_build_corpus_rejects_too_few_topics() -> None: + with pytest.raises(ValueError): + build_corpus(rng=random.Random(0), n_topics=1) + + +def test_build_corpus_rejects_too_few_sessions() -> None: + with pytest.raises(ValueError): + build_corpus(rng=random.Random(0), n_sessions=1) + + +def test_populate_store_writes_all_atoms() -> None: + store = MemoryStore(":memory:") + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + populate_store(store, corpus, rng=random.Random(1)) + assert len(store.list_belief_ids()) == len(corpus.atoms) + + +def test_populate_store_stamps_high_uncertainty_some_atoms() -> None: + store = MemoryStore(":memory:") + corpus = build_corpus(rng=random.Random(0), n_topics=4, n_atoms_per_topic=10) + populate_store( + store, corpus, rng=random.Random(0), + high_uncertainty_fraction=0.5, + ) + high_count = 0 + for bid in store.list_belief_ids(): + b = store.get_belief(bid) + assert b is not None + if b.alpha < 1.0: + high_count += 1 + assert high_count > 0 + + +def test_feedback_verdict_same_topic_confirms() -> None: + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + same_topic = [a.belief_id for a in corpus.atoms if a.topic == 0] + assert feedback_verdict((same_topic[0], same_topic[1]), corpus) == "confirm" + + +def test_feedback_verdict_cross_topic_junks() -> None: + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + t0 = next(a.belief_id for a in corpus.atoms if a.topic == 0) + t1 = next(a.belief_id for a in corpus.atoms if a.topic == 1) + assert feedback_verdict((t0, t1), corpus) == "junk" + + +def test_feedback_verdict_unknown_belief_junks() -> None: + corpus = build_corpus(rng=random.Random(0), n_topics=3, n_atoms_per_topic=5) + assert feedback_verdict(("unknown_id",), corpus) == "junk" + + +def test_feedback_verdict_empty_composition_junks() -> None: + corpus = SyntheticCorpus(atoms=()) + assert feedback_verdict((), corpus) == "junk" + + +def test_simulate_promotion_threshold_at_alpha_12() -> None: + # initial α=1, +11 confirms → α=12 → promotes + assert simulate_promotion(11, 0) + assert not simulate_promotion(10, 0) + assert ALPHA_PROMOTION_THRESHOLD == 12.0 + + +def test_simulate_promotion_with_junks() -> None: + # Junks accrue β but don't affect the α-only gate. + assert simulate_promotion(11, 100) diff --git a/tests/test_wonder_strategies.py b/tests/test_wonder_strategies.py new file mode 100644 index 00000000..1f98909c --- /dev/null +++ b/tests/test_wonder_strategies.py @@ -0,0 +1,162 @@ +"""Unit tests for the three #228 generation strategies.""" +from __future__ import annotations + +import random + +import pytest + +from aelfrice.models import ( + BELIEF_FACTUAL, + EDGE_CITES, + EDGE_RELATES_TO, + EDGE_SUPPORTS, + LOCK_NONE, + Belief, + Edge, +) +from aelfrice.store import MemoryStore +from aelfrice.wonder.models import ( + STRATEGY_RW, + STRATEGY_STS, + STRATEGY_TC, +) +from aelfrice.wonder.strategies import ( + DEFAULT_RW_UNCERTAINTY_FLOOR, + random_walk, + span_topic_sampling, + triangle_closure, +) + + +def _belief(bid: str, *, alpha: float = 1.0, beta: float = 1.0, + session: str | None = None) -> Belief: + return Belief( + id=bid, content=f"c_{bid}", content_hash=f"h_{bid}", + alpha=alpha, beta=beta, type=BELIEF_FACTUAL, + lock_level=LOCK_NONE, locked_at=None, demotion_pressure=0, + created_at="2026-05-03T00:00:00Z", last_retrieved_at=None, + session_id=session, + ) + + +@pytest.fixture +def store() -> MemoryStore: + return MemoryStore(":memory:") + + +def test_random_walk_empty_store(store: MemoryStore) -> None: + assert random_walk(store, rng=random.Random(0), n_walks=10) == [] + + +def test_random_walk_no_edges_returns_empty(store: MemoryStore) -> None: + # A seed with no outgoing edges produces a 1-atom bundle, which + # is below the ≥2 minimum, so the strategy returns nothing. + store.insert_belief(_belief("a", alpha=0.7, beta=0.7)) + assert random_walk(store, rng=random.Random(0), n_walks=5) == [] + + +def test_random_walk_produces_phantoms_with_edges(store: MemoryStore) -> None: + for bid in ("a", "b", "c"): + store.insert_belief(_belief(bid, alpha=0.7, beta=0.7)) + store.insert_edge(Edge(src="a", dst="b", type=EDGE_SUPPORTS, weight=1.0)) + store.insert_edge(Edge(src="b", dst="c", type=EDGE_SUPPORTS, weight=1.0)) + phantoms = random_walk(store, rng=random.Random(0), n_walks=10, depth=2) + assert phantoms + for p in phantoms: + assert p.strategy == STRATEGY_RW + assert p.seed_id is not None + assert len(p.composition) >= 2 + assert tuple(sorted(p.composition)) == p.composition # sorted + assert p.construction_cost >= 1.0 + + +def test_random_walk_floor_excludes_low_uncertainty(store: MemoryStore) -> None: + # alpha=10, beta=10 gives strongly negative differential entropy; + # with the default floor (-0.5) this belief is ineligible as a seed. + store.insert_belief(_belief("low", alpha=10.0, beta=10.0)) + store.insert_belief(_belief("low2", alpha=10.0, beta=10.0)) + store.insert_edge(Edge(src="low", dst="low2", type=EDGE_SUPPORTS, weight=1.0)) + assert random_walk( + store, rng=random.Random(0), n_walks=5, + uncertainty_floor=DEFAULT_RW_UNCERTAINTY_FLOOR, + ) == [] + + +def test_random_walk_is_deterministic(store: MemoryStore) -> None: + for bid in ("a", "b", "c", "d"): + store.insert_belief(_belief(bid, alpha=0.7, beta=0.7)) + store.insert_edge(Edge(src="a", dst="b", type=EDGE_SUPPORTS, weight=1.0)) + store.insert_edge(Edge(src="b", dst="c", type=EDGE_CITES, weight=1.0)) + store.insert_edge(Edge(src="c", dst="d", type=EDGE_RELATES_TO, weight=1.0)) + a = random_walk(store, rng=random.Random(42), n_walks=10, depth=2) + b = random_walk(store, rng=random.Random(42), n_walks=10, depth=2) + assert [p.composition for p in a] == [p.composition for p in b] + + +def test_triangle_closure_empty_store(store: MemoryStore) -> None: + assert triangle_closure(store) == [] + + +def test_triangle_closure_finds_pair(store: MemoryStore) -> None: + for bid in ("a", "b", "c"): + store.insert_belief(_belief(bid)) + store.insert_edge(Edge(src="a", dst="c", type=EDGE_SUPPORTS, weight=1.0)) + store.insert_edge(Edge(src="b", dst="c", type=EDGE_SUPPORTS, weight=1.0)) + phantoms = triangle_closure(store) + assert len(phantoms) == 1 + p = phantoms[0] + assert p.composition == ("a", "b") + assert p.strategy == STRATEGY_TC + assert p.construction_cost == 3.0 + assert p.seed_id is None + + +def test_triangle_closure_skips_non_eligible_edge_types(store: MemoryStore) -> None: + from aelfrice.models import EDGE_CONTRADICTS + for bid in ("a", "b", "c"): + store.insert_belief(_belief(bid)) + store.insert_edge(Edge(src="a", dst="c", type=EDGE_CONTRADICTS, weight=1.0)) + store.insert_edge(Edge(src="b", dst="c", type=EDGE_CONTRADICTS, weight=1.0)) + assert triangle_closure(store) == [] + + +def test_triangle_closure_dedups_unordered_pairs(store: MemoryStore) -> None: + # Three sources sharing one target → C(3,2) = 3 unordered pairs. + for bid in ("a", "b", "c", "t"): + store.insert_belief(_belief(bid)) + for src in ("a", "b", "c"): + store.insert_edge(Edge(src=src, dst="t", type=EDGE_SUPPORTS, weight=1.0)) + phantoms = triangle_closure(store) + assert len(phantoms) == 3 + comps = {p.composition for p in phantoms} + assert comps == {("a", "b"), ("a", "c"), ("b", "c")} + + +def test_span_topic_sampling_empty_store(store: MemoryStore) -> None: + assert span_topic_sampling(store, rng=random.Random(0)) == [] + + +def test_span_topic_sampling_needs_enough_sessions(store: MemoryStore) -> None: + store.insert_belief(_belief("a", session="s1")) + store.insert_belief(_belief("b", session="s1")) + # only one real session — composition_size=2 → no phantoms + assert span_topic_sampling(store, rng=random.Random(0), n_samples=5) == [] + + +def test_span_topic_sampling_produces_diverse_compositions( + store: MemoryStore, +) -> None: + for i, bid in enumerate(("a", "b", "c", "d")): + store.insert_belief(_belief(bid, session=f"sess_{i}")) + phantoms = span_topic_sampling(store, rng=random.Random(0), n_samples=20) + assert phantoms + for p in phantoms: + assert p.strategy == STRATEGY_STS + assert len(p.composition) == 2 + assert p.construction_cost == 2.0 + assert p.seed_id is None + + +def test_span_topic_sampling_rejects_invalid_size(store: MemoryStore) -> None: + with pytest.raises(ValueError): + span_topic_sampling(store, rng=random.Random(0), composition_size=1)