diff --git a/CHANGELOG.md b/CHANGELOG.md index edb50ebda..30eacb2d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ installable release; see the roadmap in [README.md](README.md). ### Added +- **Context-rebuilder eval harness — three integration points wired** ([#592](https://github.com/robotrocketscience/aelfrice/issues/592)). `benchmarks/context-rebuilder/eval_harness.py` was a skeleton with five `NotImplementedError` stubs since v1.2.0; this lands the three pure-Python wirings that make `--mode threshold-sweep` and `--mode budget-sweep` runnable end-to-end. `replay_to_fork` populates a fresh in-memory `MemoryStore` via `ingest_jsonl` against a tempfile holding the first `fork_turn` lines of the case JSONL. `run_rebuilder` calls `rebuild_v14()` with `floor_session=floor_l1=trigger_threshold` (mapping the harness's threshold-sweep axis onto the v1.7 per-lane composite-score floor) and returns `(rebuilt_block, latency_ms)` measured via `time.monotonic()`. `measure_token_cost` returns `estimate_tokens(rebuilt) / estimate_tokens(pre_clear)` using `benchmarks.context_rebuilder.measure.estimate_tokens` (the 4-chars-per-token heuristic that mirrors `aelfrice.context_rebuilder._CHARS_PER_TOKEN`), so harness measurements stay aligned with the rebuilder's own budget bookkeeping. `replay_post_fork` is a non-crashing stub that returns one placeholder per `eval_turn` (`{turn_idx, expected, actual="", matched=False, reason="needs_replay_client"}`) so the harness produces valid latency + token-cost numbers before the model-invocation client lands; `score_fidelity` reads `matched=False` and returns 0.0 instead of raising. Adds `debugging_session_001.meta.json` next to the bundled 16-turn synthetic fixture (midpoint fork at turn 8, eval_turns at indices 8/10/12/14) so `--corpus benchmarks/context-rebuilder/fixtures/synthetic/` resolves out of the box. 15 new tests in `tests/test_context_rebuilder_eval_harness_wiring.py` cover each wired function plus an end-to-end threshold-sweep smoke against the bundled fixture. The model-invocation client + LLM judge stages are explicit follow-ups; this commit lands the harness's runnable contract before fidelity scoring engages. + - **Session-end Stop hook prompts to lock session corrections** ([#582](https://github.com/robotrocketscience/aelfrice/issues/582)). New `aelf-stop-hook` (default-on) fires once per assistant-turn end, walks the store for unlocked correction-class beliefs created in the current `session_id`, and emits a `` block to stderr listing each candidate with a pre-filled `aelf lock --statement '<...>'` command. Candidate filter: `session_id == current AND lock_level != LOCK_USER AND (type == BELIEF_CORRECTION OR origin in {agent_inferred, agent_remembered})`. Hook is informational by default; setting `AELF_AUTOLOCK_CORRECTIONS=1` in the environment makes it auto-lock the candidates instead (logs each lock to stderr for transparency). Wired into `aelf setup` / `aelf unsetup` (`--no-stop-hook` opts out) and into `aelf doctor` as a fourth default-on auto-capture hook the v2.1 nag flags when missing. Coexists with the existing transcript-ingest Stop entry as a separate entry under the same `hooks.Stop` event key. **Note**: the issue's spec assumed a `aelf correct` CLI command and a `feedback_history.kind=correct` marker that don't exist on `main` (no historical implementation); the v0 detection signal is correction-class beliefs in the current session instead. Future work to ship `aelf correct` + `feedback_history.kind` would let this hook also surface beliefs that were *modified* (not only newly-created) in the session. - **Dep-graph render workflow — Mermaid graph of open-issue Blocks/Blocked-by links** ([#581](https://github.com/robotrocketscience/aelfrice/issues/581)). New `.github/workflows/dep-graph-render.yml` (daily cron + `workflow_dispatch`) drives `scripts/dep_graph_render.py`, which pages the GraphQL `trackedIssues` / `trackedInIssues` connections for open issues matching `--label` (default `v2.1`), renders a deterministic `graph TD` block, and posts/updates a sticky comment (`` marker) on the milestone tracker (default #474). Nodes are class-coloured: red = has open blockers, green = leaf (work-ready), yellow = in-progress (any assignee or any `author-*` label). Determinism is byte-stable across runs so unchanged data produces a noop comment edit. Closes the parallel-session work-finding gap where each session had to scan all open issues to spot a leaf node. diff --git a/benchmarks/context-rebuilder/eval_harness.py b/benchmarks/context-rebuilder/eval_harness.py index 7d721b372..d4a14ebf5 100644 --- a/benchmarks/context-rebuilder/eval_harness.py +++ b/benchmarks/context-rebuilder/eval_harness.py @@ -18,10 +18,31 @@ import argparse import json import statistics +import sys +import tempfile import time from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Literal +from typing import Final, Literal + +# Allow `python benchmarks/context-rebuilder/eval_harness.py` to import +# aelfrice and the benchmarks.context_rebuilder package. The hyphenated +# parent directory blocks `python -m`, so a sys.path shim is the +# alternative. Tests already get the repo root from pytest's testpaths +# config and skip this branch as a no-op. +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from aelfrice.context_rebuilder import ( # noqa: E402 + DEFAULT_N_RECENT_TURNS, + DEFAULT_REBUILDER_TOKEN_BUDGET, + RecentTurn, + rebuild_v14, +) +from aelfrice.ingest import ingest_jsonl # noqa: E402 +from aelfrice.store import MemoryStore # noqa: E402 +from benchmarks.context_rebuilder.measure import estimate_tokens # noqa: E402 Mode = Literal["threshold-sweep", "budget-sweep", "dynamic", "regression"] @@ -82,29 +103,120 @@ def load_corpus(corpus_dir: Path) -> list[TranscriptCase]: return cases -def replay_to_fork(case: TranscriptCase) -> "object": - """Populate a fresh aelfrice store with turns 0..fork_turn-1. +def replay_to_fork(case: TranscriptCase) -> MemoryStore: + """Populate a fresh in-memory aelfrice store with turns + 0..fork_turn-1 from `case.path`. - Returns the store handle. TODO: wire to MemoryStore + ingest_jsonl - once docs/transcript_ingest.md ships. + Returns the store handle. Caller is responsible for closing it. + + Lines are counted by file order — this matches the synthetic-fixture + convention where each line is one user/assistant turn. Compaction + markers and tool-result lines are passed through to ingest_jsonl, + which already skips them; they don't shift the fork boundary + relative to the underlying transcript file. + + Implementation note: ingest_jsonl reads a file end-to-end, so we + write the first `fork_turn` lines to a tempfile and ingest that. + Modifying ingest_jsonl to take a line-count cap would couple the + production ingest path to harness internals; keeping the truncation + here is the lighter coupling. + """ + store = MemoryStore(":memory:") + if case.fork_turn <= 0: + return store + with tempfile.NamedTemporaryFile( + mode="w", suffix=".jsonl", delete=False, encoding="utf-8" + ) as tmp: + tmp_path = Path(tmp.name) + with case.path.open("r", encoding="utf-8") as src: + for i, line in enumerate(src): + if i >= case.fork_turn: + break + tmp.write(line) + try: + ingest_jsonl(store, tmp_path, source_label="eval-harness") + finally: + tmp_path.unlink(missing_ok=True) + return store + + +def _recent_turns_pre_fork( + case: TranscriptCase, n: int = DEFAULT_N_RECENT_TURNS, +) -> list[RecentTurn]: + """Build the last `n` RecentTurn records from turns 0..fork_turn-1. + + Mirrors the production hook's adapter: each line is a JSON object + with `role` and `text`, plus an optional `session_id`. Lines that + don't carry role/text are skipped (compaction markers, malformed). """ - raise NotImplementedError("Wire to ingest_jsonl from transcript_ingest spec") + turns: list[RecentTurn] = [] + if case.fork_turn <= 0: + return turns + with case.path.open("r", encoding="utf-8") as f: + for i, line in enumerate(f): + if i >= case.fork_turn: + break + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + role = obj.get("role") + text = obj.get("text") + if not isinstance(role, str) or not isinstance(text, str): + continue + sid = obj.get("session_id") + turns.append( + RecentTurn( + role=role, text=text, + session_id=sid if isinstance(sid, str) else None, + ) + ) + return turns[-n:] def run_rebuilder( - store: object, + store: MemoryStore, case: TranscriptCase, *, trigger_threshold: float, token_budget: int, ) -> tuple[str, float]: - """Run the rebuilder against the store at the fork point. + """Run the v1.4 rebuilder against the store at the fork point. + + Returns `(rebuilt_context_block, latency_ms)`. Latency is measured + via `time.monotonic()` so it stays monotonic by construction. + + `trigger_threshold` is wired to `rebuild_v14`'s per-lane composite- + score floors (`floor_session` and `floor_l1`). The v1.7 floor knob + is the closest existing rebuild-time parameter to the harness's + "threshold-sweep" axis; sweeping it answers "at what relevance + floor does the rebuilder still pack useful content for this task?" + L0 locked beliefs are unaffected by the floor — they always pack + per the documented v1.4 contract. - Returns (rebuilt_context_block, latency_ms). TODO: wire to - aelfrice.context_rebuilder.rebuild() once context_rebuilder spec - ships an implementation. + `token_budget` is forwarded directly to `rebuild_v14`. """ - raise NotImplementedError("Wire to context_rebuilder.rebuild()") + recent = _recent_turns_pre_fork(case) + t0 = time.monotonic() + rebuilt = rebuild_v14( + recent, store, + token_budget=token_budget, + floor_session=trigger_threshold, + floor_l1=trigger_threshold, + ) + latency_ms = (time.monotonic() - t0) * 1000.0 + return rebuilt, latency_ms + + +REPLAY_PENDING_REASON: Final[str] = "needs_replay_client" +"""Marker emitted by `replay_post_fork` until the model-invocation +client lands. Surfaced in `failures[].reason` so summary readers can +distinguish "skipped, judge required" from a real fidelity miss.""" def replay_post_fork( @@ -113,14 +225,59 @@ def replay_post_fork( ) -> list[dict]: """Replay turns fork_turn..end with rebuilt_context as session start. - Returns one dict per eval_turn with keys: - {"turn_idx": int, "expected": str, "actual": str, "matched": bool}. + **Stub until the model-invocation client lands** (#592 commit-2 / -3). + Returns one placeholder dict per `case.eval_turns`: - TODO: wire to a Claude API client once we decide on the eval - invocation surface. Initially can be agent-API-driven; later may - move to a Claude-Code-harness-equivalent for higher fidelity. + {"turn_idx": int, "expected": str, "actual": "", + "matched": False, "reason": REPLAY_PENDING_REASON} + + `expected` is pulled from the captured transcript's `text` field at + the eval-turn line index (1-indexed via the file order, matching the + fork_turn convention). When the line is missing or malformed the + expected text falls back to "" so the harness still produces a + coherent record. + + Threading this stub keeps `run_one` end-to-end runnable: `score_fidelity` + reads `matched=False` and returns 0.0; `failures` populates with a + machine-readable `reason` so threshold-sweep / budget-sweep modes + surface latency + token-cost numbers without crashing on the + fidelity step. The real model client lands in a follow-up commit. """ - raise NotImplementedError("Wire to model invocation client") + expected_by_idx: dict[int, str] = {} + if case.eval_turns: + wanted = set(case.eval_turns) + try: + with case.path.open("r", encoding="utf-8") as f: + for i, line in enumerate(f): + if i not in wanted: + continue + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + t = obj.get("text") + if isinstance(t, str): + expected_by_idx[i] = t + except OSError: + # Unreadable file → return an empty list rather than partial + # placeholders. The harness's load_corpus already filtered + # this case out for the corpus-walking path; this guard is + # for direct-call use. + return [] + return [ + { + "turn_idx": idx, + "expected": expected_by_idx.get(idx, ""), + "actual": "", + "matched": False, + "reason": REPLAY_PENDING_REASON, + } + for idx in case.eval_turns + ] def score_fidelity(replay_results: list[dict]) -> float: @@ -136,16 +293,56 @@ def score_fidelity(replay_results: list[dict]) -> float: return matched / len(replay_results) +def _pre_clear_text(case: TranscriptCase) -> str: + """Concatenate the `text` fields of every turn in 0..fork_turn-1. + + This is the harness's stand-in for "the context the agent had open + just before the clear" — a deterministic, model-agnostic measure + that doesn't require replaying the transcript through a model. + """ + parts: list[str] = [] + if case.fork_turn <= 0: + return "" + with case.path.open("r", encoding="utf-8") as f: + for i, line in enumerate(f): + if i >= case.fork_turn: + break + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + t = obj.get("text") + if isinstance(t, str): + parts.append(t) + return "\n".join(parts) + + def measure_token_cost( rebuilt_context: str, case: TranscriptCase, ) -> float: - """Rebuild block size / pre-clear context size. - - TODO: needs a tokenizer (tiktoken or model-specific) and the - pre-clear context size from the captured transcript. + """Rebuild-block tokens / pre-clear-context tokens. + + Both sides use `benchmarks.context_rebuilder.measure.estimate_tokens` + — the 4-chars-per-token heuristic that mirrors the rebuilder's own + internal estimator (`aelfrice.context_rebuilder._CHARS_PER_TOKEN`). + Sharing the constant keeps harness measurements aligned with the + rebuilder's own budget bookkeeping. A real tokenizer (tiktoken / + sentencepiece) would land alongside the LLM-judge in a follow-up. + + Returns 0.0 when the pre-clear text is empty (no transcript content + pre-fork) — there's no meaningful ratio to report and division would + fail; 0.0 documents the corner without raising. """ - raise NotImplementedError("Wire to tokenizer + transcript size") + pre_clear_tokens = estimate_tokens(_pre_clear_text(case)) + if pre_clear_tokens <= 0: + return 0.0 + rebuilt_tokens = estimate_tokens(rebuilt_context) + return rebuilt_tokens / pre_clear_tokens def run_one( diff --git a/benchmarks/context-rebuilder/fixtures/synthetic/debugging_session_001.meta.json b/benchmarks/context-rebuilder/fixtures/synthetic/debugging_session_001.meta.json new file mode 100644 index 000000000..d19fe96a8 --- /dev/null +++ b/benchmarks/context-rebuilder/fixtures/synthetic/debugging_session_001.meta.json @@ -0,0 +1,6 @@ +{ + "task_type": "debug", + "fork_turn": 8, + "eval_turns": [8, 10, 12, 14], + "notes": "Midpoint fork on the 16-turn synthetic debugging-session fixture. Post-fork eval turns are the four user prompts at file indices 8, 10, 12, 14. Pairs with debugging_session_001.jsonl; load_corpus() resolves this via with_suffix('.meta.json')." +} diff --git a/tests/test_context_rebuilder_eval_harness_wiring.py b/tests/test_context_rebuilder_eval_harness_wiring.py new file mode 100644 index 000000000..b814bcda6 --- /dev/null +++ b/tests/test_context_rebuilder_eval_harness_wiring.py @@ -0,0 +1,362 @@ +"""Wire-up tests for benchmarks/context-rebuilder/eval_harness.py (#592). + +The hyphenated parent directory blocks normal `import benchmarks.context-rebuilder`, +so the module is loaded by file path via importlib. Each test boots a fresh +in-memory store and asserts the harness's integration points behave. +""" +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + +import pytest + + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_HARNESS_PATH = ( + _REPO_ROOT / "benchmarks" / "context-rebuilder" / "eval_harness.py" +) +_HARNESS_MOD_NAME = "eval_harness_under_test" + + +def _load_harness() -> ModuleType: + spec = importlib.util.spec_from_file_location( + _HARNESS_MOD_NAME, _HARNESS_PATH, + ) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + # Register before exec so dataclass annotation resolution can find + # the module via sys.modules[mod.__module__]. + sys.modules[_HARNESS_MOD_NAME] = mod + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="module") +def harness() -> ModuleType: + return _load_harness() + + +def _write_jsonl(path: Path, turns: list[dict]) -> None: + with path.open("w", encoding="utf-8") as f: + for t in turns: + f.write(json.dumps(t) + "\n") + + +def _make_turn(idx: int, role: str, text: str) -> dict: + return { + "schema_version": 1, + "ts": f"2026-04-27T15:{40 + idx:02d}:00.000Z", + "role": role, + "text": text, + "session_id": "test-session-001", + "turn_id": f"t{idx:04d}", + } + + +@pytest.fixture() +def transcript_path(tmp_path: Path) -> Path: + """Six-turn transcript: alternating user/assistant.""" + p = tmp_path / "session.jsonl" + turns = [ + _make_turn(0, "user", "How does the rebuilder pack L0 hits?"), + _make_turn(1, "assistant", "L0 locked beliefs are packed first."), + _make_turn(2, "user", "What about session-scoped retrieval?"), + _make_turn(3, "assistant", "Session-scoped runs above L2.5 and L1."), + _make_turn(4, "user", "Does the floor apply to L0?"), + _make_turn(5, "assistant", "L0 always packs without floor."), + ] + _write_jsonl(p, turns) + return p + + +# --------------------------------------------------------------------- # +# replay_to_fork # +# --------------------------------------------------------------------- # + + +def test_replay_to_fork_returns_memory_store( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=4, eval_turns=(4, 5), + ) + store = harness.replay_to_fork(case) + try: + assert store is not None + ids = list(store.list_belief_ids()) + assert len(ids) > 0 + finally: + store.close() + + +def test_replay_to_fork_respects_fork_boundary( + harness: ModuleType, transcript_path: Path +) -> None: + """Forking at turn 2 should produce strictly fewer beliefs than + forking at turn 6 — the harness only sees pre-fork content.""" + case_early = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=2, eval_turns=(), + ) + case_late = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=6, eval_turns=(), + ) + s_early = harness.replay_to_fork(case_early) + s_late = harness.replay_to_fork(case_late) + try: + n_early = len(list(s_early.list_belief_ids())) + n_late = len(list(s_late.list_belief_ids())) + assert n_early < n_late + finally: + s_early.close() + s_late.close() + + +def test_replay_to_fork_zero_fork_turn_returns_empty_store( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=0, eval_turns=(), + ) + store = harness.replay_to_fork(case) + try: + assert list(store.list_belief_ids()) == [] + finally: + store.close() + + +# --------------------------------------------------------------------- # +# run_rebuilder # +# --------------------------------------------------------------------- # + + +def test_run_rebuilder_returns_block_and_positive_latency( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=4, eval_turns=(4, 5), + ) + store = harness.replay_to_fork(case) + try: + block, latency_ms = harness.run_rebuilder( + store, case, trigger_threshold=0.0, token_budget=2000, + ) + assert isinstance(block, str) + assert latency_ms >= 0.0 + finally: + store.close() + + +def test_run_rebuilder_passes_threshold_as_floor( + harness: ModuleType, transcript_path: Path +) -> None: + """At a high enough floor, no L1 / session beliefs survive. + Without locked beliefs in the store, the block is empty per + rebuild_v14's silent-path contract.""" + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=6, eval_turns=(), + ) + store = harness.replay_to_fork(case) + try: + block_low, _ = harness.run_rebuilder( + store, case, trigger_threshold=0.0, token_budget=2000, + ) + block_hi, _ = harness.run_rebuilder( + store, case, trigger_threshold=10.0, token_budget=2000, + ) + # High floor zeroes out non-locked content. Low floor lets some + # beliefs through (or matches; tolerate equality if the store + # had no above-floor candidates either way). + assert len(block_hi) <= len(block_low) + finally: + store.close() + + +def test_run_rebuilder_empty_store_returns_empty_block( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=0, eval_turns=(), + ) + store = harness.replay_to_fork(case) + try: + block, latency_ms = harness.run_rebuilder( + store, case, trigger_threshold=0.0, token_budget=2000, + ) + # No beliefs ingested + no recent_turns + no locks → empty. + assert block == "" + assert latency_ms >= 0.0 + finally: + store.close() + + +# --------------------------------------------------------------------- # +# measure_token_cost # +# --------------------------------------------------------------------- # + + +def test_measure_token_cost_zero_pre_clear_returns_zero( + harness: ModuleType, transcript_path: Path +) -> None: + """fork_turn=0 → no pre-clear text → ratio is 0.0, not divide-by-zero.""" + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=0, eval_turns=(), + ) + assert harness.measure_token_cost("anything", case) == 0.0 + + +def test_measure_token_cost_smaller_rebuilt_means_smaller_ratio( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=4, eval_turns=(), + ) + big_block = "x" * 1000 + small_block = "x" * 10 + big = harness.measure_token_cost(big_block, case) + small = harness.measure_token_cost(small_block, case) + assert small < big + assert small >= 0.0 + + +def test_measure_token_cost_uses_shared_estimator( + harness: ModuleType, transcript_path: Path +) -> None: + """The harness ratio must match estimate_tokens(rebuilt) / + estimate_tokens(pre_clear) so the constant stays in sync with + the rebuilder's own bookkeeping.""" + from benchmarks.context_rebuilder.measure import estimate_tokens + + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=4, eval_turns=(), + ) + rebuilt = "context block of arbitrary length" + pre_clear = harness._pre_clear_text(case) + expected = estimate_tokens(rebuilt) / estimate_tokens(pre_clear) + assert harness.measure_token_cost(rebuilt, case) == pytest.approx(expected) + + +# --------------------------------------------------------------------- # +# replay_post_fork # +# --------------------------------------------------------------------- # + + +def test_replay_post_fork_returns_one_placeholder_per_eval_turn( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=2, eval_turns=(2, 4), + ) + results = harness.replay_post_fork("rebuilt-block", case) + assert len(results) == 2 + assert {r["turn_idx"] for r in results} == {2, 4} + assert all(not r["matched"] for r in results) + assert all(r["reason"] == harness.REPLAY_PENDING_REASON for r in results) + + +def test_replay_post_fork_pulls_expected_from_transcript( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=2, eval_turns=(2,), + ) + results = harness.replay_post_fork("", case) + assert results[0]["expected"] == "What about session-scoped retrieval?" + assert results[0]["actual"] == "" + + +def test_replay_post_fork_empty_eval_turns_returns_empty_list( + harness: ModuleType, transcript_path: Path +) -> None: + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=2, eval_turns=(), + ) + assert harness.replay_post_fork("rebuilt", case) == [] + + +def test_replay_post_fork_score_fidelity_returns_zero( + harness: ModuleType, transcript_path: Path +) -> None: + """Stubbed replay → score_fidelity returns 0.0, not a crash.""" + case = harness.TranscriptCase( + path=transcript_path, task_type="debug", + fork_turn=2, eval_turns=(2, 4), + ) + results = harness.replay_post_fork("", case) + assert harness.score_fidelity(results) == 0.0 + + +# --------------------------------------------------------------------- # +# End-to-end: threshold-sweep against the bundled synthetic fixture # +# --------------------------------------------------------------------- # + + +def test_threshold_sweep_runs_end_to_end_on_synthetic_fixture( + harness: ModuleType, tmp_path: Path +) -> None: + """Smoke test: load the bundled synthetic fixture, run a 3-threshold + sweep, and verify the JSON output schema. Until the model client + lands all fidelity scores will be 0.0 — the sweep still produces + valid latency + token-cost numbers.""" + corpus_dir = ( + _REPO_ROOT / "benchmarks" / "context-rebuilder" / "fixtures" / "synthetic" + ) + cases = harness.load_corpus(corpus_dir) + assert len(cases) >= 1, "synthetic fixture meta.json must resolve" + + result = harness.sweep_thresholds( + cases, thresholds=(0.0, 0.5), token_budget=2000, + ) + assert result.mode == "threshold-sweep" + assert len(result.runs) == len(cases) * 2 + for r in result.runs: + assert r.fidelity == 0.0 # stub returns 0 until model client + assert r.rebuild_latency_ms >= 0.0 + assert r.token_cost_ratio >= 0.0 + assert r.fork_turn > 0 + # summary structure: keyed by task_type, then per-threshold metrics + assert "debug" in result.summary + debug_summary = result.summary["debug"] + assert "threshold=0.0" in debug_summary + assert "median_fidelity" in debug_summary["threshold=0.0"] + assert "p99_latency_ms" in debug_summary["threshold=0.0"] + + +def test_run_one_returns_runresult_without_crashing( + harness: ModuleType, tmp_path: Path +) -> None: + """`run_one` is the per-case entry point used by both sweep modes; + it must thread store + rebuild + replay + score without raising + even with the replay stub in place.""" + corpus_dir = ( + _REPO_ROOT / "benchmarks" / "context-rebuilder" / "fixtures" / "synthetic" + ) + cases = harness.load_corpus(corpus_dir) + case = cases[0] + result = harness.run_one(case, trigger_threshold=0.0, token_budget=2000) + assert isinstance(result, harness.RunResult) + assert result.task_type == case.task_type + assert result.config == {"trigger_threshold": 0.0, "token_budget": 2000} + assert result.n_eval_turns == len(case.eval_turns) + # All eval_turns are unmatched stubs, so failures == eval_turns + assert len(result.failures) == result.n_eval_turns + assert all( + f["reason"] == harness.REPLAY_PENDING_REASON + for f in result.failures + )