diff --git a/CHANGELOG.md b/CHANGELOG.md index fec6302f7..5ae432f7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ installable release; see the roadmap in [README.md](README.md). ## [Unreleased] +### Changed + +- **Dev-only benchmark targets moved out of the `aelf` CLI** ([#342](https://github.com/robotrocketscience/aelfrice/issues/342)). `aelf bench verify-clean | longmemeval-score | posterior-residual` were always-failing in shipped wheels (the `benchmarks/` tree is dev-only and not packaged) and put dev-script knowledge in the runtime CLI surface. Replaced with module-level entry points runnable from a source checkout: `python -m benchmarks.verify_clean PATH ...`, `python -m benchmarks.longmemeval_score PREDS GT JUDGE`, `python -m benchmarks.posterior_ranking [--fixtures ... --seeds N --mrr-threshold X --ece-threshold Y --json --heat-kernel]`. The unknown-target error in `aelf bench` now points at the new entry points. The default `aelf bench` (synthetic harness in `src/aelfrice/benchmark.py`) and the `_BENCH_INERT_TARGETS` placeholders are unchanged. The deptry `DEP001 = ["benchmarks"]` ignore is removed — `src/aelfrice` no longer imports the dev-only package. + ### Added - **Belief corroboration tracking — sibling table + ingest recorder (phantom-prereqs T1)** ([#190](https://github.com/robotrocketscience/aelfrice/issues/190)). New `belief_corroborations` table records each re-ingest of identical content without disturbing the existing dedup contract. When `ingest_turn` or `ingest_triples` encounters an already-existing belief (by id), it calls `MemoryStore.record_corroboration(belief_id, source_type=...)` instead of silently discarding the duplicate. Four `source_type` constants distinguish the ingest paths: `CORROBORATION_SOURCE_COMMIT_INGEST`, `CORROBORATION_SOURCE_TRANSCRIPT_INGEST`, `CORROBORATION_SOURCE_MCP_REMEMBER`, `CORROBORATION_SOURCE_HOOK_INGEST` (all in `aelfrice.models`); `record_corroboration` validates against `CORROBORATION_SOURCE_TYPES` and raises `ValueError` on unknown values. `Belief.corroboration_count` (default 0) is populated at retrieval time via a per-belief subquery in `get_belief`, `search_beliefs`, `search_beliefs_scored`, and `list_locked_beliefs`. `session_id` and `source_path_hash` columns are nullable; T3 (#192) will wire session propagation. `ON DELETE CASCADE` removes corroboration rows when a belief is deleted. 16 new deterministic tests in `tests/test_corroborations.py` cover schema creation, idempotency, re-ingest recording, corroboration count on retrieval, cascade deletion, nullable fields, source_type enum coverage, and ValueError on unknown source_type. diff --git a/benchmarks/README.md b/benchmarks/README.md index f4f564fc1..cd5ed38e6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -72,7 +72,7 @@ PYTHONPATH=. python benchmarks/longmemeval_adapter.py \ --subset 3 --retrieve-only /tmp/lme_smoke.json # Verify no contamination in the retrieval file: -PYTHONPATH=. aelf bench verify-clean /tmp/lme_smoke.json +python -m benchmarks.verify_clean /tmp/lme_smoke.json ``` ## Missing public-surface dependencies @@ -95,10 +95,10 @@ These ports are scheduled in P2/P3/P4 of the v2.0.0 milestone plan ```bash # Contamination gate (verifies a retrieval file has no answer/gt leakage) -uv run aelf bench verify-clean path/to/retrieval.json +uv run python -m benchmarks.verify_clean path/to/retrieval.json # Score a LongMemEval predictions file (no aelfrice imports needed) -uv run aelf bench longmemeval-score preds.json gt.json +uv run python -m benchmarks.longmemeval_score preds.json gt.json ``` Anything else exits 2 with a friendly pointer back to this file. diff --git a/benchmarks/longmemeval_budget_sweep.py b/benchmarks/longmemeval_budget_sweep.py index 37ab816d0..63241969e 100644 --- a/benchmarks/longmemeval_budget_sweep.py +++ b/benchmarks/longmemeval_budget_sweep.py @@ -20,7 +20,7 @@ if _BENCH_DIR not in sys.path: sys.path.insert(0, _BENCH_DIR) -from longmemeval_adapter import ( # type: ignore[import-untyped] +from benchmarks.longmemeval_adapter import ( # type: ignore[import-untyped] LongMemEvalQuestion, load_from_huggingface, parse_questions, diff --git a/benchmarks/posterior_ranking/__main__.py b/benchmarks/posterior_ranking/__main__.py new file mode 100644 index 000000000..6b93f7ecc --- /dev/null +++ b/benchmarks/posterior_ranking/__main__.py @@ -0,0 +1,75 @@ +"""Command-line entry point for the posterior_ranking benchmark. + +Run with `python -m benchmarks.posterior_ranking [flags]` from a source +checkout. Argument surface mirrors the prior `aelf bench +posterior-residual` subcommand removed in #342. +""" +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import asdict +from pathlib import Path + +from benchmarks.posterior_ranking import run as _pr_run + + +def _default_fixtures() -> Path: + return ( + Path(__file__).parent / "fixtures" / "default.jsonl" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m benchmarks.posterior_ranking", + ) + parser.add_argument("--fixtures", default=None) + parser.add_argument("--seeds", type=int, default=5) + parser.add_argument("--mrr-threshold", type=float, default=0.05) + parser.add_argument("--ece-threshold", type=float, default=0.10) + parser.add_argument("--json", dest="json_out", action="store_true") + parser.add_argument( + "--heat-kernel", + action="store_true", + help="enable heat-kernel composition in retrieve() (slice 2 of #151)", + ) + args = parser.parse_args(argv) + + fixtures_path = Path(args.fixtures) if args.fixtures else _default_fixtures() + result = _pr_run.run( + fixtures_path, + n_seeds=args.seeds, + mrr_threshold=args.mrr_threshold, + ece_threshold=args.ece_threshold, + heat_kernel=args.heat_kernel, + ) + + if args.json_out: + print(json.dumps({ + "mrr": asdict(result["mrr"]), + "ece": asdict(result["ece"]), + "overall_pass": result["overall_pass"], + }, indent=2)) + else: + mrr = result["mrr"] + ece = result["ece"] + print( + f"posterior-residual eval\n" + f" MRR uplift: {mrr.mean_uplift:+.4f} " + f"(±2σ: [{mrr.uplift_lo:+.4f}, {mrr.uplift_hi:+.4f}]) " + f"threshold={mrr.pass_threshold:+.2f} " + f"{'PASS' if mrr.passed else 'FAIL'}\n" + f" ECE: {ece.ece:.4f} " + f"threshold={ece.pass_threshold:.2f} " + f"n={ece.n_total} " + f"{'PASS' if ece.passed else 'FAIL'}\n" + f" overall: {'PASS' if result['overall_pass'] else 'FAIL'}" + ) + + return 0 if result["overall_pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/posterior_ranking/run.py b/benchmarks/posterior_ranking/run.py index bc46dd4e4..8c9be8707 100644 --- a/benchmarks/posterior_ranking/run.py +++ b/benchmarks/posterior_ranking/run.py @@ -1,6 +1,6 @@ """Posterior-ranking eval runner: wires MRR uplift + ECE against a fixture set. -Entry point used by both tests and the `aelf bench posterior-residual` CLI. +Entry point used by tests and `python -m benchmarks.posterior_ranking`. Usage: run(fixtures_path, n_seeds=5) -> dict with keys "mrr", "ece", "overall_pass" diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index b6fd72f12..4616edb94 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -50,7 +50,7 @@ The protocol enforces: - A pre-generation contamination check is mandatory before any LLM reader touches the retrieval file: ```bash -aelf bench verify-clean /tmp/benchmark_.json +python -m benchmarks.verify_clean /tmp/benchmark_.json ``` If this fails, the run is invalid. Fix the adapter and re-run. @@ -63,7 +63,7 @@ uv run python benchmarks/.py \ --retrieve-only /tmp/benchmark_.json [--subset N] # 2. Verify the retrieval file is clean -aelf bench verify-clean /tmp/benchmark_.json +python -m benchmarks.verify_clean /tmp/benchmark_.json # 3. LLM reader generates predictions (no GT visible to it) # 4. Scoring reads predictions + GT (no retrieval context visible) diff --git a/docs/bayesian_ranking.md b/docs/bayesian_ranking.md index 255e0ecfa..2e1b5ceb1 100644 --- a/docs/bayesian_ranking.md +++ b/docs/bayesian_ranking.md @@ -228,7 +228,7 @@ In each case the rerank reverts to `partial_bayesian_score(bm25, alpha, beta, po ### Bench wedge -`aelf bench posterior-residual --heat-kernel` runs the MRR + ECE harness with the flag flipped on. Each per-seed `retrieve()` gets a fresh `GraphEigenbasisCache` built against that seed's in-memory store and rebuilt on stale (the synthetic feedback stream mutates the store after every round). Without `--heat-kernel`, output is byte-identical to today. +`python -m benchmarks.posterior_ranking --heat-kernel` runs the MRR + ECE harness with the flag flipped on. Each per-seed `retrieve()` gets a fresh `GraphEigenbasisCache` built against that seed's in-memory store and rebuilt on stale (the synthetic feedback stream mutates the store after every round). Without `--heat-kernel`, output is byte-identical to today. ### What Slice 2 still doesn't ship diff --git a/pyproject.toml b/pyproject.toml index 6a76bfbba..57a0fbd87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,9 +134,6 @@ DEP002 = [ "datasets", "huggingface_hub", ] -# DEP001: `benchmarks` is the in-repo top-level package imported via the -# pytest pythonpath shim; not a PyPI dependency. -DEP001 = ["benchmarks"] [dependency-groups] dev = [ diff --git a/src/aelfrice/cli.py b/src/aelfrice/cli.py index 1da5fa6b6..7021f5400 100644 --- a/src/aelfrice/cli.py +++ b/src/aelfrice/cli.py @@ -1033,123 +1033,18 @@ def _cmd_bench(args: argparse.Namespace, out: object) -> int: print(json.dumps(report.to_dict(), indent=2), file=out) # type: ignore[arg-type] return 0 - if target == "verify-clean": - try: - from benchmarks import verify_clean - except ModuleNotFoundError: - print( - "aelf bench verify-clean requires the source tree " - "(benchmarks/ is dev-only and not shipped in the wheel). " - "Clone the repo and run from the repo root.", - file=out, # type: ignore[arg-type] - ) - return 2 - if not args.rest: - print("usage: aelf bench verify-clean PATH [PATH ...]", file=out) # type: ignore[arg-type] - return 2 - all_clean = True - for path in args.rest: - if not verify_clean.verify_file(path): - all_clean = False - return 0 if all_clean else 1 - - if target == "longmemeval-score": - try: - from benchmarks import longmemeval_score - except ModuleNotFoundError: - print( - "aelf bench longmemeval-score requires the source tree " - "(benchmarks/ is dev-only and not shipped in the wheel). " - "Clone the repo and run from the repo root.", - file=out, # type: ignore[arg-type] - ) - return 2 - if len(args.rest) < 3: - print("usage: aelf bench longmemeval-score PREDS GT JUDGE", file=out) # type: ignore[arg-type] - return 2 - return longmemeval_score.score(args.rest[0], args.rest[1], args.rest[2]) - - if target == "posterior-residual": - try: - from benchmarks.posterior_ranking import run as _pr_run - except ModuleNotFoundError: - print( - "aelf bench posterior-residual requires the source tree " - "(benchmarks/ is dev-only and not shipped in the wheel). " - "Clone the repo and run from the repo root.", - file=out, # type: ignore[arg-type] - ) - return 2 - # The bench subparser uses nargs=REMAINDER for `rest`, which swallows - # all tokens including named flags. Parse posterior-residual flags - # here rather than via argparse pre-declared args. - import argparse as _ap - _pr_parser = _ap.ArgumentParser( - prog="aelf bench posterior-residual", - add_help=False, - ) - _pr_parser.add_argument("--fixtures", dest="pr_fixtures", default=None) - _pr_parser.add_argument("--seeds", dest="pr_seeds", type=int, default=5) - _pr_parser.add_argument( - "--mrr-threshold", dest="pr_mrr_threshold", type=float, default=0.05, - ) - _pr_parser.add_argument( - "--ece-threshold", dest="pr_ece_threshold", type=float, default=0.10, - ) - _pr_parser.add_argument( - "--json", dest="pr_json", action="store_true", - ) - _pr_parser.add_argument( - "--heat-kernel", dest="pr_heat_kernel", action="store_true", - help="enable heat-kernel composition in retrieve() (slice 2 of #151)", - ) - _pr_ns, _ = _pr_parser.parse_known_args(args.rest) - from pathlib import Path as _Path - _default_fixtures = ( - _Path(__file__).parent.parent.parent - / "benchmarks" - / "posterior_ranking" - / "fixtures" - / "default.jsonl" - ) - _fixtures_path = ( - _Path(_pr_ns.pr_fixtures) if _pr_ns.pr_fixtures else _default_fixtures - ) - result = _pr_run.run( - _fixtures_path, - n_seeds=_pr_ns.pr_seeds, - mrr_threshold=_pr_ns.pr_mrr_threshold, - ece_threshold=_pr_ns.pr_ece_threshold, - heat_kernel=_pr_ns.pr_heat_kernel, - ) - - if _pr_ns.pr_json: - from dataclasses import asdict as _asdict - print( - json.dumps({ - "mrr": _asdict(result["mrr"]), - "ece": _asdict(result["ece"]), - "overall_pass": result["overall_pass"], - }, indent=2), - file=out, # type: ignore[arg-type] - ) - else: - mrr = result["mrr"] - ece = result["ece"] - print( - f"posterior-residual eval\n" - f" MRR uplift: {mrr.mean_uplift:+.4f} " - f"(±2σ: [{mrr.uplift_lo:+.4f}, {mrr.uplift_hi:+.4f}]) " - f"threshold={mrr.pass_threshold:+.2f} " - f"{'PASS' if mrr.passed else 'FAIL'}\n" - f" ECE: {ece.ece:.4f} " - f"threshold={ece.pass_threshold:.2f} " - f"n={ece.n_total} " - f"{'PASS' if ece.passed else 'FAIL'}\n" - f" overall: {'PASS' if result['overall_pass'] else 'FAIL'}", - file=out, # type: ignore[arg-type] - ) - return 0 if result["overall_pass"] else 1 + _DEV_TARGETS_MOVED: dict[str, str] = { + "verify-clean": "python -m benchmarks.verify_clean", + "longmemeval-score": "python -m benchmarks.longmemeval_score", + "posterior-residual": "python -m benchmarks.posterior_ranking", + } + if target in _DEV_TARGETS_MOVED: + print( + f"aelf bench {target} has moved. Run " + f"`{_DEV_TARGETS_MOVED[target]} ...` from a source checkout.", + file=out, # type: ignore[arg-type] + ) + return 2 if target in _BENCH_INERT_TARGETS: phase = _BENCH_INERT_TARGETS[target] @@ -1164,9 +1059,11 @@ def _cmd_bench(args: argparse.Namespace, out: object) -> int: print( f"aelf bench: unknown target {target!r}.\n" - f"Known targets: synthetic (default), verify-clean, " - f"longmemeval-score, posterior-residual, " - f"{', '.join(sorted(_BENCH_INERT_TARGETS))}.", + f"Known targets: synthetic (default), " + f"{', '.join(sorted(_BENCH_INERT_TARGETS))}.\n" + f"Dev-only benchmarks moved to " + f"`python -m benchmarks.`: " + f"{', '.join(sorted(_DEV_TARGETS_MOVED))}.", file=out, # type: ignore[arg-type] ) return 2 diff --git a/tests/test_benchmarks_dir.py b/tests/test_benchmarks_dir.py index 725cfefd1..09c108d14 100644 --- a/tests/test_benchmarks_dir.py +++ b/tests/test_benchmarks_dir.py @@ -90,23 +90,31 @@ def test_aelf_bench_synthetic_target_explicit_runs_synthetic() -> None: assert report["corpus_size"] == 16 -def test_aelf_bench_verify_clean_dispatches_to_module(tmp_path: Path) -> None: - clean_file = tmp_path / "r.json" - clean_file.write_text(json.dumps([{"id": "q1", "question": "?"}])) - code, _ = _run_cli("bench", "verify-clean", str(clean_file)) - assert code == 0 - - -def test_aelf_bench_verify_clean_no_args_exits_2() -> None: +def test_aelf_bench_verify_clean_target_redirects_to_module() -> None: + """The dev-only target now points users at `python -m benchmarks.verify_clean`.""" code, output = _run_cli("bench", "verify-clean") assert code == 2 - assert "usage" in output.lower() + assert "python -m benchmarks.verify_clean" in output -def test_aelf_bench_longmemeval_score_no_args_exits_2() -> None: +def test_aelf_bench_longmemeval_score_target_redirects_to_module() -> None: code, output = _run_cli("bench", "longmemeval-score") assert code == 2 - assert "usage" in output.lower() + assert "python -m benchmarks.longmemeval_score" in output + + +def test_aelf_bench_posterior_residual_target_redirects_to_module() -> None: + code, output = _run_cli("bench", "posterior-residual") + assert code == 2 + assert "python -m benchmarks.posterior_ranking" in output + + +def test_aelf_bench_unknown_target_lists_moved_dev_targets() -> None: + code, output = _run_cli("bench", "no-such-target") + assert code == 2 + assert "verify-clean" in output + assert "longmemeval-score" in output + assert "posterior-residual" in output def test_benchmarks_package_imports() -> None: diff --git a/tests/test_posterior_ranking_eval.py b/tests/test_posterior_ranking_eval.py index aab68aa18..5d24be1b0 100644 --- a/tests/test_posterior_ranking_eval.py +++ b/tests/test_posterior_ranking_eval.py @@ -10,15 +10,12 @@ """ from __future__ import annotations -import io import json import math from pathlib import Path import pytest -from aelfrice.cli import main as cli_main - # --------------------------------------------------------------------------- # Helpers @@ -239,16 +236,10 @@ def test_ece_empty_observations() -> None: # --------------------------------------------------------------------------- -# Helpers for runner / CLI tests +# Helpers for runner tests # --------------------------------------------------------------------------- -def _run_cli(*argv: str) -> tuple[int, str]: - buf = io.StringIO() - code = cli_main(argv=list(argv), out=buf) - return code, buf.getvalue() - - def _write_fixtures(tmp_path: Path, fixtures: list[dict[str, object]]) -> Path: fpath = tmp_path / "fixtures.jsonl" with fpath.open("w", encoding="utf-8") as fh: @@ -373,56 +364,68 @@ def test_load_fixtures(tmp_path: Path) -> None: # --------------------------------------------------------------------------- -# CLI integration +# python -m benchmarks.posterior_ranking — module entry point # --------------------------------------------------------------------------- -def test_cli_posterior_residual_exit_0_clean(tmp_path: Path) -> None: - """aelf bench posterior-residual exits 0 when both thresholds are met. +def _run_module(*argv: str, capsys: pytest.CaptureFixture[str]) -> tuple[int, str]: + from benchmarks.posterior_ranking.__main__ import main as _module_main + + code = _module_main(list(argv)) + captured = capsys.readouterr() + return code, captured.out + + +def test_module_posterior_residual_exit_0_clean( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """`python -m benchmarks.posterior_ranking` exits 0 when both thresholds met. Uses a relaxed ECE threshold since the synthetic feedback stream is intentionally simple (only the known item gets positive feedback), which produces well-separated but not perfectly calibrated posterior_mean values - for noise items at the Jeffreys prior. The MRR fixture is designed to - produce reliable uplift. ECE calibration is validated by unit tests - via compute_ece() directly. + for noise items at the Jeffreys prior. """ fixtures = [_minimal_fixture()] fpath = _write_fixtures(tmp_path, fixtures) - code, output = _run_cli( - "bench", "posterior-residual", + code, output = _run_module( "--fixtures", str(fpath), "--seeds", "1", "--ece-threshold", "0.50", + capsys=capsys, ) assert code == 0, f"expected exit 0, got {code}. output:\n{output}" -def test_cli_posterior_residual_exit_1_tight_threshold(tmp_path: Path) -> None: - """aelf bench posterior-residual with impossibly tight MRR threshold exits 1.""" +def test_module_posterior_residual_exit_1_tight_threshold( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Impossibly tight MRR threshold exits 1.""" fixtures = [_minimal_fixture()] fpath = _write_fixtures(tmp_path, fixtures) - code, output = _run_cli( - "bench", "posterior-residual", + code, output = _run_module( "--fixtures", str(fpath), "--seeds", "1", "--mrr-threshold", "0.99", + capsys=capsys, ) assert code == 1, f"expected exit 1, got {code}. output:\n{output}" -def test_cli_posterior_residual_json_flag(tmp_path: Path) -> None: +def test_module_posterior_residual_json_flag( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: """--json flag emits machine-readable JSON with mrr, ece, overall_pass keys.""" fixtures = [_minimal_fixture()] fpath = _write_fixtures(tmp_path, fixtures) - code, output = _run_cli( - "bench", "posterior-residual", + code, output = _run_module( "--fixtures", str(fpath), "--seeds", "1", "--json", + capsys=capsys, ) parsed = json.loads(output) assert "mrr" in parsed