refactor: drop benchmarks/-importing targets from aelf bench (closes #342)

CHANGELOG.md

@@ -10,6 +10,10 @@ installable release; see the roadmap in [README.md](README.md).
 
 ## [Unreleased]
 
+### Changed
+
+- **Dev-only benchmark targets moved out of the `aelf` CLI** ([#342](https://github.com/robotrocketscience/aelfrice/issues/342)). `aelf bench verify-clean | longmemeval-score | posterior-residual` were always-failing in shipped wheels (the `benchmarks/` tree is dev-only and not packaged) and put dev-script knowledge in the runtime CLI surface. Replaced with module-level entry points runnable from a source checkout: `python -m benchmarks.verify_clean PATH ...`, `python -m benchmarks.longmemeval_score PREDS GT JUDGE`, `python -m benchmarks.posterior_ranking [--fixtures ... --seeds N --mrr-threshold X --ece-threshold Y --json --heat-kernel]`. The unknown-target error in `aelf bench` now points at the new entry points. The default `aelf bench` (synthetic harness in `src/aelfrice/benchmark.py`) and the `_BENCH_INERT_TARGETS` placeholders are unchanged. The deptry `DEP001 = ["benchmarks"]` ignore is removed — `src/aelfrice` no longer imports the dev-only package.
+
 ### Added
 
 - **Belief corroboration tracking — sibling table + ingest recorder (phantom-prereqs T1)** ([#190](https://github.com/robotrocketscience/aelfrice/issues/190)). New `belief_corroborations` table records each re-ingest of identical content without disturbing the existing dedup contract. When `ingest_turn` or `ingest_triples` encounters an already-existing belief (by id), it calls `MemoryStore.record_corroboration(belief_id, source_type=...)` instead of silently discarding the duplicate. Four `source_type` constants distinguish the ingest paths: `CORROBORATION_SOURCE_COMMIT_INGEST`, `CORROBORATION_SOURCE_TRANSCRIPT_INGEST`, `CORROBORATION_SOURCE_MCP_REMEMBER`, `CORROBORATION_SOURCE_HOOK_INGEST` (all in `aelfrice.models`); `record_corroboration` validates against `CORROBORATION_SOURCE_TYPES` and raises `ValueError` on unknown values. `Belief.corroboration_count` (default 0) is populated at retrieval time via a per-belief subquery in `get_belief`, `search_beliefs`, `search_beliefs_scored`, and `list_locked_beliefs`. `session_id` and `source_path_hash` columns are nullable; T3 (#192) will wire session propagation. `ON DELETE CASCADE` removes corroboration rows when a belief is deleted. 16 new deterministic tests in `tests/test_corroborations.py` cover schema creation, idempotency, re-ingest recording, corroboration count on retrieval, cascade deletion, nullable fields, source_type enum coverage, and ValueError on unknown source_type.

benchmarks/README.md

@@ -72,7 +72,7 @@ PYTHONPATH=. python benchmarks/longmemeval_adapter.py \
     --subset 3 --retrieve-only /tmp/lme_smoke.json
 
 # Verify no contamination in the retrieval file:
-PYTHONPATH=. aelf bench verify-clean /tmp/lme_smoke.json
+python -m benchmarks.verify_clean /tmp/lme_smoke.json
 ```
 
 ## Missing public-surface dependencies
@@ -95,10 +95,10 @@ These ports are scheduled in P2/P3/P4 of the v2.0.0 milestone plan
 
 ```bash
 # Contamination gate (verifies a retrieval file has no answer/gt leakage)
-uv run aelf bench verify-clean path/to/retrieval.json
+uv run python -m benchmarks.verify_clean path/to/retrieval.json
 
 # Score a LongMemEval predictions file (no aelfrice imports needed)
-uv run aelf bench longmemeval-score preds.json gt.json
+uv run python -m benchmarks.longmemeval_score preds.json gt.json
 ```
 
 Anything else exits 2 with a friendly pointer back to this file.

benchmarks/longmemeval_budget_sweep.py

@@ -20,7 +20,7 @@
 if _BENCH_DIR not in sys.path:
     sys.path.insert(0, _BENCH_DIR)
 
-from longmemeval_adapter import (  # type: ignore[import-untyped]
+from benchmarks.longmemeval_adapter import (  # type: ignore[import-untyped]
     LongMemEvalQuestion,
     load_from_huggingface,
     parse_questions,

benchmarks/posterior_ranking/__main__.py

@@ -0,0 +1,75 @@
+"""Command-line entry point for the posterior_ranking benchmark.
+
+Run with `python -m benchmarks.posterior_ranking [flags]` from a source
+checkout. Argument surface mirrors the prior `aelf bench
+posterior-residual` subcommand removed in #342.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict
+from pathlib import Path
+
+from benchmarks.posterior_ranking import run as _pr_run
+
+
+def _default_fixtures() -> Path:
+    return (
+        Path(__file__).parent / "fixtures" / "default.jsonl"
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m benchmarks.posterior_ranking",
+    )
+    parser.add_argument("--fixtures", default=None)
+    parser.add_argument("--seeds", type=int, default=5)
+    parser.add_argument("--mrr-threshold", type=float, default=0.05)
+    parser.add_argument("--ece-threshold", type=float, default=0.10)
+    parser.add_argument("--json", dest="json_out", action="store_true")
+    parser.add_argument(
+        "--heat-kernel",
+        action="store_true",
+        help="enable heat-kernel composition in retrieve() (slice 2 of #151)",
+    )
+    args = parser.parse_args(argv)
+
+    fixtures_path = Path(args.fixtures) if args.fixtures else _default_fixtures()
+    result = _pr_run.run(
+        fixtures_path,
+        n_seeds=args.seeds,
+        mrr_threshold=args.mrr_threshold,
+        ece_threshold=args.ece_threshold,
+        heat_kernel=args.heat_kernel,
+    )
+
+    if args.json_out:
+        print(json.dumps({
+            "mrr": asdict(result["mrr"]),
+            "ece": asdict(result["ece"]),
+            "overall_pass": result["overall_pass"],
+        }, indent=2))
+    else:
+        mrr = result["mrr"]
+        ece = result["ece"]
+        print(
+            f"posterior-residual eval\n"
+            f"  MRR uplift:  {mrr.mean_uplift:+.4f}  "
+            f"(±2σ: [{mrr.uplift_lo:+.4f}, {mrr.uplift_hi:+.4f}])  "
+            f"threshold={mrr.pass_threshold:+.2f}  "
+            f"{'PASS' if mrr.passed else 'FAIL'}\n"
+            f"  ECE:         {ece.ece:.4f}  "
+            f"threshold={ece.pass_threshold:.2f}  "
+            f"n={ece.n_total}  "
+            f"{'PASS' if ece.passed else 'FAIL'}\n"
+            f"  overall:     {'PASS' if result['overall_pass'] else 'FAIL'}"
+        )
+
+    return 0 if result["overall_pass"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

benchmarks/posterior_ranking/run.py

@@ -1,6 +1,6 @@
 """Posterior-ranking eval runner: wires MRR uplift + ECE against a fixture set.
 
-Entry point used by both tests and the `aelf bench posterior-residual` CLI.
+Entry point used by tests and `python -m benchmarks.posterior_ranking`.
 
 Usage:
     run(fixtures_path, n_seeds=5) -> dict with keys "mrr", "ece", "overall_pass"

docs/BENCHMARKS.md

@@ -50,7 +50,7 @@ The protocol enforces:
 - A pre-generation contamination check is mandatory before any LLM reader touches the retrieval file:
 
 ```bash
-aelf bench verify-clean /tmp/benchmark_<name>.json
+python -m benchmarks.verify_clean /tmp/benchmark_<name>.json
 ```
 
 If this fails, the run is invalid. Fix the adapter and re-run.
@@ -63,7 +63,7 @@ uv run python benchmarks/<adapter>.py \
     --retrieve-only /tmp/benchmark_<name>.json [--subset N]
 
 # 2. Verify the retrieval file is clean
-aelf bench verify-clean /tmp/benchmark_<name>.json
+python -m benchmarks.verify_clean /tmp/benchmark_<name>.json
 
 # 3. LLM reader generates predictions (no GT visible to it)
 # 4. Scoring reads predictions + GT (no retrieval context visible)

docs/bayesian_ranking.md

@@ -228,7 +228,7 @@ In each case the rerank reverts to `partial_bayesian_score(bm25, alpha, beta, po
 
 ### Bench wedge
 
-`aelf bench posterior-residual --heat-kernel` runs the MRR + ECE harness with the flag flipped on. Each per-seed `retrieve()` gets a fresh `GraphEigenbasisCache` built against that seed's in-memory store and rebuilt on stale (the synthetic feedback stream mutates the store after every round). Without `--heat-kernel`, output is byte-identical to today.
+`python -m benchmarks.posterior_ranking --heat-kernel` runs the MRR + ECE harness with the flag flipped on. Each per-seed `retrieve()` gets a fresh `GraphEigenbasisCache` built against that seed's in-memory store and rebuilt on stale (the synthetic feedback stream mutates the store after every round). Without `--heat-kernel`, output is byte-identical to today.
 
 ### What Slice 2 still doesn't ship
 

pyproject.toml

@@ -134,9 +134,6 @@ DEP002 = [
     "datasets",
     "huggingface_hub",
 ]
-# DEP001: `benchmarks` is the in-repo top-level package imported via the
-# pytest pythonpath shim; not a PyPI dependency.
-DEP001 = ["benchmarks"]
 
 [dependency-groups]
 dev = [

src/aelfrice/cli.py

@@ -1033,123 +1033,18 @@ def _cmd_bench(args: argparse.Namespace, out: object) -> int:
         print(json.dumps(report.to_dict(), indent=2), file=out)  # type: ignore[arg-type]
         return 0
 
-    if target == "verify-clean":
-        try:
-            from benchmarks import verify_clean
-        except ModuleNotFoundError:
-            print(
-                "aelf bench verify-clean requires the source tree "
-                "(benchmarks/ is dev-only and not shipped in the wheel). "
-                "Clone the repo and run from the repo root.",
-                file=out,  # type: ignore[arg-type]
-            )
-            return 2
-        if not args.rest:
-            print("usage: aelf bench verify-clean PATH [PATH ...]", file=out)  # type: ignore[arg-type]
-            return 2
-        all_clean = True
-        for path in args.rest:
-            if not verify_clean.verify_file(path):
-                all_clean = False
-        return 0 if all_clean else 1
-
-    if target == "longmemeval-score":
-        try:
-            from benchmarks import longmemeval_score
-        except ModuleNotFoundError:
-            print(
-                "aelf bench longmemeval-score requires the source tree "
-                "(benchmarks/ is dev-only and not shipped in the wheel). "
-                "Clone the repo and run from the repo root.",
-                file=out,  # type: ignore[arg-type]
-            )
-            return 2
-        if len(args.rest) < 3:
-            print("usage: aelf bench longmemeval-score PREDS GT JUDGE", file=out)  # type: ignore[arg-type]
-            return 2
-        return longmemeval_score.score(args.rest[0], args.rest[1], args.rest[2])
-
-    if target == "posterior-residual":
-        try:
-            from benchmarks.posterior_ranking import run as _pr_run
-        except ModuleNotFoundError:
-            print(
-                "aelf bench posterior-residual requires the source tree "
-                "(benchmarks/ is dev-only and not shipped in the wheel). "
-                "Clone the repo and run from the repo root.",
-                file=out,  # type: ignore[arg-type]
-            )
-            return 2
-        # The bench subparser uses nargs=REMAINDER for `rest`, which swallows
-        # all tokens including named flags. Parse posterior-residual flags
-        # here rather than via argparse pre-declared args.
-        import argparse as _ap
-        _pr_parser = _ap.ArgumentParser(
-            prog="aelf bench posterior-residual",
-            add_help=False,
-        )
-        _pr_parser.add_argument("--fixtures", dest="pr_fixtures", default=None)
-        _pr_parser.add_argument("--seeds", dest="pr_seeds", type=int, default=5)
-        _pr_parser.add_argument(
-            "--mrr-threshold", dest="pr_mrr_threshold", type=float, default=0.05,
-        )
-        _pr_parser.add_argument(
-            "--ece-threshold", dest="pr_ece_threshold", type=float, default=0.10,
-        )
-        _pr_parser.add_argument(
-            "--json", dest="pr_json", action="store_true",
-        )
-        _pr_parser.add_argument(
-            "--heat-kernel", dest="pr_heat_kernel", action="store_true",
-            help="enable heat-kernel composition in retrieve() (slice 2 of #151)",
-        )
-        _pr_ns, _ = _pr_parser.parse_known_args(args.rest)
-        from pathlib import Path as _Path
-        _default_fixtures = (
-            _Path(__file__).parent.parent.parent
-            / "benchmarks"
-            / "posterior_ranking"
-            / "fixtures"
-            / "default.jsonl"
-        )
-        _fixtures_path = (
-            _Path(_pr_ns.pr_fixtures) if _pr_ns.pr_fixtures else _default_fixtures
-        )
-        result = _pr_run.run(
-            _fixtures_path,
-            n_seeds=_pr_ns.pr_seeds,
-            mrr_threshold=_pr_ns.pr_mrr_threshold,
-            ece_threshold=_pr_ns.pr_ece_threshold,
-            heat_kernel=_pr_ns.pr_heat_kernel,
-        )
-
-        if _pr_ns.pr_json:
-            from dataclasses import asdict as _asdict
-            print(
-                json.dumps({
-                    "mrr": _asdict(result["mrr"]),
-                    "ece": _asdict(result["ece"]),
-                    "overall_pass": result["overall_pass"],
-                }, indent=2),
-                file=out,  # type: ignore[arg-type]
-            )
-        else:
-            mrr = result["mrr"]
-            ece = result["ece"]
-            print(
-                f"posterior-residual eval\n"
-                f"  MRR uplift:  {mrr.mean_uplift:+.4f}  "
-                f"(±2σ: [{mrr.uplift_lo:+.4f}, {mrr.uplift_hi:+.4f}])  "
-                f"threshold={mrr.pass_threshold:+.2f}  "
-                f"{'PASS' if mrr.passed else 'FAIL'}\n"
-                f"  ECE:         {ece.ece:.4f}  "
-                f"threshold={ece.pass_threshold:.2f}  "
-                f"n={ece.n_total}  "
-                f"{'PASS' if ece.passed else 'FAIL'}\n"
-                f"  overall:     {'PASS' if result['overall_pass'] else 'FAIL'}",
-                file=out,  # type: ignore[arg-type]
-            )
-        return 0 if result["overall_pass"] else 1
+    _DEV_TARGETS_MOVED: dict[str, str] = {
+        "verify-clean": "python -m benchmarks.verify_clean",
+        "longmemeval-score": "python -m benchmarks.longmemeval_score",
+        "posterior-residual": "python -m benchmarks.posterior_ranking",
+    }
+    if target in _DEV_TARGETS_MOVED:
+        print(
+            f"aelf bench {target} has moved. Run "
+            f"`{_DEV_TARGETS_MOVED[target]} ...` from a source checkout.",
+            file=out,  # type: ignore[arg-type]
+        )
+        return 2
 
     if target in _BENCH_INERT_TARGETS:
         phase = _BENCH_INERT_TARGETS[target]
@@ -1164,9 +1059,11 @@ def _cmd_bench(args: argparse.Namespace, out: object) -> int:
 
     print(
         f"aelf bench: unknown target {target!r}.\n"
-        f"Known targets: synthetic (default), verify-clean, "
-        f"longmemeval-score, posterior-residual, "
-        f"{', '.join(sorted(_BENCH_INERT_TARGETS))}.",
+        f"Known targets: synthetic (default), "
+        f"{', '.join(sorted(_BENCH_INERT_TARGETS))}.\n"
+        f"Dev-only benchmarks moved to "
+        f"`python -m benchmarks.<name>`: "
+        f"{', '.join(sorted(_DEV_TARGETS_MOVED))}.",
         file=out,  # type: ignore[arg-type]
     )
     return 2

tests/test_benchmarks_dir.py

@@ -90,23 +90,31 @@ def test_aelf_bench_synthetic_target_explicit_runs_synthetic() -> None:
     assert report["corpus_size"] == 16
 
 
-def test_aelf_bench_verify_clean_dispatches_to_module(tmp_path: Path) -> None:
-    clean_file = tmp_path / "r.json"
-    clean_file.write_text(json.dumps([{"id": "q1", "question": "?"}]))
-    code, _ = _run_cli("bench", "verify-clean", str(clean_file))
-    assert code == 0
-
-
-def test_aelf_bench_verify_clean_no_args_exits_2() -> None:
+def test_aelf_bench_verify_clean_target_redirects_to_module() -> None:
+    """The dev-only target now points users at `python -m benchmarks.verify_clean`."""
     code, output = _run_cli("bench", "verify-clean")
     assert code == 2
-    assert "usage" in output.lower()
+    assert "python -m benchmarks.verify_clean" in output
 
 
-def test_aelf_bench_longmemeval_score_no_args_exits_2() -> None:
+def test_aelf_bench_longmemeval_score_target_redirects_to_module() -> None:
     code, output = _run_cli("bench", "longmemeval-score")
     assert code == 2
-    assert "usage" in output.lower()
+    assert "python -m benchmarks.longmemeval_score" in output
+
+
+def test_aelf_bench_posterior_residual_target_redirects_to_module() -> None:
+    code, output = _run_cli("bench", "posterior-residual")
+    assert code == 2
+    assert "python -m benchmarks.posterior_ranking" in output
+
+
+def test_aelf_bench_unknown_target_lists_moved_dev_targets() -> None:
+    code, output = _run_cli("bench", "no-such-target")
+    assert code == 2
+    assert "verify-clean" in output
+    assert "longmemeval-score" in output
+    assert "posterior-residual" in output
 
 
 def test_benchmarks_package_imports() -> None: