From 392e16a6c4a0b13dfe17e6c316504c0d4865652f Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 30 Apr 2026 09:16:34 -0700 Subject: [PATCH 1/5] feat(bench-gate): add corpus-root fixture + bench_gated marker (#319) Public-side scaffold for the v2.0 bench-gate harness. Resolves AELFRICE_CORPUS_ROOT to a directory and exposes load_corpus_module() to bench-gate tests. Public CI passes without corpus access; lab runners export the env var to point at ~/projects/aelfrice-lab/tests/corpus/v2_0. No corpus content moves; the locked rule on ~/.claude/-derived content is respected. --- pyproject.toml | 1 + tests/conftest.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/conftest.py diff --git a/pyproject.toml b/pyproject.toml index 1042d7115..62f11e630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ strict_markers = true markers = [ "regression: cumulative integration scenarios run alongside the unit suite", "uat: v1.0 acceptance-criteria tests, run as the final pre-tag gate", + "bench_gated: v2.0 bench-gate harness (#319) — skips when AELFRICE_CORPUS_ROOT is unset", ] [tool.pyright] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..2a46b752e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,62 @@ +"""Shared pytest fixtures (#319 v2.0 bench-gate harness). + +The bench-gate harness consumes the v2.0 evaluation corpus from #307. Corpus +content lives in the private lab repo only; the public repo carries the +schema contract and harness scaffold. The `AELFRICE_CORPUS_ROOT` env var +points the harness at a mounted corpus. When the var is unset, or when a +specific module directory is empty, bench-gate tests skip cleanly so public +CI passes without corpus access. + +See `tests/corpus/v2_0/README.md` for the schema contract. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +CORPUS_ENV_VAR = "AELFRICE_CORPUS_ROOT" + + +def _corpus_root() -> Path | None: + raw = os.environ.get(CORPUS_ENV_VAR) + if not raw: + return None + p = Path(raw).expanduser() + return p if p.is_dir() else None + + +@pytest.fixture(scope="session") +def aelfrice_corpus_root() -> Path: + """Resolve `AELFRICE_CORPUS_ROOT` to a directory; skip the test otherwise. + + Tests that depend on labeled corpus rows should request this fixture and + will skip on public CI where the env var is unset. + """ + root = _corpus_root() + if root is None: + pytest.skip( + f"{CORPUS_ENV_VAR} not set or not a directory; " + "skipping bench-gate test (lab corpus absent)" + ) + return root + + +def load_corpus_module(root: Path, module: str) -> list[dict]: + """Load every `*.jsonl` row under `root//`. Skip if empty.""" + mod_dir = root / module + if not mod_dir.is_dir(): + pytest.skip(f"corpus module {module!r} missing under {root}") + rows: list[dict] = [] + for p in sorted(mod_dir.glob("*.jsonl")): + with p.open() as f: + for line in f: + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + if not rows: + pytest.skip(f"corpus module {module!r} empty under {root}") + return rows From b7b9cf9071e6a4c57eee998645c8ddd022457b04 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 30 Apr 2026 09:17:34 -0700 Subject: [PATCH 2/5] feat(bench-gate): six per-module bench-gate test stubs (#319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One pytest entry-point per bench-gated module — #197 dedup, #199 enforcement, #201 contradiction, #228 wonder_consolidation, #229 promotion_trigger, #193 sentiment. Each loads the lab corpus via the AELFRICE_CORPUS_ROOT fixture and skips when the detector module isn't yet implemented. Public CI: 6 skipped (corpus absent). Lab CI with corpus mounted: 6 skipped (detectors not yet shipped) — flips to real assertions as each detector lands. --- tests/bench_gate/__init__.py | 0 tests/bench_gate/test_contradiction.py | 27 ++++++++++++++++ tests/bench_gate/test_dedup.py | 32 +++++++++++++++++++ tests/bench_gate/test_enforcement.py | 27 ++++++++++++++++ tests/bench_gate/test_promotion_trigger.py | 27 ++++++++++++++++ tests/bench_gate/test_sentiment.py | 27 ++++++++++++++++ tests/bench_gate/test_wonder_consolidation.py | 27 ++++++++++++++++ 7 files changed, 167 insertions(+) create mode 100644 tests/bench_gate/__init__.py create mode 100644 tests/bench_gate/test_contradiction.py create mode 100644 tests/bench_gate/test_dedup.py create mode 100644 tests/bench_gate/test_enforcement.py create mode 100644 tests/bench_gate/test_promotion_trigger.py create mode 100644 tests/bench_gate/test_sentiment.py create mode 100644 tests/bench_gate/test_wonder_consolidation.py diff --git a/tests/bench_gate/__init__.py b/tests/bench_gate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/bench_gate/test_contradiction.py b/tests/bench_gate/test_contradiction.py new file mode 100644 index 000000000..e7a2b553b --- /dev/null +++ b/tests/bench_gate/test_contradiction.py @@ -0,0 +1,27 @@ +"""Bench gate for #201 semantic contradiction detector.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_contradiction_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "contradiction") + try: + from aelfrice import relationship_detector # type: ignore[attr-defined] + except ImportError: + pytest.skip("contradiction detector not yet implemented (#201)") + + correct = 0 + for row in rows: + predicted = relationship_detector.classify(row["belief_a"], row["belief_b"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"contradiction accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_dedup.py b/tests/bench_gate/test_dedup.py new file mode 100644 index 000000000..8e4057985 --- /dev/null +++ b/tests/bench_gate/test_dedup.py @@ -0,0 +1,32 @@ +"""Bench gate for #197 deduplication module. + +Loads the lab-mounted corpus and runs the dedup detector against the +labels. Skips on public CI (corpus absent) and skips again here until +the detector module from #197 ships. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_dedup_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "dedup") + try: + from aelfrice import dedup # type: ignore[attr-defined] + except ImportError: + pytest.skip("dedup detector not yet implemented (#197)") + + correct = 0 + for row in rows: + predicted = dedup.classify(row["belief_a"], row["belief_b"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"dedup accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_enforcement.py b/tests/bench_gate/test_enforcement.py new file mode 100644 index 000000000..09fba0f35 --- /dev/null +++ b/tests/bench_gate/test_enforcement.py @@ -0,0 +1,27 @@ +"""Bench gate for #199 enforcement module.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_enforcement_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "enforcement") + try: + from aelfrice import enforcement # type: ignore[attr-defined] + except ImportError: + pytest.skip("enforcement detector not yet implemented (#199)") + + correct = 0 + for row in rows: + predicted = enforcement.classify(row["user_directive"], row["agent_output"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"enforcement accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_promotion_trigger.py b/tests/bench_gate/test_promotion_trigger.py new file mode 100644 index 000000000..5193186d6 --- /dev/null +++ b/tests/bench_gate/test_promotion_trigger.py @@ -0,0 +1,27 @@ +"""Bench gate for #229 phantom promotion-trigger rule.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_promotion_trigger_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "promotion_trigger") + try: + from aelfrice import promotion_trigger # type: ignore[attr-defined] + except ImportError: + pytest.skip("promotion_trigger rule not yet implemented (#229)") + + correct = 0 + for row in rows: + predicted = promotion_trigger.decide(row["belief_sequence"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"promotion_trigger accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_sentiment.py b/tests/bench_gate/test_sentiment.py new file mode 100644 index 000000000..68d1f0bd5 --- /dev/null +++ b/tests/bench_gate/test_sentiment.py @@ -0,0 +1,27 @@ +"""Bench gate for #193 sentiment-from-prose feedback evaluation.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_sentiment_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "sentiment") + try: + from aelfrice import sentiment # type: ignore[attr-defined] + except ImportError: + pytest.skip("sentiment detector not yet implemented (#193)") + + correct = 0 + for row in rows: + predicted = sentiment.classify(row["user_message"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"sentiment accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_wonder_consolidation.py b/tests/bench_gate/test_wonder_consolidation.py new file mode 100644 index 000000000..dfe83f150 --- /dev/null +++ b/tests/bench_gate/test_wonder_consolidation.py @@ -0,0 +1,27 @@ +"""Bench gate for #228 wonder-consolidation strategy bake-off.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_wonder_consolidation_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "wonder_consolidation") + try: + from aelfrice import wonder_consolidation # type: ignore[attr-defined] + except ImportError: + pytest.skip("wonder_consolidation strategy not yet implemented (#228)") + + # Expected human ratings 1-5; strategy outputs a generated phantom + # which the bake-off rates. v0.1 acceptance is correlation-shaped, + # not exact-match — leave the metric to #228 to define. + scored = 0 + for row in rows: + rating = wonder_consolidation.score(row["seed_belief"], row["retrieved_neighbors"]) + assert isinstance(rating, (int, float)) + scored += 1 + assert scored == len(rows) From 9d3705588f2e08430fdffae2ccdc1e686f9ccbee Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 30 Apr 2026 09:17:54 -0700 Subject: [PATCH 3/5] feat(bench-gate): add lab-side runner script run_bench_gate.sh (#319) Defaults AELFRICE_CORPUS_ROOT to ~/projects/aelfrice-lab/tests/corpus/v2_0 matching the two-repo layout in CLAUDE.md. Errors clearly when the corpus dir is missing rather than silently skipping. The script lives public-side (path string only, no derived content). --- scripts/run_bench_gate.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 scripts/run_bench_gate.sh diff --git a/scripts/run_bench_gate.sh b/scripts/run_bench_gate.sh new file mode 100755 index 000000000..c0467b905 --- /dev/null +++ b/scripts/run_bench_gate.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Run the v2.0 bench-gate harness against a mounted lab corpus (#319). +# +# The corpus content lives in the private lab repo (see #307); this +# script just points the public harness at it. Default expects the +# standard two-repo layout: +# +# ~/projects/aelfrice <- public, this repo +# ~/projects/aelfrice-lab <- private, holds the corpus +# +# Override via AELFRICE_CORPUS_ROOT. +set -euo pipefail + +: "${AELFRICE_CORPUS_ROOT:=$HOME/projects/aelfrice-lab/tests/corpus/v2_0}" +export AELFRICE_CORPUS_ROOT + +if [[ ! -d "$AELFRICE_CORPUS_ROOT" ]]; then + echo "error: AELFRICE_CORPUS_ROOT does not exist: $AELFRICE_CORPUS_ROOT" >&2 + echo " (mount the lab corpus or set the env var)" >&2 + exit 1 +fi + +echo "bench-gate corpus root: $AELFRICE_CORPUS_ROOT" +exec uv run pytest tests/bench_gate/ -v -m bench_gated "$@" From 3fd820fd2cbffd7939560bf249c9b475d9370a3e Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 30 Apr 2026 09:18:07 -0700 Subject: [PATCH 4/5] docs(bench-gate): point README at #319 harness, add lab-mount instructions Removes the stale #288 cross-reference (#288 is the rebuilder-precision harness, not the v2.0 bench-gate harness). Documents AELFRICE_CORPUS_ROOT and the run_bench_gate.sh entrypoint for lab-side runs. --- tests/corpus/v2_0/README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/corpus/v2_0/README.md b/tests/corpus/v2_0/README.md index 1dd40f565..0cff2fe7a 100644 --- a/tests/corpus/v2_0/README.md +++ b/tests/corpus/v2_0/README.md @@ -1,8 +1,20 @@ # v2.0 evaluation corpus — schema (#307) Six bench-gated v2.0 modules ship/no-ship on positive impact against a labeled -corpus. This directory holds that corpus. The harness in #288 reads it; the -modules in #193, #197, #199, #201, #228, #229 are evaluated against it. +corpus. This directory holds that corpus. The bench-gate harness (#319) reads it +via `AELFRICE_CORPUS_ROOT`; the modules in #193, #197, #199, #201, #228, #229 +are evaluated against it. (#288 is the **rebuilder**-precision harness — a +different consumer.) + +## Mounting on the lab side + +Corpus content lives in the private lab repo only. Public CI runs with the +env var unset; bench-gate tests skip cleanly. Lab runs: + +```bash +export AELFRICE_CORPUS_ROOT="$HOME/projects/aelfrice-lab/tests/corpus/v2_0" +./scripts/run_bench_gate.sh +``` ## Layout From c176ea03bbcf50a312c8ccb8ce1640d930b0eea1 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 30 Apr 2026 09:52:35 -0700 Subject: [PATCH 5/5] fix(bench-gate): autouse skip guard + narrow ModuleNotFoundError (#319) CodeRabbit review on #320: - conftest.py grows an autouse fixture keyed on the bench_gated marker. Backstops the aelfrice_corpus_root fixture so a marker-only test that forgets to request the fixture still skips on public CI. - All six bench-gate tests narrow except ImportError to ModuleNotFoundError scoped to the missing detector module name; any other ImportError (broken submodule import, missing transitive dep) now surfaces instead of being masked as a skip. - test_dedup.py adds an explicit non-empty rows assertion before the accuracy division. uv run pytest tests/bench_gate/ -> 6 skipped, 0 failed (corpus absent). --- tests/bench_gate/test_contradiction.py | 6 ++++-- tests/bench_gate/test_dedup.py | 7 +++++-- tests/bench_gate/test_enforcement.py | 6 ++++-- tests/bench_gate/test_promotion_trigger.py | 6 ++++-- tests/bench_gate/test_sentiment.py | 6 ++++-- tests/bench_gate/test_wonder_consolidation.py | 6 ++++-- tests/conftest.py | 16 ++++++++++++++++ 7 files changed, 41 insertions(+), 12 deletions(-) diff --git a/tests/bench_gate/test_contradiction.py b/tests/bench_gate/test_contradiction.py index e7a2b553b..1b13431ab 100644 --- a/tests/bench_gate/test_contradiction.py +++ b/tests/bench_gate/test_contradiction.py @@ -13,8 +13,10 @@ def test_contradiction_detector_against_corpus(aelfrice_corpus_root: Path) -> No rows = load_corpus_module(aelfrice_corpus_root, "contradiction") try: from aelfrice import relationship_detector # type: ignore[attr-defined] - except ImportError: - pytest.skip("contradiction detector not yet implemented (#201)") + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.relationship_detector"}: + pytest.skip("contradiction detector not yet implemented (#201)") + raise correct = 0 for row in rows: diff --git a/tests/bench_gate/test_dedup.py b/tests/bench_gate/test_dedup.py index 8e4057985..499f4ecb0 100644 --- a/tests/bench_gate/test_dedup.py +++ b/tests/bench_gate/test_dedup.py @@ -18,14 +18,17 @@ def test_dedup_detector_against_corpus(aelfrice_corpus_root: Path) -> None: rows = load_corpus_module(aelfrice_corpus_root, "dedup") try: from aelfrice import dedup # type: ignore[attr-defined] - except ImportError: - pytest.skip("dedup detector not yet implemented (#197)") + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.dedup"}: + pytest.skip("dedup detector not yet implemented (#197)") + raise correct = 0 for row in rows: predicted = dedup.classify(row["belief_a"], row["belief_b"]) if predicted == row["label"]: correct += 1 + assert rows, "dedup corpus produced zero rows; cannot compute accuracy" accuracy = correct / len(rows) assert accuracy >= 0.5, ( f"dedup accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" diff --git a/tests/bench_gate/test_enforcement.py b/tests/bench_gate/test_enforcement.py index 09fba0f35..d3c38fde0 100644 --- a/tests/bench_gate/test_enforcement.py +++ b/tests/bench_gate/test_enforcement.py @@ -13,8 +13,10 @@ def test_enforcement_detector_against_corpus(aelfrice_corpus_root: Path) -> None rows = load_corpus_module(aelfrice_corpus_root, "enforcement") try: from aelfrice import enforcement # type: ignore[attr-defined] - except ImportError: - pytest.skip("enforcement detector not yet implemented (#199)") + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.enforcement"}: + pytest.skip("enforcement detector not yet implemented (#199)") + raise correct = 0 for row in rows: diff --git a/tests/bench_gate/test_promotion_trigger.py b/tests/bench_gate/test_promotion_trigger.py index 5193186d6..d25e2b7f3 100644 --- a/tests/bench_gate/test_promotion_trigger.py +++ b/tests/bench_gate/test_promotion_trigger.py @@ -13,8 +13,10 @@ def test_promotion_trigger_against_corpus(aelfrice_corpus_root: Path) -> None: rows = load_corpus_module(aelfrice_corpus_root, "promotion_trigger") try: from aelfrice import promotion_trigger # type: ignore[attr-defined] - except ImportError: - pytest.skip("promotion_trigger rule not yet implemented (#229)") + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.promotion_trigger"}: + pytest.skip("promotion_trigger rule not yet implemented (#229)") + raise correct = 0 for row in rows: diff --git a/tests/bench_gate/test_sentiment.py b/tests/bench_gate/test_sentiment.py index 68d1f0bd5..e8e0cb9b5 100644 --- a/tests/bench_gate/test_sentiment.py +++ b/tests/bench_gate/test_sentiment.py @@ -13,8 +13,10 @@ def test_sentiment_detector_against_corpus(aelfrice_corpus_root: Path) -> None: rows = load_corpus_module(aelfrice_corpus_root, "sentiment") try: from aelfrice import sentiment # type: ignore[attr-defined] - except ImportError: - pytest.skip("sentiment detector not yet implemented (#193)") + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.sentiment"}: + pytest.skip("sentiment detector not yet implemented (#193)") + raise correct = 0 for row in rows: diff --git a/tests/bench_gate/test_wonder_consolidation.py b/tests/bench_gate/test_wonder_consolidation.py index dfe83f150..15bcd1d4a 100644 --- a/tests/bench_gate/test_wonder_consolidation.py +++ b/tests/bench_gate/test_wonder_consolidation.py @@ -13,8 +13,10 @@ def test_wonder_consolidation_against_corpus(aelfrice_corpus_root: Path) -> None rows = load_corpus_module(aelfrice_corpus_root, "wonder_consolidation") try: from aelfrice import wonder_consolidation # type: ignore[attr-defined] - except ImportError: - pytest.skip("wonder_consolidation strategy not yet implemented (#228)") + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.wonder_consolidation"}: + pytest.skip("wonder_consolidation strategy not yet implemented (#228)") + raise # Expected human ratings 1-5; strategy outputs a generated phantom # which the bake-off rates. v0.1 acceptance is correlation-shaped, diff --git a/tests/conftest.py b/tests/conftest.py index 2a46b752e..0f3461732 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,6 +44,22 @@ def aelfrice_corpus_root() -> Path: return root +@pytest.fixture(autouse=True) +def _skip_bench_gated_without_corpus(request: pytest.FixtureRequest) -> None: + """Autouse guard: any test marked `bench_gated` skips when corpus is absent. + + Backstops the `aelfrice_corpus_root` fixture for marker-only tests that + forget to request it explicitly. + """ + if "bench_gated" not in request.keywords: + return + if _corpus_root() is None: + pytest.skip( + f"{CORPUS_ENV_VAR} not set or not a directory; " + "skipping bench-gate test (lab corpus absent)" + ) + + def load_corpus_module(root: Path, module: str) -> list[dict]: """Load every `*.jsonl` row under `root//`. Skip if empty.""" mod_dir = root / module