diff --git a/pyproject.toml b/pyproject.toml index 1042d7115..62f11e630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ strict_markers = true markers = [ "regression: cumulative integration scenarios run alongside the unit suite", "uat: v1.0 acceptance-criteria tests, run as the final pre-tag gate", + "bench_gated: v2.0 bench-gate harness (#319) — skips when AELFRICE_CORPUS_ROOT is unset", ] [tool.pyright] diff --git a/scripts/run_bench_gate.sh b/scripts/run_bench_gate.sh new file mode 100755 index 000000000..c0467b905 --- /dev/null +++ b/scripts/run_bench_gate.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Run the v2.0 bench-gate harness against a mounted lab corpus (#319). +# +# The corpus content lives in the private lab repo (see #307); this +# script just points the public harness at it. Default expects the +# standard two-repo layout: +# +# ~/projects/aelfrice <- public, this repo +# ~/projects/aelfrice-lab <- private, holds the corpus +# +# Override via AELFRICE_CORPUS_ROOT. +set -euo pipefail + +: "${AELFRICE_CORPUS_ROOT:=$HOME/projects/aelfrice-lab/tests/corpus/v2_0}" +export AELFRICE_CORPUS_ROOT + +if [[ ! -d "$AELFRICE_CORPUS_ROOT" ]]; then + echo "error: AELFRICE_CORPUS_ROOT does not exist: $AELFRICE_CORPUS_ROOT" >&2 + echo " (mount the lab corpus or set the env var)" >&2 + exit 1 +fi + +echo "bench-gate corpus root: $AELFRICE_CORPUS_ROOT" +exec uv run pytest tests/bench_gate/ -v -m bench_gated "$@" diff --git a/tests/bench_gate/__init__.py b/tests/bench_gate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/bench_gate/test_contradiction.py b/tests/bench_gate/test_contradiction.py new file mode 100644 index 000000000..1b13431ab --- /dev/null +++ b/tests/bench_gate/test_contradiction.py @@ -0,0 +1,29 @@ +"""Bench gate for #201 semantic contradiction detector.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_contradiction_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "contradiction") + try: + from aelfrice import relationship_detector # type: ignore[attr-defined] + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.relationship_detector"}: + pytest.skip("contradiction detector not yet implemented (#201)") + raise + + correct = 0 + for row in rows: + predicted = relationship_detector.classify(row["belief_a"], row["belief_b"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"contradiction accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_dedup.py b/tests/bench_gate/test_dedup.py new file mode 100644 index 000000000..499f4ecb0 --- /dev/null +++ b/tests/bench_gate/test_dedup.py @@ -0,0 +1,35 @@ +"""Bench gate for #197 deduplication module. + +Loads the lab-mounted corpus and runs the dedup detector against the +labels. Skips on public CI (corpus absent) and skips again here until +the detector module from #197 ships. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_dedup_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "dedup") + try: + from aelfrice import dedup # type: ignore[attr-defined] + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.dedup"}: + pytest.skip("dedup detector not yet implemented (#197)") + raise + + correct = 0 + for row in rows: + predicted = dedup.classify(row["belief_a"], row["belief_b"]) + if predicted == row["label"]: + correct += 1 + assert rows, "dedup corpus produced zero rows; cannot compute accuracy" + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"dedup accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_enforcement.py b/tests/bench_gate/test_enforcement.py new file mode 100644 index 000000000..d3c38fde0 --- /dev/null +++ b/tests/bench_gate/test_enforcement.py @@ -0,0 +1,29 @@ +"""Bench gate for #199 enforcement module.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_enforcement_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "enforcement") + try: + from aelfrice import enforcement # type: ignore[attr-defined] + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.enforcement"}: + pytest.skip("enforcement detector not yet implemented (#199)") + raise + + correct = 0 + for row in rows: + predicted = enforcement.classify(row["user_directive"], row["agent_output"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"enforcement accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_promotion_trigger.py b/tests/bench_gate/test_promotion_trigger.py new file mode 100644 index 000000000..d25e2b7f3 --- /dev/null +++ b/tests/bench_gate/test_promotion_trigger.py @@ -0,0 +1,29 @@ +"""Bench gate for #229 phantom promotion-trigger rule.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_promotion_trigger_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "promotion_trigger") + try: + from aelfrice import promotion_trigger # type: ignore[attr-defined] + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.promotion_trigger"}: + pytest.skip("promotion_trigger rule not yet implemented (#229)") + raise + + correct = 0 + for row in rows: + predicted = promotion_trigger.decide(row["belief_sequence"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"promotion_trigger accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_sentiment.py b/tests/bench_gate/test_sentiment.py new file mode 100644 index 000000000..e8e0cb9b5 --- /dev/null +++ b/tests/bench_gate/test_sentiment.py @@ -0,0 +1,29 @@ +"""Bench gate for #193 sentiment-from-prose feedback evaluation.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_sentiment_detector_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "sentiment") + try: + from aelfrice import sentiment # type: ignore[attr-defined] + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.sentiment"}: + pytest.skip("sentiment detector not yet implemented (#193)") + raise + + correct = 0 + for row in rows: + predicted = sentiment.classify(row["user_message"]) + if predicted == row["label"]: + correct += 1 + accuracy = correct / len(rows) + assert accuracy >= 0.5, ( + f"sentiment accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows" + ) diff --git a/tests/bench_gate/test_wonder_consolidation.py b/tests/bench_gate/test_wonder_consolidation.py new file mode 100644 index 000000000..15bcd1d4a --- /dev/null +++ b/tests/bench_gate/test_wonder_consolidation.py @@ -0,0 +1,29 @@ +"""Bench gate for #228 wonder-consolidation strategy bake-off.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + + +@pytest.mark.bench_gated +def test_wonder_consolidation_against_corpus(aelfrice_corpus_root: Path) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "wonder_consolidation") + try: + from aelfrice import wonder_consolidation # type: ignore[attr-defined] + except ModuleNotFoundError as exc: + if exc.name in {"aelfrice", "aelfrice.wonder_consolidation"}: + pytest.skip("wonder_consolidation strategy not yet implemented (#228)") + raise + + # Expected human ratings 1-5; strategy outputs a generated phantom + # which the bake-off rates. v0.1 acceptance is correlation-shaped, + # not exact-match — leave the metric to #228 to define. + scored = 0 + for row in rows: + rating = wonder_consolidation.score(row["seed_belief"], row["retrieved_neighbors"]) + assert isinstance(rating, (int, float)) + scored += 1 + assert scored == len(rows) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..0f3461732 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,78 @@ +"""Shared pytest fixtures (#319 v2.0 bench-gate harness). + +The bench-gate harness consumes the v2.0 evaluation corpus from #307. Corpus +content lives in the private lab repo only; the public repo carries the +schema contract and harness scaffold. The `AELFRICE_CORPUS_ROOT` env var +points the harness at a mounted corpus. When the var is unset, or when a +specific module directory is empty, bench-gate tests skip cleanly so public +CI passes without corpus access. + +See `tests/corpus/v2_0/README.md` for the schema contract. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +CORPUS_ENV_VAR = "AELFRICE_CORPUS_ROOT" + + +def _corpus_root() -> Path | None: + raw = os.environ.get(CORPUS_ENV_VAR) + if not raw: + return None + p = Path(raw).expanduser() + return p if p.is_dir() else None + + +@pytest.fixture(scope="session") +def aelfrice_corpus_root() -> Path: + """Resolve `AELFRICE_CORPUS_ROOT` to a directory; skip the test otherwise. + + Tests that depend on labeled corpus rows should request this fixture and + will skip on public CI where the env var is unset. + """ + root = _corpus_root() + if root is None: + pytest.skip( + f"{CORPUS_ENV_VAR} not set or not a directory; " + "skipping bench-gate test (lab corpus absent)" + ) + return root + + +@pytest.fixture(autouse=True) +def _skip_bench_gated_without_corpus(request: pytest.FixtureRequest) -> None: + """Autouse guard: any test marked `bench_gated` skips when corpus is absent. + + Backstops the `aelfrice_corpus_root` fixture for marker-only tests that + forget to request it explicitly. + """ + if "bench_gated" not in request.keywords: + return + if _corpus_root() is None: + pytest.skip( + f"{CORPUS_ENV_VAR} not set or not a directory; " + "skipping bench-gate test (lab corpus absent)" + ) + + +def load_corpus_module(root: Path, module: str) -> list[dict]: + """Load every `*.jsonl` row under `root//`. Skip if empty.""" + mod_dir = root / module + if not mod_dir.is_dir(): + pytest.skip(f"corpus module {module!r} missing under {root}") + rows: list[dict] = [] + for p in sorted(mod_dir.glob("*.jsonl")): + with p.open() as f: + for line in f: + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + if not rows: + pytest.skip(f"corpus module {module!r} empty under {root}") + return rows diff --git a/tests/corpus/v2_0/README.md b/tests/corpus/v2_0/README.md index 1dd40f565..0cff2fe7a 100644 --- a/tests/corpus/v2_0/README.md +++ b/tests/corpus/v2_0/README.md @@ -1,8 +1,20 @@ # v2.0 evaluation corpus — schema (#307) Six bench-gated v2.0 modules ship/no-ship on positive impact against a labeled -corpus. This directory holds that corpus. The harness in #288 reads it; the -modules in #193, #197, #199, #201, #228, #229 are evaluated against it. +corpus. This directory holds that corpus. The bench-gate harness (#319) reads it +via `AELFRICE_CORPUS_ROOT`; the modules in #193, #197, #199, #201, #228, #229 +are evaluated against it. (#288 is the **rebuilder**-precision harness — a +different consumer.) + +## Mounting on the lab side + +Corpus content lives in the private lab repo only. Public CI runs with the +env var unset; bench-gate tests skip cleanly. Lab runs: + +```bash +export AELFRICE_CORPUS_ROOT="$HOME/projects/aelfrice-lab/tests/corpus/v2_0" +./scripts/run_bench_gate.sh +``` ## Layout