feat: bench-gate harness — corpus-root fixture, six module stubs, lab runner (#319)

pyproject.toml

@@ -116,6 +116,7 @@ strict_markers = true
 markers = [
     "regression: cumulative integration scenarios run alongside the unit suite",
     "uat: v1.0 acceptance-criteria tests, run as the final pre-tag gate",
+    "bench_gated: v2.0 bench-gate harness (#319) — skips when AELFRICE_CORPUS_ROOT is unset",
 ]
 
 [tool.pyright]

scripts/run_bench_gate.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Run the v2.0 bench-gate harness against a mounted lab corpus (#319).
+#
+# The corpus content lives in the private lab repo (see #307); this
+# script just points the public harness at it. Default expects the
+# standard two-repo layout:
+#
+#   ~/projects/aelfrice         <- public, this repo
+#   ~/projects/aelfrice-lab     <- private, holds the corpus
+#
+# Override via AELFRICE_CORPUS_ROOT.
+set -euo pipefail
+
+: "${AELFRICE_CORPUS_ROOT:=$HOME/projects/aelfrice-lab/tests/corpus/v2_0}"
+export AELFRICE_CORPUS_ROOT
+
+if [[ ! -d "$AELFRICE_CORPUS_ROOT" ]]; then
+    echo "error: AELFRICE_CORPUS_ROOT does not exist: $AELFRICE_CORPUS_ROOT" >&2
+    echo "       (mount the lab corpus or set the env var)" >&2
+    exit 1
+fi
+
+echo "bench-gate corpus root: $AELFRICE_CORPUS_ROOT"
+exec uv run pytest tests/bench_gate/ -v -m bench_gated "$@"

tests/bench_gate/test_contradiction.py

@@ -0,0 +1,29 @@
+"""Bench gate for #201 semantic contradiction detector."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import load_corpus_module
+
+
+@pytest.mark.bench_gated
+def test_contradiction_detector_against_corpus(aelfrice_corpus_root: Path) -> None:
+    rows = load_corpus_module(aelfrice_corpus_root, "contradiction")
+    try:
+        from aelfrice import relationship_detector  # type: ignore[attr-defined]
+    except ModuleNotFoundError as exc:
+        if exc.name in {"aelfrice", "aelfrice.relationship_detector"}:
+            pytest.skip("contradiction detector not yet implemented (#201)")
+        raise
+
+    correct = 0
+    for row in rows:
+        predicted = relationship_detector.classify(row["belief_a"], row["belief_b"])
+        if predicted == row["label"]:
+            correct += 1
+    accuracy = correct / len(rows)
+    assert accuracy >= 0.5, (
+        f"contradiction accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows"
+    )

tests/bench_gate/test_dedup.py

@@ -0,0 +1,35 @@
+"""Bench gate for #197 deduplication module.
+
+Loads the lab-mounted corpus and runs the dedup detector against the
+labels. Skips on public CI (corpus absent) and skips again here until
+the detector module from #197 ships.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import load_corpus_module
+
+
+@pytest.mark.bench_gated
+def test_dedup_detector_against_corpus(aelfrice_corpus_root: Path) -> None:
+    rows = load_corpus_module(aelfrice_corpus_root, "dedup")
+    try:
+        from aelfrice import dedup  # type: ignore[attr-defined]
+    except ModuleNotFoundError as exc:
+        if exc.name in {"aelfrice", "aelfrice.dedup"}:
+            pytest.skip("dedup detector not yet implemented (#197)")
+        raise
+
+    correct = 0
+    for row in rows:
+        predicted = dedup.classify(row["belief_a"], row["belief_b"])
+        if predicted == row["label"]:
+            correct += 1
+    assert rows, "dedup corpus produced zero rows; cannot compute accuracy"
+    accuracy = correct / len(rows)
+    assert accuracy >= 0.5, (
+        f"dedup accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows"
+    )

tests/bench_gate/test_enforcement.py

@@ -0,0 +1,29 @@
+"""Bench gate for #199 enforcement module."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import load_corpus_module
+
+
+@pytest.mark.bench_gated
+def test_enforcement_detector_against_corpus(aelfrice_corpus_root: Path) -> None:
+    rows = load_corpus_module(aelfrice_corpus_root, "enforcement")
+    try:
+        from aelfrice import enforcement  # type: ignore[attr-defined]
+    except ModuleNotFoundError as exc:
+        if exc.name in {"aelfrice", "aelfrice.enforcement"}:
+            pytest.skip("enforcement detector not yet implemented (#199)")
+        raise
+
+    correct = 0
+    for row in rows:
+        predicted = enforcement.classify(row["user_directive"], row["agent_output"])
+        if predicted == row["label"]:
+            correct += 1
+    accuracy = correct / len(rows)
+    assert accuracy >= 0.5, (
+        f"enforcement accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows"
+    )

tests/bench_gate/test_promotion_trigger.py

@@ -0,0 +1,29 @@
+"""Bench gate for #229 phantom promotion-trigger rule."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import load_corpus_module
+
+
+@pytest.mark.bench_gated
+def test_promotion_trigger_against_corpus(aelfrice_corpus_root: Path) -> None:
+    rows = load_corpus_module(aelfrice_corpus_root, "promotion_trigger")
+    try:
+        from aelfrice import promotion_trigger  # type: ignore[attr-defined]
+    except ModuleNotFoundError as exc:
+        if exc.name in {"aelfrice", "aelfrice.promotion_trigger"}:
+            pytest.skip("promotion_trigger rule not yet implemented (#229)")
+        raise
+
+    correct = 0
+    for row in rows:
+        predicted = promotion_trigger.decide(row["belief_sequence"])
+        if predicted == row["label"]:
+            correct += 1
+    accuracy = correct / len(rows)
+    assert accuracy >= 0.5, (
+        f"promotion_trigger accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows"
+    )

tests/bench_gate/test_sentiment.py

@@ -0,0 +1,29 @@
+"""Bench gate for #193 sentiment-from-prose feedback evaluation."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import load_corpus_module
+
+
+@pytest.mark.bench_gated
+def test_sentiment_detector_against_corpus(aelfrice_corpus_root: Path) -> None:
+    rows = load_corpus_module(aelfrice_corpus_root, "sentiment")
+    try:
+        from aelfrice import sentiment  # type: ignore[attr-defined]
+    except ModuleNotFoundError as exc:
+        if exc.name in {"aelfrice", "aelfrice.sentiment"}:
+            pytest.skip("sentiment detector not yet implemented (#193)")
+        raise
+
+    correct = 0
+    for row in rows:
+        predicted = sentiment.classify(row["user_message"])
+        if predicted == row["label"]:
+            correct += 1
+    accuracy = correct / len(rows)
+    assert accuracy >= 0.5, (
+        f"sentiment accuracy {accuracy:.3f} below 0.5 floor on {len(rows)} rows"
+    )

tests/bench_gate/test_wonder_consolidation.py

@@ -0,0 +1,29 @@
+"""Bench gate for #228 wonder-consolidation strategy bake-off."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import load_corpus_module
+
+
+@pytest.mark.bench_gated
+def test_wonder_consolidation_against_corpus(aelfrice_corpus_root: Path) -> None:
+    rows = load_corpus_module(aelfrice_corpus_root, "wonder_consolidation")
+    try:
+        from aelfrice import wonder_consolidation  # type: ignore[attr-defined]
+    except ModuleNotFoundError as exc:
+        if exc.name in {"aelfrice", "aelfrice.wonder_consolidation"}:
+            pytest.skip("wonder_consolidation strategy not yet implemented (#228)")
+        raise
+
+    # Expected human ratings 1-5; strategy outputs a generated phantom
+    # which the bake-off rates. v0.1 acceptance is correlation-shaped,
+    # not exact-match — leave the metric to #228 to define.
+    scored = 0
+    for row in rows:
+        rating = wonder_consolidation.score(row["seed_belief"], row["retrieved_neighbors"])
+        assert isinstance(rating, (int, float))
+        scored += 1
+    assert scored == len(rows)

tests/conftest.py

@@ -0,0 +1,78 @@
+"""Shared pytest fixtures (#319 v2.0 bench-gate harness).
+
+The bench-gate harness consumes the v2.0 evaluation corpus from #307. Corpus
+content lives in the private lab repo only; the public repo carries the
+schema contract and harness scaffold. The `AELFRICE_CORPUS_ROOT` env var
+points the harness at a mounted corpus. When the var is unset, or when a
+specific module directory is empty, bench-gate tests skip cleanly so public
+CI passes without corpus access.
+
+See `tests/corpus/v2_0/README.md` for the schema contract.
+"""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+CORPUS_ENV_VAR = "AELFRICE_CORPUS_ROOT"
+
+
+def _corpus_root() -> Path | None:
+    raw = os.environ.get(CORPUS_ENV_VAR)
+    if not raw:
+        return None
+    p = Path(raw).expanduser()
+    return p if p.is_dir() else None
+
+
+@pytest.fixture(scope="session")
+def aelfrice_corpus_root() -> Path:
+    """Resolve `AELFRICE_CORPUS_ROOT` to a directory; skip the test otherwise.
+
+    Tests that depend on labeled corpus rows should request this fixture and
+    will skip on public CI where the env var is unset.
+    """
+    root = _corpus_root()
+    if root is None:
+        pytest.skip(
+            f"{CORPUS_ENV_VAR} not set or not a directory; "
+            "skipping bench-gate test (lab corpus absent)"
+        )
+    return root
+
+
+@pytest.fixture(autouse=True)
+def _skip_bench_gated_without_corpus(request: pytest.FixtureRequest) -> None:
+    """Autouse guard: any test marked `bench_gated` skips when corpus is absent.
+
+    Backstops the `aelfrice_corpus_root` fixture for marker-only tests that
+    forget to request it explicitly.
+    """
+    if "bench_gated" not in request.keywords:
+        return
+    if _corpus_root() is None:
+        pytest.skip(
+            f"{CORPUS_ENV_VAR} not set or not a directory; "
+            "skipping bench-gate test (lab corpus absent)"
+        )
+
+
+def load_corpus_module(root: Path, module: str) -> list[dict]:
+    """Load every `*.jsonl` row under `root/<module>/`. Skip if empty."""
+    mod_dir = root / module
+    if not mod_dir.is_dir():
+        pytest.skip(f"corpus module {module!r} missing under {root}")
+    rows: list[dict] = []
+    for p in sorted(mod_dir.glob("*.jsonl")):
+        with p.open() as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                rows.append(json.loads(line))
+    if not rows:
+        pytest.skip(f"corpus module {module!r} empty under {root}")
+    return rows

tests/corpus/v2_0/README.md

@@ -1,8 +1,20 @@
 # v2.0 evaluation corpus — schema (#307)
 
 Six bench-gated v2.0 modules ship/no-ship on positive impact against a labeled
-corpus. This directory holds that corpus. The harness in #288 reads it; the
-modules in #193, #197, #199, #201, #228, #229 are evaluated against it.
+corpus. This directory holds that corpus. The bench-gate harness (#319) reads it
+via `AELFRICE_CORPUS_ROOT`; the modules in #193, #197, #199, #201, #228, #229
+are evaluated against it. (#288 is the **rebuilder**-precision harness — a
+different consumer.)
+
+## Mounting on the lab side
+
+Corpus content lives in the private lab repo only. Public CI runs with the
+env var unset; bench-gate tests skip cleanly. Lab runs:
+
+```bash
+export AELFRICE_CORPUS_ROOT="$HOME/projects/aelfrice-lab/tests/corpus/v2_0"
+./scripts/run_bench_gate.sh
+```
 
 ## Layout