From 776f2f8e56380eb1e7976a719b269b99454831bc Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:43:55 -0700 Subject: [PATCH 1/2] test(bench): make the channel-3 sweeper check falsifiable (#1267) The sweep ran against an empty deferred-feedback queue, so sweep_deferred_feedback never entered its classification loop and the audit could not tell the audit-only sweeper from the pre-#1162 mutating one. Its other assertion read result.mutated, a dataclass field default that is never assigned anywhere in src/. Restoring the mutating branch and re-running left the script at exit 0. Enqueue one exposure row and backdate it past the grace window so the row is eligible, then assert would_apply == 1 alongside the unmoved posterior: the count is what proves the ladder ran and elected to apply, which is the state the old sweeper mutated in. Also assert no feedback_history row and that alpha_withheld accounts for the withheld epsilon. Against the same restored mutating branch this now exits 1. --- benchmarks/posterior_channel_audit.py | 53 +++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/benchmarks/posterior_channel_audit.py b/benchmarks/posterior_channel_audit.py index 01d2514a1..bf9361132 100644 --- a/benchmarks/posterior_channel_audit.py +++ b/benchmarks/posterior_channel_audit.py @@ -43,6 +43,7 @@ import os import sys +from datetime import UTC, datetime, timedelta # Clear ambient opt-ins BEFORE importing aelfrice: several of these # resolvers read the environment at import time or per call, and the @@ -52,7 +53,10 @@ del os.environ[_k] from aelfrice.deferred_feedback import ( # noqa: E402 + enqueue_retrieval_exposures, is_enqueue_on_retrieve_enabled, + resolve_epsilon, + resolve_grace_seconds, sweep_deferred_feedback, ) from aelfrice.hook_search import ( # noqa: E402 @@ -199,21 +203,54 @@ def channel_3_sweeper() -> list[str]: failures.append("retrieval enqueues exposures by default") store = _seed("b3") + # The queue has to be non-empty for this to test anything. A sweep + # over an empty queue never enters its classification loop, so it is + # a no-op for the audit-only sweeper and for the pre-#1162 mutating + # one alike — the two are indistinguishable and the check below + # passes either way. Bank a real row and backdate it past the grace + # window so the row is eligible and the loop actually runs on it. + grace = resolve_grace_seconds() + epsilon = resolve_epsilon() + enqueued_at = ( + datetime.now(UTC) - timedelta(seconds=grace + 60) + ).strftime("%Y-%m-%dT%H:%M:%SZ") + enqueue_retrieval_exposures(store, ["b3"], now=enqueued_at) + before = _ab(store, "b3") result = sweep_deferred_feedback(store) after = _ab(store, "b3") + audit_rows = int(store.count_feedback_events("b3")) store.close() - print(f" sweep mutated={result.mutated} " + print(f" enqueued 1 row, backdated {grace + 60}s (grace={grace}s) " + f"-> eligible") + print(f" sweep would_apply={result.would_apply} " f"alpha_withheld={result.alpha_withheld} " - f"would_apply={result.would_apply}") - print(f" a/b {before} -> {after} moved={before != after}") - - if result.mutated or before != after: + f"epsilon={result.epsilon_used}") + print(f" a/b {before} -> {after} moved={before != after} " + f"feedback_history rows={audit_rows}") + + # `would_apply == 1` is what makes the rest of this load-bearing: it + # proves the eligibility ladder ran and elected to apply. The pre- + # #1162 sweeper would have moved alpha by exactly `epsilon` on this + # row and written a `feedback_history` entry; audit-only does neither. + if result.would_apply != 1: + failures.append( + f"the eligible row was not classified would_apply " + f"(got {result.would_apply}) — this check proves nothing" + ) + if before != after: failures.append("the sweeper mutated a posterior") - - print(" => no residual exposure-as-evidence path: the sweep writes " - "nothing and the enqueue is off.") + if audit_rows != 0: + failures.append("the sweeper wrote a feedback_history row") + if result.alpha_withheld != round(epsilon, 6): + failures.append( + f"alpha_withheld {result.alpha_withheld} does not account for " + f"the one withheld epsilon ({epsilon})" + ) + + print(" => no residual exposure-as-evidence path: an eligible row is " + "classified and then withheld, not applied.") return failures From 49b9d5b76d0f9e99530a8900e3f393a2c1e1dbc8 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:43:55 -0700 Subject: [PATCH 2/2] test(bench): run the posterior-channel audit from the pytest job (#1267) Nothing invoked benchmarks/posterior_channel_audit.py. bench-smoke has a benchmarks/** path filter but only runs test_bench_smoke.py and test_recalibrate.py, so a default could drift and the LIMITATIONS entry the script backs would go stale exactly as the last one did. Drive it as a subprocess from tests/test_benchmarks_dir.py, which the required pytest (3.12)/(3.13) checks already collect. Subprocess rather than import because the script deletes every ambient AELFRICE_ variable at import time to measure defaults, which must not leak into the rest of the session. --- tests/test_benchmarks_dir.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_benchmarks_dir.py b/tests/test_benchmarks_dir.py index 7e4deadc5..65975fff6 100644 --- a/tests/test_benchmarks_dir.py +++ b/tests/test_benchmarks_dir.py @@ -133,3 +133,39 @@ def test_benchmarks_package_imports() -> None: """The benchmarks/ directory is a valid (empty) package.""" import benchmarks assert benchmarks is not None + + +@pytest.mark.timeout(120) +def test_posterior_channel_audit_holds_the_documented_defaults() -> None: + """`benchmarks/posterior_channel_audit.py` exits 0 on the shipped defaults. + + The script is the regression guard behind the LIMITATIONS entry that + says no automatic channel moves a posterior (#1267). A guard nothing + invokes cannot keep a doc entry honest, and no workflow runs + `benchmarks/*.py` — `bench-smoke` only runs two named test modules. + Running it from the required `pytest` job is what makes the claim + enforceable. + + Driven as a subprocess rather than imported: the script deletes every + ambient `AELFRICE_*` variable at import time so it measures defaults + rather than the developer's opt-ins, and that must not leak into the + rest of the test session. + """ + import subprocess + import sys + + repo_root = Path(__file__).resolve().parent.parent + script = repo_root / "benchmarks" / "posterior_channel_audit.py" + assert script.is_file(), f"missing {script}" + + proc = subprocess.run( + [sys.executable, str(script)], + cwd=repo_root, + capture_output=True, + text=True, + timeout=110, + ) + assert proc.returncode == 0, ( + f"posterior-channel audit failed (exit {proc.returncode}); a " + f"documented default moved:\n{proc.stdout}\n{proc.stderr}" + )