diff --git a/tests/test_r3_idf_clip_reachability.py b/tests/test_r3_idf_clip_reachability.py index bee956bf1..02cd2dca9 100644 --- a/tests/test_r3_idf_clip_reachability.py +++ b/tests/test_r3_idf_clip_reachability.py @@ -31,6 +31,8 @@ """ from __future__ import annotations +import json +import math from collections.abc import Iterator from pathlib import Path @@ -46,6 +48,9 @@ compute_idf_quantile_thresholds, ) from aelfrice.store import MemoryStore +from benchmarks.r3_idf_clip_bound import load_prompts +from benchmarks.r3_idf_clip_bound import main as bound_main +from benchmarks.r3_idf_clip_bound import reachability # A Zipfian-shaped corpus: every document shares a small common core and # carries two document-unique terms. That puts the hapax share well above @@ -143,6 +148,133 @@ def test_a_reachable_high_quantile_does_boost(index: BM25Index) -> None: assert len(out) > len(set(out)) +def test_low_cutoff_is_reported_as_an_exact_document_frequency( + index: BM25Index, +) -> None: + """`df_at_low_cutoff` must invert the IDF the index actually uses. + + The harness ships to be re-run on other stores, so this has to hold away + from the development store's operating point. Inverting + ``log(1 + (N + 0.5) / (df + 0.5))`` — dropping the ``- df`` from the + numerator of the shipped form — agrees to 0.02% where ``exp(low) >> 1`` + and is off by 58% at ``idf == 1.0``, so a smaller or less Zipfian corpus + would be reported wrongly with no visible symptom. + + Feeding back the IDF of a known `df` must return that `df`. The dropped + form fails this across the whole range on this fixture — 1.0380 against + 1.0 at df = 1, and 3280.0 against 40.0 at df = 40 — so the assertion + measures the inversion rather than restating it. Note the error grows + with `df`: it is mildest exactly where the development store's cutoff + sits, which is why the defect survived a review that checked only there. + """ + n_docs = len(index.belief_ids) + idf_max = float(index.idf.max()) + for df in range(1, n_docs + 1): + idf_at_df = float(np.log(1.0 + (n_docs - df + 0.5) / (df + 0.5))) + reported = reachability(index, idf_at_df, idf_max)["df_at_low_cutoff"] + assert reported == pytest.approx(df, rel=1e-9) + + +def test_low_cutoff_is_non_finite_exactly_when_the_cutoff_is_zero( + index: BM25Index, +) -> None: + """Pins the precondition `main`'s null-conversion exists for. + + A `low` of 0.0 admits every term, so there is no document frequency to + report and the field is NaN. `main` depends on that being the only + non-finite case when it serialises; this fails loudly if the + representation changes to a sentinel number instead. + """ + idf_max = float(index.idf.max()) + assert math.isnan(reachability(index, 0.0, idf_max)["df_at_low_cutoff"]) + assert math.isfinite(reachability(index, 1.0, idf_max)["df_at_low_cutoff"]) + + +def test_json_out_is_parseable_by_a_strict_rfc_8259_reader( + tmp_path: Path, +) -> None: + """End-to-end: `--json-out` must never emit a bare `NaN` or `Infinity`. + + `json.dumps` writes those tokens by default and RFC 8259 does not permit + them, so a strict parser rejects the file. `parse_constant` fires on + exactly those three tokens, so this guards the whole payload against a + future non-finite field, not `df_at_low_cutoff` alone. + + Scope, stated plainly: this does **not** exercise `main`'s NaN-to-null + conversion, and would still pass with it deleted. Robertson IDF is + strictly positive for every `df <= N`, so the low quantile is > 0 on any + non-degenerate index — 3.3081 on this fixture — and `df_at_low_cutoff` + is therefore finite on the whole reachable input range. That conversion + is defensive rather than live, which is the same shape of finding as the + unreachable boost arm this module exists to pin. The NaN branch itself + is covered directly against `reachability` above. + """ + store_path = tmp_path / "e2e.db" + store = MemoryStore(str(store_path)) + for i, text in enumerate(_corpus()): + out = derive( + DerivationInput( + source_kind=INGEST_SOURCE_FILESYSTEM, + raw_text=text, + source_path=f"doc{i}.md", + session_id=None, + ts="2026-01-01T00:00:00+00:00", + ), + ) + assert out.belief is not None + store.insert_or_corroborate(out.belief, source_type="filesystem_ingest") + store.close() + + audit = tmp_path / "hook_audit.jsonl" + audit.write_text( + "\n".join( + json.dumps({"hook": "user_prompt_submit", "prompt_prefix": text}) + for text in _corpus() + ) + + "\n", + encoding="utf-8", + ) + + json_out = tmp_path / "bound.json" + assert bound_main( + [ + "--store", str(store_path), + "--audit", str(audit), + "--json-out", str(json_out), + ], + ) == 0 + + def _reject(token: str) -> float: + raise AssertionError(f"non-RFC-8259 token in --json-out: {token}") + + payload = json.loads(json_out.read_text(), parse_constant=_reject) + assert "reachability" in payload + + +def test_a_missing_audit_path_warns_instead_of_measuring_a_partial_corpus( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """A misspelt `--audit` path must not silently shrink the corpus. + + `load_prompts` takes several paths; skipping a bad one without a word + yields numbers over whatever survived, with nothing in the output to say + the measurement is partial. + """ + good = tmp_path / "good.jsonl" + good.write_text( + json.dumps({"hook": "user_prompt_submit", "prompt_prefix": _corpus()[0]}) + + "\n", + encoding="utf-8", + ) + missing = tmp_path / "typo.jsonl" + + prompts = load_prompts([good, missing]) + + assert len(prompts) == 1 + assert str(missing) in capsys.readouterr().err + + def test_idf_is_monotone_decreasing_in_document_frequency( index: BM25Index, ) -> None: