From dafd812a7b2325f102b7ea508266d056d55c0265 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sat, 9 May 2026 19:39:29 -0700 Subject: [PATCH 1/3] feat(test): run_vocab_bridge_uplift driver for #433 bench-gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors run_doc_linker_uplift / run_query_strategy_uplift / run_clustering_uplift: per row, runs retrieve_v2 twice on fresh stores seeded from the row's store_beliefs (with anchors), once with use_vocab_bridge=False (baseline) and once with =True. NDCG@k against expected_top_k, averaged across rows. Result type is VocabBridgeUplift{n_rows, mean_ndcg_off, mean_ndcg_on}; uplift property mirrors the other module result types so the bench-gate failure formatter doesn't need per-runner branching. Adds _seed_store_for_vocab_bridge to translate the existing precondition row shape (store_beliefs[i].anchors as inbound CITES edges) into store state — same logic as the precondition gate's local seeder, hoisted into the runner so the OFF/ON arms see byte-identical store contents and any NDCG delta is attributable to the bridge alone. Rows missing expected_top_k are skipped so precondition-only rows (those carrying just expected_canonicals) don't dilute the mean. --- tests/retrieve_uplift_runner.py | 174 ++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/tests/retrieve_uplift_runner.py b/tests/retrieve_uplift_runner.py index 91daf33ae..0f67a811d 100644 --- a/tests/retrieve_uplift_runner.py +++ b/tests/retrieve_uplift_runner.py @@ -609,6 +609,180 @@ def run_query_strategy_uplift( ) +# --------------------------------------------------------------------- +# Vocabulary bridge (#433) — use_vocab_bridge bench gate (spec § A2). +# +# Consumed by tests/bench_gate/test_vocab_bridge_uplift.py. +# Row schema (`tests/corpus/v2_0/vocab_bridge/*.jsonl`) — extends the +# precondition row schema with `expected_top_k` and optional `k`: +# +# { +# "id": "row-id", +# "query": "raw query string", +# "k": 10, +# "store_beliefs": [ +# {"id": "b1", "content": "...", "anchors": ["text", "..."]}, +# ... +# ], +# "expected_canonicals": ["sqlite", ...], # used by precondition +# "expected_top_k": ["b1", "b2", ...] # ground-truth ranking +# } +# +# `expected_canonicals` stays the precondition surface — it lets the +# appends-at-least-one gate fire without naming specific belief ids. +# `expected_top_k` is the strict-NDCG ground truth: the bridge wins +# only if rewriting the query surfaces these ids in this order vs the +# default-OFF baseline. +# --------------------------------------------------------------------- + +_BENCH_TS = "2026-05-08T00:00:00Z" + + +@dataclass(frozen=True) +class VocabBridgeUplift: + """#433 result shape. + + Field names mirror ``DocLinkerUpliftResults`` / ``QueryStrategyUplift`` + so the bench-gate failure-message formatter reads + ``mean_ndcg_off`` / ``mean_ndcg_on`` / ``uplift`` without + per-runner branching. + """ + + n_rows: int + mean_ndcg_off: float + mean_ndcg_on: float + + @property + def uplift(self) -> float: + return self.mean_ndcg_on - self.mean_ndcg_off + + +def _seed_store_for_vocab_bridge( + store: MemoryStore, row: dict, # type: ignore[type-arg] +) -> None: + """Seed a store from the vocab_bridge row shape. + + Row carries ``store_beliefs`` (not ``beliefs``) where each entry + optionally lists ``anchors`` — surface-form strings written as + incoming-edge ``anchor_text`` from synthetic citing beliefs. This + mirrors the precondition seeder in + ``tests/bench_gate/test_vocab_bridge_uplift.py`` so the runner + sees an identical store shape; the bridge harvest/rewrite path + then operates on the same anchor-text universe under both arms. + """ + for entry in row.get("store_beliefs", []): + bid = str(entry["id"]) + store.insert_belief( + Belief( + id=bid, + content=str(entry["content"]), + content_hash=f"corpus:{bid}", + alpha=float(entry.get("alpha", 1.0)), + beta=float(entry.get("beta", 1.0)), + type=entry.get("type", BELIEF_FACTUAL), + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=_BENCH_TS, + last_retrieved_at=None, + origin=ORIGIN_AGENT_INFERRED, + ) + ) + for entry in row.get("store_beliefs", []): + for j, anchor in enumerate(entry.get("anchors") or []): + citer_id = f"_a_{entry['id']}_{j}" + if not store.get_belief(citer_id): + store.insert_belief( + Belief( + id=citer_id, + content="anchor citer", + content_hash=f"corpus:{citer_id}", + alpha=1.0, + beta=1.0, + type=BELIEF_FACTUAL, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=_BENCH_TS, + last_retrieved_at=None, + origin=ORIGIN_AGENT_INFERRED, + ) + ) + store.insert_edge( + Edge( + src=citer_id, + dst=str(entry["id"]), + type="CITES", + weight=1.0, + anchor_text=str(anchor), + ) + ) + + +def run_vocab_bridge_uplift( + rows: list[dict], # type: ignore[type-arg] +) -> VocabBridgeUplift: + """Spec § A2 vocabulary-bridge uplift driver. + + Per row, runs ``retrieve_v2`` twice on fresh stores seeded from + ``store_beliefs`` (with anchors): once with + ``use_vocab_bridge=False`` (baseline) and once with ``=True``. + Same query, same store shape, same ``k``. NDCG@k is scored against + ``expected_top_k`` and averaged across rows. + + Rows missing ``expected_top_k`` are skipped (the precondition gate + only needs ``expected_canonicals``). The shipped substrate is + default-OFF: the bridge appends canonical-entity tokens to the + query, never substitutes — so on a corpus where BM25 alone + already ranks the relevant beliefs above noise, ``uplift`` will + be ~0 and the strict ``> 0`` ship-gate assertion will fail. That + is the correct gate-broken signal until labelled rows expose + surface-form gaps the bridge can close. + """ + scoreable = [r for r in rows if r.get("expected_top_k")] + n = len(scoreable) + if n == 0: + return VocabBridgeUplift(0, 0.0, 0.0) + + off_total = 0.0 + on_total = 0.0 + with tempfile.TemporaryDirectory() as tmp: + tmp_root = Path(tmp) + for row in scoreable: + k = _default_k(row) + expected = list(row["expected_top_k"]) + + for bridge_on in (False, True): + _db_counter[0] += 1 + db = ( + tmp_root + / f"vb_{row['id']}_{int(bridge_on)}_{_db_counter[0]}.db" + ) + store = MemoryStore(str(db)) + try: + _seed_store_for_vocab_bridge(store, row) + result = retrieve_v2( + store, row["query"], + budget=DEFAULT_TOKEN_BUDGET, + use_entity_index=False, + use_vocab_bridge=bridge_on, + ) + returned = [b.id for b in result.beliefs[:k]] + finally: + store.close() + ndcg = ndcg_at_k(returned, expected, k) + if bridge_on: + on_total += ndcg + else: + off_total += ndcg + + return VocabBridgeUplift( + n_rows=n, + mean_ndcg_off=off_total / n, + mean_ndcg_on=on_total / n, + ) + + def _format_table(results: list[FlagUplift]) -> str: lines = [ f"{'flag':<28} {'n':>4} {'NDCG_off':>10} {'NDCG_on':>10} {'uplift':>10}", From 20f48751c9d1f140a18ba249957c18125e141de8 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sat, 9 May 2026 19:39:34 -0700 Subject: [PATCH 2/3] test(uplift): unit tests for run_vocab_bridge_uplift (#433) Three tests mirroring the doc_linker / query_strategy harness coverage: - empty_input: zero rows -> zero metrics; the contract is total silence on a no-op call, not a divide-by-zero. - skips_rows_without_expected_top_k: a precondition-only row (expected_canonicals but no expected_top_k) must not be counted in n_rows. Falsifiable if the runner silently treats missing ground truth as ndcg=0 and dilutes the mean. - runs_on_synthetic_row: shape + bounded-metric smoke. The contract under test is that OFF/ON arms execute end-to-end and produce in-range NDCG values; whether the bridge actually wins on a hand-crafted two-belief store is the lab-side gate. --- tests/test_retrieve_uplift_runner.py | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/test_retrieve_uplift_runner.py b/tests/test_retrieve_uplift_runner.py index 246faefb1..7e7e6afd7 100644 --- a/tests/test_retrieve_uplift_runner.py +++ b/tests/test_retrieve_uplift_runner.py @@ -282,6 +282,73 @@ def test_query_strategy_uplift_lowercase_no_extremes_means_zero_uplift() -> None assert r.uplift == 0.0 +def test_vocab_bridge_uplift_empty_input() -> None: + from tests.retrieve_uplift_runner import run_vocab_bridge_uplift + + r = run_vocab_bridge_uplift([]) + assert r.n_rows == 0 + assert r.mean_ndcg_off == 0.0 + assert r.mean_ndcg_on == 0.0 + assert r.uplift == 0.0 + + +def test_vocab_bridge_uplift_skips_rows_without_expected_top_k() -> None: + """Precondition rows (those carrying only ``expected_canonicals``) + have no ground-truth ranking, so they must be skipped — not silently + scored as zero, which would dilute the mean and falsely shrink the + uplift signal. ``n_rows`` reflects the scoreable subset.""" + from tests.retrieve_uplift_runner import run_vocab_bridge_uplift + + rows = [ + { + "id": "vb-precondition-only", + "query": "alpha", + "store_beliefs": [{"id": "a", "content": "alpha alpha"}], + "expected_canonicals": ["alpha"], + }, + ] + r = run_vocab_bridge_uplift(rows) + assert r.n_rows == 0 + assert r.uplift == 0.0 + + +def test_vocab_bridge_uplift_runs_on_synthetic_row() -> None: + """OFF/ON arms run end-to-end without raising; metrics are bounded. + + Real uplift on a synthetic row is unlikely (the bridge harvests + canonical-entity superpositions from corpus surface forms; a + two-belief store doesn't expose a meaningful canonical universe). + The contract under test is shape + bounded metric, not that the + rewrite influences ranking — the corpus uplift is the lab-side + gate.""" + from tests.retrieve_uplift_runner import run_vocab_bridge_uplift + + row = { + "id": "vb-test-001", + "query": "memory store", + "k": 3, + "store_beliefs": [ + { + "id": "b1", + "content": "the memory store persists beliefs", + "anchors": ["MemoryStore"], + }, + {"id": "b2", "content": "the configuration file lives at /etc"}, + { + "id": "b3", + "content": "the memory store uses sqlite", + "anchors": ["sqlite"], + }, + ], + "expected_canonicals": ["sqlite"], + "expected_top_k": ["b1", "b3"], + } + r = run_vocab_bridge_uplift([row]) + assert r.n_rows == 1 + assert 0.0 <= r.mean_ndcg_off <= 1.0 + assert 0.0 <= r.mean_ndcg_on <= 1.0 + + def test_run_per_flag_uplift_covers_all_flags() -> None: """Hypothesis: the harness reports one row per registered flag. Falsifiable if a flag is silently dropped.""" From 37e076407c28d9914e8799e3fc3f0cbe8c24b090 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sat, 9 May 2026 19:39:41 -0700 Subject: [PATCH 3/3] test(bench-gate): #433 vocab_bridge strict-NDCG ship gate scaffold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds test_vocab_bridge_ship_gate_runner_present, mirroring the #436 intentional-clustering / #435 doc-linker pattern: skip-if-runner-absent plus skip-if-no-row-has-expected_top_k, so the gate fires only when both the harness and labelled rows are present. Operator-side gate for flipping use_vocab_bridge to default-on once lab evidence clears. Module docstring updated to document the extended row schema: expected_top_k joins expected_canonicals as an optional field; the precondition gate reads expected_canonicals, the strict gate reads expected_top_k. Rows can carry either or both. No code change to the precondition gate — its 50% appends-at-least-one threshold remains the regression tripwire that fires before the heavier NDCG run. --- tests/bench_gate/test_vocab_bridge_uplift.py | 50 ++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/bench_gate/test_vocab_bridge_uplift.py b/tests/bench_gate/test_vocab_bridge_uplift.py index 7a4dd59bf..06f38a1bd 100644 --- a/tests/bench_gate/test_vocab_bridge_uplift.py +++ b/tests/bench_gate/test_vocab_bridge_uplift.py @@ -22,19 +22,26 @@ { "id": "row-id", "query": "raw query string passed to retrieve", + "k": 10, # optional, default 10 "store_beliefs": [ {"id": "b1", "content": "...", "anchors": ["text", "..."]}, ... ], - "expected_canonicals": ["sqlite", "python", "..."] + "expected_canonicals": ["sqlite", "python", "..."], + "expected_top_k": ["b1", "b2", ...] # optional; A2 ship gate } `store_beliefs[i].anchors` is optional; when present, each anchor string is added as an inbound edge from a synthetic citing belief to seed the bridge with anchor-source surface forms (#148 parity). `expected_canonicals` is the set of canonical-entity tokens that -the bridge SHOULD append for this query; the test asserts at least -one is present in the rewritten output. +the bridge SHOULD append for this query; the precondition test +asserts at least one is present in the rewritten output. +`expected_top_k` is the ground-truth belief-id ranking; the strict +A2 ship gate (``test_vocab_bridge_ship_gate_runner_present``) runs +``retrieve_v2`` with the bridge OFF then ON and asserts NDCG@k goes +strictly up. Rows without ``expected_top_k`` are skipped by the +ship gate but still exercised by the precondition gate. """ from __future__ import annotations @@ -133,3 +140,40 @@ def test_bridge_appends_at_least_one_expected_canonical( f"{n_hits}/{n_rows} rows ({coverage:.1%}); harvest/rewrite " f"regression suspected (was the surface-form pipeline broken?)" ) + + +@pytest.mark.bench_gated +def test_vocab_bridge_ship_gate_runner_present( + aelfrice_corpus_root: Path, +) -> None: + """The full A2 NDCG@k ship gate runs from + ``tests.retrieve_uplift_runner.run_vocab_bridge_uplift``. This test + skips when the runner is absent or when no row carries + ``expected_top_k`` — the runner is the operator-side gate for + flipping ``use_vocab_bridge`` to default-on (#433 Phase 2). + """ + rows = load_corpus_module(aelfrice_corpus_root, "vocab_bridge") + assert rows, "vocab_bridge corpus produced zero rows" + + runner_mod = pytest.importorskip( + "tests.retrieve_uplift_runner", + reason=( + "vocab_bridge uplift runner not yet wired " + "(operator gate; spec § A2 — pending lab-side corpus + scorer)" + ), + ) + + if not any(r.get("expected_top_k") for r in rows): + pytest.skip( + "vocab_bridge corpus has no rows with expected_top_k; " + "annotate at least one row before this gate can fire" + ) + + results = runner_mod.run_vocab_bridge_uplift(rows) + assert results.uplift > 0, ( + "use_vocab_bridge must show strictly positive NDCG@k uplift\n" + f" ON={results.mean_ndcg_on:.4f} " + f"OFF={results.mean_ndcg_off:.4f} " + f"uplift={results.uplift:+.4f} " + f"n_rows={results.n_rows}" + )