diff --git a/docs/CONFIG.md b/docs/CONFIG.md index fdbd55f76..f170322e8 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -298,9 +298,31 @@ Precedence (first decisive wins): env var `AELFRICE_HEAT_KERNEL=0`/`1` > explici ### `use_hrr_structural` -Boolean, default `false`, opt-in. Enables the HRR structural-query lane (#152). Like `use_heat_kernel`, the lane is implemented and stays opt-in pending the #154 benchmark gate. +Boolean, default `false`, opt-in. Enables the HRR structural-query lane (#152). Wired into `retrieve_v2` as a parallel routing branch (per spec: not blended with the textual lane). When on, `retrieve_v2` parses the query for a structural marker before any other rewrite or lane fans out: -Precedence (first decisive wins): env var `AELFRICE_HRR_STRUCTURAL=0`/`1` > explicit Python kwarg > TOML `[retrieval] use_hrr_structural` > default `false`. +``` +query string -> parse_structural_marker + hit: HRRStructIndex.probe(kind, target_id) -> RetrievalResult + miss: textual lane (vocab-bridge rewrite, then BM25F + heat-kernel + BFS) +``` + +A marker is a leading uppercase edge-type token followed by `:` and a non-empty target belief id. Recognised kinds match `aelfrice.models.EDGE_TYPES` (currently `SUPPORTS`, `CITES`, `RELATES_TO`, `SUPERSEDES`, `CONTRADICTS`, `DERIVED_FROM`). Case-sensitive: `contradicts:b/abc` does not match and falls through to the textual lane on the literal string. Whitespace inside the target is preserved; leading/trailing whitespace on the query is stripped. + +Examples: + +| Query | Routes to | Returns | +|---|---|---| +| `CONTRADICTS:b/abc` | structural lane | beliefs whose outgoing edge of kind `CONTRADICTS` targets `b/abc`, ranked by HRR probe score | +| `SUPPORTS:b/xyz` | structural lane | beliefs that `SUPPORTS` `b/xyz` | +| `contradicts everything` | textual lane | BM25 over the literal string | +| `CONTRADICTS: ` (empty target) | textual lane (marker rejected by regex) | BM25 over the literal string | +| `CONTRADICTS:nonexistent_id` | textual lane (marker parsed but probe finds no edges) | BM25 over the literal string | + +On structural lane hit, locked beliefs (when `include_locked=True`) pin to the head of the result and bypass the budget per the existing public-API contract; HRR-ranked beliefs are appended in score-descending order until the token budget is exhausted. Beliefs already in the locked set are de-duped from the HRR tail. + +Long-running consumers should pass an explicit `hrr_struct_index_cache: HRRStructIndexCache | None` to amortise the per-belief HRR encode cost across queries. None falls through to a fresh build per call. The cache subscribes to the store's invalidation registry so any belief / edge mutation drops the index transparently. + +Precedence (first decisive wins): env var `AELFRICE_HRR_STRUCTURAL=0`/`1` > explicit Python kwarg `use_hrr_structural=` > TOML `[retrieval] use_hrr_structural` > default `false`. The default-on flip is gated on the #154 composition-tracker bench (currently 7/11 per #474). ### `use_type_aware_compression` diff --git a/src/aelfrice/hrr_index.py b/src/aelfrice/hrr_index.py index 19a92d711..fa529a0e6 100644 --- a/src/aelfrice/hrr_index.py +++ b/src/aelfrice/hrr_index.py @@ -19,7 +19,15 @@ queries to this lane and falls through to the textual lane otherwise. Default-OFF at v1.7.0 behind ``use_hrr_structural`` per the #154 -composition tracker. +composition tracker. Wired into :func:`aelfrice.retrieval.retrieve_v2` +as a parallel routing branch that fires before vocab-bridge rewrite — +on a structural-marker hit the textual lane is bypassed entirely; +on miss the call falls through to BM25F + heat kernel as before. + +Long-running callers should pass an explicit +:class:`HRRStructIndexCache` to amortise the build cost across +queries. The cache subscribes to the store's invalidation registry +so any belief / edge mutation drops the index transparently. """ from __future__ import annotations @@ -262,3 +270,41 @@ def load(cls, path: str | Path) -> "HRRStructIndex": n: role_matrix[i] for i, n in enumerate(role_names) } return idx + + +@dataclass +class HRRStructIndexCache: + """Lazy, invalidation-aware wrapper around a single ``HRRStructIndex``. + + Subscribes to the store's invalidation callback registry on + construction, so any belief / edge mutation drops the cached + index. The next ``get()`` rebuilds. + + Mirrors :class:`aelfrice.vocab_bridge.VocabBridgeCache`. Per- + instance: two caches pointing at different stores never share + state. Thread safety is the caller's responsibility. + """ + + store: MemoryStore + dim: int = DEFAULT_DIM + store_path: str | None = None + seed: int | None = None + _index: HRRStructIndex | None = field(default=None, init=False, repr=False) + _subscribed: bool = field(default=False, init=False, repr=False) + + def __post_init__(self) -> None: + if not self._subscribed: + self.store.add_invalidation_callback(self.invalidate) + self._subscribed = True + + def get(self) -> HRRStructIndex: + """Return the current index, building or rebuilding as needed.""" + if self._index is None: + idx = HRRStructIndex(dim=self.dim) + idx.build(self.store, store_path=self.store_path, seed=self.seed) + self._index = idx + return self._index + + def invalidate(self) -> None: + """Drop the cached index. Wired to the store mutation hook.""" + self._index = None diff --git a/src/aelfrice/retrieval.py b/src/aelfrice/retrieval.py index 10ea9b4d0..350f2e1bc 100644 --- a/src/aelfrice/retrieval.py +++ b/src/aelfrice/retrieval.py @@ -75,6 +75,11 @@ ) from aelfrice.compression import CompressedBelief, compress_for_retrieval from aelfrice.doc_linker import DocAnchor +from aelfrice.hrr_index import ( + HRRStructIndex, + HRRStructIndexCache, + parse_structural_marker, +) from aelfrice.vocab_bridge import VocabBridge, VocabBridgeCache from aelfrice.entity_extractor import extract_entities from aelfrice.graph_spectral import ( @@ -798,6 +803,72 @@ def is_hrr_structural_enabled( return False +def _route_structural_query( + store: MemoryStore, + query: str, + cache: HRRStructIndexCache | None, + *, + top_k: int, + include_locked: bool, + budget: int, +) -> RetrievalResult | None: + """Probe the HRR structural lane and pack results to budget. + + Returns ``None`` when the query is not a structural marker, or + when the marker resolves to an unknown ``(kind, target)`` pair on + the index. The caller must fall through to the textual lane in + both cases — the structural lane is parallel, never blended. + + On hit, locks (when ``include_locked=True``) are pinned at the + head of the result and bypass the budget per the existing public- + API contract; HRR-ranked beliefs are appended in score-descending + order until the budget is exhausted. Beliefs already present + among the locks are de-duped from the HRR tail so the locked + pin-to-head invariant is preserved. + """ + parsed = parse_structural_marker(query) + if parsed is None: + return None + kind, target_id = parsed + idx: HRRStructIndex + if cache is None: + idx = HRRStructIndex() + idx.build(store) + else: + idx = cache.get() + hits = idx.probe(kind, target_id, top_k=top_k) + if not hits: + # Marker parsed but the (kind, target) pair is unknown to the + # index (no edges of that type touch target_id). Fall through + # so the caller can try the textual lane on the literal + # marker string — better than returning an empty result. + return None + + locked: list[Belief] = ( + list(store.list_locked_beliefs()) if include_locked else [] + ) + locked_ids: set[str] = {b.id for b in locked} + used: int = sum(_belief_tokens(b) for b in locked) + out: list[Belief] = list(locked) + + for belief_id, _score in hits: + if belief_id in locked_ids: + continue + belief = store.get_belief(belief_id) + if belief is None: + continue + cost = _belief_tokens(belief) + if used + cost > budget: + break + out.append(belief) + used += cost + + return RetrievalResult( + beliefs=out, + locked_ids=[b.id for b in locked], + ) + + def resolve_use_type_aware_compression( explicit: bool | None = None, *, @@ -1644,6 +1715,8 @@ def retrieve_v2( use_vocab_bridge: bool | None = None, vocab_bridge_cache: VocabBridgeCache | None = None, use_intentional_clustering: bool | None = None, + use_hrr_structural: bool | None = None, + hrr_struct_index_cache: HRRStructIndexCache | None = None, with_doc_anchors: bool = False, ) -> RetrievalResult: """Lab-compatible retrieval wrapper for academic-suite adapters. @@ -1695,10 +1768,39 @@ def retrieve_v2( the half-life when `temporal_sort=True`. None falls through to `resolve_temporal_half_life()`'s precedence chain. Ignored when `temporal_sort=False`. + - `use_hrr_structural` (#152) — when True AND the query parses as + a `:` structural marker, the HRR structural + lane fires and returns instead of the textual lane. Parallel, + not blended (per spec): on marker hit the BM25F + heat-kernel + stack is bypassed entirely; on miss the call falls through to + the textual lane unchanged. Default-OFF until the #437 + reproducibility harness clears, per the #154 composition + tracker. + - `hrr_struct_index_cache` (#152) — explicit + `HRRStructIndexCache` to reuse an already-built index across + calls. None falls through to a fresh build per call. + Long-running consumers (interactive shells, bench harnesses) + should pass an explicit cache to amortise the per-belief HRR + encode cost. - Returns a `RetrievalResult` wrapper so adapters can read `result.beliefs` (and stub diagnostics fields, plus the new v1.3 `entity_hits` and `bfs_chains`). """ + # v2.1 #152 HRR structural-query routing. Fires BEFORE the vocab- + # bridge rewrite so a `CONTRADICTS:b/abc` marker is not munged + # into bag-of-words rewrites. Returns early on marker hit; + # falls through on miss (non-marker query, marker-with-unknown- + # target, or flag OFF) so the textual lane handles the call. + if is_hrr_structural_enabled(use_hrr_structural): + struct_result = _route_structural_query( + store, query, hrr_struct_index_cache, + top_k=l1_limit, + include_locked=include_locked, + budget=budget, + ) + if struct_result is not None: + return struct_result + # v2.1 #433 vocabulary-bridge query rewrite. Runs before lane # fan-out so every lane sees the widened query string. The bridge # appends canonical-entity rewrites; the original tokens are diff --git a/tests/test_hrr_struct_index.py b/tests/test_hrr_struct_index.py index 890996cf9..d6f826792 100644 --- a/tests/test_hrr_struct_index.py +++ b/tests/test_hrr_struct_index.py @@ -25,6 +25,7 @@ from aelfrice.hrr_index import ( HRRStructIndex, + HRRStructIndexCache, _seed_from_path, parse_structural_marker, ) @@ -256,6 +257,35 @@ def test_save_load_round_trip(tmp_path: Path) -> None: assert a == b +# --- HRRStructIndexCache -------------------------------------------------- + + +def test_cache_lazy_build_then_reuse() -> None: + s = _toy_store() + cache = HRRStructIndexCache(store=s, dim=256, seed=7) + a = cache.get() + b = cache.get() + assert a is b, "second get() must return the cached instance" + + +def test_cache_invalidates_on_store_mutation() -> None: + s = _toy_store() + cache = HRRStructIndexCache(store=s, dim=256, seed=7) + first = cache.get() + s.insert_belief(_mk("b6")) + second = cache.get() + assert first is not second, "store mutation must drop the cache" + assert "b6" in second.belief_ids + + +def test_cache_explicit_invalidate_drops_index() -> None: + s = _toy_store() + cache = HRRStructIndexCache(store=s, dim=256, seed=7) + cache.get() + cache.invalidate() + assert cache._index is None + + # --- AC6 / AC7 (perf-gated) ---------------------------------------------- diff --git a/tests/test_retrieve_v2_hrr_structural.py b/tests/test_retrieve_v2_hrr_structural.py new file mode 100644 index 000000000..14ee7f479 --- /dev/null +++ b/tests/test_retrieve_v2_hrr_structural.py @@ -0,0 +1,204 @@ +"""Integration tests for the HRR structural-query lane wiring (#152). + +The substrate (HRRStructIndex, parse_structural_marker) was shipped +at v1.7.0 but never connected to retrieve_v2 — the flag was a +phantom until this PR. These tests assert the wiring is real: + +- IT1: structural marker + flag ON returns HRR-ranked beliefs +- IT2: structural marker + flag OFF falls through to textual + (byte-identical to pre-PR behavior on marker queries) +- IT3: non-marker query + flag ON behaves like flag OFF + (byte-identical default-OFF posture for normal text queries) +- IT4: marker with unknown target falls through to textual + (graceful miss — better than empty result) +- IT5: explicit cache reuses the index across calls +- IT6: locked beliefs still pin to head when structural lane fires +""" +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from aelfrice.hrr_index import HRRStructIndexCache +from aelfrice.models import ( + BELIEF_FACTUAL, + EDGE_CITES, + EDGE_CONTRADICTS, + EDGE_SUPPORTS, + LOCK_NONE, + LOCK_USER, + Belief, + Edge, +) +from aelfrice.retrieval import RetrievalResult, retrieve_v2 +from aelfrice.store import MemoryStore + + +@pytest.fixture +def store(tmp_path: Path) -> Iterator[MemoryStore]: + s = MemoryStore(str(tmp_path / "rv2_hrr.db")) + yield s + s.close() + + +def _mk(bid: str, *, locked: bool = False) -> Belief: + return Belief( + id=bid, + content=f"content of {bid}", + content_hash=f"h_{bid}", + alpha=1.0, + beta=1.0, + type=BELIEF_FACTUAL, + lock_level=LOCK_USER if locked else LOCK_NONE, + locked_at="2026-05-08T00:00:00+00:00" if locked else None, + demotion_pressure=0, + created_at="2026-05-08T00:00:00+00:00", + last_retrieved_at=None, + ) + + +def _populate(s: MemoryStore) -> None: + """Topology: b1 -CONTRADICTS-> b2; b3 -SUPPORTS-> b2; + b4 -CITES-> b5; b1 -CITES-> b5.""" + for i in range(1, 6): + s.insert_belief(_mk(f"b{i}")) + s.insert_edge(Edge(src="b1", dst="b2", type=EDGE_CONTRADICTS, weight=1.0)) + s.insert_edge(Edge(src="b3", dst="b2", type=EDGE_SUPPORTS, weight=1.0)) + s.insert_edge(Edge(src="b4", dst="b5", type=EDGE_CITES, weight=1.0)) + s.insert_edge(Edge(src="b1", dst="b5", type=EDGE_CITES, weight=1.0)) + + +# --- IT1 ----------------------------------------------------------------- + + +def test_structural_marker_with_flag_on_returns_hrr_results( + store: MemoryStore, +) -> None: + _populate(store) + cache = HRRStructIndexCache(store=store, dim=512, seed=42) + result = retrieve_v2( + store, "CONTRADICTS:b2", + use_hrr_structural=True, + hrr_struct_index_cache=cache, + budget=10_000, + ) + assert isinstance(result, RetrievalResult) + ids = [b.id for b in result.beliefs] + # b1 -CONTRADICTS-> b2: b1 must be in the result. + assert "b1" in ids, f"expected b1 (CONTRADICTS source), got {ids}" + # b3 only SUPPORTS b2 — must NOT lead the CONTRADICTS probe. + assert ids.index("b1") <= ids.index("b3") if "b3" in ids else True + + +# --- IT2 ----------------------------------------------------------------- + + +def test_structural_marker_with_flag_off_falls_through_to_textual( + store: MemoryStore, +) -> None: + _populate(store) + # Same query string, flag explicitly OFF: routes through textual + # lane (BM25 over the literal "CONTRADICTS:b2" string). The + # textual lane will likely return nothing matching, but the key + # point is that the HRR lane is bypassed — assert by absence of + # the structural-only-derivable result. + result = retrieve_v2( + store, "CONTRADICTS:b2", + use_hrr_structural=False, + budget=10_000, + ) + # Textual lane on "CONTRADICTS:b2" string with content like + # "content of b1" cannot match b1; b1 in results would only come + # from the structural lane. + ids = [b.id for b in result.beliefs] + assert "b1" not in ids + + +# --- IT3 ----------------------------------------------------------------- + + +def test_non_marker_query_with_flag_on_behaves_like_flag_off( + store: MemoryStore, +) -> None: + _populate(store) + on = retrieve_v2( + store, "content of b1", + use_hrr_structural=True, + budget=10_000, + ) + off = retrieve_v2( + store, "content of b1", + use_hrr_structural=False, + budget=10_000, + ) + # Non-marker query: flag is no-op. Result lists must match. + assert [b.id for b in on.beliefs] == [b.id for b in off.beliefs] + + +# --- IT4 ----------------------------------------------------------------- + + +def test_marker_with_unknown_target_falls_through( + store: MemoryStore, +) -> None: + _populate(store) + # Valid marker syntax but target b_does_not_exist isn't in the + # store. _route_structural_query returns None on empty hits so + # the textual lane handles the literal string. + result = retrieve_v2( + store, "CONTRADICTS:b_does_not_exist", + use_hrr_structural=True, + budget=10_000, + ) + # No assertion on content — just that the call succeeds without + # raising and returns a well-formed RetrievalResult. + assert isinstance(result, RetrievalResult) + + +# --- IT5 ----------------------------------------------------------------- + + +def test_explicit_cache_reuses_index_across_calls( + store: MemoryStore, +) -> None: + _populate(store) + cache = HRRStructIndexCache(store=store, dim=256, seed=7) + # First call builds the index; second call must reuse the same + # instance (verified by the cache's identity-preservation). + retrieve_v2( + store, "CONTRADICTS:b2", + use_hrr_structural=True, + hrr_struct_index_cache=cache, + ) + first = cache._index + retrieve_v2( + store, "CITES:b5", + use_hrr_structural=True, + hrr_struct_index_cache=cache, + ) + second = cache._index + assert first is not None and first is second + + +# --- IT6 ----------------------------------------------------------------- + + +def test_locked_beliefs_pin_to_head_under_structural_lane( + store: MemoryStore, +) -> None: + # b0 is a locked belief unrelated to the structural query. + store.insert_belief(_mk("b0", locked=True)) + _populate(store) + cache = HRRStructIndexCache(store=store, dim=256, seed=7) + result = retrieve_v2( + store, "CONTRADICTS:b2", + use_hrr_structural=True, + hrr_struct_index_cache=cache, + include_locked=True, + budget=10_000, + ) + assert result.beliefs, "result must be non-empty" + assert result.beliefs[0].lock_level == LOCK_USER + assert result.locked_ids == ["b0"]