feat(retrieval): flip use_bm25f_anchors default-on at v1.7.0 (#154)

src/aelfrice/bm25.py

@@ -40,6 +40,7 @@
 import io
 import re
 from dataclasses import dataclass, field
+from functools import lru_cache
 from typing import Final
 
 import numpy as np
@@ -54,10 +55,23 @@
 # which matches SQLite FTS5's `tokenize='porter unicode61'` behavior
 # closely enough that q="banana" against content "bananas" hits in
 # both lanes. Constructed once at module load (cheap) and shared
-# across `tokenize()` calls (the stemmer is stateless on each
-# `stemWord` call).
+# across `tokenize_stemmed()` calls.
 _PORTER_STEMMER = snowballstemmer.stemmer("porter")
 
+
+@lru_cache(maxsize=65_536)
+def _stem(token: str) -> str:
+    """LRU-memoised Porter stem.
+
+    snowballstemmer's `stemWord` is pure-Python and slow per-call
+    (~10-30 µs); at 10k+ beliefs the per-doc tokenisation dominates
+    the BM25Index build. Real corpora have small vocabulary
+    relative to total tokens (Zipfian), so a 64K-entry LRU has very
+    high hit rate after warm-up. Cache is module-global; reset on
+    process exit.
+    """
+    return _PORTER_STEMMER.stemWord(token)
+
 # Default weight for the incoming-anchor token stream, per the #148
 # spec. Synthetic-graph evaluation at N=50k under a 15%-vocab-shifted
 # regime: rank of vocab-shifted relevant beliefs drops from ~132 to
@@ -123,7 +137,7 @@ def tokenize_stemmed(text: str) -> list[str]:
     if not text:
         return []
     return [
-        _PORTER_STEMMER.stemWord(m.group(0).lower())
+        _stem(m.group(0).lower())
         for m in _TOKEN_PATTERN.finditer(text)
     ]
 

src/aelfrice/retrieval.py

@@ -505,12 +505,20 @@ def resolve_use_bm25f_anchors(
       1. AELFRICE_BM25F env var (truthy / falsy normalised).
       2. Explicit `explicit` kwarg from the caller.
       3. `[retrieval] use_bm25f_anchors` in `.aelfrice.toml`.
-      4. Default: False (v1.5.0 default-OFF).
-
-    The default-off contract means the v1.4 FTS5 path remains
-    byte-identical for callers that do not opt in. The composition
-    tracker (#154) flips the default at v1.5.x once benchmarks
-    confirm the quality lift on captured-corpus data.
+      4. Default: True (v1.7.0 default-ON per #154 bench evidence).
+
+    The composition-tracker (#154) bench gate ran on the
+    `tests/corpus/v2_0/retrieve_uplift/v0_1.jsonl` lab fixture and
+    measured **+0.6650 NDCG@k uplift** for `use_bm25f_anchors=True`
+    versus the all-flags-off baseline (30 rows, 6 categories) under
+    Porter stemming. No regression on any row. See #154 for the
+    per-flag table; the stemming addition (#428) closed the
+    `q="banana"` vs content `"bananas"` gap that briefly blocked
+    the flip.
+
+    Callers that need the v1.5/v1.6 FTS5 path can still set
+    `AELFRICE_BM25F=0`, pass `use_bm25f_anchors=False`, or write
+    `[retrieval] use_bm25f_anchors = false` in `.aelfrice.toml`.
     """
     env = _env_bm25f_override()
     if env is not None:
@@ -520,7 +528,7 @@ def resolve_use_bm25f_anchors(
     toml_value = _read_toml_flag_for(BM25F_FLAG, start)
     if toml_value is not None:
         return toml_value
-    return False
+    return True
 
 
 def is_hrr_structural_enabled(

tests/test_bayesian_ranking.py

@@ -97,13 +97,22 @@ def test_ac1_retrieve_and_retrieve_v2_accept_posterior_weight() -> None:
 
 
 def test_ac2_weight_zero_byte_identical_to_v10x() -> None:
-    """The most important regression test: at weight 0 the result
-    list is identical to what `store.search_beliefs(...)` returns
+    """At weight 0 AND with the legacy plain-BM25 (non-BM25F) path,
+    retrieve() result equals what `store.search_beliefs(...)` returns
     for the L1 portion. (L0 prefix is unaffected by weight.)
+
+    Per #154: BM25F is now the default lane (default-on flip at v1.7.0
+    ratified on +0.6650 NDCG@k uplift evidence post-stemming).
+    `use_bm25f_anchors=False` here pins the contract to the legacy
+    v1.0.x BM25 path so this regression test still asserts what it
+    was meant to assert.
     """
     s = _equal_bm25_store()
     direct = s.search_beliefs("widget", limit=50)
-    weighted = retrieve(s, "widget", token_budget=10_000, posterior_weight=0.0)
+    weighted = retrieve(
+        s, "widget", token_budget=10_000,
+        posterior_weight=0.0, use_bm25f_anchors=False,
+    )
     # The retrieve() output may include an L0 prefix; here the
     # store has no locked beliefs, so the lists must match
     # byte-for-byte.

tests/test_bm25_index.py

@@ -296,15 +296,22 @@ def test_retrieve_with_use_bm25f_anchors_returns_anchor_recovered_belief() -> No
     assert "shifted" not in fts_hits
 
 
-def test_retrieve_default_off_byte_identical_to_pre_v15_path() -> None:
-    """Default-off contract: with use_bm25f_anchors unset (None), the
-    output equals the explicit use_bm25f_anchors=False path. Guards
-    against accidental flip in a future commit."""
+def test_retrieve_default_on_byte_identical_to_explicit_on() -> None:
+    """Default-on contract (#154 v1.7.0): with use_bm25f_anchors unset
+    (None), the output equals the explicit use_bm25f_anchors=True path.
+
+    Replaces the v1.5/v1.6 default-off byte-identity check. Default
+    flipped on +0.6650 NDCG@k uplift evidence on the
+    `tests/corpus/v2_0/retrieve_uplift/v0_1.jsonl` lab fixture
+    post-stemming (see #154 comment 4380967901). Guards against
+    accidental flip-back in a future commit; the legacy FTS5 path
+    remains reachable via explicit `use_bm25f_anchors=False`.
+    """
     s = MemoryStore(":memory:")
     for i in range(8):
         s.insert_belief(_mk(f"b{i}", f"token{i} content blob"))
     default = [b.id for b in retrieve(s, "token3 content")]
-    explicit_off = [b.id for b in retrieve(
-        s, "token3 content", use_bm25f_anchors=False,
+    explicit_on = [b.id for b in retrieve(
+        s, "token3 content", use_bm25f_anchors=True,
     )]
-    assert default == explicit_off
+    assert default == explicit_on

tests/test_composition_tracker.py

@@ -116,13 +116,18 @@ def test_placeholder_flags_constant_lists_unwired_lanes() -> None:
     }
 
 
-def test_lane_telemetry_records_fts5_lane_by_default() -> None:
-    """A retrieve() call with no opt-in flag populates LaneTelemetry
-    with bm25f_used=False."""
+def test_lane_telemetry_records_fts5_lane_when_opted_out() -> None:
+    """Explicit `use_bm25f_anchors=False` populates LaneTelemetry with
+    bm25f_used=False — the legacy FTS5 path remains reachable.
+
+    Replaces the v1.5/v1.6 default-off check. Per #154 the default
+    flipped to ON at v1.7.0 on +0.6650 NDCG@k uplift evidence; this
+    test now asserts the opt-out path is intact.
+    """
     s = MemoryStore(":memory:")
     s.insert_belief(_mk("b1", "alpha beta"))
     s.insert_belief(_mk("b2", "gamma delta"))
-    retrieve(s, "alpha")
+    retrieve(s, "alpha", use_bm25f_anchors=False)
     t = last_lane_telemetry()
     assert isinstance(t, LaneTelemetry)
     assert t.bm25f_used is False
@@ -131,8 +136,19 @@ def test_lane_telemetry_records_fts5_lane_by_default() -> None:
     assert t.locked == 0
 
 
+def test_lane_telemetry_records_bm25f_lane_by_default() -> None:
+    """Default-on contract (#154 v1.7.0): retrieve() with no explicit
+    flag populates LaneTelemetry with bm25f_used=True."""
+    s = MemoryStore(":memory:")
+    s.insert_belief(_mk("b1", "alpha beta"))
+    retrieve(s, "alpha")
+    t = last_lane_telemetry()
+    assert t.bm25f_used is True
+
+
 def test_lane_telemetry_records_bm25f_lane_when_opted_in() -> None:
-    """Setting use_bm25f_anchors=True flips the bm25f_used flag."""
+    """Setting use_bm25f_anchors=True keeps the bm25f_used flag on
+    (matches the default at v1.7.0+)."""
     s = MemoryStore(":memory:")
     s.insert_belief(_mk("b1", "alpha beta"))
     retrieve(s, "alpha", use_bm25f_anchors=True)