From a805e2cfa3b86b69026afe49cae7fda70b2edad7 Mon Sep 17 00:00:00 2001
From: rrs <276464689+robotrocketscience@users.noreply.github.com>
Date: Tue, 5 May 2026 09:02:50 -0700
Subject: [PATCH] feat(bm25): Porter stemming on the BM25F lane (#154)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds `snowballstemmer` runtime dep + `tokenize_stemmed()` helper.
`BM25Index.build` and `BM25Index.score` now stem at index/query time
so the BM25F lane has FTS5-equivalent matching behavior. Without
this, switching the L1 lane to BM25F by default would silently
regress queries like q="banana" against content "bananas" — FTS5's
Porter stemming caught those for free; stemless BM25F missed them.

`tokenize()` (without stem) stays for callers like
`relationship_detector` that depend on word-form-preserving tokens
to match against unstemmed quantifier vocabulary like "always" /
"rarely". Splitting the helpers keeps each call site
semantically correct.

Bench evidence: re-running the per-flag NDCG@k harness on the
v0.1 retrieve_uplift fixture under stemming raises
use_bm25f_anchors uplift from +0.6010 to +0.6650 (other four flags
unchanged at 0.0000). Stemming unblocks the v1.7 default-on flip
for use_bm25f_anchors per #154.
---
 pyproject.toml       |  7 ++++++
 src/aelfrice/bm25.py | 56 +++++++++++++++++++++++++++++++++++++-------
 uv.lock              | 11 +++++++++
 3 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 6feb5c194..19f56a54c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -44,6 +44,13 @@ dependencies = [
     # longer stdlib-only. CHANGELOG records the change.
     "numpy>=2.0",
     "scipy>=1.11",
+    # v1.7.0 added snowballstemmer for BM25F tokenisation parity with
+    # FTS5's Porter stemmer (#154). Pure-Python, ~150 KB; deterministic
+    # for the English Porter algorithm. Required so `q="banana"`
+    # against content `"bananas"` matches in both lanes — flipping the
+    # BM25F default-on without it loses the stemming win FTS5 gives
+    # for free.
+    "snowballstemmer>=2.2",
 ]
 
 [project.urls]
diff --git a/src/aelfrice/bm25.py b/src/aelfrice/bm25.py
index 34210046e..91fe10ccf 100644
--- a/src/aelfrice/bm25.py
+++ b/src/aelfrice/bm25.py
@@ -44,10 +44,20 @@
 
 import numpy as np
 import scipy.sparse as sp
+import snowballstemmer
 
 from aelfrice.models import Belief
 from aelfrice.store import MemoryStore
 
+# v1.7.0 #154: Porter stemmer for FTS5 parity. snowballstemmer's
+# "porter" implementation is the original Porter (1980) algorithm,
+# which matches SQLite FTS5's `tokenize='porter unicode61'` behavior
+# closely enough that q="banana" against content "bananas" hits in
+# both lanes. Constructed once at module load (cheap) and shared
+# across `tokenize()` calls (the stemmer is stateless on each
+# `stemWord` call).
+_PORTER_STEMMER = snowballstemmer.stemmer("porter")
+
 # Default weight for the incoming-anchor token stream, per the #148
 # spec. Synthetic-graph evaluation at N=50k under a 15%-vocab-shifted
 # regime: rank of vocab-shifted relevant beliefs drops from ~132 to
@@ -78,18 +88,46 @@
 
 
 def tokenize(text: str) -> list[str]:
-    """Lowercase + Unicode-word tokenisation.
-
-    Returned tokens are the canonical form used by `BM25Index.build`
-    and `BM25Index.score`. Two pieces of text that differ only in
-    case or punctuation tokenise identically. Empty / whitespace-only
-    input returns ``[]``.
+    """Lowercase + Unicode-word tokenisation. No stemming.
+
+    Returned tokens are the canonical form used by callers that need
+    word-form-preserving tokens (e.g.,
+    `aelfrice.relationship_detector` matches against unstemmed
+    quantifier tokens like ``"always"`` and ``"rarely"``). Two pieces
+    of text that differ only in case or punctuation tokenise
+    identically. Empty / whitespace-only input returns ``[]``.
+
+    BM25 indexing uses `tokenize_stemmed()` instead — that's where
+    Porter stemming lives so `q="banana"` matches content `"bananas"`
+    on the BM25F path (FTS5 already stems).
     """
     if not text:
         return []
     return [m.group(0).lower() for m in _TOKEN_PATTERN.finditer(text)]
 
 
+def tokenize_stemmed(text: str) -> list[str]:
+    """Lowercase + Unicode-word tokenisation + Porter stemming.
+
+    Used by `BM25Index.build` and `BM25Index.score` so the BM25F
+    lane has FTS5-equivalent stemming. SQLite FTS5 uses Porter by
+    default; without stemming on the BM25F path,
+    `q="banana"` against content `"bananas"` would miss matches that
+    the legacy FTS5 lane catches. Added at v1.7.0 (#154) when the
+    default-on flip was prepared.
+
+    Non-BM25 callers (relationship_detector, scoring helpers, etc.)
+    that depend on word-form-preserving tokens should keep using
+    `tokenize()`; stemming is BM25-specific.
+    """
+    if not text:
+        return []
+    return [
+        _PORTER_STEMMER.stemWord(m.group(0).lower())
+        for m in _TOKEN_PATTERN.finditer(text)
+    ]
+
+
 @dataclass
 class BM25Index:
     """Precomputed BM25F sparse term-frequency index.
@@ -185,9 +223,9 @@ def build(
         tokens_per_doc: list[list[str]] = []
         for bid in belief_ids:
             content = contents.get(bid, "")
-            doc_tokens = tokenize(content)
+            doc_tokens = tokenize_stemmed(content)
             for anchor in incoming.get(bid, ()):
-                anchor_tokens = tokenize(anchor)
+                anchor_tokens = tokenize_stemmed(anchor)
                 for _ in range(anchor_weight):
                     doc_tokens.extend(anchor_tokens)
             tokens_per_doc.append(doc_tokens)
@@ -275,7 +313,7 @@ def score(
             return []
         if self.tf.shape[0] == 0 or self.tf.shape[1] == 0:
             return []
-        q_tokens = tokenize(query)
+        q_tokens = tokenize_stemmed(query)
         if not q_tokens:
             return []
         # Build the query indicator * idf vector in dense form
diff --git a/uv.lock b/uv.lock
index 66d4faecd..37ae60a1c 100644
--- a/uv.lock
+++ b/uv.lock
@@ -17,6 +17,7 @@ source = { editable = "." }
 dependencies = [
     { name = "numpy" },
     { name = "scipy" },
+    { name = "snowballstemmer" },
 ]
 
 [package.optional-dependencies]
@@ -54,6 +55,7 @@ requires-dist = [
     { name = "nltk", marker = "extra == 'benchmarks'", specifier = ">=3.9" },
     { name = "numpy", specifier = ">=2.0" },
     { name = "scipy", specifier = ">=1.11" },
+    { name = "snowballstemmer", specifier = ">=2.2" },
     { name = "tiktoken", marker = "extra == 'benchmarks'", specifier = ">=0.7" },
 ]
 provides-extras = ["mcp", "onboard-llm", "archive", "benchmarks"]
@@ -2253,6 +2255,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
 ]
 
+[[package]]
+name = "snowballstemmer"
+version = "3.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" },
+]
+
 [[package]]
 name = "sortedcontainers"
 version = "2.4.0"