fix(retrieval): reserve a relevance budget floor so locks can't starve query-relevant hits (#1014)

CHANGELOG/v3.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - **Conversational facts are captured on a session cadence, not only at compaction ([#1011](https://github.com/robotrocketscience/aelfrice/issues/1011)).** Belief ingestion previously fired only on the PreCompact rotation (`transcript_logger._handle_pre_compact`), so a session that ended *without* compacting logged its turns to `turns.jsonl` but never folded them into beliefs — a fresh session could not recall what the user had stated, the core-capture promise. The `Stop` hook now flushes the live `turns.jsonl` through `aelf ingest-transcript` once `AELFRICE_INGEST_STOP_FLUSH_TURNS` (default `12`) new turns have accumulated since the last flush. Ingestion is idempotent per `(source_label, sentence)`, so re-ingesting the live file captures only new statements without inflating the store, and the file is **not** rotated — the rebuilder / retrieval recent-turns window is preserved. Set the env var to `0` to restore PreCompact-only capture.
+- **Locked beliefs no longer starve query-relevant retrieval under budget saturation ([#1014](https://github.com/robotrocketscience/aelfrice/issues/1014)).** L0 locks are injected unconditionally and never trimmed (#379), and their tokens are subtracted from the L2.5/L1 budget — so a store whose locks alone met or exceeded the token budget returned **only the locks for every prompt**, with zero query-relevant content (observed on a real store: 12 locks = 2485 tokens vs a 2400 budget → 0 relevance tokens). Retrieval now reserves a relevance floor (`relevance_budget = max(effective_budget × 0.25, effective_budget − locked_used)`), capping L2.5+L1 at `locked_used + relevance_budget`. It is **byte-identical** outside the lock-saturated regime — lock-free corpora (e.g. LoCoMo) are unaffected — and locks are still never trimmed; in the saturated regime total output may exceed the nominal budget by up to the floor. On the real store, query-relevant results went from 0 → 20–27 per query.
 
 ## [3.7.0] - 2026-06-23
 

src/aelfrice/retrieval.py

@@ -118,6 +118,20 @@
 # behaviour byte-for-byte on queries where L2.5 returns nothing.
 DEFAULT_TOKEN_BUDGET: Final[int] = 2400
 
+# Reserved relevance-budget floor. L0 locked beliefs are injected
+# unconditionally and never trimmed (#379), so a store whose locks alone
+# meet or exceed `effective_budget` left ZERO budget for query-relevant
+# L2.5/L1 hits — every retrieval returned only the locks, regardless of the
+# prompt (observed on a real 12-lock store: 2485 lock tokens vs a 2400
+# budget → 0 relevance tokens). This reserves at least
+# `floor(effective_budget * RELEVANCE_BUDGET_FLOOR_FRACTION)` tokens for
+# L2.5+L1. It is a strict no-op (byte-identical) whenever the locks leave at
+# least that much room — i.e. it only fires in the lock-saturated regime —
+# so lock-free corpora (e.g. LoCoMo) are unaffected. Locks are still never
+# trimmed; in the saturated regime total output may exceed the nominal
+# budget by up to the floor, the intended trade for never going blind.
+RELEVANCE_BUDGET_FLOOR_FRACTION: Final[float] = 0.25
+
 _CHARS_PER_TOKEN: Final[float] = 4.0
 DEFAULT_L1_LIMIT: Final[int] = 50
 
@@ -2510,6 +2524,13 @@ def retrieve(
     until the estimated total token count is at or below
     `token_budget`. L0 beliefs are never trimmed.
 
+    Exception (#1014): because L0 locks are never trimmed, a store whose
+    locks alone meet or exceed `token_budget` reserves a relevance floor
+    (`RELEVANCE_BUDGET_FLOOR_FRACTION` of the budget) for L2.5/L1 so locks
+    can't starve query-relevant hits to zero. In that lock-saturated
+    regime the returned total may exceed `token_budget` by up to that
+    floor; outside it the budget cap holds exactly (byte-identical).
+
     L2.5 (v1.3.0): entity-index lookup. Default-on; gated by
     `is_entity_index_enabled()` (env override → kwarg → TOML →
     default True). When disabled the path collapses to v1.2's L0+L1
@@ -2635,7 +2656,14 @@ def _cost(b: Belief) -> int:
     # tokens back than they asked for, while still letting the
     # default 2400-budget caller see the full 400-token L2.5 slice.
     locked_used: int = sum(_belief_tokens(b) for b in locked)
-    l25_room: int = max(0, effective_budget - locked_used)
+    # #379 locks are uncapped + never trimmed; reserve a relevance floor so
+    # they can't starve L2.5/L1 to zero. No-op (byte-identical) unless locks
+    # leave less than the floor — see RELEVANCE_BUDGET_FLOOR_FRACTION.
+    relevance_budget: int = max(
+        int(effective_budget * RELEVANCE_BUDGET_FLOOR_FRACTION),
+        effective_budget - locked_used,
+    )
+    l25_room: int = max(0, relevance_budget)
     effective_l25_subbudget: int = min(l25_token_subbudget, l25_room)
 
     l25: list[Belief]
@@ -2674,7 +2702,7 @@ def _cost(b: Belief) -> int:
     l1_packed: list[Belief] = []
     for b in l1:
         cost: int = _cost(b)
-        if used + cost > effective_budget:
+        if used + cost > locked_used + relevance_budget:
             break
         out.append(b)
         l1_packed.append(b)
@@ -2700,7 +2728,7 @@ def _cost(b: Belief) -> int:
                 if hop.belief.id in seen_ids:
                     continue
                 cost = _cost(hop.belief)
-                if used + cost > effective_budget:
+                if used + cost > locked_used + relevance_budget:
                     break
                 out.append(hop.belief)
                 seen_ids.add(hop.belief.id)
@@ -2856,7 +2884,14 @@ def _cost(b: Belief) -> int:
         else LEGACY_TOKEN_BUDGET
     )
     locked_used: int = sum(_belief_tokens(b) for b in locked)
-    l25_room: int = max(0, effective_budget - locked_used)
+    # #379 locks are uncapped + never trimmed; reserve a relevance floor so
+    # they can't starve L2.5/L1 to zero. No-op (byte-identical) unless locks
+    # leave less than the floor — see RELEVANCE_BUDGET_FLOOR_FRACTION.
+    relevance_budget: int = max(
+        int(effective_budget * RELEVANCE_BUDGET_FLOOR_FRACTION),
+        effective_budget - locked_used,
+    )
+    l25_room: int = max(0, relevance_budget)
     effective_l25_subbudget: int = min(l25_token_subbudget, l25_room)
 
     if enabled and query.strip():
@@ -2906,7 +2941,7 @@ def _cost(b: Belief) -> int:
             edge_weight_floor=DEFAULT_CLUSTER_EDGE_FLOOR,
         )
         l1_by_id: dict[str, Belief] = {b.id: b for b in l1}
-        l1_remaining_budget = max(0, effective_budget - used)
+        l1_remaining_budget = max(0, locked_used + relevance_budget - used)
         l1_packed = pack_with_clusters(
             clusters, l1_by_id,
             token_budget=l1_remaining_budget,
@@ -2920,7 +2955,7 @@ def _cost(b: Belief) -> int:
     else:
         for b in l1:
             cost: int = _cost(b)
-            if used + cost > effective_budget:
+            if used + cost > locked_used + relevance_budget:
                 break
             out.append(b)
             l1_packed.append(b)
@@ -2953,7 +2988,7 @@ def _cost(b: Belief) -> int:
             if b.id in seen_pre:
                 continue
             cost = _cost(b)
-            if used + cost > effective_budget:
+            if used + cost > locked_used + relevance_budget:
                 break
             out.append(b)
             hrr_expanded.append(b)
@@ -2983,7 +3018,7 @@ def _cost(b: Belief) -> int:
                 if hop.belief.id in seen_ids:
                     continue
                 cost = _cost(hop.belief)
-                if used + cost > effective_budget:
+                if used + cost > locked_used + relevance_budget:
                     break
                 out.append(hop.belief)
                 bfs_chains.append(list(hop.path))

tests/test_relevance_budget_floor.py

@@ -0,0 +1,67 @@
+"""Reserved relevance-budget floor: locked beliefs (uncapped, never
+trimmed per #379) must not starve query-relevant L2.5/L1 hits to zero.
+
+Regression for the lock-saturation bug: a store whose locks alone meet or
+exceed the token budget returned ONLY the locks for every prompt (observed
+on a real 12-lock store: 2485 lock tokens vs a 2400 budget -> 0 relevance
+tokens). The floor reserves a slice for relevance; it is a strict no-op
+when the locks leave at least that much room.
+"""
+from __future__ import annotations
+
+from aelfrice.models import BELIEF_FACTUAL, LOCK_NONE, LOCK_USER, Belief
+from aelfrice.retrieval import (
+    DEFAULT_TOKEN_BUDGET,
+    RELEVANCE_BUDGET_FLOOR_FRACTION,
+    retrieve,
+)
+from aelfrice.store import MemoryStore
+
+_PAD = " ".join(["context"] * 120)  # ~120 tokens -> realistic ~150-token beliefs
+
+
+def _mk(bid: str, content: str, *, locked: bool, pad: bool = True) -> Belief:
+    return Belief(
+        id=bid,
+        content=(content + " " + _PAD) if pad else content,
+        content_hash=f"h_{bid}",
+        alpha=1.0,
+        beta=1.0,
+        type=BELIEF_FACTUAL,
+        lock_level=LOCK_USER if locked else LOCK_NONE,
+        locked_at="2026-01-01T00:00:00Z" if locked else None,
+        created_at="2026-05-11T00:00:00Z",
+        last_retrieved_at=None,
+    )
+
+
+def test_saturating_locks_do_not_starve_relevance() -> None:
+    """20 long locks overflow the default budget; a query-relevant
+    non-lock belief must still surface (was 0 before the floor)."""
+    s = MemoryStore(":memory:")
+    for i in range(20):
+        s.insert_belief(_mk(f"L{i}", f"unrelated locked fact about topic alpha {i}", locked=True))
+    for i in range(3):
+        s.insert_belief(_mk(f"T{i}", f"kubernetes deployment rollout pods replicas note {i}", locked=False))
+    hits = retrieve(s, "kubernetes deployment rollout", token_budget=DEFAULT_TOKEN_BUDGET)
+    relevant = [b for b in hits if b.id.startswith("T")]
+    assert relevant, "relevance floor must surface >=1 query-relevant belief under lock saturation"
+
+
+def test_floor_is_noop_when_locks_fit() -> None:
+    """With locks that fit comfortably, every query-relevant belief
+    surfaces exactly as before — the floor must not change the fit
+    regime."""
+    s = MemoryStore(":memory:")
+    # 2 short locks (tiny token cost), 3 relevant non-lock beliefs.
+    for i in range(2):
+        s.insert_belief(_mk(f"L{i}", f"short lock {i}", locked=True, pad=False))
+    for i in range(3):
+        s.insert_belief(_mk(f"T{i}", f"kubernetes deployment rollout pods note {i}", locked=False, pad=False))
+    hits = retrieve(s, "kubernetes deployment rollout", token_budget=DEFAULT_TOKEN_BUDGET)
+    relevant = [b for b in hits if b.id.startswith("T")]
+    assert len(relevant) == 3, "all relevant beliefs should surface when locks fit"
+
+
+def test_floor_fraction_sane() -> None:
+    assert 0.0 < RELEVANCE_BUDGET_FLOOR_FRACTION < 1.0