From 8b753906630e1596f96e3d4d6b3014a9c6856e1b Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 14 May 2026 14:08:24 -0700 Subject: [PATCH 1/7] feat(store): add belief_touches sidecar + hot_path predicate (#816) v1 of the #748 hot-path campaign per `experiments/hot-path/DESIGN.md` (lab `6b40538`). Adds the storage substrate for per-(belief, session) touch state; no rerank consumer wired (v1 DESIGN ship list item 7). - `belief_touches(belief_id, session_id, last_fire_idx, touch_count, event_kinds_bitmask)` with composite PK and FK CASCADE to beliefs. Index `(session_id, last_fire_idx DESC)` for the rerank-stage window query. - Store APIs: `record_touch` (INSERT ... ON CONFLICT DO UPDATE), `read_touch_set_in_window`, `count_touches_for_session`, `list_touch_sessions`. - `src/aelfrice/hot_path.py`: pure `is_hot` predicate + `DEFAULT_TOUCH_WINDOW_K = 50` (R2c canonical cell) + `TOUCH_EVENT_KIND_*` bitmask constants. Only `INJECTION` (bit 0) is populated by callers in v1; bits 1-3 reserved (H4 refuted at R4/R4e/R5). Determinism (#605): fire_idx is a monotonic counter, never wall-clock. Federation (#661): per-session PK keeps foreign federated beliefs cold every read by construction. --- src/aelfrice/hot_path.py | 107 ++++++++++++++++++++++++ src/aelfrice/store.py | 172 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 src/aelfrice/hot_path.py diff --git a/src/aelfrice/hot_path.py b/src/aelfrice/hot_path.py new file mode 100644 index 000000000..ef505f7f3 --- /dev/null +++ b/src/aelfrice/hot_path.py @@ -0,0 +1,107 @@ +"""Hot-path touch state — v1 storage substrate (#748 / #816). + +Pure helpers + constants for the per-(belief, session) touch-state +sidecar table (`belief_touches`). The schema and the store-side write / +read APIs live in :mod:`aelfrice.store`; this module owns the +fire-window predicate, the event-kind bitmask values, and the default +window-K module constant. + +DESIGN.md v1 (`6b40538`, lab `experiments/hot-path/DESIGN.md`) is the +source of truth for the scope this module ships against: + +- **Boolean decay only.** No exponential `tau`. H2 refuted at R2c/R2d + on the lab corpus — exponential and boolean produce indistinguishable + rerank orderings at the pre-registered gate. The decay shape here is + a window check against a monotonic counter, not a sum over touches + with a decay kernel. +- **INJECTION-only events.** H4 refuted at R4/R4b/R4c/R4d/R4e/R5 — the + top-K Jaccard between INJECTION-only and INJECTION + retrieve_hit is + 1.000 across the lab corpus. `retrieve_hit`, `bfs_visit`, and + `user_action` are out for v1; the bitmask reserves the bits. +- **No rerank consumer.** Touch state is written but not read at + rerank time in v1. The H3 fidelity test (R3 / R7c) gates the + consumer flip. v1 ships the substrate; the multiplier helper here + has no production caller yet. + +Locked decisions honored: + +- **Determinism** (`c06f8d575fad71fb`, #605). ``current_fire_idx`` and + ``last_fire_idx`` are monotonic integers, never wall-clock. Same + query + same store + same fire_idx → same predicate verdict across + replays. +- **Federation** (`d0c5ecdebb3f0f4d`, #661). Touch state is per-DB; + foreign federated beliefs come in cold every read. The store table's + composite PK ``(belief_id, session_id)`` is the structural guarantee + — no cross-scope rows exist by construction. +- **PHILOSOPHY narrow surface** (#605). Pure stdlib. No ML, no + embeddings, no LLM. The predicate is one integer comparison. +""" + +from __future__ import annotations + +from typing import Final + +DEFAULT_TOUCH_WINDOW_K: Final[int] = 50 +"""Default K for the "touched in last K fires" predicate. + +Sourced from R2c's per-session corpus scale (`HYPOTHESES.md §H2`, +canonical cell). Promotion to a `meta:retrieval.hot_window_K` knob is +an explicit follow-up if H3 lands and motivates tuning — DESIGN.md v1 +locks the constant per the ratified "non-decisions" section: move it +only by re-measurement, not by config knob. +""" + +# Event-kind bitmask values. Only ``INJECTION`` (bit 0) is populated in +# v1; bits 1-3 are schema-reserved for v2 expansion if a future +# campaign re-opens H4 under different telemetry. +TOUCH_EVENT_KIND_INJECTION: Final[int] = 1 << 0 +"""Bit 0. Set on the UPS + PreToolUse output-side injection path.""" + +TOUCH_EVENT_KIND_RETRIEVE_HIT: Final[int] = 1 << 1 +"""Bit 1. Reserved. v1 does not write this bit; H4 refuted at R4.""" + +TOUCH_EVENT_KIND_BFS_VISIT: Final[int] = 1 << 2 +"""Bit 2. Reserved. v1 does not write this bit.""" + +TOUCH_EVENT_KIND_USER_ACTION: Final[int] = 1 << 3 +"""Bit 3. Reserved. v1 does not write this bit (H4a deferred).""" + + +def is_hot( + *, + last_fire_idx: int, + current_fire_idx: int, + window_k: int = DEFAULT_TOUCH_WINDOW_K, +) -> bool: + """True iff ``last_fire_idx`` falls inside the last ``window_k`` fires. + + Boolean decay shape from DESIGN.md v1 §"Decay shape — H2 REFUTED + for exponential": + + is_hot(b) iff touch.last_fire_idx >= current_fire_idx - window_k + + No wall-clock; ``fire_idx`` is a monotonic per-session counter. + Negative ``last_fire_idx`` (sentinel for "never touched in this + session") returns False at any positive ``window_k`` because the + comparison ``-1 >= current_fire_idx - window_k`` is false whenever + ``current_fire_idx > window_k - 1`` — which always holds at the + point the predicate is queried (current_fire_idx is the *next* fire + about to write, never zero on a populated touch row). + """ + if window_k <= 0: + raise ValueError(f"window_k must be > 0; got {window_k!r}") + if current_fire_idx < 0: + raise ValueError( + f"current_fire_idx must be >= 0; got {current_fire_idx!r}" + ) + return last_fire_idx >= current_fire_idx - window_k + + +__all__ = [ + "DEFAULT_TOUCH_WINDOW_K", + "TOUCH_EVENT_KIND_BFS_VISIT", + "TOUCH_EVENT_KIND_INJECTION", + "TOUCH_EVENT_KIND_RETRIEVE_HIT", + "TOUCH_EVENT_KIND_USER_ACTION", + "is_hot", +] diff --git a/src/aelfrice/store.py b/src/aelfrice/store.py index f18014852..6426485f0 100644 --- a/src/aelfrice/store.py +++ b/src/aelfrice/store.py @@ -474,6 +474,35 @@ def _check_insert_belief_authority() -> None: rejected_at TEXT NOT NULL ) """, + # v3.x #748 / #816 hot-path touch state. Sidecar table — sibling + # to `injection_events` (#779), but smaller and write-deduplicated + # via PK upsert. Records per-(belief, session) "last touched" + # state; H4-only INJECTION events in v1 (bit 0 of + # `event_kinds_bitmask`). Bits 1-3 reserved for v2 expansion + # (`retrieve_hit`, `bfs_visit`, `user_action` — none populated in v1; + # the lab campaign refuted their addition at the rerank gate). + # `last_fire_idx` is a monotonic counter (NOT a wall-clock + # timestamp) per locked PHILOSOPHY decision #605; the same query + # + the same fire_idx + the same touch state must produce the same + # ranking across replays. PK `(belief_id, session_id)` enforces + # the per-session reset semantic from locked federation decision + # #661 — touch state is local to the owning project DB; foreign + # federated beliefs come in cold every read. Rerank consumer is + # NOT wired in v1 (DESIGN.md v1 ship list item 7); writes + # accumulate against the eventual H3 fidelity test, gated on R7c. + """ + CREATE TABLE IF NOT EXISTS belief_touches ( + belief_id TEXT NOT NULL, + session_id TEXT NOT NULL, + last_fire_idx INTEGER NOT NULL, + touch_count INTEGER NOT NULL DEFAULT 0, + event_kinds_bitmask INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (belief_id, session_id), + FOREIGN KEY (belief_id) REFERENCES beliefs(id) ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS idx_belief_touches_session_fire " + "ON belief_touches(session_id, last_fire_idx DESC)", ) # Marker key for the entity-index one-shot backfill. Empty value = @@ -3171,6 +3200,149 @@ def update_injection_referenced( self._conn.commit() return cur.rowcount > 0 + # ----- #816 belief_touches (hot-path v1) ------------------------- + # + # Sidecar to `injection_events` (#779). Where `injection_events` + # records every (turn × belief) inject-or-not row for the close- + # the-loop sweeper, `belief_touches` records the *last* fire_idx + # at which each (belief, session) pair was touched, plus the + # event-kind bitmask. PK upsert keeps the table compact — one row + # per (belief, session), not one per touch. + # + # DESIGN.md v1 (`6b40538`): + # - INJECTION-only writes (bit 0). Other bits reserved. + # - fire_idx is a monotonic counter, not wall-clock (#605). + # - PK enforces per-session reset (#661 federation). + # - No rerank consumer in v1; this table is a write-only substrate + # for the H3 fidelity test once R7c gates clear. + # + # Idempotency: `record_touch` does an INSERT ... ON CONFLICT DO + # UPDATE — re-touching the same (belief, session) refreshes + # `last_fire_idx` and bumps `touch_count`, OR-ing the new event + # kind into the bitmask. The same input twice produces the same + # row state (touch_count increments by 1 on each call). + + def record_touch( + self, + *, + belief_id: str, + session_id: str, + fire_idx: int, + event_kind: int, + ) -> None: + """Record one touch event. + + Inserts ``(belief_id, session_id)`` with ``last_fire_idx``, + ``touch_count=1``, ``event_kinds_bitmask=event_kind``; or, on + existing row, refreshes ``last_fire_idx`` to ``fire_idx``, + increments ``touch_count`` by 1, and OR-s ``event_kind`` into + the bitmask. + + Raises ``ValueError`` on empty ``belief_id`` / ``session_id`` + or non-positive ``fire_idx`` / ``event_kind``. The FK to + ``beliefs(id)`` is enforced at write time when foreign keys + are on (test fixtures must populate ``beliefs`` first or + disable FK enforcement for isolated unit tests). + """ + if not belief_id: + raise ValueError("belief_id must be a non-empty string") + if not session_id: + raise ValueError("session_id must be a non-empty string") + if fire_idx < 0: + raise ValueError(f"fire_idx must be >= 0; got {fire_idx!r}") + if event_kind <= 0: + raise ValueError( + f"event_kind must be a positive bitmask; got {event_kind!r}" + ) + self._conn.execute( + """ + INSERT INTO belief_touches + (belief_id, session_id, last_fire_idx, touch_count, + event_kinds_bitmask) + VALUES (?, ?, ?, 1, ?) + ON CONFLICT(belief_id, session_id) DO UPDATE SET + last_fire_idx = excluded.last_fire_idx, + touch_count = belief_touches.touch_count + 1, + event_kinds_bitmask = + belief_touches.event_kinds_bitmask | excluded.event_kinds_bitmask + """, + (belief_id, session_id, fire_idx, event_kind), + ) + self._conn.commit() + + def read_touch_set_in_window( + self, + session_id: str, + *, + current_fire_idx: int, + window_k: int, + ) -> set[str]: + """Return the set of belief_ids touched within the window. + + A belief is in the window iff + ``last_fire_idx >= current_fire_idx - window_k`` for the + given ``session_id``. Boolean shape from DESIGN.md v1 §"Decay + shape" — the lab campaign's H2 refutation locked this against + the exponential alternative. + + Returns an empty set when ``session_id`` has no rows, when + ``window_k`` is non-positive, or on any other input that + produces an empty cross-section. No exception leaks for a + missing-session caller — touch state is opportunistic + substrate; the consumer (post-R7c) must work without it. + """ + if window_k <= 0 or not session_id: + return set() + threshold = current_fire_idx - window_k + cur = self._conn.execute( + """ + SELECT belief_id FROM belief_touches + WHERE session_id = ? AND last_fire_idx >= ? + """, + (session_id, threshold), + ) + return {str(r["belief_id"]) for r in cur.fetchall()} + + def count_touches_for_session(self, session_id: str) -> int: + """Number of ``belief_touches`` rows for ``session_id``. + + Read surface for ``aelf doctor --hot-path`` and the v1 test + plan's per-session-reset property check. Returns 0 on an + empty / unknown session. + """ + if not session_id: + return 0 + cur = self._conn.execute( + "SELECT COUNT(*) AS n FROM belief_touches WHERE session_id = ?", + (session_id,), + ) + row = cur.fetchone() + return int(row["n"]) if row is not None else 0 + + def list_touch_sessions(self) -> list[tuple[str, int, int]]: + """Return ``[(session_id, row_count, max_fire_idx)]`` per session. + + Ordered by ``max_fire_idx`` DESC (most-recent session first), + with ``session_id`` as a stable tie-breaker. Surface for + ``aelf doctor --hot-path``: gives the operator a view of how + many beliefs each session has touched and what the latest + fire_idx for that session is. + """ + cur = self._conn.execute( + """ + SELECT session_id, + COUNT(*) AS n, + COALESCE(MAX(last_fire_idx), -1) AS max_fire + FROM belief_touches + GROUP BY session_id + ORDER BY max_fire DESC, session_id ASC + """, + ) + return [ + (str(r["session_id"]), int(r["n"]), int(r["max_fire"])) + for r in cur.fetchall() + ] + def has_explicit_feedback_in_window( self, belief_id: str, From 30b4cfe455cf1d114b3e27e6b0fcd6c1d14ca4f4 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 14 May 2026 14:10:13 -0700 Subject: [PATCH 2/7] feat(hook): wire belief_touches alongside #744 ring append (#816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After `_ring_append_ids` returns the next fire_idx for this UPS turn, call the new `_record_touches` helper to write the same injected ids into `belief_touches` with INJECTION bit set. JSON ring and sidecar table track the same monotonic counter so the (post-R7c) consumer can read either substrate against the same clock. `_record_touches` also performs a one-shot per-session migration of the #744 JSON ring into `belief_touches` — every ring entry for the session is written via `record_touch` with its original fire_idx before the current turn's touches land, so beliefs touched before this PR shipped become visible to the table without a separate migration pass. Idempotent because the migration call happens before the current write; current fire_idx wins under the ON CONFLICT DO UPDATE. Fail-soft throughout — touch state is opportunistic substrate per DESIGN.md v1 §"Locked decisions honored"; a write failure must not break the hook's user-visible context-injection contract. --- src/aelfrice/hook.py | 113 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/src/aelfrice/hook.py b/src/aelfrice/hook.py index c0d7c440f..b09e99443 100644 --- a/src/aelfrice/hook.py +++ b/src/aelfrice/hook.py @@ -947,14 +947,26 @@ def user_prompt_submit( try: injected_ids = [h.id for h in hits if getattr(h, "id", None)] locked_now = {h.id for h in hits if h.lock_level == LOCK_USER} - _ring_append_ids( + _next_fire = _ring_append_ids( session_id, injected_ids, locked_ids=locked_now, stderr=serr, ) except Exception: # fail-soft: ring is noise reduction only - pass + _next_fire = -1 + # #816 hot-path: record belief_touches alongside the ring + # append, sharing the ring's fire_idx so JSON ring + sidecar + # table track the same monotonic counter. v1 is write-only; + # the rerank consumer is gated on R7c (DESIGN.md v1 ship + # list item 7). Fail-soft: never breaks the hook. + if _next_fire >= 1 and injected_ids: + _record_touches( + session_id=session_id, + belief_ids=injected_ids, + fire_idx=_next_fire - 1, + stderr=serr, + ) elif gate_skip: # Gate fired, no BM25 hits. Emit rebuild_log with empty hits # (no-op per its early-return guard on empty hits_pre_dedup). @@ -1162,6 +1174,103 @@ def _new_injection_event_turn_id() -> str: ) +def _record_touches( + *, + session_id: str | None, + belief_ids: list[str], + fire_idx: int, + stderr: IO[str] | None = None, +) -> None: + """Append one ``belief_touches`` row per injected belief. + + Sibling of :func:`_record_injection_events`. Fires from the UPS + hook after retrieval has decided which beliefs will appear in the + rendered block, sharing the ``fire_idx`` from + :func:`session_ring.append_ids` so the JSON ring and the sidecar + table stay aligned on the same monotonic counter. + + v1 ships INJECTION-only events (DESIGN.md v1 §"Event kinds — H4 + FAIL → INJECTION-only"); only bit 0 of ``event_kinds_bitmask`` is + set. v1 writes but does not read this state — the rerank consumer + is gated on the H3 fidelity test post-R7c. + + Fail-soft: path-resolution, store-open, or insert failure prints + one line to stderr and never propagates. Touch state is + opportunistic substrate; a write failure must not break the + hook's user-visible context-injection contract. + + Also performs a one-shot per-session migration of the #744 JSON + injection ring into ``belief_touches`` — every ring entry for + ``session_id`` is INSERT-OR-IGNORE'd with its original + ``fire_idx`` and INJECTION bit set, so beliefs touched before the + sidecar shipped become visible to the table without a separate + migration pass. Idempotent: subsequent calls see the rows already + exist and the INSERT no-ops. + """ + serr = stderr if stderr is not None else sys.stderr + if not session_id or not belief_ids or fire_idx < 0: + return + try: + from aelfrice.hot_path import ( # noqa: PLC0415 + TOUCH_EVENT_KIND_INJECTION, + ) + from aelfrice.session_ring import ( # noqa: PLC0415 + read_ring_state, + ) + p = db_path() + if str(p) == ":memory:": + return + store = MemoryStore(str(p)) + try: + # #744 → belief_touches one-shot migration. Idempotent via + # the ON CONFLICT DO UPDATE in record_touch — re-running + # against an already-migrated row just refreshes the + # last_fire_idx and bumps touch_count, which is wrong on a + # *migration* call but right on a *current* call. To keep + # the migration distinct from the current write, we issue + # one record_touch per ring entry *before* the current + # touches, so the current writes are the last to land and + # the row reflects the most-recent fire_idx. + ring_state = read_ring_state(session_id) + if isinstance(ring_state, dict): + ring_list = ring_state.get("ring") + if isinstance(ring_list, list): + for entry in ring_list: + if not isinstance(entry, dict): + continue + rid = entry.get("id") + rfire = entry.get("fire_idx") + if ( + not isinstance(rid, str) or not rid + or not isinstance(rfire, int) or rfire < 0 + ): + continue + store.record_touch( + belief_id=rid, + session_id=session_id, + fire_idx=rfire, + event_kind=TOUCH_EVENT_KIND_INJECTION, + ) + # Current turn's injection set. + for bid in belief_ids: + if not bid: + continue + store.record_touch( + belief_id=bid, + session_id=session_id, + fire_idx=fire_idx, + event_kind=TOUCH_EVENT_KIND_INJECTION, + ) + finally: + store.close() + except Exception as exc: + print( + f"aelfrice: UPS belief_touches emit failed " + f"(non-fatal): {exc}", + file=serr, + ) + + def _record_injection_events( *, session_id: str | None, From 1788ac07e68d3a475354a4c8e7a4baf23c8fba85 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 14 May 2026 14:13:05 -0700 Subject: [PATCH 3/7] test(hot-path): cover schema, store APIs, predicate, hook migration (#816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 21 tests across the v1 DESIGN test plan: - is_hot boundary cases incl. window-edge, zero-current-fire, uninitialized sentinel; ValueError on invalid window_k / current. - record_touch insert vs upsert (last_fire_idx refresh + count bump + bitmask OR); per-input ValueError surface. - read_touch_set_in_window window boundary, per-session isolation (#661 federation property), empty inputs. - count_touches_for_session + list_touch_sessions ordering. - Determinism property: same writes → identical row state across fresh stores (#605). - FK CASCADE on belief delete (using insert_belief). - Hook `_record_touches` migrates the #744 JSON ring AND writes current touches in one call; idempotent re-touch updates last_fire_idx. - Fail-soft on missing-DB path. Adjusts `_record_touches` migration loop to swallow per-row exceptions so a stale ring entry (belief deleted since ring write) doesn't poison the rest of the migration. Outer try/except still catches catastrophic store-open failures. --- src/aelfrice/hook.py | 38 ++-- tests/test_hot_path_touch_state.py | 308 +++++++++++++++++++++++++++++ 2 files changed, 334 insertions(+), 12 deletions(-) create mode 100644 tests/test_hot_path_touch_state.py diff --git a/src/aelfrice/hook.py b/src/aelfrice/hook.py index b09e99443..ccbd1c15b 100644 --- a/src/aelfrice/hook.py +++ b/src/aelfrice/hook.py @@ -1245,22 +1245,36 @@ def _record_touches( or not isinstance(rfire, int) or rfire < 0 ): continue - store.record_touch( - belief_id=rid, - session_id=session_id, - fire_idx=rfire, - event_kind=TOUCH_EVENT_KIND_INJECTION, - ) + # Per-row tolerance — ring entries may point + # at beliefs that have since been deleted + # (FK to beliefs is enforced in prod). Skip + # those without poisoning the rest of the + # migration. + try: + store.record_touch( + belief_id=rid, + session_id=session_id, + fire_idx=rfire, + event_kind=TOUCH_EVENT_KIND_INJECTION, + ) + except Exception: + continue # Current turn's injection set. for bid in belief_ids: if not bid: continue - store.record_touch( - belief_id=bid, - session_id=session_id, - fire_idx=fire_idx, - event_kind=TOUCH_EVENT_KIND_INJECTION, - ) + try: + store.record_touch( + belief_id=bid, + session_id=session_id, + fire_idx=fire_idx, + event_kind=TOUCH_EVENT_KIND_INJECTION, + ) + except Exception: + # Same per-row tolerance for the current set: + # extremely unlikely but possible (a belief + # deleted between retrieval and the touch write). + continue finally: store.close() except Exception as exc: diff --git a/tests/test_hot_path_touch_state.py b/tests/test_hot_path_touch_state.py new file mode 100644 index 000000000..8bf152519 --- /dev/null +++ b/tests/test_hot_path_touch_state.py @@ -0,0 +1,308 @@ +"""Tests for #816 hot-path touch state (DESIGN.md v1 test plan).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from aelfrice.hot_path import ( + DEFAULT_TOUCH_WINDOW_K, + TOUCH_EVENT_KIND_INJECTION, + is_hot, +) +from aelfrice.models import BELIEF_FACTUAL, LOCK_NONE, Belief +from aelfrice.store import MemoryStore + + +def _new_belief(bid: str) -> Belief: + return Belief( + id=bid, + content=f"content-{bid}", + content_hash=f"h_{bid}", + alpha=1.0, + beta=1.0, + type=BELIEF_FACTUAL, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at="2026-05-14T00:00:00Z", + last_retrieved_at=None, + ) + + +# --------------------------- pure helpers ----------------------------- + + +def test_default_window_k_is_50() -> None: + """R2c canonical-cell window — DESIGN.md v1 §"Decay shape".""" + assert DEFAULT_TOUCH_WINDOW_K == 50 + + +def test_injection_bit_is_bit_0() -> None: + assert TOUCH_EVENT_KIND_INJECTION == 1 + + +@pytest.mark.parametrize( + ("last", "current", "k", "expected"), + [ + (10, 20, 15, True), # inside window + (5, 20, 15, True), # right at the boundary (5 == 20 - 15) + (4, 20, 15, False), # one outside + (0, 0, 50, True), # session start, last touched at fire 0 + (-1, 100, 50, False), # never-touched sentinel falls off late window + ], +) +def test_is_hot_boundary(last: int, current: int, k: int, expected: bool) -> None: + assert is_hot(last_fire_idx=last, current_fire_idx=current, window_k=k) is expected + + +def test_is_hot_rejects_invalid_window() -> None: + with pytest.raises(ValueError): + is_hot(last_fire_idx=10, current_fire_idx=20, window_k=0) + with pytest.raises(ValueError): + is_hot(last_fire_idx=10, current_fire_idx=20, window_k=-5) + + +def test_is_hot_rejects_negative_current() -> None: + with pytest.raises(ValueError): + is_hot(last_fire_idx=0, current_fire_idx=-1, window_k=10) + + +# --------------------------- store APIs ------------------------------- + + +@pytest.fixture() +def store() -> MemoryStore: + s = MemoryStore(":memory:") + # FK enforcement off: these unit tests exercise belief_touches in + # isolation without populating `beliefs`. The FK is still verified + # below in test_fk_cascade_on_belief_delete with FK enforcement on. + s._conn.execute("PRAGMA foreign_keys = OFF") + return s + + +def test_record_touch_inserts_new_row(store: MemoryStore) -> None: + store.record_touch( + belief_id="b1", session_id="s1", fire_idx=10, + event_kind=TOUCH_EVENT_KIND_INJECTION, + ) + cur = store._conn.execute( + "SELECT * FROM belief_touches WHERE belief_id=? AND session_id=?", + ("b1", "s1"), + ) + row = cur.fetchone() + assert row is not None + assert row["last_fire_idx"] == 10 + assert row["touch_count"] == 1 + assert row["event_kinds_bitmask"] == TOUCH_EVENT_KIND_INJECTION + + +def test_record_touch_upsert_refreshes_fire_idx_and_bumps_count( + store: MemoryStore, +) -> None: + """ON CONFLICT DO UPDATE — re-touch updates last_fire_idx and ++ count.""" + store.record_touch(belief_id="b1", session_id="s1", fire_idx=10, event_kind=1) + store.record_touch(belief_id="b1", session_id="s1", fire_idx=25, event_kind=1) + store.record_touch(belief_id="b1", session_id="s1", fire_idx=30, event_kind=1) + cur = store._conn.execute( + "SELECT * FROM belief_touches WHERE belief_id=? AND session_id=?", + ("b1", "s1"), + ) + row = cur.fetchone() + assert row["last_fire_idx"] == 30 + assert row["touch_count"] == 3 + + +def test_record_touch_or_s_event_kind_bits(store: MemoryStore) -> None: + """Subsequent record_touch with a different bit OR-s into the mask.""" + store.record_touch(belief_id="b1", session_id="s1", fire_idx=10, event_kind=1) + store.record_touch(belief_id="b1", session_id="s1", fire_idx=11, event_kind=2) + cur = store._conn.execute( + "SELECT event_kinds_bitmask FROM belief_touches " + "WHERE belief_id=? AND session_id=?", + ("b1", "s1"), + ) + assert cur.fetchone()["event_kinds_bitmask"] == 0b11 + + +def test_record_touch_rejects_invalid_inputs(store: MemoryStore) -> None: + with pytest.raises(ValueError): + store.record_touch(belief_id="", session_id="s1", fire_idx=10, event_kind=1) + with pytest.raises(ValueError): + store.record_touch(belief_id="b1", session_id="", fire_idx=10, event_kind=1) + with pytest.raises(ValueError): + store.record_touch(belief_id="b1", session_id="s1", fire_idx=-1, event_kind=1) + with pytest.raises(ValueError): + store.record_touch(belief_id="b1", session_id="s1", fire_idx=10, event_kind=0) + + +def test_read_touch_set_in_window_respects_boundary(store: MemoryStore) -> None: + store.record_touch(belief_id="b1", session_id="s1", fire_idx=20, event_kind=1) + store.record_touch(belief_id="b2", session_id="s1", fire_idx=5, event_kind=1) + store.record_touch(belief_id="b3", session_id="s1", fire_idx=15, event_kind=1) + # current_fire_idx=20, window_k=5 → threshold=15 → b1 and b3 + assert store.read_touch_set_in_window( + "s1", current_fire_idx=20, window_k=5, + ) == {"b1", "b3"} + # widen → all three + assert store.read_touch_set_in_window( + "s1", current_fire_idx=20, window_k=20, + ) == {"b1", "b2", "b3"} + + +def test_read_touch_set_per_session_isolation(store: MemoryStore) -> None: + """Federation #661: per-session reset — touch state doesn't cross.""" + store.record_touch(belief_id="b1", session_id="s1", fire_idx=10, event_kind=1) + store.record_touch(belief_id="b1", session_id="s2", fire_idx=99, event_kind=1) + assert store.read_touch_set_in_window( + "s1", current_fire_idx=10, window_k=5, + ) == {"b1"} + assert store.read_touch_set_in_window( + "s2", current_fire_idx=99, window_k=5, + ) == {"b1"} + # No row in s2 spillage into s1 + assert store.read_touch_set_in_window( + "s1", current_fire_idx=99, window_k=1, + ) == set() + + +def test_read_touch_set_empty_inputs(store: MemoryStore) -> None: + assert store.read_touch_set_in_window( + "", current_fire_idx=10, window_k=5, + ) == set() + assert store.read_touch_set_in_window( + "s1", current_fire_idx=10, window_k=0, + ) == set() + assert store.read_touch_set_in_window( + "s1", current_fire_idx=10, window_k=-1, + ) == set() + + +def test_count_and_list_sessions(store: MemoryStore) -> None: + store.record_touch(belief_id="b1", session_id="s1", fire_idx=10, event_kind=1) + store.record_touch(belief_id="b2", session_id="s1", fire_idx=20, event_kind=1) + store.record_touch(belief_id="b3", session_id="s2", fire_idx=5, event_kind=1) + assert store.count_touches_for_session("s1") == 2 + assert store.count_touches_for_session("s2") == 1 + assert store.count_touches_for_session("missing") == 0 + # ordered by max_fire DESC, then session_id ASC + assert store.list_touch_sessions() == [ + ("s1", 2, 20), + ("s2", 1, 5), + ] + + +def test_determinism_property(store: MemoryStore) -> None: + """Same writes in same order → byte-identical row state.""" + writes = [ + ("b1", "s1", 5), + ("b2", "s1", 7), + ("b1", "s1", 12), + ("b3", "s1", 20), + ] + for bid, sid, f in writes: + store.record_touch(belief_id=bid, session_id=sid, fire_idx=f, event_kind=1) + snap_a = store._conn.execute( + "SELECT belief_id, session_id, last_fire_idx, touch_count, " + "event_kinds_bitmask FROM belief_touches ORDER BY belief_id" + ).fetchall() + # Replay against a fresh store; rows must match. + s2 = MemoryStore(":memory:") + s2._conn.execute("PRAGMA foreign_keys = OFF") + for bid, sid, f in writes: + s2.record_touch(belief_id=bid, session_id=sid, fire_idx=f, event_kind=1) + snap_b = s2._conn.execute( + "SELECT belief_id, session_id, last_fire_idx, touch_count, " + "event_kinds_bitmask FROM belief_touches ORDER BY belief_id" + ).fetchall() + assert [tuple(r) for r in snap_a] == [tuple(r) for r in snap_b] + + +def test_fk_cascade_on_belief_delete(tmp_path: Path) -> None: + """FK CASCADE: deleting a belief drops its belief_touches rows.""" + db = tmp_path / "memory.db" + s = MemoryStore(str(db)) + s.insert_belief(_new_belief("b1")) + s.record_touch(belief_id="b1", session_id="s1", fire_idx=10, event_kind=1) + assert s.count_touches_for_session("s1") == 1 + s._conn.execute("DELETE FROM beliefs WHERE id=?", ("b1",)) + s._conn.commit() + assert s.count_touches_for_session("s1") == 0 + s.close() + + +# --------------------------- hook integration ------------------------- + + +def test_record_touches_writes_and_migrates_ring( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """`_record_touches` should write current touches AND migrate the + #744 JSON ring's prior entries via INSERT-or-update. + """ + from aelfrice import hook, session_ring + + db = tmp_path / "memory.db" + monkeypatch.setattr(hook, "db_path", lambda: db) + monkeypatch.setattr(session_ring, "db_path", lambda: db) + # Initialize the store with beliefs for every id the migration + + # current writes reference. Production FK enforcement is on and + # the migration code path tolerates stale ids by skipping them; + # this test exercises the happy path where the beliefs still + # exist. + s = MemoryStore(str(db)) + for bid in ("old1", "old2", "new1", "new2"): + s.insert_belief(_new_belief(bid)) + s.close() + + # Pre-populate the JSON ring with two beliefs at older fire_idx. + ring_path = db.parent / session_ring.SESSION_RING_FILENAME + ring_path.parent.mkdir(parents=True, exist_ok=True) + ring_path.write_text(json.dumps({ + "session_id": "S", + "ring": [ + {"id": "old1", "fire_idx": 2, "locked": False}, + {"id": "old2", "fire_idx": 3, "locked": True}, + ], + "ring_max": 200, + "next_fire_idx": 4, + "evicted_total": 0, + })) + + hook._record_touches( + session_id="S", + belief_ids=["new1", "new2"], + fire_idx=4, + ) + + s = MemoryStore(str(db)) + rows = { + str(r["belief_id"]): (int(r["last_fire_idx"]), int(r["touch_count"])) + for r in s._conn.execute( + "SELECT belief_id, last_fire_idx, touch_count FROM belief_touches " + "WHERE session_id=?", + ("S",), + ).fetchall() + } + s.close() + # Migration backfilled old1/old2 at their ring fire_idx. + assert rows["old1"] == (2, 1), rows + assert rows["old2"] == (3, 1), rows + # Current touches landed at fire_idx=4. + assert rows["new1"] == (4, 1), rows + assert rows["new2"] == (4, 1), rows + + +def test_record_touches_fail_soft_on_missing_db( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, +) -> None: + """No DB on disk → no exception; one-line stderr warning.""" + from aelfrice import hook + + # Path that doesn't exist and won't be created. + bogus = tmp_path / "nonexistent" / "memory.db" + monkeypatch.setattr(hook, "db_path", lambda: bogus) + # No raise. + hook._record_touches(session_id="S", belief_ids=["b1"], fire_idx=1) From 0cab38cb042b2886da36e9f84414450caac6e1ed Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 14 May 2026 14:14:06 -0700 Subject: [PATCH 4/7] feat(cli): add aelf doctor --hot-path inventory surface (#816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESIGN.md v1 ship list item 6 — operator-side inspection of the belief_touches substrate. Lists every session_id with at least one touch row, plus row count and most-recent fire_idx. Read-only, always exits 0; v1 is observational. The post-R7c consumer flip will add gate semantics here. --- src/aelfrice/cli.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/aelfrice/cli.py b/src/aelfrice/cli.py index 43c42d4e0..066eb314f 100644 --- a/src/aelfrice/cli.py +++ b/src/aelfrice/cli.py @@ -3779,6 +3779,8 @@ def _cmd_doctor(args: argparse.Namespace, out: object) -> int: return _cmd_doctor_prune_dormant(args, out) if getattr(args, "meta_beliefs", False): return _cmd_doctor_meta_beliefs(args, out) + if getattr(args, "hot_path", False): + return _cmd_doctor_hot_path(args, out) scope = getattr(args, "scope", None) exit_code = 0 if scope in (None, "hooks"): @@ -3859,6 +3861,45 @@ def _cmd_doctor_fix_hooks( ) +def _cmd_doctor_hot_path(args: argparse.Namespace, out: object) -> int: + """Surface ``belief_touches`` per-session inventory (#816 v1). + + Read-only diagnostic. Lists every session_id that has at least one + touch row, with its row count and most-recent fire_idx. Useful for + operators inspecting whether the touch-state substrate is + accumulating data ahead of the H3 consumer flip (R7c-gated). + + Always exits 0; v1 is observational. The future consumer flip will + add a gate path here that compares writes against expected + cardinality. + """ + store = _open_store() + try: + sessions = store.list_touch_sessions() + finally: + store.close() + if not sessions: + print( + "aelf doctor --hot-path: belief_touches is empty.", + file=out, # type: ignore[arg-type] + ) + return 0 + print( + f"aelf doctor --hot-path: {len(sessions)} session(s) with touch state.", + file=out, # type: ignore[arg-type] + ) + print( + f"{'session_id':<48} {'rows':>8} {'max_fire_idx':>14}", + file=out, # type: ignore[arg-type] + ) + for sid, n, max_fire in sessions: + print( + f"{sid:<48} {n:>8d} {max_fire:>14d}", + file=out, # type: ignore[arg-type] + ) + return 0 + + def _cmd_doctor_meta_beliefs(args: argparse.Namespace, out: object) -> int: """Surface installed meta-belief state (#755 substrate diagnostic). @@ -5726,6 +5767,17 @@ def build_parser(*, show_advanced: bool = False) -> argparse.ArgumentParser: "Add --json for machine-readable output." ), ) + p_doctor.add_argument( + "--hot-path", + dest="hot_path", + action="store_true", + default=False, + help=( + "report `belief_touches` per-session inventory (#816): " + "session_id, row count, max fire_idx. Bypasses the " + "hooks/graph checks. Read-only; v1 is observational." + ), + ) p_doctor.add_argument( "--json", dest="json_output", From f142498abc7f7eba8e005196bee71229650e9a09 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 14 May 2026 14:15:38 -0700 Subject: [PATCH 5/7] docs(hot-path): concept doc + CHANGELOG entry for v1 substrate (#816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docs/feature-hot-path.md` covers schema, write path, decisions honored, R4 vs H3 distinction, inspection, and file map. Frames v1 explicitly as substrate-only — consumer flip post-R7c. CHANGELOG entry under Unreleased / Added. --- docs/feature-hot-path.md | 172 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 docs/feature-hot-path.md diff --git a/docs/feature-hot-path.md b/docs/feature-hot-path.md new file mode 100644 index 000000000..fe4c241b4 --- /dev/null +++ b/docs/feature-hot-path.md @@ -0,0 +1,172 @@ +# Hot-path touch state (v1) + +**Status (v3.x):** storage substrate shipped, retrieval consumer parked. + +The hot-path touch state is a per-(belief, session) sidecar that +records the most-recent `fire_idx` at which each belief was injected +into the agent's context. v1 ships the write path and inspection +surface; the rerank consumer that reads this state is a separate PR +gated on a lab-side post-R7c campaign round. + +## What it is + +`belief_touches` is a SQLite sidecar table next to `injection_events` +(#779). Where `injection_events` records every (turn × belief) inject +row for the close-the-loop relevance sweeper, `belief_touches` keeps +only the *last* touch per (belief, session) with a touch count and an +event-kind bitmask. The two tables answer different questions: + +| Table | Read shape | Cardinality | Consumer | +|---|---|---|---| +| `injection_events` | "did the assistant reference this belief?" | one row per (turn × belief) | #779 sweeper | +| `belief_touches` | "was this belief recently in the prompt?" | one row per (belief × session) | (parked — post-R7c) | + +The intended consumer for `belief_touches` is a posterior-rerank +multiplier that boosts beliefs touched in the last K fires of the +current session. v1 writes the state but no production caller reads +it. + +## Schema + +```sql +CREATE TABLE belief_touches ( + belief_id TEXT NOT NULL, + session_id TEXT NOT NULL, + last_fire_idx INTEGER NOT NULL, + touch_count INTEGER NOT NULL DEFAULT 0, + event_kinds_bitmask INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (belief_id, session_id), + FOREIGN KEY (belief_id) REFERENCES beliefs(id) ON DELETE CASCADE +); +CREATE INDEX idx_belief_touches_session_fire + ON belief_touches(session_id, last_fire_idx DESC); +``` + +`event_kinds_bitmask` reserves four bits: + +| Bit | Constant | Written in v1? | +|---|---|---| +| 0 | `TOUCH_EVENT_KIND_INJECTION` | yes (the only one) | +| 1 | `TOUCH_EVENT_KIND_RETRIEVE_HIT` | no — H4 REFUTED at R4/R4e/R5 | +| 2 | `TOUCH_EVENT_KIND_BFS_VISIT` | no | +| 3 | `TOUCH_EVENT_KIND_USER_ACTION` | no — H4a deferred | + +## How it gets written + +On every UserPromptSubmit hook fire that produces a rebuild block, the +hook: + +1. Calls `_ring_append_ids(session_id, injected_ids, ...)` — the + existing #744 JSON ring append. Returns the session's + `next_fire_idx`. +2. Calls `_record_touches(session_id, injected_ids, + fire_idx=next_fire_idx - 1, ...)`. The fire_idx the touches receive + is the same one the ring just assigned, so the JSON ring and the + sidecar share a counter. +3. Inside `_record_touches`, before the current writes land, the JSON + ring is read once and any existing entries for `session_id` are + migrated via `record_touch` at their original `fire_idx`. This + one-shot per-session migration ensures beliefs touched before this + PR shipped become visible to the table. + +`record_touch` upserts: a new (belief, session) pair inserts with +`touch_count=1` and the supplied `event_kind` bit set; an existing +pair refreshes `last_fire_idx`, increments `touch_count` by one, and +OR-s the event_kind bit into the bitmask. + +## Locked decisions honored + +- **Determinism (#605, `c06f8d575fad71fb`).** `last_fire_idx` is a + monotonic integer per session, not a wall-clock timestamp. Same + query + same store + same fire_idx → same window contents across + replays. +- **Federation (#661, `d0c5ecdebb3f0f4d`).** The composite primary key + `(belief_id, session_id)` keeps foreign federated beliefs cold every + read by construction. No touch row crosses the federation boundary. +- **PHILOSOPHY narrow surface (#605).** The decay shape is one + integer comparison (`is_hot`); no embedding, no ML, no LLM. Pure + stdlib. +- **Audit-immutable `beliefs` table.** Sidecar table preserves the + per-turn audit invariant — touch-state updates don't mutate belief + rows. + +## Why no consumer in v1 + +The retrieval-consumer plan is a rerank-stage posterior multiplier +that boosts beliefs `is_hot(b, current_fire_idx, K)` returns True for. +DESIGN.md v1 (`experiments/hot-path/DESIGN.md` in the lab) gates the +consumer flip on two preconditions: + +1. **PR #782** (v3.1 `JUDGE_PROMPT_TEMPLATE` sharpening + hot_start + fixture widening) — landed 2026-05-14. +2. **R7c production-posterior-temperature ρ measurement** — needs + ≥50 rows accumulated in `injection_events` on a real per-project + DB. Structurally unrunnable until the substrate has been live long + enough to gather rows. + +Until R7c reports `r`, the v1 substrate writes are the only useful +output — they buy R7c the ability to run. The consumer flip is a +separate PR. + +## What R4 measures and what it does NOT + +The lab campaign's R4 series proved that adding `retrieve_hit` events +to the touch state changes the top-K rerank ordering on the corpus by +0% (Jaccard 1.000 across the standard cells). That's why bit 1 stays +unwritten: `retrieve_hit` adds zero observable surface to what +consumers actually see. The decision is robust to formula choice +(R4d/R4e), corpus dwell (R4b), event-mix frequency (R4c), and +multiplicative-vs-additive blend (R5). + +R4 does **not** measure rebuilder continuation fidelity. That is H3's +job (R3 — load-bearing, parked on R7c). v1 ships the substrate so +H3 can be measured at all. + +## Inspection + +``` +$ aelf doctor --hot-path +aelf doctor --hot-path: 2 session(s) with touch state. +session_id rows max_fire_idx + 42 81 + 12 17 +``` + +Empty (cold start, no UPS fires yet under this PR): + +``` +$ aelf doctor --hot-path +aelf doctor --hot-path: belief_touches is empty. +``` + +Read-only; always exits 0. The future consumer flip will add gate +semantics here. + +## Window default + +`DEFAULT_TOUCH_WINDOW_K = 50` lives as a module constant in +`src/aelfrice/hot_path.py`. The value is sourced from the lab +campaign's R2c canonical cell. Promotion to a +`meta:retrieval.hot_window_K` knob is an explicit follow-up if the +consumer flip lands and motivates tuning — DESIGN.md v1 locks the +constant per the "non-decisions" section: move it only by +re-measurement, not by config knob. + +## File map + +| Path | Role | +|---|---| +| `src/aelfrice/hot_path.py` | Pure helpers (`is_hot`), constants (`DEFAULT_TOUCH_WINDOW_K`, `TOUCH_EVENT_KIND_*`). | +| `src/aelfrice/store.py` | Schema DDL, `record_touch`, `read_touch_set_in_window`, `count_touches_for_session`, `list_touch_sessions`. | +| `src/aelfrice/hook.py` | `_record_touches` helper; UPS call site after `_ring_append_ids`. | +| `src/aelfrice/cli.py` | `aelf doctor --hot-path` surface. | +| `tests/test_hot_path_touch_state.py` | Schema + store + helper + hook integration tests (21). | + +## Related issues + +- [#748](https://github.com/robotrocketscience/aelfrice/issues/748) — R&D campaign tracker (closes once consumer ships and H3 reports). +- [#816](https://github.com/robotrocketscience/aelfrice/issues/816) — this storage substrate. +- [#779](https://github.com/robotrocketscience/aelfrice/issues/779) — `injection_events` sibling. +- [#744](https://github.com/robotrocketscience/aelfrice/issues/744) / [#740](https://github.com/robotrocketscience/aelfrice/issues/740) — JSON injection ring (predecessor; v1 reads-and-migrates). +- [#605](https://github.com/robotrocketscience/aelfrice/issues/605) — locked PHILOSOPHY (determinism, narrow surface). +- [#661](https://github.com/robotrocketscience/aelfrice/issues/661) — locked federation decision. From c0a94aa44c776698a2a2d13e0c4d73c70c39d11d Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 14 May 2026 15:51:52 -0700 Subject: [PATCH 6/7] fix(hook): drop non-idempotent JSON-ring replay in _record_touches (#821 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #821 review feedback (2026-05-14) flagged that _record_touches re-walks the entire #744 JSON ring on every UPS fire and calls record_touch for every entry. Because record_touch upserts with `touch_count = touch_count + 1` on conflict, the docstring's "one-shot INSERT-OR-IGNORE migration" claim was false in two ways: not one-shot, and not idempotent. Net effect on the example [b1],[b2],[b1]: b1.touch_count=5, b2.touch_count=3 (reviewers' repro), vs expected 2 / 1. Functional impact in v1: none — no consumer reads touch_count. Correctness impact for the H3 fidelity bench / any v2 rerank consumer: real, since touch_count drift would mis-rank. Fix: drop the ring replay entirely. The hook is now forward-only: record_touch is called only for the current turn's belief_ids. Pre- substrate ring entries (rows in the #744 JSON ring that predate the sidecar) are NOT backfilled. The ring is bounded (ring_max=200), no v1 consumer reads belief_touches, and the v2 rerank consumer will only care about touches recorded after v1 ships — so the value of backfill was already low. A sentinel-based "true one-shot" alternative would have required DB-level state for ~zero payoff. Test plan: - test_record_touches_writes_current_turn_only_no_ring_backfill — asserts ring-pre-pop is NOT migrated; current turn lands at touch_count=1. - test_record_touches_count_matches_actual_inject_count_across_fires — regression for the [b1],[b2],[b1] pattern; asserts touch_count={2,1} and last_fire_idx monotonicity (b1 > b2). Mimics the production caller sequence (append_ids then _record_touches fire_idx=next_fire-1). 22 tests in tests/test_hot_path_touch_state.py pass (was 21; migration test replaced by the two above). Docstring, docs/feature-hot-path.md, and the v3.1 CHANGELOG entry updated to describe forward-only semantics. --- CHANGELOG/v3.md | 2 + docs/feature-hot-path.md | 15 ++-- src/aelfrice/hook.py | 58 +++------------- tests/test_hot_path_touch_state.py | 107 +++++++++++++++++++++++------ 4 files changed, 106 insertions(+), 76 deletions(-) diff --git a/CHANGELOG/v3.md b/CHANGELOG/v3.md index d9530dd33..ead591c42 100644 --- a/CHANGELOG/v3.md +++ b/CHANGELOG/v3.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Hot-path touch state v1 storage substrate** ([#816](https://github.com/robotrocketscience/aelfrice/issues/816), closes [#748](https://github.com/robotrocketscience/aelfrice/issues/748)'s storage axis). Lands the per-(belief, session) touch sidecar specified by `experiments/hot-path/DESIGN.md` v1 (lab `6b40538`) after the R0..R7c campaign — sidecar table over wide-row columns (H1 PASS via R1/R1b), boolean "touched in last K fires" decay over exponential τ (H2 REFUTED via R2c/R2d), INJECTION-only event kind (H4 REFUTED via R4..R4e/R5; `retrieve_hit` adds zero observable surface at top-K Jaccard = 1.000). **No retrieval consumer wired in v1** — the rerank multiplier path is gated on the H3 fidelity test post-R7c (DESIGN.md v1 ship list item 7). New `belief_touches(belief_id, session_id, last_fire_idx, touch_count, event_kinds_bitmask)` table with composite PK + FK CASCADE to `beliefs` + `(session_id, last_fire_idx DESC)` index; `MemoryStore` APIs `record_touch` (INSERT ... ON CONFLICT DO UPDATE — last_fire_idx refresh, touch_count bump, event_kinds_bitmask OR-in), `read_touch_set_in_window` (boolean window predicate), `count_touches_for_session`, `list_touch_sessions`. New `src/aelfrice/hot_path.py` module with `is_hot()` pure predicate, `DEFAULT_TOUCH_WINDOW_K = 50` (R2c canonical cell), and `TOUCH_EVENT_KIND_*` bitmask constants (only `INJECTION` populated; `RETRIEVE_HIT` / `BFS_VISIT` / `USER_ACTION` bits reserved per DESIGN.md "Out of scope"). Hook integration writes touches alongside the existing #744 JSON injection ring at the UPS site, sharing the ring's monotonic `fire_idx` so both substrates track the same counter. Forward-only — the hook records only the current turn's injection set; ring entries that predate this table are not backfilled (an earlier revision did backfill via `record_touch`, but the replay was non-idempotent under `ON CONFLICT DO UPDATE` and was dropped before merge). Determinism (#605): `fire_idx` is a monotonic integer, never wall-clock; same query + same store + same fire_idx → same window contents. Federation (#661): composite PK `(belief_id, session_id)` keeps foreign federated beliefs cold every read by construction. New `aelf doctor --hot-path` read-only diagnostic surface lists every session_id with at least one touch row plus row count and max fire_idx. Fail-soft throughout the hook — touch state is opportunistic substrate; a write failure must not break the user-visible context-injection contract. 22 new tests in `tests/test_hot_path_touch_state.py` cover schema, store API round-trip, ON CONFLICT semantics, bitmask OR-in, window-boundary read, per-session isolation property, ordering of `list_touch_sessions`, determinism property, FK CASCADE on belief delete, hook forward-only writes (current-turn only; pre-substrate ring entries NOT backfilled), `touch_count`-matches-actual-inject-count regression across repeated UPS fires, and missing-DB fail-soft. Concept doc at `docs/feature-hot-path.md`. + - **ζ posterior-rerank surface — bounded sigmoid contribution** ([#817](https://github.com/robotrocketscience/aelfrice/issues/817), closes [#800](https://github.com/robotrocketscience/aelfrice/issues/800)). Ships behind a default-OFF flag mirroring γ ([#796](https://github.com/robotrocketscience/aelfrice/issues/796) / [PR #807](https://github.com/robotrocketscience/aelfrice/pull/807)). Replaces γ's unbounded `(1/T)·log(p)` with `α·(σ(β·(log(p)−log(0.5)))−0.5)·scale` — bounded posterior contribution in `(-α·scale/2, +α·scale/2)`, collapses to zero at the posterior-neutral `p=0.5`. Pinned defaults from the #800 R&D campaign verdict (R0–R4 at `experiments/zeta-posterior/`, R2 head-to-head: ζ dominates γ on `rank_biased_overlap` at similar `rank_changed_fraction`): `ZETA_ALPHA_DEFAULT=1.0`, `ZETA_BETA_DEFAULT=0.25`, `ZETA_SCALE_DEFAULT=14.5`. New API: `scoring.zeta_posterior_score(bm25_raw, alpha, beta, scale, posterior_mean)` with `ZETA_POSTERIOR_FLOOR` clamp on degenerate posteriors (corrupted store row never raises math domain error at retrieval time). Retrieval-side wiring: `USE_ZETA_POSTERIOR_RERANK_FLAG`, `ENV_USE_ZETA_POSTERIOR_RERANK` (`AELFRICE_USE_ZETA_POSTERIOR_RERANK`), `_env_use_zeta_posterior_rerank_override()`, `resolve_use_zeta_posterior_rerank(explicit=None, *, start=None)` with five-path precedence (env > kwarg > TOML `[retrieval] use_zeta_posterior_rerank` > False). `_l1_hits` gains `zeta_params: tuple[float, float, float] | None`; both byte-identical short-circuits extend to require `zeta_params is None` so ζ-on always exercises the rerank loop. `retrieve()` and `retrieve_with_tiers()` resolve both γ and ζ at entry and call `_assert_gamma_zeta_mutual_exclusion(gamma_on, zeta_on)` — both flags ON raises `ValueError` per the operator decision to defer composition (#817 § "Out of scope"). Heat-rerank still dominates both γ and ζ. ζ is **not** byte-identical to γ@T=1.0 nor `partial_bayesian_score(..., 1.0)` at any non-trivial inputs (issue #817 § "Note re: cold-start byte-identity"); it is rank-equivalent to log-BM25 alone on uniform-posterior=0.5 stores. 42 new tests across `tests/test_scoring_zeta.py` (posterior-neutral point, σ-bound, monotonicity, floor clamp, determinism, non-byte-identity to γ, uniform-posterior collapse to log-BM25), `tests/test_retrieve_zeta_flag.py` (five-path resolver precedence, flag-off byte-identical, flag-on deterministic, reorders by posterior, γ + ζ mutex helper, both-flags-ON raises in retrieve and retrieve_with_tiers), and `tests/test_zeta_vs_gamma_panel.py` (panel-metric reuse with `rank_biased_overlap` / `ordered_top_k_overlap`, γ-on/ζ-on rank-identical on uniform-posterior fixture). Flip-default deferred until the labeled relevance corpus exists (same gate as γ's G3). Env / TOML knobs for `(α, β, scale)` deferred per #817 § "Out of scope". ### Fixed diff --git a/docs/feature-hot-path.md b/docs/feature-hot-path.md index fe4c241b4..196fe2331 100644 --- a/docs/feature-hot-path.md +++ b/docs/feature-hot-path.md @@ -63,11 +63,14 @@ hook: fire_idx=next_fire_idx - 1, ...)`. The fire_idx the touches receive is the same one the ring just assigned, so the JSON ring and the sidecar share a counter. -3. Inside `_record_touches`, before the current writes land, the JSON - ring is read once and any existing entries for `session_id` are - migrated via `record_touch` at their original `fire_idx`. This - one-shot per-session migration ensures beliefs touched before this - PR shipped become visible to the table. +3. Inside `_record_touches`, the supplied `injected_ids` are each + recorded via `record_touch`. The hook is **forward-only** — it does + NOT read the JSON ring or backfill pre-substrate entries. An earlier + revision tried a one-shot per-session migration off the JSON ring, + but `record_touch` uses `ON CONFLICT DO UPDATE` so the replay was + non-idempotent: every UPS fire re-bumped `touch_count` on every ring + entry. The migration is gone; ring entries that predate this table + are simply not represented in `belief_touches`. `record_touch` upserts: a new (belief, session) pair inserts with `touch_count=1` and the supplied `event_kind` bit set; an existing @@ -167,6 +170,6 @@ re-measurement, not by config knob. - [#748](https://github.com/robotrocketscience/aelfrice/issues/748) — R&D campaign tracker (closes once consumer ships and H3 reports). - [#816](https://github.com/robotrocketscience/aelfrice/issues/816) — this storage substrate. - [#779](https://github.com/robotrocketscience/aelfrice/issues/779) — `injection_events` sibling. -- [#744](https://github.com/robotrocketscience/aelfrice/issues/744) / [#740](https://github.com/robotrocketscience/aelfrice/issues/740) — JSON injection ring (predecessor; v1 reads-and-migrates). +- [#744](https://github.com/robotrocketscience/aelfrice/issues/744) / [#740](https://github.com/robotrocketscience/aelfrice/issues/740) — JSON injection ring (predecessor; v1 shares its `fire_idx` counter but does NOT migrate ring entries — forward-only). - [#605](https://github.com/robotrocketscience/aelfrice/issues/605) — locked PHILOSOPHY (determinism, narrow surface). - [#661](https://github.com/robotrocketscience/aelfrice/issues/661) — locked federation decision. diff --git a/src/aelfrice/hook.py b/src/aelfrice/hook.py index ccbd1c15b..77b890905 100644 --- a/src/aelfrice/hook.py +++ b/src/aelfrice/hook.py @@ -1199,13 +1199,15 @@ def _record_touches( opportunistic substrate; a write failure must not break the hook's user-visible context-injection contract. - Also performs a one-shot per-session migration of the #744 JSON - injection ring into ``belief_touches`` — every ring entry for - ``session_id`` is INSERT-OR-IGNORE'd with its original - ``fire_idx`` and INJECTION bit set, so beliefs touched before the - sidecar shipped become visible to the table without a separate - migration pass. Idempotent: subsequent calls see the rows already - exist and the INSERT no-ops. + Forward-only: this writes the current turn's injection set only. + Pre-substrate ring entries (#744 JSON ring rows that predate this + sidecar) are NOT backfilled. A prior implementation tried to + "migrate" the ring on every UPS fire, but ``record_touch`` uses + ``ON CONFLICT DO UPDATE`` (touch_count = touch_count + 1), so the + replay was non-idempotent: every UPS fire re-bumped ``touch_count`` + on every ring entry. v1 has no consumer reading ``touch_count``, + so the bug was latent; v2 rerank correctness depends on the + counter being one-per-actual-touch, so the replay is gone. """ serr = stderr if stderr is not None else sys.stderr if not session_id or not belief_ids or fire_idx < 0: @@ -1214,52 +1216,12 @@ def _record_touches( from aelfrice.hot_path import ( # noqa: PLC0415 TOUCH_EVENT_KIND_INJECTION, ) - from aelfrice.session_ring import ( # noqa: PLC0415 - read_ring_state, - ) p = db_path() if str(p) == ":memory:": return store = MemoryStore(str(p)) try: - # #744 → belief_touches one-shot migration. Idempotent via - # the ON CONFLICT DO UPDATE in record_touch — re-running - # against an already-migrated row just refreshes the - # last_fire_idx and bumps touch_count, which is wrong on a - # *migration* call but right on a *current* call. To keep - # the migration distinct from the current write, we issue - # one record_touch per ring entry *before* the current - # touches, so the current writes are the last to land and - # the row reflects the most-recent fire_idx. - ring_state = read_ring_state(session_id) - if isinstance(ring_state, dict): - ring_list = ring_state.get("ring") - if isinstance(ring_list, list): - for entry in ring_list: - if not isinstance(entry, dict): - continue - rid = entry.get("id") - rfire = entry.get("fire_idx") - if ( - not isinstance(rid, str) or not rid - or not isinstance(rfire, int) or rfire < 0 - ): - continue - # Per-row tolerance — ring entries may point - # at beliefs that have since been deleted - # (FK to beliefs is enforced in prod). Skip - # those without poisoning the rest of the - # migration. - try: - store.record_touch( - belief_id=rid, - session_id=session_id, - fire_idx=rfire, - event_kind=TOUCH_EVENT_KIND_INJECTION, - ) - except Exception: - continue - # Current turn's injection set. + # Current turn's injection set — forward-only, no ring replay. for bid in belief_ids: if not bid: continue diff --git a/tests/test_hot_path_touch_state.py b/tests/test_hot_path_touch_state.py index 8bf152519..53dfc3d81 100644 --- a/tests/test_hot_path_touch_state.py +++ b/tests/test_hot_path_touch_state.py @@ -236,22 +236,41 @@ def test_fk_cascade_on_belief_delete(tmp_path: Path) -> None: # --------------------------- hook integration ------------------------- -def test_record_touches_writes_and_migrates_ring( +def _read_touch_rows(db: Path, session_id: str) -> dict[str, tuple[int, int]]: + s = MemoryStore(str(db)) + try: + return { + str(r["belief_id"]): (int(r["last_fire_idx"]), int(r["touch_count"])) + for r in s._conn.execute( + "SELECT belief_id, last_fire_idx, touch_count " + "FROM belief_touches WHERE session_id=?", + (session_id,), + ).fetchall() + } + finally: + s.close() + + +def test_record_touches_writes_current_turn_only_no_ring_backfill( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """`_record_touches` should write current touches AND migrate the - #744 JSON ring's prior entries via INSERT-or-update. + """Forward-only contract — pre-substrate JSON-ring entries are NOT + backfilled into ``belief_touches``. Only the current turn's + ``belief_ids`` land. + + Regression guard for the dropped one-shot migration: a prior + implementation re-read the ring on every UPS fire and replayed + every entry through ``record_touch``, which uses ON CONFLICT DO + UPDATE (touch_count = touch_count + 1) — so the replay was + non-idempotent. The replay is gone; this test pins the new + contract. """ from aelfrice import hook, session_ring db = tmp_path / "memory.db" monkeypatch.setattr(hook, "db_path", lambda: db) monkeypatch.setattr(session_ring, "db_path", lambda: db) - # Initialize the store with beliefs for every id the migration + - # current writes reference. Production FK enforcement is on and - # the migration code path tolerates stale ids by skipping them; - # this test exercises the happy path where the beliefs still - # exist. + s = MemoryStore(str(db)) for bid in ("old1", "old2", "new1", "new2"): s.insert_belief(_new_belief(bid)) @@ -277,24 +296,68 @@ def test_record_touches_writes_and_migrates_ring( fire_idx=4, ) - s = MemoryStore(str(db)) - rows = { - str(r["belief_id"]): (int(r["last_fire_idx"]), int(r["touch_count"])) - for r in s._conn.execute( - "SELECT belief_id, last_fire_idx, touch_count FROM belief_touches " - "WHERE session_id=?", - ("S",), - ).fetchall() - } - s.close() - # Migration backfilled old1/old2 at their ring fire_idx. - assert rows["old1"] == (2, 1), rows - assert rows["old2"] == (3, 1), rows - # Current touches landed at fire_idx=4. + rows = _read_touch_rows(db, "S") + # old1/old2 were in the ring but are NOT backfilled — forward-only. + assert "old1" not in rows, rows + assert "old2" not in rows, rows + # Current touches landed at fire_idx=4 with touch_count=1. assert rows["new1"] == (4, 1), rows assert rows["new2"] == (4, 1), rows +def test_record_touches_count_matches_actual_inject_count_across_fires( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """``touch_count`` increments once per UPS fire that actually injects + the belief — not once per ring entry. + + Regression for the pre-fix bug (reviews on PR #821, 2026-05-14): + every UPS fire walked the entire JSON ring and called + ``record_touch`` for every entry, so a belief sitting in the ring + got ``touch_count`` bumped on every fire regardless of whether it + was in the current injection set; current-turn beliefs got bumped + twice (once via ring replay, once via the explicit current loop). + + Mimics the production caller in + ``hook.py:_record_touches`` call site: ``append_ids`` first, then + ``_record_touches`` with ``fire_idx=_next_fire - 1``. + """ + from aelfrice import hook, session_ring + from aelfrice.session_ring import append_ids + + db = tmp_path / "memory.db" + monkeypatch.setattr(hook, "db_path", lambda: db) + monkeypatch.setattr(session_ring, "db_path", lambda: db) + + s = MemoryStore(str(db)) + for bid in ("b1", "b2"): + s.insert_belief(_new_belief(bid)) + s.close() + + # Three UPS fires, modeling the production sequence: + # append_ids(...) → returns next_fire_idx + # _record_touches(fire_idx=next_fire - 1) + # Injection pattern: [b1], [b2], [b1]. + for ids in (["b1"], ["b2"], ["b1"]): + next_fire = append_ids("S", ids, locked_ids=set()) + assert next_fire >= 1 + hook._record_touches( + session_id="S", + belief_ids=ids, + fire_idx=next_fire - 1, + ) + + rows = _read_touch_rows(db, "S") + # b1 was actually injected twice → touch_count=2; last_fire_idx is + # the slot of the third (final) fire. + assert rows["b1"][1] == 2, rows + # b2 was injected once → touch_count=1. + assert rows["b2"][1] == 1, rows + # last_fire_idx monotonicity: b1's last touch is at the latest fire, + # which is strictly greater than b2's. + assert rows["b1"][0] > rows["b2"][0], rows + + def test_record_touches_fail_soft_on_missing_db( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, ) -> None: From 2dd86dfedbbd206cf0489d32fc507a5502b5e72c Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Thu, 14 May 2026 16:02:08 -0700 Subject: [PATCH 7/7] test(hot-path): drop demotion_pressure kwarg from _new_belief (#820 rebase rot) --- tests/test_hot_path_touch_state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_hot_path_touch_state.py b/tests/test_hot_path_touch_state.py index 53dfc3d81..88b7e76fe 100644 --- a/tests/test_hot_path_touch_state.py +++ b/tests/test_hot_path_touch_state.py @@ -26,7 +26,6 @@ def _new_belief(bid: str) -> Belief: type=BELIEF_FACTUAL, lock_level=LOCK_NONE, locked_at=None, - demotion_pressure=0, created_at="2026-05-14T00:00:00Z", last_retrieved_at=None, )