feat(doc-linker): belief↔document anchor v2.0 (#435)

docs/feature-doc-linker.md

@@ -1,6 +1,6 @@
 # Feature spec: Document / semantic linker (#435)
 
-**Status:** spec, no implementation
+**Status:** implementation shipped (PR for #435), bench-gate pending lab-side corpus
 **Issue:** #435
 **Recovery-inventory line:** [`docs/ROADMAP.md`](ROADMAP.md) — *"Doc / semantic linker | v2.0.0"*
 **Substrate prereqs:** belief schema (foundation), `ingest_log` (#205, v1.6.0), `belief_corroborations` (#190, v1.5.0), `DerivationInput.source_path` (`src/aelfrice/derivation.py:64`)

src/aelfrice/cli.py

@@ -41,6 +41,7 @@
 from pathlib import Path
 from typing import Any, Final, Sequence
 
+from aelfrice.doc_linker import ANCHOR_MANUAL, link_belief_to_document
 from aelfrice.auditor import (
     CORPUS_MIN_DEFAULT as AUDIT_CORPUS_MIN_DEFAULT,
     SEVERITY_FAIL as AUDIT_SEVERITY_FAIL,
@@ -1099,6 +1100,21 @@ def _cmd_lock(args: argparse.Namespace, out: object) -> int:
             print(f"locked: {actual_id} (corroborated existing)", file=out)  # type: ignore[arg-type]
         else:
             print(f"locked: {actual_id}", file=out)  # type: ignore[arg-type]
+
+        # #435 doc-linker manual anchor. Idempotent on (belief_id,
+        # doc_uri); subsequent lock --doc with the same URI is a no-op
+        # write. The lock entry-point passes source_path=None to the
+        # worker (cli_remember has no canonical document), so the
+        # ingest-time hook does NOT fire — manual is the only path that
+        # writes anchors here.
+        doc_uri = getattr(args, "doc_uri", None)
+        if doc_uri:
+            link_belief_to_document(
+                store,
+                actual_id,
+                doc_uri,
+                anchor_type=ANCHOR_MANUAL,
+            )
     finally:
         store.close()
     return 0
@@ -3667,6 +3683,14 @@ def build_parser(*, show_advanced: bool = False) -> argparse.ArgumentParser:
             "if neither is set (#192)."
         ),
     )
+    p_lock.add_argument(
+        "--doc", dest="doc_uri", default=None,
+        help=(
+            "optional doc URI to anchor on this belief (#435). Stored as "
+            "anchor_type='manual' on belief_documents; opaque to the "
+            "linker beyond non-empty (file:// or https:// recommended)."
+        ),
+    )
     p_lock.set_defaults(func=_cmd_lock)
 
     p_locked = sub.add_parser("locked", help="list locked beliefs")

src/aelfrice/derivation_worker.py

@@ -45,6 +45,7 @@
 from typing import Final
 
 from aelfrice.derivation import DerivationInput, RouteOverrides, derive
+from aelfrice.doc_linker import ANCHOR_INGEST, file_uri_from_path
 from aelfrice.models import (
     CORROBORATION_SOURCE_CLI_REMEMBER,
     CORROBORATION_SOURCE_COMMIT_INGEST,
@@ -299,6 +300,19 @@
         store.insert_edge(edge)
         derived_edge_ids.append((edge.src, edge.dst, edge.type))
 
+    # #435 doc-linker. When source_path is materialised on the ingest
+    # row, write a belief↔document anchor. Idempotent on (belief_id,
+    # doc_uri) at the storage layer, so re-derive of the same row is a
+    # no-op. Skip when source_path is None (transcript-ingest, lock /
+    # remember without --doc, etc.) per spec § "Linker invocation point".
+    if inp.source_path:
+        store.link_belief_to_document(
+            belief_id=actual_id,
+            doc_uri=file_uri_from_path(inp.source_path),
+            anchor_type=ANCHOR_INGEST,
+            position_hint=None,
+        )
+
     store.update_ingest_derived_ids(
         log_id,
         derived_belief_ids=[actual_id],

src/aelfrice/doc_linker.py

@@ -0,0 +1,134 @@
+"""Document / semantic linker (#435).
+
+Connects a belief to the document anchor it describes — a file path with a
+line range, a URL with a section fragment, etc. — so retrieval can return the
+canonical reference alongside the bare belief snippet. Spec:
+``docs/feature-doc-linker.md``.
+
+The linker stores opaque URI strings in a sibling table ``belief_documents``
+keyed on ``(belief_id, doc_uri)``. Idempotent re-ingest: ``INSERT OR IGNORE``
+collapses repeats to the first row. Ingest-time writes happen inside the
+derivation worker when ``DerivationInput.source_path`` is set; manual writes
+happen via ``aelf lock --doc=URI``. Retrieval consumers opt in via the
+``with_doc_anchors=True`` kwarg on ``retrieve()`` / ``retrieve_v2()``.
+
+Out of scope at v2.0.0 (per spec):
+
+- ``anchor_type='derived'`` writers (retrieval-time inference). Enum value is
+  reserved; no writer in this module.
+- URI validation beyond non-empty.
+- ``source_path`` normalisation. The ingest writer stores whatever path the
+  caller passes; consumers normalise on the way out.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Final
+
+if TYPE_CHECKING:
+    from aelfrice.store import MemoryStore
+
+ANCHOR_INGEST: Final[str] = "ingest"
+ANCHOR_MANUAL: Final[str] = "manual"
+ANCHOR_DERIVED: Final[str] = "derived"
+
+ANCHOR_TYPES: Final[frozenset[str]] = frozenset(
+    {ANCHOR_INGEST, ANCHOR_MANUAL, ANCHOR_DERIVED},
+)
+
+
+@dataclass(frozen=True)
+class DocAnchor:
+    """One stored anchor row from ``belief_documents``.
+
+    ``doc_uri`` is opaque to the linker. Recommended encodings at v2.0.0:
+
+    - ``file:///abs/path/to/source.py#Lstart-Lend`` — local-source ingest.
+    - ``https://host/path#fragment`` — external doc / web ingest.
+
+    Other forms are accepted; the linker only rejects empty input.
+
+    ``position_hint`` is a free-form string (e.g. ``"L42-L60"``,
+    ``"#section"``); ``None`` when the URI itself encodes the position or
+    no hint is available.
+
+    ``created_at`` is a unix timestamp (seconds since epoch).
+    """
+
+    belief_id: str
+    doc_uri: str
+    anchor_type: str
+    position_hint: str | None
+    created_at: float
+
+
+def file_uri_from_path(
+    source_path: str,
+    *,
+    project_root: Path | None = None,
+    position_hint: str | None = None,
+) -> str:
+    """Build a ``file://`` URI from a local source path.
+
+    When ``project_root`` is set and ``source_path`` is inside it, the URI
+    encodes the path relative to the project root (avoids leaking the
+    operator's filesystem layout into the store). Otherwise the absolute
+    path is used.
+
+    ``position_hint`` is appended as a fragment when provided; the linker
+    stores the same value separately so consumers can read it without
+    parsing the URI.
+    """
+    p = Path(source_path)
+    rel: str
+    if project_root is not None:
+        try:
+            rel = str(p.resolve().relative_to(project_root.resolve()))
+        except ValueError:
+            rel = str(p)
+    else:
+        rel = str(p)
+    uri = f"file:{rel}" if not rel.startswith("/") else f"file://{rel}"
+    if position_hint:
+        uri = f"{uri}#{position_hint}"
+    return uri
+
+
+def link_belief_to_document(
+    store: "MemoryStore",
+    belief_id: str,
+    doc_uri: str,
+    *,
+    anchor_type: str = ANCHOR_INGEST,
+    position_hint: str | None = None,
+) -> DocAnchor:
+    """Persist a ``belief_id ↔ doc_uri`` anchor and return the row.
+
+    Idempotent on ``(belief_id, doc_uri)`` via ``INSERT OR IGNORE`` in the
+    storage layer: re-calling with the same pair is a no-op write but always
+    returns a ``DocAnchor`` reflecting the canonical (first-write) row.
+
+    Raises ``ValueError`` on empty ``doc_uri`` or unknown ``anchor_type``.
+    """
+    if not doc_uri:
+        raise ValueError("doc_uri must be non-empty")
+    if anchor_type not in ANCHOR_TYPES:
+        raise ValueError(
+            f"Unknown anchor_type {anchor_type!r}. "
+            f"Must be one of {sorted(ANCHOR_TYPES)}"
+        )
+    return store.link_belief_to_document(
+        belief_id=belief_id,
+        doc_uri=doc_uri,
+        anchor_type=anchor_type,
+        position_hint=position_hint,
+    )
+
+
+def get_doc_anchors(
+    store: "MemoryStore",
+    belief_id: str,
+) -> list[DocAnchor]:
+    """Return every anchor for a belief, ordered by ``created_at`` ASC."""
+    return store.get_doc_anchors(belief_id)

src/aelfrice/retrieval.py

@@ -74,6 +74,7 @@
     pack_with_clusters,
 )
 from aelfrice.compression import CompressedBelief, compress_for_retrieval
+from aelfrice.doc_linker import DocAnchor
 from aelfrice.vocab_bridge import VocabBridge, VocabBridgeCache
 from aelfrice.entity_extractor import extract_entities
 from aelfrice.graph_spectral import (
@@ -240,6 +241,12 @@ class RetrievalResult:
     last call. The benchmark adapter consumes it for the L0/L1/L2.5
     counts surface; default `[]` for backwards-compat with adapters
     that only inspect `beliefs`.
+
+    `doc_anchors` (#435) is a parallel list to `beliefs`: same length,
+    same order. `doc_anchors[i]` lists every `belief_documents` row for
+    `beliefs[i]`. Empty when the caller did not opt in via
+    `with_doc_anchors=True`; also empty for beliefs that have no
+    anchors.
     """
 
     beliefs: list[Belief]
@@ -248,6 +255,7 @@ class RetrievalResult:
     entity_hits: list[str] = field(default_factory=lambda: [])
     locked_ids: list[str] = field(default_factory=lambda: [])
     l1_ids: list[str] = field(default_factory=lambda: [])
+    doc_anchors: list[list[DocAnchor]] = field(default_factory=lambda: [])
     # v2.1 #434 type-aware compression. Populated when
     # use_type_aware_compression resolves True. Same length and order as
     # `beliefs` (parallel field — consumers that want compressed render
@@ -1636,6 +1644,7 @@ def retrieve_v2(
     use_vocab_bridge: bool | None = None,
     vocab_bridge_cache: VocabBridgeCache | None = None,
     use_intentional_clustering: bool | None = None,
+    with_doc_anchors: bool = False,
 ) -> RetrievalResult:
     """Lab-compatible retrieval wrapper for academic-suite adapters.
 
@@ -1740,13 +1749,24 @@ def retrieve_v2(
             for b in beliefs
         ]
 
+    # #435 doc-linker post-rank, pre-pack projection. Default OFF keeps
+    # the adapter wire bytes-identical for callers that don't opt in.
+    # When ON, one batched `belief_id IN (...)` SELECT joins anchors
+    # onto the result. Anchors are metadata for the consumer; they do
+    # NOT count against the token budget pack.
+    doc_anchors_list: list[list[DocAnchor]] = []
+    if with_doc_anchors and beliefs:
+        anchors_by_id = store.get_doc_anchors_batch([b.id for b in beliefs])
+        doc_anchors_list = [anchors_by_id.get(b.id, []) for b in beliefs]
+
     return RetrievalResult(
         beliefs=beliefs,
         entity_hits=l25_ids_list,
         locked_ids=locked_ids_list,
         l1_ids=l1_ids_list,
         bfs_chains=bfs_chains,
         compressed_beliefs=compressed,
+        doc_anchors=doc_anchors_list,
     )