runtime: add RLS-scoped pgvector CandidateIndex

adapters/http/app.py

@@ -93,6 +93,7 @@
     CitationAuthorityUnavailable,
 )
 from engine.runtime.construction import required_kernel_dependencies
+from engine.runtime.content_io import CandidateIndexUnavailable
 from engine.runtime.context_run import ContextRunPersistenceUnavailable
 from engine.runtime.contracts import (
     Acquire,
@@ -722,6 +723,8 @@ def resolve_context(
             raise TrustedAuthorityUnavailable from None
         except ActiveReleaseUnavailable:
             raise TrustedAuthorityUnavailable from None
+        except CandidateIndexUnavailable:
+            raise TrustedAuthorityUnavailable from None
         except InvalidTrustedScopeSnapshot:
             raise TrustedAuthorityUnavailable from None
 

adapters/pgvector.py

@@ -0,0 +1,86 @@
+"""PostgreSQL ANN candidate discovery using one retained Runtime transaction."""
+
+from __future__ import annotations
+
+from engine.runtime.content_io import CandidateIndexUnavailable
+from engine.runtime.contracts import Acquire
+from engine.runtime.evidence import CandidateRef
+from engine.runtime.materialized import (
+    MaterializedProjectionSession,
+    _discover_materialized_vector,
+)
+from engine.supply import (
+    CONTEXT_FRAGMENT_EMBEDDING_DIMENSION,
+    EmbeddingProfile,
+    EmbeddingProvider,
+    EmbeddingProviderUnavailable,
+    validate_embedding_batch,
+)
+
+DEFAULT_VECTOR_CANDIDATE_LIMIT = 16
+MAX_VECTOR_CANDIDATE_LIMIT = 64
+
+
+class VectorCandidateIndexUnavailable(CandidateIndexUnavailable):
+    """Content-free failure of query embedding or ANN candidate discovery."""
+
+
+class PostgreSQLVectorCandidateIndex:
+    """Return bounded content-free pgvector candidates for one Acquire."""
+
+    __slots__ = ("_embedding_profile", "_embedding_provider", "_limit")
+
+    def __init__(
+        self,
+        embedding_provider: EmbeddingProvider,
+        *,
+        limit: int = DEFAULT_VECTOR_CANDIDATE_LIMIT,
+    ) -> None:
+        try:
+            embedding_profile = embedding_provider.profile
+        except (AttributeError, TypeError, ValueError):
+            raise TypeError(
+                "Vector candidate index requires an embedding provider"
+            ) from None
+        if type(embedding_profile) is not EmbeddingProfile:
+            raise TypeError("Vector candidate index requires an embedding provider")
+        if embedding_profile.dimension != CONTEXT_FRAGMENT_EMBEDDING_DIMENSION:
+            raise ValueError("Vector candidate dimension does not match storage")
+        if type(limit) is not int or not 1 <= limit <= MAX_VECTOR_CANDIDATE_LIMIT:
+            raise ValueError("Vector candidate limit is not available")
+        self._embedding_provider = embedding_provider
+        self._embedding_profile = embedding_profile
+        self._limit = limit
+
+    def discover(
+        self,
+        request: Acquire,
+        projection_session: MaterializedProjectionSession,
+    ) -> tuple[CandidateRef, ...]:
+        if type(request) is not Acquire:
+            raise TypeError("Vector candidate discovery requires Acquire")
+        try:
+            query_embedding = validate_embedding_batch(
+                (request.need.query,),
+                self._embedding_provider.embed((request.need.query,)),
+                self._embedding_profile,
+            )[0]
+            return _discover_materialized_vector(
+                projection_session,
+                query_embedding,
+                self._limit,
+                source_refs=(
+                    request.narrowing.source_refs
+                    if request.narrowing is not None
+                    else None
+                ),
+                resource_refs=(
+                    request.narrowing.resource_refs
+                    if request.narrowing is not None
+                    else None
+                ),
+            )
+        except EmbeddingProviderUnavailable:
+            raise VectorCandidateIndexUnavailable(
+                "Vector candidate discovery is unavailable"
+            ) from None

docs/decisions/0067-discover-vector-candidates-in-the-current-runtime-transaction.md

@@ -0,0 +1,140 @@
+---
+name: adr-0067-discover-vector-candidates-in-the-current-runtime-transaction
+version: "1.0.0"
+description: >
+  Embed Acquire queries with the publication-compatible profile and discover
+  bounded content-free pgvector candidates in the retained UserActor transaction.
+---
+
+# 0067. Discover vector candidates in the current Runtime transaction
+
+- Status: accepted
+- Date: 2026-07-27
+- Refines: ADR-0009, ADR-0025, ADR-0066
+
+## Context
+
+ADR-0066 published fixed-dimension Fragment vectors and reserved query-time
+embedding, compatibility, and filtered-ANN evidence for the first vector
+CandidateIndex decision. Runtime already owns one current UserActor transaction
+whose transaction-local settings drive FORCE RLS and whose nominal projection
+session remains live through the sealed AuthorizationKernel. Opening a second
+database session for retrieval would detach discovery from that trusted context.
+
+HNSW applies post-scan filters, including RLS and active-resource predicates.
+With selective Organization, Membership, source, or policy filters, a bounded
+one-pass approximate scan can underfill even when visible rows exist. Candidate
+ordering cannot be authorization input, and ADR-0025 deliberately makes current
+deterministic assembly independent of untrusted candidate order.
+
+## Decision
+
+`PostgreSQLVectorCandidateIndex` uses the explicit `EmbeddingProvider` seam and
+requires exact dimensional compatibility with the stored 384-dimensional
+vectors. Each
+`Acquire` embeds only its exact query, validates the provider response through
+the same float32 contract used by publication, and collapses provider failure
+to one content-free availability category. HTTP maps that category to the
+existing generic service-unavailable response.
+
+The index calls a narrow vector-discovery operation on the current
+`MaterializedProjectionSession`. Its PostgreSQL port executes on the retained
+UserActor connection; it opens no connection and creates no transaction. The
+query selects only Organization/source/resource/revision/fragment lineage,
+orders by cosine distance with `<=>`, and applies an exact 1–64 limit. Body,
+path, title, provider metadata, and distance score never leave persistence.
+Every result remains an untrusted `CandidateRef` and still passes the unchanged
+AuthorizationKernel before projection.
+
+Caller-supplied source and resource narrowing is passed through only as an ANN
+pre-filter before the candidate limit. It can remove candidates but cannot
+grant access; the Kernel independently recomputes the authoritative
+`EffectiveScope` intersection and exactly reauthorizes every surviving
+`CandidateRef`.
+
+Before the ANN query, the port sets pgvector `hnsw.iterative_scan` to strict
+order and a bounded maximum scan-tuple ceiling with transaction-local settings.
+This allows pgvector to continue scanning after RLS and live-resource filters
+remove approximate neighbors while retaining bounded work and exact distance
+order at the index seam. The pinned harness uses the pgvector version recorded
+by the served database topology.
+
+Candidate rank is intentionally not preserved through the current Kernel.
+ADR-0025 makes assembly independent of candidate order so index rank cannot
+affect access or the deterministic Package. This slice therefore provides
+bounded vector recall, not relevance-ranked budget selection. Changing that
+behavior requires a separately designed authorized ranking stage and is not
+smuggled into this product-lane index change.
+
+The served composition does not activate the index here. Its default remains
+reject-all and reports Runtime delivery as not active.
+
+The current `EmbeddingProfile` records dimension only. It does not prove that
+publication and query vectors share one immutable model and input profile, so
+this slice does not claim semantic embedding-profile compatibility. A served
+external provider is prohibited until storage or immutable Release lineage
+binds that identity and Runtime rejects mismatches. The bounded dogfood
+activation may use only the deterministic network-free twin over a freshly
+reimported corpus produced by that same twin.
+
+An external query-embedding provider is also not eligible for served
+composition while Runtime cannot meter and enforce that call against the
+effective PackageBudget. The follow-up activation must either add that usage
+propagation or explicitly compose only the deterministic network-free twin,
+classified as zero external provider calls and zero provider cost. External
+provider configuration must fail closed until the metered path exists.
+
+The follow-up served activation must also pass the Kernel-computed
+`EffectiveScope` to discovery as a removal-only pre-filter before the ANN
+limit. Caller narrowing alone is insufficient when trusted policy operands are
+stricter than the RLS-visible set. Every returned candidate must still pass the
+Kernel's exact authorization and projection; the pre-filter never grants
+authority. Until that propagation exists, the served vector carrier remains
+`NOT_ACTIVE`.
+
+## Rationale
+
+Keeping discovery on the retained UserActor transaction preserves the exact
+FORCE-RLS and Membership lifetime already established for resolve; a second
+connection would require reconstructing trusted context and create a drift-prone
+authorization boundary. Strict-order iterative HNSW scanning bounds work while
+continuing past rows removed by tenant, lifecycle, ACL, and narrowing filters.
+Keeping rank outside the current Kernel preserves ADR-0025's deterministic,
+rank-independent assembly until a separately measured authorized-ranking stage
+can justify changing the sealed path.
+
+## Consequences
+
+- Query embedding and publication share one dimension contract; semantic
+  model/input-profile identity is not yet persisted or release-bound.
+- FORCE RLS narrows discovery as defense in depth; it never grants authority.
+- Filtered HNSW scans can continue within a fixed work ceiling instead of
+  silently stopping after the first filtered approximate batch.
+- Provider diagnostics, distances, bodies, paths, titles, and denied lineage do
+  not enter the candidate output or HTTP error response.
+- Current PackageBudget selection remains deterministic and rank-independent;
+  semantic ranking quality is not claimed by this slice.
+- Served external query embedding remains `NOT_ACTIVE` until its provider call,
+  cost, and elapsed usage can be enforced and recorded by Runtime.
+- Served vector discovery remains `NOT_ACTIVE` until the Kernel-computed
+  EffectiveScope can restrict candidates before the ANN limit without becoming
+  an authorization decision at the index.
+
+## Revisit trigger
+
+Revisit after the real golden set and dogfood corpus exist, or before activating
+rank-sensitive budget selection. The exit evidence for tuning or replacing the
+current strategy includes exact-search recall delta, underfilled-result rate,
+`EXPLAIN ANALYZE`, corpus size, filter selectivity, hardware, iterative-scan or
+oversampling settings, latency, and cost. Hybrid fusion and content-bearing
+reranking remain separate evidence-triggered decisions.
+
+Revisit before any served composition admits an external query-embedding
+provider without exact PackageBudget usage propagation.
+
+Revisit before any served external provider or mixed-origin corpus can be used
+without immutable publication/query embedding-profile identity and mismatch
+rejection.
+
+Revisit before any served composition can limit ANN candidates without first
+applying the current Kernel-computed EffectiveScope as a removal-only filter.

docs/decisions/README.md

@@ -52,6 +52,7 @@ kernel, capability separation, and publication visibility model.
 | Bounded File reclaim | [0060 — Reclaim expired File imports with bounded retries](0060-reclaim-expired-file-imports-with-bounded-retries.md) | The same function-only scheduler prefers database-timed expired work, revalidates exact current authority, and advances at most three higher WorkerLease generations through durable-boundary recovery | Caller-selected retry routing/timing/generation, terminal-failure retry, dead-letter or requeue authority, delete execution, Runtime authority, or a new process/queue |
 | Recursive File discovery | [0065 — Recurse File discovery with anchored descriptors](0065-recurse-file-discovery-with-anchored-descriptors.md) | Canonical nested Markdown paths are discovered through stable descriptor-relative no-follow traversal under one server-owned 1–64 MiB byte ceiling | Path-string traversal, symlink following, truncated baselines, unbounded or caller-owned read limits, non-Markdown discovery, or new polling/resync authority |
 | Fragment embedding publication | [0066 — Embed Fragments before publication](0066-embed-fragments-before-publication.md) | New immutable Fragments receive one validated 384-dimensional vector before activation through an explicit external provider or network-free CI twin | Implicit twin fallback, query-time retrieval, backfill authority, missing-vector activation, or vector-based authorization |
+| Vector candidate discovery | [0067 — Discover vector candidates in the current Runtime transaction](0067-discover-vector-candidates-in-the-current-runtime-transaction.md) | Query embedding uses the stored profile and bounded filtered HNSW discovery stays inside the retained UserActor transaction, returning content-free CandidateRefs only | Second retrieval connections, vector rank as authority, content/score-bearing candidates, implicit provider fallback, or served carrier activation |
 | Private delivery ingress | [0045 — Redeem private delivery evidence at ingress](0045-redeem-private-delivery-evidence-at-ingress.md) | One digest-only service/request/asker/audience/epoch-bound DeliveryEvidenceRef constructs private TrustedDeliveryContext inside the current UserActor transaction before content work | Raw trusted delivery facts on the wire, bearer persistence, application-role minting/table reads, alternate Runtime paths, or claiming later M2 carriers |
 | Exact Package egress | [0046 — Bind egress to one exact Package hop](0046-bind-egress-to-one-exact-package-hop.md) | One digest-only grant binds one exact audience-bound Package to one model or channel preflight hop and redeems atomically | Treating Package construction as disclosure authority, arbitrary content at egress, cross-hop reuse, or bypassing final policy |
 | Public OpenAPI v0 | [0047 — Freeze OpenAPI v0 through one Runtime path](0047-freeze-openapi-v0-through-one-runtime-path.md) | One public `/v0/resolve` schema and a hidden provisional v1 bridge share the same sealed Runtime; Package release lineage is read-only from the Learning-published active manifest | Two authorization compositions, caller-authored release facts, Runtime publication/fallback, or in-place mutation of historical snapshots |
@@ -166,3 +167,4 @@ touched:
 - [0064 — Split process ceremony along the kernel-seam boundary](0064-split-process-ceremony-along-the-kernel-seam-boundary.md)
 - [0065 — Recursive File discovery with anchored descriptors](0065-recurse-file-discovery-with-anchored-descriptors.md)
 - [0066 — Embed Fragments before publication](0066-embed-fragments-before-publication.md)
+- [0067 — Discover vector candidates in the current Runtime transaction](0067-discover-vector-candidates-in-the-current-runtime-transaction.md)

engine/persistence/membership_context.py

@@ -164,6 +164,35 @@ def _canonical_candidate_revision(value: str) -> UUID | None:
     return parsed
 
 
+_VECTOR_ITERATIVE_SCAN = "strict_order"
+_VECTOR_MAX_SCAN_TUPLES = 20_000
+_VECTOR_CANDIDATE_SQL = """
+    SELECT
+        resource.organization_id,
+        resource.source_ref,
+        fragment.resource_ref,
+        fragment.revision_id,
+        fragment.fragment_ref
+    FROM context_fragment AS fragment
+    JOIN context_resource AS resource
+      ON resource.organization_id = fragment.organization_id
+     AND resource.resource_ref = fragment.resource_ref
+     AND resource.active_revision_id = fragment.revision_id
+     AND resource.tombstoned IS FALSE
+    WHERE fragment.embedding IS NOT NULL
+      AND (
+        CAST(:source_refs AS text[]) IS NULL
+        OR resource.source_ref = ANY(CAST(:source_refs AS text[]))
+      )
+      AND (
+        CAST(:resource_refs AS text[]) IS NULL
+        OR fragment.resource_ref = ANY(CAST(:resource_refs AS text[]))
+      )
+    ORDER BY fragment.embedding <=> CAST(:query_embedding AS vector)
+    LIMIT :limit
+"""
+
+
 class _PostgreSQLMaterializedProjectionPort:
     """Two-stage Fragment reads on the owning current-UserActor transaction."""
 
@@ -184,6 +213,51 @@ def source_is_active(self, source_ref: UUID) -> bool:
         ).scalar_one()
         return observed is True
 
+    def discover_vector(
+        self,
+        query_embedding: tuple[float, ...],
+        limit: int,
+        source_refs: tuple[str, ...] | None,
+        resource_refs: tuple[str, ...] | None,
+    ) -> tuple[CandidateRef, ...]:
+        self._connection.execute(
+            text(
+                "SELECT set_config('hnsw.iterative_scan', :iterative_scan, true), "
+                "set_config('hnsw.max_scan_tuples', :max_scan_tuples, true)"
+            ),
+            {
+                "iterative_scan": _VECTOR_ITERATIVE_SCAN,
+                "max_scan_tuples": str(_VECTOR_MAX_SCAN_TUPLES),
+            },
+        )
+        rows = self._connection.execute(
+            text(_VECTOR_CANDIDATE_SQL),
+            {
+                "query_embedding": "["
+                + ",".join(repr(value) for value in query_embedding)
+                + "]",
+                "limit": limit,
+                "source_refs": list(source_refs) if source_refs is not None else None,
+                "resource_refs": (
+                    list(resource_refs) if resource_refs is not None else None
+                ),
+            },
+        )
+        candidates: list[CandidateRef] = []
+        for row in rows:
+            if type(row.revision_id) is not UUID:
+                raise TypeError("vector discovery returned invalid revision lineage")
+            candidates.append(
+                CandidateRef(
+                    organization_id=row.organization_id,
+                    source_ref=row.source_ref,
+                    resource_ref=row.resource_ref,
+                    revision_ref=str(row.revision_id),
+                    fragment_ref=row.fragment_ref,
+                )
+            )
+        return tuple(candidates)
+
     def discover_exact_phrase(
         self,
         phrase_digest: str,

engine/runtime/content_io.py

@@ -8,6 +8,16 @@
 from engine.runtime.evidence import CandidateRef
 from engine.runtime.materialized import MaterializedProjectionSession
 
+__all__ = [
+    "CandidateIndex",
+    "CandidateIndexUnavailable",
+    "ContextProvider",
+    "RuntimeContentIo",
+    "SourceContentReader",
+    "exact_phrase_digest",
+    "prohibited_empty_path_content_io",
+]
+
 _EXACT_PHRASE_DIGEST_DOMAIN = b"context-engine.exact-phrase.v1\x00"
 
 
@@ -29,6 +39,10 @@ def discover(
     ) -> tuple[CandidateRef, ...]: ...
 
 
+class CandidateIndexUnavailable(RuntimeError):
+    """Content-free transient failure of one configured candidate index."""
+
+
 class ContextProvider(Protocol):
     """Future provider projection seam."""