From e4aec83bcafce38abaa70669931984c9613e1ddb Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 23 Jul 2026 00:35:50 +0800 Subject: [PATCH 1/3] feat: publish one file through authorized runtime --- adapters/exact_phrase.py | 27 + adapters/file_source.py | 151 ++ ...h-first-file-through-exact-worker-lease.md | 94 + engine/control/__init__.py | 14 + engine/control/authority.py | 1 + engine/control/contracts.py | 85 +- engine/control/file_imports.py | 138 ++ engine/control/module.py | 53 +- engine/persistence/__init__.py | 10 + engine/persistence/control_sources.py | 130 +- engine/persistence/file_imports.py | 343 ++++ engine/persistence/membership_context.py | 88 + .../persistence/schema_security_manifest.yaml | 373 +++- engine/persistence/worker_jobs.py | 62 + engine/runtime/construction.py | 2 +- engine/runtime/content_io.py | 24 +- engine/runtime/materialized.py | 65 +- engine/supply/__init__.py | 14 + engine/supply/jobs.py | 72 +- eval/catalogs/m0-security-evidence.yaml | 11 +- .../20260722_0011_file_import_tracer.py | 998 +++++++++++ scripts/security_gate/rls.py | 5 + .../test_authorized_field_schema.py | 5 + tests/integration/test_file_import_tracer.py | 1554 +++++++++++++++++ .../integration/test_m0_security_gate_rls.py | 20 +- ...membership_field_projection_integration.py | 5 +- tests/integration/test_membership_schema.py | 1 + tests/integration/test_migrations.py | 196 +-- ...runtime_authorized_evidence_integration.py | 5 +- .../test_runtime_empty_package_integration.py | 3 +- ...est_runtime_non_enumeration_integration.py | 5 +- tests/unit/test_context_control.py | 64 +- tests/unit/test_effective_scope_runtime.py | 4 +- tests/unit/test_file_import.py | 156 ++ tests/unit/test_http_trust_boundary.py | 4 +- tests/unit/test_m0_rls_inventory.py | 13 +- tests/unit/test_materialized_projection.py | 7 + .../unit/test_membership_field_projection.py | 7 + .../unit/test_runtime_authorized_evidence.py | 13 +- tests/unit/test_runtime_empty_package.py | 6 +- .../test_runtime_unavailable_capabilities.py | 8 +- tests/unit/test_schema_security_manifest.py | 73 +- tests/unit/test_worker_lease.py | 47 + 43 files changed, 4706 insertions(+), 250 deletions(-) create mode 100644 adapters/exact_phrase.py create mode 100644 adapters/file_source.py create mode 100644 docs/decisions/0037-publish-first-file-through-exact-worker-lease.md create mode 100644 engine/control/file_imports.py create mode 100644 engine/persistence/file_imports.py create mode 100644 migrations/versions/20260722_0011_file_import_tracer.py create mode 100644 tests/integration/test_file_import_tracer.py create mode 100644 tests/unit/test_file_import.py diff --git a/adapters/exact_phrase.py b/adapters/exact_phrase.py new file mode 100644 index 00000000..6dca4088 --- /dev/null +++ b/adapters/exact_phrase.py @@ -0,0 +1,27 @@ +"""PostgreSQL-backed content-free exact-phrase candidate discovery.""" + +from __future__ import annotations + +from engine.runtime.content_io import exact_phrase_digest +from engine.runtime.contracts import Acquire +from engine.runtime.evidence import CandidateRef +from engine.runtime.materialized import ( + MaterializedProjectionSession, + _discover_materialized_exact_phrase, +) + + +class PostgreSQLExactPhraseCandidateIndex: + """Discover content-free candidates within one trusted Organization.""" + + def discover( + self, + request: Acquire, + projection_session: MaterializedProjectionSession, + ) -> tuple[CandidateRef, ...]: + if type(request) is not Acquire: + raise TypeError("exact phrase discovery requires Acquire") + return _discover_materialized_exact_phrase( + projection_session, + exact_phrase_digest(request.need.query), + ) diff --git a/adapters/file_source.py b/adapters/file_source.py new file mode 100644 index 00000000..1357e0a2 --- /dev/null +++ b/adapters/file_source.py @@ -0,0 +1,151 @@ +"""Explicit host bindings for registered logical File roots.""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType + +from engine.control import FileImportPath, FileRootRef + +MAX_CONFIGURED_FILE_BYTES = 64 * 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class FileReadLimits: + """Server-owned hard ceiling for one acquired File payload.""" + + max_file_bytes: int + + def __post_init__(self) -> None: + if ( + type(self.max_file_bytes) is not int + or not 1 <= self.max_file_bytes <= MAX_CONFIGURED_FILE_BYTES + ): + raise ValueError("File byte ceiling must be a bounded positive integer") + + +@dataclass(frozen=True, slots=True) +class _AnchoredRoot: + display_path: Path + descriptor: int + + +def _open_anchored_directory(path: Path) -> tuple[Path, int]: + """Open every absolute path component without following any symlink.""" + + absolute = Path(os.path.abspath(path)) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + no_follow = getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(absolute.anchor, flags | no_follow) + try: + for component in absolute.parts[1:]: + next_descriptor = os.open( + component, + flags | no_follow, + dir_fd=descriptor, + ) + os.close(descriptor) + descriptor = next_descriptor + if not stat.S_ISDIR(os.fstat(descriptor).st_mode): + raise NotADirectoryError + return absolute, descriptor + except Exception: + os.close(descriptor) + raise + + +class FileRootRegistry: + """Resolve a logical root and closed filename without discovering files.""" + + __slots__ = ("_limits", "_roots") + + def __init__( + self, + roots: Mapping[FileRootRef, Path], + *, + limits: FileReadLimits, + ) -> None: + if not isinstance(roots, Mapping) or not roots: + raise ValueError("File root registry requires explicit bindings") + if type(limits) is not FileReadLimits: + raise TypeError("File root registry requires FileReadLimits") + copied: dict[FileRootRef, _AnchoredRoot] = {} + try: + for root_ref, root_path in roots.items(): + if type(root_ref) is not FileRootRef or not isinstance( + root_path, Path + ): + raise TypeError( + "File root bindings require FileRootRef and Path" + ) + try: + display_path, descriptor = _open_anchored_directory(root_path) + except OSError: + raise ValueError( + "File root must be an existing non-symlink directory" + ) from None + copied[root_ref] = _AnchoredRoot(display_path, descriptor) + except Exception: + for root in copied.values(): + os.close(root.descriptor) + raise + self._roots = MappingProxyType(copied) + self._limits = limits + + def resolve(self, root_ref: FileRootRef, path: FileImportPath) -> Path: + if type(root_ref) is not FileRootRef or type(path) is not FileImportPath: + raise TypeError("File root resolution requires exact contracts") + anchored = self._roots.get(root_ref) + if anchored is None: + raise LookupError("File root is not configured") + target = anchored.display_path / path.value + if target.parent != anchored.display_path: + raise LookupError("File target is outside the configured root") + return target + + def read(self, root_ref: FileRootRef, path: FileImportPath) -> bytes: + """Read one regular file without following a final symlink.""" + + self.resolve(root_ref, path) + anchored = self._roots[root_ref] + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path.value, flags, dir_fd=anchored.descriptor) + except OSError: + raise LookupError( + "File target is not a regular configured-root file" + ) from None + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_size > self._limits.max_file_bytes + ): + raise LookupError( + "File target is not a regular configured-root file" + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + payload = stream.read(self._limits.max_file_bytes + 1) + if len(payload) > self._limits.max_file_bytes: + raise LookupError("File target exceeds the configured byte ceiling") + return payload + finally: + os.close(descriptor) + + def close(self) -> None: + """Release the server-owned directory capabilities.""" + + roots = self._roots + self._roots = MappingProxyType({}) + for root in roots.values(): + os.close(root.descriptor) + + def __enter__(self) -> FileRootRegistry: + return self + + def __exit__(self, *args: object) -> None: + self.close() diff --git a/docs/decisions/0037-publish-first-file-through-exact-worker-lease.md b/docs/decisions/0037-publish-first-file-through-exact-worker-lease.md new file mode 100644 index 00000000..95856645 --- /dev/null +++ b/docs/decisions/0037-publish-first-file-through-exact-worker-lease.md @@ -0,0 +1,94 @@ +--- +name: adr-0037-publish-first-file-through-exact-worker-lease +version: "1.0.0" +description: > + Activate one trusted Markdown File import through an exact durable WorkerLease, + atomic immutable publication, and a content-free exact-phrase CandidateIndex. +--- + +# 0037. Publish the first File through an exact WorkerLease + +- Status: accepted +- Date: 2026-07-22 +- Refines: ADR-0018, ADR-0029, ADR-0035, ADR-0036 + +## Context + +Issue #23 is the first end-to-end Supply-to-Runtime tracer. It must turn one +registered File Source and one trusted Markdown filename into an authorized +`ContextPackage` without promoting the Issue #17 no-op carrier into an implicit +content authority. The worker needs filesystem access, while Control and Runtime +must remain unable to accept a host path or caller-authored tenant facts. Initial +publication also needs a Resource row before its immutable Revision can satisfy +the deferred active pointer. + +## Decision + +File Source registration remains the unavailable version-1 declaration from +Issue #21. A trusted `prepare_file_import` Control call validates one basename +ending in `.md`, revalidates the current audience Membership and registered File +import ServicePrincipal, creates an immutable version-2 SourceVersion declaring +only `fileSourceAccess` and `ingestionJobs` available, atomically switches the +Source active pointer, and creates one immutable acquisition plus one durable +job. Control stores only a logical root reference and relative filename; it never +opens the filesystem. + +The version-2 WorkerLease preserves the version-1 no-op token bytes and binds +Organization, exact durable job, Source, receiver, workload +`supply.file-import`, audience `context-engine-worker`, operation `file.import`, +database-owned issue/expiry times, key version, and nonce. Redemption and +publication each revalidate those exact current row values, database time, and +the enabled ServicePrincipal. The content-free terminal failure transition has +the same checks, so expiry or receiver revocation cannot retain even a failed +state mutation. The worker receives no user impersonation authority. + +The File adapter opens every component of a server-owned logical root as a +no-follow directory capability, then opens exactly one validated filename +relative to that retained descriptor. It accepts only a regular file below an +explicit server-owned byte ceiling. Bytes pass unchanged into the Issue #22 +compiler. One successful compilation is published in one database +transaction as Resource, immutable Revision, immutable compilation snapshot, +one paragraph Fragment, mirrored Resource access, exact Membership body right, +content-free exact-phrase candidate, ordered `prepared -> indexed -> active` +events, active pointer, and completed job. Any failure rolls the entire effect +back; a post-redemption acquisition or compilation failure leaves only a +content-free terminal failed-job marker so it cannot remain runnable. The +reversible migration removes only Issue #23-owned rows and schema +before restoring the Issue #21 and Issue #17 constraints. + +The exact-phrase index stores a SHA-256 query digest and lineage references, not +content. Its Runtime SELECT requires the complete current UserActor transaction +context and runs inside the same retained projection transaction used by the +Kernel; returned `CandidateRef` values remain untrusted discovery output. +Every candidate still crosses the sealed AuthorizationKernel and +`AuthorizedProjection` gates before content-bearing assembly. Cross-Organization +or scope-denied resolution returns the canonical empty package. + +## Rationale + +This is the smallest production-shaped tracer that exercises real acquisition, +durable job authority, immutable publication, retrieval, authorization, +provenance, and HTTP delivery. Logical roots keep deployment paths out of +contracts. A content-free deterministic index proves that retrieval is not +authorization. Atomic publication prevents Runtime from observing prepared or +indexed content before the active pointer and access rights agree. + +## Consequences + +- One explicitly prepared Markdown basename can be imported and resolved by an + exact phrase through the public HTTP seam. +- Direct worker table mutation remains unavailable; the shared definer role has + only operation-specific policies, grants, and functions. +- Directory discovery, watchers, symlinks, traversal, multiple files, update, + delete, hash no-op, retry/recovery, replacement, and approximate retrieval + remain unavailable. +- The complete WorkerLease contract is still not proven for Policy Epoch, + generation, outbox, or arbitrary Source operations. + +## Revisit trigger + +Revisit before accepting directory trees, links, remote files, repeat imports, +updates/deletes, checkpointing, recovery, approximate search, more compiler +shapes, or a broader ServiceActor operation set. Each expansion must retain +exact durable-job binding, server-owned roots, atomic immutable publication, and +the `CandidateRef -> AuthorizationKernel -> AuthorizedProjection` boundary. diff --git a/engine/control/__init__.py b/engine/control/__init__.py index 0e4e5eb4..1f63b6f4 100644 --- a/engine/control/__init__.py +++ b/engine/control/__init__.py @@ -11,6 +11,7 @@ ) from engine.control.contracts import ( FILE_CAPABILITY_MANIFEST, + FILE_IMPORT_CAPABILITY_MANIFEST, CapabilityStatus, FileCapabilityManifest, FileRootRef, @@ -26,10 +27,18 @@ SourceResourceKind, SourceVersion, ) +from engine.control.file_imports import ( + FileImportAudience, + FileImportPath, + FileImportReceiver, + PreparedFileImport, + PrepareFileImport, +) from engine.control.module import ContextControl, ControlStorePort __all__ = [ "FILE_CAPABILITY_MANIFEST", + "FILE_IMPORT_CAPABILITY_MANIFEST", "CapabilityStatus", "ContextControl", "ControlOperation", @@ -39,8 +48,13 @@ "ControlOperatorAuthorityUnavailable", "ControlStorePort", "FileCapabilityManifest", + "FileImportAudience", + "FileImportPath", + "FileImportReceiver", "FileRootRef", "RegisterFileSource", + "PrepareFileImport", + "PreparedFileImport", "SourceAclEvidenceMode", "SourceControlUnavailable", "SourceContentKind", diff --git a/engine/control/authority.py b/engine/control/authority.py index 57d1c23d..4ba2cd36 100644 --- a/engine/control/authority.py +++ b/engine/control/authority.py @@ -19,6 +19,7 @@ class ControlOperation(StrEnum): + IMPORT_FILE = "import_file" REGISTER_SOURCE = "register_source" READ_SOURCE = "read_source" diff --git a/engine/control/contracts.py b/engine/control/contracts.py index 6f754c34..c1171077 100644 --- a/engine/control/contracts.py +++ b/engine/control/contracts.py @@ -73,7 +73,7 @@ class CapabilityStatus(StrEnum): @dataclass(frozen=True, slots=True) class FileCapabilityManifest: - """Exact Issue #21 declaration; registration is not acquisition readiness.""" + """Immutable File capability snapshot for registration or manual import.""" declaration_version: str = "file-capabilities-v1" source_mode: SourceMode = SourceMode.MATERIALIZED @@ -98,33 +98,47 @@ class FileCapabilityManifest: ingestion_jobs: CapabilityStatus = CapabilityStatus.UNAVAILABLE def __post_init__(self) -> None: + unavailable_statuses = ( + self.cursor_semantics, + self.checkpoint_semantics, + self.batch_limits, + self.freshness, + self.consistency_guarantees, + self.describe_capabilities, + self.read_changes, + self.discover, + self.authorize_and_project, + self.checkpoint, + self.deletion, + ) if ( - self.declaration_version != "file-capabilities-v1" - or self.source_mode is not SourceMode.MATERIALIZED + self.source_mode is not SourceMode.MATERIALIZED or self.content_kinds != (SourceContentKind.MARKDOWN,) or self.resource_kinds != (SourceResourceKind.MARKDOWN_DOCUMENT,) or self.acl_evidence_mode is not SourceAclEvidenceMode.MIRRORED or self.projection_fields != () or any( status is not CapabilityStatus.UNAVAILABLE - for status in ( - self.cursor_semantics, - self.checkpoint_semantics, - self.batch_limits, - self.freshness, - self.consistency_guarantees, - self.describe_capabilities, - self.read_changes, - self.discover, - self.authorize_and_project, - self.checkpoint, - self.deletion, - self.file_source_access, - self.ingestion_jobs, + for status in unavailable_statuses + ) + or ( + self.declaration_version == "file-capabilities-v1" + and ( + self.file_source_access is not CapabilityStatus.UNAVAILABLE + or self.ingestion_jobs is not CapabilityStatus.UNAVAILABLE + ) + ) + or ( + self.declaration_version == "file-capabilities-v2" + and ( + self.file_source_access is not CapabilityStatus.AVAILABLE + or self.ingestion_jobs is not CapabilityStatus.AVAILABLE ) ) + or self.declaration_version + not in {"file-capabilities-v1", "file-capabilities-v2"} ): - raise ValueError("File capability manifest is closed at Issue #21") + raise ValueError("File capability manifest is not a recognized snapshot") def document(self) -> dict[str, object]: """Return the exact persisted/public declaration without activation claims.""" @@ -153,6 +167,11 @@ def document(self) -> dict[str, object]: FILE_CAPABILITY_MANIFEST = FileCapabilityManifest() +FILE_IMPORT_CAPABILITY_MANIFEST = FileCapabilityManifest( + declaration_version="file-capabilities-v2", + file_source_access=CapabilityStatus.AVAILABLE, + ingestion_jobs=CapabilityStatus.AVAILABLE, +) @dataclass(frozen=True, slots=True) @@ -261,7 +280,7 @@ def __post_init__(self) -> None: _require_utc("SourceManifest created_at", self.created_at) @classmethod - def issue_21_file( + def registered_file( cls, *, source_ref: SourceRef, @@ -269,6 +288,8 @@ def issue_21_file( display_name: str, root_ref: FileRootRef, created_at: datetime, + version_created_at: datetime | None = None, + capabilities: FileCapabilityManifest = FILE_CAPABILITY_MANIFEST, ) -> SourceManifest: """Construct the exact first File manifest from trusted stored facts.""" @@ -277,8 +298,10 @@ def issue_21_file( version_ref=version_ref, kind=SourceKind.FILE, root_ref=root_ref, - capabilities=FILE_CAPABILITY_MANIFEST, - created_at=created_at, + capabilities=capabilities, + created_at=( + created_at if version_created_at is None else version_created_at + ), ) return cls( source_ref=source_ref, @@ -288,6 +311,26 @@ def issue_21_file( created_at=created_at, ) + @classmethod + def issue_21_file( + cls, + *, + source_ref: SourceRef, + version_ref: UUID, + display_name: str, + root_ref: FileRootRef, + created_at: datetime, + ) -> SourceManifest: + """Construct the original registration-only v1 snapshot.""" + + return cls.registered_file( + source_ref=source_ref, + version_ref=version_ref, + display_name=display_name, + root_ref=root_ref, + created_at=created_at, + ) + class SourceNotAvailable(Exception): """One generic result for unauthorized, unknown, or unavailable sources.""" diff --git a/engine/control/file_imports.py b/engine/control/file_imports.py new file mode 100644 index 00000000..3e3c586d --- /dev/null +++ b/engine/control/file_imports.py @@ -0,0 +1,138 @@ +"""Closed contracts for the first trusted one-file import operation.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import NoReturn +from uuid import UUID + +from engine.control.contracts import SourceRef + +FILE_IMPORT_WORKLOAD = "supply.file-import" +FILE_IMPORT_WORKER_AUDIENCE = "context-engine-worker" +FILE_IMPORT_OPERATION = "file.import" + + +def _require_token(field_name: str, value: object, *, maximum: int = 255) -> str: + if ( + type(value) is not str + or not value + or value.isspace() + or value != value.strip() + or len(value) > maximum + or any(character.isspace() for character in value) + ): + raise ValueError(f"{field_name} must be a bounded nonblank token") + return value + + +@dataclass(frozen=True, slots=True) +class FileImportPath: + """One relative Markdown filename; directories and traversal are closed.""" + + value: str = field(repr=False) + + def __post_init__(self) -> None: + value = self.value + if ( + type(value) is not str + or not value + or value != value.strip() + or value in {".", ".."} + or "/" in value + or "\\" in value + or not value.casefold().endswith(".md") + or len(value) > 255 + or any(ord(character) < 0x20 for character in value) + ): + raise ValueError("File import path must be one bounded Markdown filename") + + +@dataclass(frozen=True, slots=True) +class FileImportAudience: + """Mirrored first-publication grant target; never worker identity.""" + + principal_ref: str = field(repr=False) + membership_id: UUID = field(repr=False) + membership_version: int = field(repr=False) + + def __post_init__(self) -> None: + _require_token("File import principal_ref", self.principal_ref) + if type(self.membership_id) is not UUID: + raise TypeError("File import membership_id must be UUID") + if ( + type(self.membership_version) is not int + or not 1 <= self.membership_version <= 2**63 - 1 + ): + raise ValueError("File import Membership version must be positive") + + +@dataclass(frozen=True, slots=True) +class PrepareFileImport: + """Trusted Control request for one registered-source import envelope.""" + + source_ref: SourceRef + path: FileImportPath = field(repr=False) + audience: FileImportAudience = field(repr=False) + idempotency_key: str = field(repr=False) + + def __post_init__(self) -> None: + if type(self.source_ref) is not SourceRef: + raise TypeError("File import source_ref must be SourceRef") + if type(self.path) is not FileImportPath: + raise TypeError("File import path must be FileImportPath") + if type(self.audience) is not FileImportAudience: + raise TypeError("File import audience must be FileImportAudience") + _require_token("File import idempotency_key", self.idempotency_key) + + def __reduce__(self) -> NoReturn: + raise TypeError("File import command is not serializable") + + +@dataclass(frozen=True, slots=True) +class PreparedFileImport: + """Content-free exact-job locator returned by trusted ContextControl.""" + + organization_id: UUID = field(repr=False) + job_id: UUID = field(repr=False) + source_ref: SourceRef = field(repr=False) + service_principal_id: UUID = field(repr=False) + workload: str = field(default=FILE_IMPORT_WORKLOAD, init=False) + worker_audience: str = field( + default=FILE_IMPORT_WORKER_AUDIENCE, + init=False, + ) + operation: str = field( + default=FILE_IMPORT_OPERATION, + init=False, + ) + + def __post_init__(self) -> None: + if type(self.organization_id) is not UUID or type(self.job_id) is not UUID: + raise TypeError("Prepared File import identifiers must be UUID") + if type(self.source_ref) is not SourceRef: + raise TypeError("Prepared File import source_ref must be SourceRef") + if type(self.service_principal_id) is not UUID: + raise TypeError("Prepared File import service principal must be UUID") + + +@dataclass(frozen=True, slots=True) +class FileImportReceiver: + """Trusted registered worker identity injected into Control composition.""" + + service_principal_id: UUID = field(repr=False) + workload: str = field(default=FILE_IMPORT_WORKLOAD, init=False, repr=False) + worker_audience: str = field( + default=FILE_IMPORT_WORKER_AUDIENCE, + init=False, + repr=False, + ) + operation: str = field( + default=FILE_IMPORT_OPERATION, + init=False, + repr=False, + ) + + def __post_init__(self) -> None: + if type(self.service_principal_id) is not UUID: + raise TypeError("File import receiver identity must be UUID") diff --git a/engine/control/module.py b/engine/control/module.py index 41d8e1bc..a833f0bc 100644 --- a/engine/control/module.py +++ b/engine/control/module.py @@ -20,6 +20,7 @@ SourceNotAvailable, SourceRef, ) +from engine.control.file_imports import PreparedFileImport, PrepareFileImport class ControlStorePort(Protocol): @@ -37,9 +38,15 @@ def read_source( source_ref: SourceRef, ) -> SourceManifest: ... + def prepare_file_import( + self, + call: TrustedControlCall, + command: PrepareFileImport, + ) -> PreparedFileImport: ... + class ContextControl: - """Own File source enrollment and read-back, but no acquisition behavior.""" + """Own trusted File enrollment, read-back, and import preparation.""" __slots__ = ("_authority", "_clock", "_store") @@ -50,7 +57,11 @@ def __init__( authority: ControlOperatorAuthority, clock: Callable[[], datetime], ) -> None: - for method_name in ("register_file_source", "read_source"): + for method_name in ( + "prepare_file_import", + "register_file_source", + "read_source", + ): if not callable(getattr(store, method_name, None)): raise TypeError("ContextControl store is incomplete") if type(authority) is not ControlOperatorAuthority: @@ -119,6 +130,44 @@ def read_source( except Exception: raise SourceControlUnavailable("source read is unavailable") from None + def prepare_file_import( + self, + call: TrustedControlCall, + command: PrepareFileImport, + ) -> PreparedFileImport: + """Create one durable acquisition/job under trusted Control authority.""" + + if type(command) is not PrepareFileImport: + raise TypeError("prepare_file_import requires PrepareFileImport") + try: + _validate_and_consume_control_call( + call, + authority=self._authority, + expected_operation=ControlOperation.IMPORT_FILE, + checked_at=self._clock(), + ) + prepared = self._store.prepare_file_import(call, command) + if type(prepared) is not PreparedFileImport: + raise SourceControlUnavailable( + "source store returned an invalid File import" + ) + if ( + prepared.organization_id != call.organization_id + or prepared.source_ref != command.source_ref + ): + raise SourceControlUnavailable( + "source store returned a mismatched File import" + ) + return prepared + except (ControlOperatorAuthenticationRejected, SourceNotAvailable): + raise SourceNotAvailable from None + except SourceControlUnavailable: + raise + except Exception: + raise SourceControlUnavailable( + "File import preparation is unavailable" + ) from None + @staticmethod def _require_manifest(manifest: object) -> None: if type(manifest) is not SourceManifest: diff --git a/engine/persistence/__init__.py b/engine/persistence/__init__.py index 54f5db6b..5bc5871a 100644 --- a/engine/persistence/__init__.py +++ b/engine/persistence/__init__.py @@ -33,6 +33,12 @@ ) from engine.persistence.control_sources import PostgreSQLControlStore from engine.persistence.database import create_database_engine +from engine.persistence.file_imports import ( + FileImportLeaseRedemption, + FileImportUnavailable, + PostgreSQLFileImportWorker, + PublishedFileImport, +) from engine.persistence.membership_context import ( MembershipAuthorityUnavailable, MembershipIdentity, @@ -92,6 +98,10 @@ "OperatorAuthorizationProvenance", "PostgreSQLContextRunReader", "PostgreSQLControlStore", + "FileImportLeaseRedemption", + "FileImportUnavailable", + "PostgreSQLFileImportWorker", + "PublishedFileImport", "VerifiedContextRunOperatorIdentity", "ResourceAccessRevocation", "OrganizationContextBindingError", diff --git a/engine/persistence/control_sources.py b/engine/persistence/control_sources.py index ad3d4394..cf7ef099 100644 --- a/engine/persistence/control_sources.py +++ b/engine/persistence/control_sources.py @@ -14,6 +14,7 @@ from engine.control import ( FILE_CAPABILITY_MANIFEST, + FILE_IMPORT_CAPABILITY_MANIFEST, FileRootRef, RegisterFileSource, SourceControlUnavailable, @@ -23,6 +24,11 @@ TrustedControlCall, ) from engine.persistence.role_guard import assert_control_role +from engine.supply import ( + FileImportReceiver, + PreparedFileImport, + PrepareFileImport, +) _REGISTRATION_OPERATION = "register_source" _ACTIVE_SOURCE_SELECT = """ @@ -49,7 +55,13 @@ def _capability_document() -> dict[str, object]: return FILE_CAPABILITY_MANIFEST.document() -_CAPABILITY_DOCUMENT = _capability_document() +_REGISTRATION_CAPABILITY_DOCUMENT = FILE_CAPABILITY_MANIFEST.document() +_KNOWN_CAPABILITY_DOCUMENTS = { + FILE_CAPABILITY_MANIFEST.declaration_version: FILE_CAPABILITY_MANIFEST, + FILE_IMPORT_CAPABILITY_MANIFEST.declaration_version: ( + FILE_IMPORT_CAPABILITY_MANIFEST + ), +} def _registration_digest(command: RegisterFileSource) -> str: @@ -89,12 +101,19 @@ def __init__( *, clock: Callable[[], datetime], uuid_factory: Callable[[], UUID] = uuid4, + file_import_receiver: FileImportReceiver | None = None, ) -> None: if not callable(clock) or not callable(uuid_factory): raise TypeError("PostgreSQLControlStore requires clock and UUID factory") self._engine = engine self._clock = clock self._uuid_factory = uuid_factory + if ( + file_import_receiver is not None + and type(file_import_receiver) is not FileImportReceiver + ): + raise TypeError("file_import_receiver must be FileImportReceiver") + self._file_import_receiver = file_import_receiver def register_file_source( self, @@ -166,7 +185,7 @@ def register_file_source( "version_id": version_id, "root_ref": command.root_ref.value, "capabilities": rfc8785.dumps( - cast(Any, _CAPABILITY_DOCUMENT) + cast(Any, _REGISTRATION_CAPABILITY_DOCUMENT) ).decode("utf-8"), "created_at": created_at, }, @@ -220,6 +239,91 @@ def read_source( "File source read database authority is unavailable" ) from None + def prepare_file_import( + self, + call: TrustedControlCall, + command: PrepareFileImport, + ) -> PreparedFileImport: + """Atomically persist one acquisition and its exact worker job.""" + + receiver = self._file_import_receiver + if ( + type(call) is not TrustedControlCall + or type(command) is not PrepareFileImport + or receiver is None + ): + raise SourceNotAvailable + job_id = self._uuid_factory() + acquisition_id = self._uuid_factory() + activated_version_id = self._uuid_factory() + document = { + "audience_membership_id": str(command.audience.membership_id), + "audience_membership_version": command.audience.membership_version, + "audience_principal_ref": command.audience.principal_ref, + "idempotency_key": command.idempotency_key, + "operation": receiver.operation, + "path": command.path.value, + "source_id": str(command.source_ref.value), + } + digest = hashlib.sha256( + b"context-engine.prepare-file-import.v1\x00" + + rfc8785.dumps(cast(Any, document)) + ).hexdigest() + try: + with self._engine.begin() as connection: + assert_control_role(connection) + row = connection.execute( + text( + """ + SELECT job_id, service_principal_id + FROM public.context_control_prepare_file_import( + :organization_id, + :acquisition_id, + :job_id, + :activated_version_id, + :source_id, + :relative_path, + :audience_principal_ref, + :audience_membership_id, + :audience_membership_version, + :idempotency_key, + :request_digest, + :service_principal_id + ) + """ + ), + { + "organization_id": call.organization_id, + "acquisition_id": acquisition_id, + "job_id": job_id, + "activated_version_id": activated_version_id, + "source_id": command.source_ref.value, + "relative_path": command.path.value, + "audience_principal_ref": command.audience.principal_ref, + "audience_membership_id": command.audience.membership_id, + "audience_membership_version": ( + command.audience.membership_version + ), + "idempotency_key": command.idempotency_key, + "request_digest": digest, + "service_principal_id": receiver.service_principal_id, + }, + ).one_or_none() + if row is None: + raise SourceNotAvailable + return PreparedFileImport( + organization_id=call.organization_id, + job_id=row.job_id, + source_ref=command.source_ref, + service_principal_id=row.service_principal_id, + ) + except SourceNotAvailable: + raise + except (DBAPIError, SQLAlchemyError, AssertionError): + raise SourceControlUnavailable( + "File import Control database authority is unavailable" + ) from None + @staticmethod def _select_registration( connection: Any, @@ -249,7 +353,21 @@ def _select_registration( @staticmethod def _manifest(row: Mapping[str, object]) -> SourceManifest: capabilities = row["capability_manifest"] - if capabilities != _CAPABILITY_DOCUMENT: + if type(capabilities) is not dict: + raise SourceControlUnavailable( + "stored File capability declaration is not recognized" + ) + declaration_version_value = capabilities.get("declarationVersion") + declaration_version = ( + declaration_version_value + if type(declaration_version_value) is str + else "" + ) + capability_manifest = _KNOWN_CAPABILITY_DOCUMENTS.get(declaration_version) + if ( + capability_manifest is None + or capabilities != capability_manifest.document() + ): raise SourceControlUnavailable( "stored File capability declaration is not recognized" ) @@ -270,13 +388,15 @@ def _manifest(row: Mapping[str, object]) -> SourceManifest: or type(root_ref) is not str or type(source_created_at) is not datetime or type(version_created_at) is not datetime - or source_created_at != version_created_at + or version_created_at < source_created_at ): raise SourceControlUnavailable("stored File source manifest is invalid") - return SourceManifest.issue_21_file( + return SourceManifest.registered_file( source_ref=SourceRef(source_id), version_ref=version_id, display_name=display_name, root_ref=FileRootRef(root_ref), created_at=source_created_at, + version_created_at=version_created_at, + capabilities=capability_manifest, ) diff --git a/engine/persistence/file_imports.py b/engine/persistence/file_imports.py new file mode 100644 index 00000000..3d3055de --- /dev/null +++ b/engine/persistence/file_imports.py @@ -0,0 +1,343 @@ +"""Exact WorkerLease execution path for one registered Markdown file.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from hashlib import sha256 +from typing import Literal +from uuid import UUID, uuid4 + +from sqlalchemy import Engine, text +from sqlalchemy.exc import SQLAlchemyError + +from adapters.file_source import FileRootRegistry +from adapters.parsers.markdown import compile_markdown +from engine.control import ( + FileImportPath, + FileImportReceiver, + FileRootRef, + SourceRef, +) +from engine.persistence.role_guard import assert_worker_role +from engine.runtime.content_io import exact_phrase_digest +from engine.runtime.evidence import CandidateRef +from engine.supply import ( + FILE_IMPORT_WORKER_LEASE_OPERATION, + CompilationFailure, + MarkdownCompilerConfig, + ParsedDocument, + WorkerLeaseClaims, + WorkerLeaseCodec, + WorkerLeaseRejectionAuditReceipt, + WorkerLeaseToken, + WorkNotAvailable, + worker_lease_digest, +) +from engine.supply.jobs import _require_utc + + +@dataclass(frozen=True, slots=True) +class FileImportLeaseRedemption: + """Untrusted queue carrier with one opaque lease and routing locators.""" + + token: WorkerLeaseToken = field(repr=False) + expected_organization_id: UUID = field(repr=False) + expected_job_id: UUID = field(repr=False) + expected_source_ref: SourceRef = field(repr=False) + + def __post_init__(self) -> None: + if type(self.token) is not WorkerLeaseToken: + raise TypeError("File import redemption requires WorkerLeaseToken") + if ( + type(self.expected_organization_id) is not UUID + or type(self.expected_job_id) is not UUID + ): + raise TypeError("File import redemption identifiers must be UUID") + if type(self.expected_source_ref) is not SourceRef: + raise TypeError("File import redemption source must be SourceRef") + + +@dataclass(frozen=True, slots=True) +class PublishedFileImport: + """Complete immutable lineage for the one committed publication effect.""" + + candidate_ref: CandidateRef + acquisition_id: UUID = field(repr=False) + publication_states: tuple[str, str, str] = ( + "prepared", + "indexed", + "active", + ) + effect_count: Literal[1] = 1 + + def __post_init__(self) -> None: + if type(self.candidate_ref) is not CandidateRef: + raise TypeError("published File import requires CandidateRef") + if type(self.acquisition_id) is not UUID: + raise TypeError("published File import acquisition must be UUID") + if self.publication_states != ("prepared", "indexed", "active"): + raise ValueError("File publication state sequence must remain closed") + if self.effect_count != 1: + raise ValueError("published File import has exactly one effect") + + +class FileImportUnavailable(RuntimeError): + """Generic failure after a valid lease reaches acquisition/publication.""" + + +@dataclass(frozen=True, slots=True) +class _RedeemedFileImport: + source_ref: SourceRef + root_ref: FileRootRef + path: FileImportPath + acquisition_id: UUID + + +def _rejection(token: WorkerLeaseToken) -> WorkNotAvailable: + return WorkNotAvailable( + WorkerLeaseRejectionAuditReceipt(worker_lease_digest(token)) + ) + + +def _resource_ref(source_ref: SourceRef, path: FileImportPath) -> str: + identity = sha256( + b"context-engine.file-resource.v1\x00" + + source_ref.value.bytes + + path.value.encode("utf-8") + ).hexdigest() + return f"resource:file:{identity}" + + +class PostgreSQLFileImportWorker: + """Verify, redeem, acquire, compile, and atomically publish one file.""" + + __slots__ = ( + "_clock", + "_codec", + "_config", + "_engine", + "_identity", + "_roots", + "_uuid_factory", + ) + + def __init__( + self, + engine: Engine, + codec: WorkerLeaseCodec, + identity: FileImportReceiver, + roots: FileRootRegistry, + config: MarkdownCompilerConfig, + *, + clock: Callable[[], object], + uuid_factory: Callable[[], UUID] = uuid4, + ) -> None: + if type(codec) is not WorkerLeaseCodec: + raise TypeError("File import worker requires WorkerLeaseCodec") + if type(identity) is not FileImportReceiver: + raise TypeError("File import worker requires FileImportReceiver") + if type(roots) is not FileRootRegistry: + raise TypeError("File import worker requires FileRootRegistry") + if type(config) is not MarkdownCompilerConfig: + raise TypeError("File import worker requires MarkdownCompilerConfig") + if not callable(clock) or not callable(uuid_factory): + raise TypeError("File import worker requires clock and UUID factory") + self._engine = engine + self._codec = codec + self._identity = identity + self._roots = roots + self._config = config + self._clock = clock + self._uuid_factory = uuid_factory + + def run(self, redemption: FileImportLeaseRedemption) -> PublishedFileImport: + """Perform file I/O only after signature and durable lease redemption.""" + + if type(redemption) is not FileImportLeaseRedemption: + raise TypeError("File import worker requires exact redemption") + checked_at = _require_utc("File import worker clock", self._clock()) + identity = self._identity + claims = self._codec.verify( + redemption.token, + expected_organization_id=redemption.expected_organization_id, + expected_job_id=redemption.expected_job_id, + expected_service_principal_id=identity.service_principal_id, + expected_workload=identity.workload, + expected_operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + expected_worker_audience=identity.worker_audience, + expected_source_ref=str(redemption.expected_source_ref.value), + now=checked_at, + ) + redeemed = self._redeem(redemption.token, claims) + try: + source = self._roots.read(redeemed.root_ref, redeemed.path) + outcome = compile_markdown(source, self._config) + except LookupError: + self._fail(redemption.token, claims) + raise FileImportUnavailable("File import is unavailable") from None + if type(outcome) is CompilationFailure: + self._fail(redemption.token, claims) + raise FileImportUnavailable("File import is unavailable") + if type(outcome) is not ParsedDocument: # pragma: no cover - closed union + self._fail(redemption.token, claims) + raise FileImportUnavailable("File import is unavailable") + try: + return self._publish(redemption.token, claims, redeemed, outcome) + except (FileImportUnavailable, WorkNotAvailable): + self._fail(redemption.token, claims) + raise + + def _redeem( + self, + token: WorkerLeaseToken, + claims: WorkerLeaseClaims, + ) -> _RedeemedFileImport: + if type(claims) is not WorkerLeaseClaims: + raise _rejection(token) + try: + with self._engine.begin() as connection: + assert_worker_role(connection) + row = connection.execute( + text( + """ + SELECT * FROM public.context_worker_redeem_file_import( + :organization_id, :job_id, :service_principal_id, + :source_ref, :signing_key_version, :nonce, + :issued_at, :expires_at + ) + """ + ), + { + "organization_id": claims.organization_id, + "job_id": claims.job_id, + "service_principal_id": claims.service_principal_id, + "source_ref": claims.source_ref, + "signing_key_version": claims.signing_key_version, + "nonce": claims.nonce, + "issued_at": claims.issued_at, + "expires_at": claims.expires_at, + }, + ).one_or_none() + if row is None or row.source_ref != claims.source_ref: + raise _rejection(token) + return _RedeemedFileImport( + source_ref=SourceRef(UUID(row.source_ref)), + root_ref=FileRootRef(row.root_ref), + path=FileImportPath(row.relative_path), + acquisition_id=row.acquisition_id, + ) + except WorkNotAvailable: + raise + except (SQLAlchemyError, AssertionError, ValueError): + raise FileImportUnavailable( + "File import redemption is unavailable" + ) from None + + def _publish( + self, + token: WorkerLeaseToken, + claims: WorkerLeaseClaims, + redeemed: _RedeemedFileImport, + document: ParsedDocument, + ) -> PublishedFileImport: + if type(claims) is not WorkerLeaseClaims: + raise _rejection(token) + revision_id = self._uuid_factory() + resource_ref = _resource_ref(redeemed.source_ref, redeemed.path) + fragment_ref = "fragment:paragraph:1" + paragraph = document.sections[1].text + try: + with self._engine.begin() as connection: + assert_worker_role(connection) + row = connection.execute( + text( + """ + SELECT effect_count + FROM public.context_worker_publish_file_import( + :organization_id, :job_id, :service_principal_id, + :source_ref, :resource_ref, + :revision_id, :fragment_ref, :canonical_text, + :paragraph, :content_hash, :compilation_digest, + :compiler_version, :config_version, :phrase_digest, + :signing_key_version, :nonce, :issued_at, :expires_at + ) + """ + ), + { + "organization_id": claims.organization_id, + "job_id": claims.job_id, + "service_principal_id": claims.service_principal_id, + "source_ref": claims.source_ref, + "resource_ref": resource_ref, + "revision_id": revision_id, + "fragment_ref": fragment_ref, + "canonical_text": document.canonical_text, + "paragraph": paragraph, + "content_hash": document.content_hash, + "compilation_digest": document.compilation_digest, + "compiler_version": document.provenance.compiler_version, + "config_version": document.provenance.config_version, + "phrase_digest": exact_phrase_digest(paragraph), + "signing_key_version": claims.signing_key_version, + "nonce": claims.nonce, + "issued_at": claims.issued_at, + "expires_at": claims.expires_at, + }, + ).one_or_none() + if row is None or row.effect_count != 1: + raise _rejection(token) + except WorkNotAvailable: + raise + except (SQLAlchemyError, AssertionError): + raise FileImportUnavailable("File publication is unavailable") from None + return PublishedFileImport( + candidate_ref=CandidateRef( + organization_id=claims.organization_id, + source_ref=str(redeemed.source_ref.value), + resource_ref=resource_ref, + revision_ref=str(revision_id), + fragment_ref=fragment_ref, + ), + acquisition_id=redeemed.acquisition_id, + ) + + def _fail( + self, + token: WorkerLeaseToken, + claims: WorkerLeaseClaims, + ) -> None: + """Seal a redeemed job as failed without retaining content or reason.""" + + try: + with self._engine.begin() as connection: + assert_worker_role(connection) + changed = connection.execute( + text( + """ + SELECT public.context_worker_fail_file_import( + :organization_id, :job_id, :service_principal_id, + :source_ref, :signing_key_version, :nonce, + :issued_at, :expires_at + ) + """ + ), + { + "organization_id": claims.organization_id, + "job_id": claims.job_id, + "service_principal_id": claims.service_principal_id, + "source_ref": claims.source_ref, + "signing_key_version": claims.signing_key_version, + "nonce": claims.nonce, + "issued_at": claims.issued_at, + "expires_at": claims.expires_at, + }, + ).scalar_one() + if changed is not True: + raise _rejection(token) + except WorkNotAvailable: + raise + except (SQLAlchemyError, AssertionError): + raise FileImportUnavailable( + "File import failure recording is unavailable" + ) from None diff --git a/engine/persistence/membership_context.py b/engine/persistence/membership_context.py index 744fad48..36ced7ff 100644 --- a/engine/persistence/membership_context.py +++ b/engine/persistence/membership_context.py @@ -35,6 +35,8 @@ MaterializedFragmentLocator, MaterializedFragmentProjection, MaterializedProjectionKind, + MaterializedProjectionSession, + MaterializedPublicationTrace, _close_materialized_projection_scope, _construct_materialized_projection_session, _open_materialized_projection_scope, @@ -138,6 +140,77 @@ class _PostgreSQLMaterializedProjectionPort: def __init__(self, connection: Connection) -> None: self._connection = connection + def discover_exact_phrase( + self, + phrase_digest: str, + ) -> tuple[CandidateRef, ...]: + rows = self._connection.execute( + text( + """ + SELECT + organization_id, + source_ref, + resource_ref, + revision_id, + fragment_ref + FROM exact_phrase_candidate + WHERE phrase_digest = :phrase_digest + ORDER BY resource_ref, revision_id, fragment_ref + LIMIT 64 + """ + ), + {"phrase_digest": phrase_digest}, + ) + return tuple( + CandidateRef( + organization_id=row.organization_id, + source_ref=row.source_ref, + resource_ref=row.resource_ref, + revision_ref=str(row.revision_id), + fragment_ref=row.fragment_ref, + ) + for row in rows + ) + + def observe_publication( + self, + candidate_ref: CandidateRef, + ) -> MaterializedPublicationTrace | None: + revision_id = _canonical_candidate_revision(candidate_ref.revision_ref) + if revision_id is None: + return None + row = self._connection.execute( + text( + """ + SELECT + resource.active_revision_id, + array_agg(event.state ORDER BY event.ordinal) AS states + FROM context_resource AS resource + JOIN revision_publication_event AS event + ON event.organization_id = resource.organization_id + AND event.resource_ref = resource.resource_ref + AND event.revision_id = resource.active_revision_id + WHERE resource.organization_id = :organization_id + AND resource.source_ref = :source_ref + AND resource.resource_ref = :resource_ref + AND resource.active_revision_id = :revision_id + GROUP BY resource.active_revision_id + """ + ), + { + "organization_id": candidate_ref.organization_id, + "source_ref": candidate_ref.source_ref, + "resource_ref": candidate_ref.resource_ref, + "revision_id": revision_id, + }, + ).one_or_none() + if row is None: + return None + return MaterializedPublicationTrace( + states=tuple(row.states), + active_revision_ref=str(row.active_revision_id), + ) + def locate( self, candidate_ref: CandidateRef, @@ -466,6 +539,21 @@ class PostgreSQLMembershipAuthority: def __init__(self, engine: Engine) -> None: self._engine = engine + @contextmanager + def current_projection_session( + self, + identity: MembershipIdentity, + ) -> Iterator[MaterializedProjectionSession]: + """Expose the retained authorized projection seam to trusted callers.""" + + with self.current_user_actor(identity) as verification: + session = verification.materialized_projection_session + if session is None: # pragma: no cover - closed PostgreSQL composition + raise MembershipAuthorityUnavailable( + "materialized projection session is unavailable" + ) + yield session + @contextmanager def current_user_actor( self, diff --git a/engine/persistence/schema_security_manifest.yaml b/engine/persistence/schema_security_manifest.yaml index e2f78685..94bfe865 100644 --- a/engine/persistence/schema_security_manifest.yaml +++ b/engine/persistence/schema_security_manifest.yaml @@ -1,5 +1,5 @@ { - "manifestVersion": "10.0.0", + "manifestVersion": "11.0.0", "controlOperations": [ { "name": "register_file_source", @@ -51,6 +51,69 @@ "callerSuppliedReceiverDimensions": [], "atomicWrites": ["worker_noop_job"] }, + { + "name": "prepare_file_import", + "databaseFunction": "context_control_prepare_file_import", + "role": "context_engine_control", + "definerRole": "context_engine_worker_lease_definer", + "directTableMutationAllowed": false, + "trustedOrganizationSource": "TrustedControlCall", + "organizationScopedIdempotency": true, + "databaseOwnedTime": true, + "filesystemAccessAllowed": false, + "durableJobCreationAllowed": true, + "atomicWrites": ["source_version", "context_source", "file_acquisition", "file_import_job"] + }, + { + "name": "issue_file_import_lease", + "databaseFunction": "context_worker_issue_file_import_lease", + "role": "context_engine_control", + "definerRole": "context_engine_worker_lease_definer", + "directTableMutationAllowed": false, + "databaseOwnedTime": true, + "maxTtlSeconds": 3600, + "atomicWrites": ["file_import_job"] + }, + { + "name": "redeem_file_import_lease", + "databaseFunction": "context_worker_redeem_file_import", + "role": "context_engine_worker", + "definerRole": "context_engine_worker_lease_definer", + "directTableMutationAllowed": false, + "rawNonceComparedAsSha256": true, + "fixedReceiver": { + "databaseRole": "context_engine_worker", + "workload": "supply.file-import", + "workerAudience": "context-engine-worker", + "operation": "file.import" + }, + "atomicWrites": ["file_import_job"] + }, + { + "name": "publish_file_import", + "databaseFunction": "context_worker_publish_file_import", + "role": "context_engine_worker", + "definerRole": "context_engine_worker_lease_definer", + "directTableMutationAllowed": false, + "rawNonceComparedAsSha256": true, + "atomicWrites": [ + "context_resource", "context_revision", "file_revision_snapshot", + "context_fragment", "revision_publication_event", + "exact_phrase_candidate", "resource_access_policy", + "membership_resource_field_right", "file_import_job" + ] + }, + { + "name": "fail_file_import", + "databaseFunction": "context_worker_fail_file_import", + "role": "context_engine_worker", + "definerRole": "context_engine_worker_lease_definer", + "directTableMutationAllowed": false, + "databaseOwnedTime": true, + "rawNonceComparedAsSha256": true, + "reasonOrContentPersistenceAllowed": false, + "atomicWrites": ["file_import_job"] + }, { "name": "context_learning_promote_release", "databaseFunction": "context_learning_promote_release", @@ -206,12 +269,19 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "membership_file_import_definer_select", + "command": "SELECT", + "roles": ["context_engine_worker_lease_definer"], + "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, "permittedOperations": { "context_engine_runtime": ["SELECT"], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["SELECT"] }, "partitions": [], "securityInvariantIds": [ @@ -401,6 +471,19 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "context_source_file_import_definer_select", + "command": "SELECT", + "roles": ["context_engine_worker_lease_definer"], + "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "context_source_file_import_definer_update", + "command": "UPDATE", + "roles": ["context_engine_worker_lease_definer"], + "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid", + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, @@ -409,7 +492,8 @@ "context_engine_learning": [], "context_engine_runtime": [], "context_engine_security_operator": [], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["SELECT", "UPDATE"] }, "partitions": [], "securityInvariantIds": [ @@ -457,8 +541,8 @@ "expression": "root_ref ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' AND root_ref NOT IN ('.', '..')" }, { - "name": "ck_source_version_issue_21_capabilities", - "expression": "capability_manifest = '{\"aclEvidenceMode\": \"mirrored\", \"authorizeAndProject\": \"unavailable\", \"batchLimits\": \"unavailable\", \"checkpoint\": \"unavailable\", \"checkpointSemantics\": \"unavailable\", \"contentKinds\": [\"markdown\"], \"consistencyGuarantees\": \"unavailable\", \"cursorSemantics\": \"unavailable\", \"declarationVersion\": \"file-capabilities-v1\", \"deletion\": \"unavailable\", \"describeCapabilities\": \"unavailable\", \"discover\": \"unavailable\", \"fileSourceAccess\": \"unavailable\", \"freshness\": \"unavailable\", \"ingestionJobs\": \"unavailable\", \"projectionFields\": [], \"readChanges\": \"unavailable\", \"resourceKinds\": [\"markdown_document\"], \"sourceMode\": \"materialized\"}'::jsonb" + "name": "ck_source_version_file_capabilities", + "expression": "capability_manifest IN ('{\"aclEvidenceMode\": \"mirrored\", \"authorizeAndProject\": \"unavailable\", \"batchLimits\": \"unavailable\", \"checkpoint\": \"unavailable\", \"checkpointSemantics\": \"unavailable\", \"contentKinds\": [\"markdown\"], \"consistencyGuarantees\": \"unavailable\", \"cursorSemantics\": \"unavailable\", \"declarationVersion\": \"file-capabilities-v1\", \"deletion\": \"unavailable\", \"describeCapabilities\": \"unavailable\", \"discover\": \"unavailable\", \"fileSourceAccess\": \"unavailable\", \"freshness\": \"unavailable\", \"ingestionJobs\": \"unavailable\", \"projectionFields\": [], \"readChanges\": \"unavailable\", \"resourceKinds\": [\"markdown_document\"], \"sourceMode\": \"materialized\"}'::jsonb, '{\"aclEvidenceMode\": \"mirrored\", \"authorizeAndProject\": \"unavailable\", \"batchLimits\": \"unavailable\", \"checkpoint\": \"unavailable\", \"checkpointSemantics\": \"unavailable\", \"contentKinds\": [\"markdown\"], \"consistencyGuarantees\": \"unavailable\", \"cursorSemantics\": \"unavailable\", \"declarationVersion\": \"file-capabilities-v2\", \"deletion\": \"unavailable\", \"describeCapabilities\": \"unavailable\", \"discover\": \"unavailable\", \"fileSourceAccess\": \"available\", \"freshness\": \"unavailable\", \"ingestionJobs\": \"available\", \"projectionFields\": [], \"readChanges\": \"unavailable\", \"resourceKinds\": [\"markdown_document\"], \"sourceMode\": \"materialized\"}'::jsonb)" } ], "rowLevelSecurity": { @@ -483,6 +567,18 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "source_version_file_import_definer_select", + "command": "SELECT", + "roles": ["context_engine_worker_lease_definer"], + "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "source_version_file_import_definer_insert", + "command": "INSERT", + "roles": ["context_engine_worker_lease_definer"], + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, @@ -497,7 +593,8 @@ "context_engine_learning": [], "context_engine_runtime": [], "context_engine_security_operator": [], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["SELECT", "INSERT"] }, "partitions": [], "securityInvariantIds": [ @@ -559,12 +656,32 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "context_resource_file_import_definer_select", + "command": "SELECT", + "roles": ["context_engine_worker_lease_definer"], + "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "context_resource_file_import_definer_insert", + "command": "INSERT", + "roles": ["context_engine_worker_lease_definer"], + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "context_resource_file_import_definer_update", + "command": "UPDATE", + "roles": ["context_engine_worker_lease_definer"], + "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid", + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, "permittedOperations": { "context_engine_runtime": ["SELECT"], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["SELECT", "INSERT", "UPDATE"] }, "partitions": [], "securityInvariantIds": [ @@ -624,6 +741,12 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "context_revision_file_import_definer_insert", + "command": "INSERT", + "roles": ["context_engine_worker_lease_definer"], + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, @@ -635,7 +758,8 @@ }, "permittedOperations": { "context_engine_runtime": ["SELECT"], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["INSERT"] }, "partitions": [], "securityInvariantIds": [ @@ -818,6 +942,12 @@ "roles": ["context_engine_access_policy_definer"], "using": "resource_access_policy.organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid", "withCheck": "resource_access_policy.organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "resource_access_policy_file_import_definer_insert", + "command": "INSERT", + "roles": ["context_engine_worker_lease_definer"], + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, @@ -825,7 +955,8 @@ "context_engine_access_policy_definer": ["SELECT", "UPDATE"], "context_engine_control": ["EXECUTE change_resource_access"], "context_engine_runtime": ["SELECT"], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["INSERT"] }, "partitions": [], "securityInvariantIds": [ @@ -1245,7 +1376,7 @@ "evidenceId": "PG-WORKER-LEASE-007", "selector": {"table": "service_principal"} }, - "purpose": "Bounded registered receiver principal for the Issue #17 no-op carrier; not the full canonical ServiceActor", + "purpose": "Bounded registered receiver principal for exact no-op and File import WorkerLease operations; not the full canonical ServiceActor", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ { @@ -1277,7 +1408,7 @@ }, { "name": "ck_service_principal_workload_issue17", - "expression": "workload = 'supply.noop'" + "expression": "workload IN ('supply.noop', 'supply.file-import')" }, { "name": "ck_service_principal_worker_audience_bounds", @@ -1289,7 +1420,11 @@ }, { "name": "ck_service_principal_operation_noop_complete", - "expression": "operation = 'noop.complete'" + "expression": "operation IN ('noop.complete', 'file.import')" + }, + { + "name": "ck_service_principal_workload_operation_binding", + "expression": "(workload = 'supply.noop' AND operation = 'noop.complete') OR (workload = 'supply.file-import' AND operation = 'file.import')" } ], "rowLevelSecurity": { @@ -1308,6 +1443,12 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "service_principal_file_import_definer_select", + "command": "SELECT", + "roles": ["context_engine_worker_lease_definer"], + "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import' AND enabled IS TRUE" } ] }, @@ -1543,6 +1684,12 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "context_fragment_file_import_definer_insert", + "command": "INSERT", + "roles": ["context_engine_worker_lease_definer"], + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, @@ -1554,7 +1701,8 @@ }, "permittedOperations": { "context_engine_runtime": ["SELECT"], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["INSERT"] }, "partitions": [], "securityInvariantIds": [ @@ -1715,12 +1863,19 @@ "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true" + }, + { + "name": "membership_resource_field_right_file_import_definer_insert", + "command": "INSERT", + "roles": ["context_engine_worker_lease_definer"], + "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" } ] }, "permittedOperations": { "context_engine_runtime": ["SELECT"], - "context_engine_worker": [] + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["INSERT"] }, "partitions": [], "securityInvariantIds": ["TENANT-OWNERSHIP-001", "TENANT-FK-002", "RLS-FAIL-CLOSED-003", "SCOPE-INTERSECTION-004", "INDEX-NOT-AUTHORITY-005"], @@ -2213,6 +2368,196 @@ "securityInvariantIds": ["TENANT-OWNERSHIP-001", "TENANT-FK-002", "RLS-FAIL-CLOSED-003", "RELEASE-OWNER-019"], "negativeTestIds": ["DB-001", "DB-004", "DB-005", "DB-008", "LEARN-004", "LEARN-006", "LEARN-007"] }, + { + "name": "file_acquisition", + "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-FILE-IMPORT-023", + "selector": {"table": "file_acquisition"} + }, + "purpose": "Immutable trusted request to acquire one Markdown filename from one registered File Source version", + "organizationColumn": "organization_id", + "organizationInclusiveKeys": [ + {"name": "pk_file_acquisition", "kind": "primary_key", "columns": ["organization_id", "acquisition_id"]}, + {"name": "uq_file_acquisition_source_idempotency", "kind": "unique", "columns": ["organization_id", "source_id", "idempotency_key"]} + ], + "foreignKeys": [ + { + "name": "fk_file_acquisition_source_version_same_organization", + "columns": ["organization_id", "source_id", "source_version_id"], + "references": {"table": "source_version", "columns": ["organization_id", "source_id", "version_id"]} + }, + { + "name": "fk_file_acquisition_membership_version_same_organization", + "columns": ["organization_id", "audience_membership_id", "audience_membership_version"], + "references": {"table": "membership", "columns": ["organization_id", "membership_id", "membership_version"]} + } + ], + "checkConstraints": [ + {"name": "ck_file_acquisition_one_markdown_filename", "expression": "relative_path ~ '^[^/\\\\]+\\.[mM][dD]$' AND relative_path NOT IN ('.', '..')"}, + {"name": "ck_file_acquisition_principal_nonblank", "expression": "btrim(audience_principal_ref) <> ''"}, + {"name": "ck_file_acquisition_membership_version_positive", "expression": "audience_membership_version > 0"}, + {"name": "ck_file_acquisition_idempotency_key", "expression": "idempotency_key ~ '^[^[:space:]]{1,255}$'"}, + {"name": "ck_file_acquisition_request_digest", "expression": "request_digest ~ '^[0-9a-f]{64}$'"} + ], + "rowLevelSecurity": { + "enabled": true, + "forced": true, + "policies": [ + {"name": "file_acquisition_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, + {"name": "file_acquisition_file_import_definer_select", "command": "SELECT", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"}, + {"name": "file_acquisition_file_import_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"} + ] + }, + "immutableRows": {"trigger": "file_acquisition_immutable", "function": "context_content_reject_mutation", "events": ["UPDATE", "DELETE"], "sqlstate": "55000"}, + "functionOnlyMutation": {"databaseFunction": "context_control_prepare_file_import", "role": "context_engine_control", "definerRole": "context_engine_worker_lease_definer", "directTableMutationAllowed": false}, + "permittedOperations": {"context_engine_control": ["EXECUTE context_control_prepare_file_import"], "context_engine_runtime": [], "context_engine_worker": [], "context_engine_worker_lease_definer": ["SELECT", "INSERT"]}, + "partitions": [], + "securityInvariantIds": ["TENANT-OWNERSHIP-001", "TENANT-FK-002", "RLS-FAIL-CLOSED-003", "WORKER-LEASE-007"], + "negativeTestIds": ["DB-001", "DB-004", "DB-008", "JOB-001", "WORKER-LEASE-007"] + }, + { + "name": "file_import_job", + "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-FILE-IMPORT-023", + "selector": {"table": "file_import_job"} + }, + "purpose": "Durable one-shot File import job and exact WorkerLease redemption state", + "organizationColumn": "organization_id", + "organizationInclusiveKeys": [ + {"name": "pk_file_import_job", "kind": "primary_key", "columns": ["organization_id", "job_id"]}, + {"name": "uq_file_import_job_acquisition", "kind": "unique", "columns": ["organization_id", "acquisition_id"]} + ], + "foreignKeys": [ + { + "name": "fk_file_import_job_acquisition_same_organization", + "columns": ["organization_id", "acquisition_id"], + "references": {"table": "file_acquisition", "columns": ["organization_id", "acquisition_id"]} + }, + { + "name": "fk_file_import_job_service_principal_binding", + "columns": ["organization_id", "service_principal_id", "workload", "worker_audience", "operation"], + "references": {"table": "service_principal", "columns": ["organization_id", "service_principal_id", "workload", "worker_audience", "operation"]} + } + ], + "checkConstraints": [ + {"name": "ck_file_import_job_workload", "expression": "workload = 'supply.file-import'"}, + {"name": "ck_file_import_job_worker_audience", "expression": "worker_audience = 'context-engine-worker'"}, + {"name": "ck_file_import_job_actor_kind", "expression": "actor_kind = 'service'"}, + {"name": "ck_file_import_job_operation", "expression": "operation = 'file.import'"}, + {"name": "ck_file_import_job_state", "expression": "state IN ('available', 'leased', 'running', 'failed', 'completed')"}, + {"name": "ck_file_import_job_state_consistency", "expression": "state transitions bind one lease nonce, exact time window, one completion lineage, and effect_count IN (0, 1)"} + ], + "rowLevelSecurity": { + "enabled": true, + "forced": true, + "policies": [ + {"name": "file_import_job_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, + {"name": "file_import_job_definer_select", "command": "SELECT", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'"}, + {"name": "file_import_job_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'"}, + {"name": "file_import_job_definer_update", "command": "UPDATE", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'", "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'"} + ] + }, + "functionOnlyMutation": {"databaseFunctions": ["context_control_prepare_file_import", "context_worker_issue_file_import_lease", "context_worker_redeem_file_import", "context_worker_fail_file_import", "context_worker_publish_file_import"], "definerRole": "context_engine_worker_lease_definer", "directTableMutationAllowed": false}, + "permittedOperations": {"context_engine_control": ["EXECUTE context_control_prepare_file_import", "EXECUTE context_worker_issue_file_import_lease"], "context_engine_runtime": [], "context_engine_worker": ["EXECUTE context_worker_redeem_file_import", "EXECUTE context_worker_fail_file_import", "EXECUTE context_worker_publish_file_import"], "context_engine_worker_lease_definer": ["SELECT", "INSERT", "UPDATE"]}, + "partitions": [], + "securityInvariantIds": ["TENANT-OWNERSHIP-001", "TENANT-FK-002", "RLS-FAIL-CLOSED-003", "WORKER-LEASE-007"], + "negativeTestIds": ["DB-001", "DB-004", "DB-008", "JOB-001", "JOB-005", "WORKER-LEASE-007"] + }, + { + "name": "file_revision_snapshot", + "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-FILE-IMPORT-023", + "selector": {"table": "file_revision_snapshot"} + }, + "purpose": "Immutable canonical Markdown compilation payload and digest lineage for one Revision", + "organizationColumn": "organization_id", + "organizationInclusiveKeys": [ + {"name": "pk_file_revision_snapshot", "kind": "primary_key", "columns": ["organization_id", "resource_ref", "revision_id"]} + ], + "foreignKeys": [ + {"name": "fk_file_revision_snapshot_revision_same_organization", "columns": ["organization_id", "resource_ref", "revision_id"], "references": {"table": "context_revision", "columns": ["organization_id", "resource_ref", "revision_id"]}}, + {"name": "fk_file_revision_snapshot_acquisition_same_organization", "columns": ["organization_id", "acquisition_id"], "references": {"table": "file_acquisition", "columns": ["organization_id", "acquisition_id"]}} + ], + "checkConstraints": [ + {"name": "ck_file_revision_snapshot_content_hash", "expression": "content_hash ~ '^[0-9a-f]{64}$'"}, + {"name": "ck_file_revision_snapshot_compilation_digest", "expression": "compilation_digest ~ '^[0-9a-f]{64}$'"} + ], + "rowLevelSecurity": {"enabled": true, "forced": true, "policies": [ + {"name": "file_revision_snapshot_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, + {"name": "file_revision_snapshot_file_import_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"} + ]}, + "immutableRows": {"trigger": "file_revision_snapshot_immutable", "function": "context_content_reject_mutation", "events": ["UPDATE", "DELETE"], "sqlstate": "55000"}, + "permittedOperations": {"context_engine_control": [], "context_engine_runtime": [], "context_engine_worker": ["EXECUTE context_worker_publish_file_import"], "context_engine_worker_lease_definer": ["INSERT"]}, + "partitions": [], + "securityInvariantIds": ["TENANT-OWNERSHIP-001", "TENANT-FK-002", "RLS-FAIL-CLOSED-003", "TRACE-REDACTION-012"], + "negativeTestIds": ["DB-001", "DB-004", "DB-008", "WORKER-LEASE-007"] + }, + { + "name": "revision_publication_event", + "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-FILE-IMPORT-023", + "selector": {"table": "revision_publication_event"} + }, + "purpose": "Immutable ordered prepared-indexed-active publication evidence for one Revision", + "organizationColumn": "organization_id", + "organizationInclusiveKeys": [ + {"name": "pk_revision_publication_event", "kind": "primary_key", "columns": ["organization_id", "resource_ref", "revision_id", "ordinal"]}, + {"name": "uq_revision_publication_event_state", "kind": "unique", "columns": ["organization_id", "resource_ref", "revision_id", "state"]} + ], + "foreignKeys": [ + {"name": "fk_revision_publication_event_revision_same_organization", "columns": ["organization_id", "resource_ref", "revision_id"], "references": {"table": "context_revision", "columns": ["organization_id", "resource_ref", "revision_id"]}} + ], + "checkConstraints": [ + {"name": "ck_revision_publication_event_order", "expression": "(ordinal, state) IN ((0, 'prepared'), (1, 'indexed'), (2, 'active'))"} + ], + "rowLevelSecurity": {"enabled": true, "forced": true, "policies": [ + {"name": "revision_publication_event_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, + {"name": "revision_publication_event_file_import_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"}, + {"name": "revision_publication_event_current_user_actor", "command": "SELECT", "roles": ["context_engine_runtime"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND current_setting('app.actor_kind', true) = 'user' AND EXISTS (SELECT 1 FROM public.membership AS actor_membership WHERE actor_membership.organization_id = revision_publication_event.organization_id AND actor_membership.user_id = NULLIF(current_setting('app.user_id', true), '')::uuid AND actor_membership.membership_id = NULLIF(current_setting('app.membership_id', true), '')::uuid AND actor_membership.membership_version = NULLIF(current_setting('app.membership_version', true), '')::bigint AND actor_membership.status = 'active' AND actor_membership.valid_from <= NULLIF(current_setting('app.checked_at', true), '')::timestamptz AND (actor_membership.valid_until IS NULL OR actor_membership.valid_until > NULLIF(current_setting('app.checked_at', true), '')::timestamptz)) AND EXISTS (SELECT 1 FROM public.resource_access_policy AS access_policy WHERE access_policy.organization_id = revision_publication_event.organization_id AND access_policy.resource_ref = revision_publication_event.resource_ref AND access_policy.principal_ref = current_setting('app.principal_ref', true) AND access_policy.access_state = 'allowed')"} + ]}, + "immutableRows": {"trigger": "revision_publication_event_immutable", "function": "context_content_reject_mutation", "events": ["UPDATE", "DELETE"], "sqlstate": "55000"}, + "permittedOperations": {"context_engine_control": [], "context_engine_runtime": ["SELECT"], "context_engine_worker": ["EXECUTE context_worker_publish_file_import"], "context_engine_worker_lease_definer": ["INSERT"]}, + "partitions": [], + "securityInvariantIds": ["TENANT-OWNERSHIP-001", "TENANT-FK-002", "RLS-FAIL-CLOSED-003", "TRACE-REDACTION-012"], + "negativeTestIds": ["DB-001", "DB-004", "DB-008", "WORKER-LEASE-007"] + }, + { + "name": "exact_phrase_candidate", + "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-FILE-IMPORT-023", + "selector": {"table": "exact_phrase_candidate"} + }, + "purpose": "Content-free deterministic exact-phrase CandidateIndex; never an authorization authority", + "organizationColumn": "organization_id", + "organizationInclusiveKeys": [ + {"name": "pk_exact_phrase_candidate", "kind": "primary_key", "columns": ["organization_id", "phrase_digest", "resource_ref", "revision_id", "fragment_ref"]} + ], + "foreignKeys": [ + {"name": "fk_exact_phrase_candidate_fragment_same_organization", "columns": ["organization_id", "resource_ref", "revision_id", "fragment_ref"], "references": {"table": "context_fragment", "columns": ["organization_id", "resource_ref", "revision_id", "fragment_ref"]}} + ], + "checkConstraints": [ + {"name": "ck_exact_phrase_candidate_digest", "expression": "phrase_digest ~ '^[0-9a-f]{64}$'"} + ], + "rowLevelSecurity": { + "enabled": true, + "forced": true, + "policies": [ + {"name": "exact_phrase_candidate_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, + {"name": "exact_phrase_candidate_file_import_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"}, + {"name": "exact_phrase_candidate_runtime", "command": "SELECT", "roles": ["context_engine_runtime"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND current_setting('app.actor_kind', true) = 'user' AND NULLIF(current_setting('app.principal_ref', true), '') IS NOT NULL AND NULLIF(current_setting('app.request_id', true), '') IS NOT NULL AND NULLIF(current_setting('app.authentication_binding_ref', true), '') IS NOT NULL AND NULLIF(current_setting('app.checked_at', true), '') IS NOT NULL AND EXISTS (SELECT 1 FROM public.membership AS actor_membership WHERE actor_membership.organization_id = exact_phrase_candidate.organization_id AND actor_membership.user_id = NULLIF(current_setting('app.user_id', true), '')::uuid AND actor_membership.membership_id = NULLIF(current_setting('app.membership_id', true), '')::uuid AND actor_membership.membership_version = NULLIF(current_setting('app.membership_version', true), '')::bigint AND actor_membership.status = 'active' AND actor_membership.valid_from <= NULLIF(current_setting('app.checked_at', true), '')::timestamptz AND (actor_membership.valid_until IS NULL OR actor_membership.valid_until > NULLIF(current_setting('app.checked_at', true), '')::timestamptz))"} + ] + }, + "immutableRows": {"trigger": "exact_phrase_candidate_immutable", "function": "context_content_reject_mutation", "events": ["UPDATE", "DELETE"], "sqlstate": "55000"}, + "permittedOperations": {"context_engine_control": [], "context_engine_runtime": ["SELECT"], "context_engine_worker": ["EXECUTE context_worker_publish_file_import"], "context_engine_worker_lease_definer": ["INSERT"]}, + "partitions": [], + "securityInvariantIds": ["TENANT-OWNERSHIP-001", "TENANT-FK-002", "RLS-FAIL-CLOSED-003", "INDEX-NOT-AUTHORITY-005"], + "negativeTestIds": ["DB-001", "DB-002", "DB-004", "DB-008", "DB-009", "DB-010", "PG-INDEX-NOT-AUTHORITY-005"] + }, { "name": "release_promotion_audit", "classification": "tenant_owned", diff --git a/engine/persistence/worker_jobs.py b/engine/persistence/worker_jobs.py index e025a4c7..89dcf040 100644 --- a/engine/persistence/worker_jobs.py +++ b/engine/persistence/worker_jobs.py @@ -12,8 +12,10 @@ from sqlalchemy import Connection, Engine, text from sqlalchemy.exc import SQLAlchemyError +from engine.control import PreparedFileImport from engine.persistence.role_guard import assert_control_role, assert_worker_role from engine.supply.jobs import ( + FILE_IMPORT_WORKER_LEASE_OPERATION, WORKER_LEASE_ACTOR_KIND, WORKER_LEASE_OPERATION, WorkerLeaseClaims, @@ -235,6 +237,66 @@ def issue_noop_lease( "worker lease issuance database work failed" ) from None + def issue_file_import_lease( + self, + prepared: PreparedFileImport, + ) -> WorkerLeaseToken: + """Lease one prepared File import with exact Source binding.""" + + if type(prepared) is not PreparedFileImport: + raise TypeError("File import lease requires PreparedFileImport") + nonce = generate_worker_lease_nonce() + try: + with self._control_engine.begin() as connection: + self._require_control_role(connection) + row = connection.execute( + text( + """ + SELECT issued_at, expires_at + FROM public.context_worker_issue_file_import_lease( + :organization_id, :job_id, + :service_principal_id, :source_ref, + :signing_key_version, :nonce, + :lease_ttl_seconds + ) + """ + ), + { + "organization_id": prepared.organization_id, + "job_id": prepared.job_id, + "service_principal_id": prepared.service_principal_id, + "source_ref": str(prepared.source_ref.value), + "signing_key_version": ( + self._codec.active_signing_key_version + ), + "nonce": nonce, + "lease_ttl_seconds": self._lease_ttl_seconds, + }, + ).one_or_none() + if row is None: + raise WorkerLeaseIssueNotAvailable + claims = WorkerLeaseClaims( + signing_key_version=self._codec.active_signing_key_version, + organization_id=prepared.organization_id, + job_id=prepared.job_id, + service_principal_id=prepared.service_principal_id, + workload=prepared.workload, + worker_audience=prepared.worker_audience, + issued_at=_require_utc("issued_at", row.issued_at), + expires_at=_require_utc("expires_at", row.expires_at), + nonce=nonce, + operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + source_ref=str(prepared.source_ref.value), + ) + token = self._codec.mint(claims) + return token + except (WorkerLeaseIssueNotAvailable, WorkerLeaseAuthorityUnavailable): + raise + except SQLAlchemyError: + raise WorkerLeaseAuthorityUnavailable( + "File import lease issuance database work failed" + ) from None + @staticmethod def _require_control_role(connection: Connection) -> None: try: diff --git a/engine/runtime/construction.py b/engine/runtime/construction.py index 5e851890..052ff333 100644 --- a/engine/runtime/construction.py +++ b/engine/runtime/construction.py @@ -456,7 +456,7 @@ def authorize_acquire( raise RuntimeConfigurationError( "candidate discovery requires same-transaction projection session" ) - discovered = candidate_index.discover(request) + discovered = candidate_index.discover(request, projection_session) if type(discovered) is not tuple or any( type(candidate) is not CandidateRef for candidate in discovered ): diff --git a/engine/runtime/content_io.py b/engine/runtime/content_io.py index b2a1af3f..aac58ee3 100644 --- a/engine/runtime/content_io.py +++ b/engine/runtime/content_io.py @@ -1,16 +1,32 @@ """Content-free discovery plus prohibited legacy content seams.""" from dataclasses import dataclass +from hashlib import sha256 from typing import Protocol from engine.runtime.contracts import Acquire from engine.runtime.evidence import CandidateRef +from engine.runtime.materialized import MaterializedProjectionSession + +_EXACT_PHRASE_DIGEST_DOMAIN = b"context-engine.exact-phrase.v1\x00" + + +def exact_phrase_digest(value: str) -> str: + """Digest exact UTF-8 query text for the content-free candidate index.""" + + if type(value) is not str or not value or value.isspace(): + raise ValueError("exact phrase must be nonblank") + return sha256(_EXACT_PHRASE_DIGEST_DOMAIN + value.encode("utf-8")).hexdigest() class CandidateIndex(Protocol): """Content-free candidate discovery seam; never an authorization source.""" - def discover(self, request: Acquire) -> tuple[CandidateRef, ...]: ... + def discover( + self, + request: Acquire, + projection_session: MaterializedProjectionSession, + ) -> tuple[CandidateRef, ...]: ... class ContextProvider(Protocol): @@ -35,7 +51,11 @@ class RuntimeContentIo: class _ProhibitedCandidateIndex: - def discover(self, request: Acquire) -> tuple[()]: + def discover( + self, + request: Acquire, + projection_session: MaterializedProjectionSession, + ) -> tuple[()]: raise RuntimeError("candidate index is prohibited on the empty Package path") diff --git a/engine/runtime/materialized.py b/engine/runtime/materialized.py index b1a3d4b3..6b959b59 100644 --- a/engine/runtime/materialized.py +++ b/engine/runtime/materialized.py @@ -19,6 +19,7 @@ "MaterializedProjectionKind", "MaterializedProjectionPort", "MaterializedProjectionSession", + "MaterializedPublicationTrace", ] _STRUCTURED_FIELD_LINE_BREAKS: Final = frozenset( @@ -179,9 +180,32 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True, slots=True) +class MaterializedPublicationTrace: + """Authorized, content-free observation of one active publication lineage.""" + + states: tuple[str, ...] + active_revision_ref: str + + def __post_init__(self) -> None: + if self.states != ("prepared", "indexed", "active"): + raise ValueError("publication trace must have the closed initial sequence") + _require_nonblank_ref("active revision ref", self.active_revision_ref) + + class MaterializedProjectionPort(Protocol): """Narrow operations executed by the owning current database transaction.""" + def discover_exact_phrase( + self, + phrase_digest: str, + ) -> tuple[CandidateRef, ...]: ... + + def observe_publication( + self, + candidate_ref: CandidateRef, + ) -> MaterializedPublicationTrace | None: ... + def locate( self, candidate_ref: CandidateRef, @@ -277,8 +301,11 @@ def _construct_materialized_projection_session( "materialized projection requires an active materialized projection " "scope" ) - if not callable(getattr(port, "locate", None)) or not callable( - getattr(port, "project", None) + if ( + not callable(getattr(port, "locate", None)) + or not callable(getattr(port, "project", None)) + or not callable(getattr(port, "discover_exact_phrase", None)) + or not callable(getattr(port, "observe_publication", None)) ): raise TypeError("materialized projection port is incomplete") session = object.__new__(MaterializedProjectionSession) @@ -300,6 +327,40 @@ def _locate_materialized_fragment( return locator +def _discover_materialized_exact_phrase( + session: MaterializedProjectionSession, + phrase_digest: str, +) -> tuple[CandidateRef, ...]: + """Discover content-free lineage on the retained current-UserActor transaction.""" + + _require_active_materialized_projection_session(session) + if type(phrase_digest) is not str or not phrase_digest: + raise ValueError("exact phrase digest must be nonblank") + candidates = session._port.discover_exact_phrase(phrase_digest) + if type(candidates) is not tuple or any( + type(candidate) is not CandidateRef for candidate in candidates + ): + raise TypeError( + "materialized exact discovery must return exact CandidateRef values" + ) + return candidates + + +def _observe_materialized_publication( + session: MaterializedProjectionSession, + candidate_ref: CandidateRef, +) -> MaterializedPublicationTrace | None: + """Read initial publication state on the current UserActor transaction.""" + + _require_active_materialized_projection_session(session) + if type(candidate_ref) is not CandidateRef: + raise TypeError("publication observation requires CandidateRef") + observed = session._port.observe_publication(candidate_ref) + if observed is not None and type(observed) is not MaterializedPublicationTrace: + raise TypeError("publication observation returned the wrong nominal type") + return observed + + def _project_materialized_fragment( session: MaterializedProjectionSession, locator: MaterializedFragmentLocator, diff --git a/engine/supply/__init__.py b/engine/supply/__init__.py index 6cfe7f46..de322796 100644 --- a/engine/supply/__init__.py +++ b/engine/supply/__init__.py @@ -1,6 +1,14 @@ """Public Supply domain contracts.""" +from engine.control.file_imports import ( + FileImportAudience, + FileImportPath, + FileImportReceiver, + PreparedFileImport, + PrepareFileImport, +) from engine.supply.jobs import ( + FILE_IMPORT_WORKER_LEASE_OPERATION, WORKER_LEASE_ACTOR_KIND, WORKER_LEASE_OPERATION, WorkerLeaseClaims, @@ -49,9 +57,15 @@ "CompilationProvenance", "CompilationWarning", "CompilationWarningCode", + "FileImportAudience", + "FileImportPath", + "FileImportReceiver", + "FILE_IMPORT_WORKER_LEASE_OPERATION", "MarkdownCompilerConfig", "ParsedDocument", "ParsedSection", + "PrepareFileImport", + "PreparedFileImport", "SectionKind", "SourcePoint", "SourceSpan", diff --git a/engine/supply/jobs.py b/engine/supply/jobs.py index 02d786ab..0f5e78a8 100644 --- a/engine/supply/jobs.py +++ b/engine/supply/jobs.py @@ -18,9 +18,11 @@ WORKER_LEASE_ACTOR_KIND: Final = "service" WORKER_LEASE_OPERATION: Final = "noop.complete" +FILE_IMPORT_WORKER_LEASE_OPERATION: Final = "file.import" _ALGORITHM: Final = "HS256" _TOKEN_TYPE: Final = "CE-WorkerLease" _TOKEN_VERSION: Final = 1 +_FILE_IMPORT_TOKEN_VERSION: Final = 2 _DOMAIN: Final = "context-engine.worker-lease" _MAX_KEY_VERSION: Final = (1 << 63) - 1 _MINIMUM_SECRET_BYTES: Final = 32 @@ -42,6 +44,7 @@ "workload", } ) +_FILE_IMPORT_CLAIM_FIELDS: Final = _CLAIM_FIELDS | frozenset({"source_ref"}) def _require_key_version(value: object) -> int: @@ -95,12 +98,11 @@ class WorkerLeaseClaims: issued_at: datetime = field(repr=False) expires_at: datetime = field(repr=False) nonce: bytes = field(repr=False) + operation: str = field(default=WORKER_LEASE_OPERATION, repr=False) + source_ref: str | None = field(default=None, repr=False) actor_kind: Literal["service"] = field( default=WORKER_LEASE_ACTOR_KIND, init=False, repr=False ) - operation: Literal["noop.complete"] = field( - default=WORKER_LEASE_OPERATION, init=False, repr=False - ) def __post_init__(self) -> None: _require_key_version(self.signing_key_version) @@ -117,6 +119,16 @@ def __post_init__(self) -> None: raise ValueError("WorkerLease expiry must follow issuance") if type(self.nonce) is not bytes or len(self.nonce) != _NONCE_BYTES: raise ValueError("WorkerLease nonce must contain exactly 256 bits") + if self.operation not in { + WORKER_LEASE_OPERATION, + FILE_IMPORT_WORKER_LEASE_OPERATION, + }: + raise ValueError("WorkerLease operation must be closed") + if self.operation == WORKER_LEASE_OPERATION: + if self.source_ref is not None: + raise ValueError("no-op WorkerLease cannot bind a source") + else: + _require_identifier("source_ref", self.source_ref, maximum_length=255) def __reduce__(self) -> NoReturn: raise TypeError("WorkerLeaseClaims are not serializable") @@ -257,7 +269,7 @@ def _timestamp(value: datetime) -> str: def _claims_document(claims: WorkerLeaseClaims) -> dict[str, object]: - return { + document: dict[str, object] = { "actor_kind": claims.actor_kind, "expires_at": _timestamp(claims.expires_at), "issued_at": _timestamp(claims.issued_at), @@ -270,6 +282,9 @@ def _claims_document(claims: WorkerLeaseClaims) -> dict[str, object]: "worker_audience": claims.worker_audience, "workload": claims.workload, } + if claims.operation == FILE_IMPORT_WORKER_LEASE_OPERATION: + document["source_ref"] = claims.source_ref + return document class WorkerLeaseCodec: @@ -296,12 +311,17 @@ def mint(self, claims: WorkerLeaseClaims) -> WorkerLeaseToken: key = self._keyring._key_for(claims.signing_key_version) if key is None: # pragma: no cover - keyring construction proves this raise ValueError("active WorkerLease signing key is unavailable") + token_version = ( + _TOKEN_VERSION + if claims.operation == WORKER_LEASE_OPERATION + else _FILE_IMPORT_TOKEN_VERSION + ) header = { "alg": _ALGORITHM, "dom": _DOMAIN, "kid": claims.signing_key_version, "typ": _TOKEN_TYPE, - "v": _TOKEN_VERSION, + "v": token_version, } encoded_header = _base64url_encode(_canonical_json(header)) encoded_claims = _base64url_encode(_canonical_json(_claims_document(claims))) @@ -319,9 +339,10 @@ def verify( expected_job_id: UUID, expected_service_principal_id: UUID, expected_workload: str, - expected_operation: Literal["noop.complete"], + expected_operation: str, expected_worker_audience: str, now: datetime, + expected_source_ref: str | None = None, ) -> WorkerLeaseClaims: if type(token) is not WorkerLeaseToken: raise TypeError("token must be WorkerLeaseToken") @@ -336,6 +357,19 @@ def verify( expected_worker_audience, maximum_length=255, ) + if expected_operation not in { + WORKER_LEASE_OPERATION, + FILE_IMPORT_WORKER_LEASE_OPERATION, + }: + raise WorkNotAvailable( + WorkerLeaseRejectionAuditReceipt(worker_lease_digest(token)) + ) + if expected_operation == FILE_IMPORT_WORKER_LEASE_OPERATION: + _require_identifier( + "expected_source_ref", expected_source_ref, maximum_length=255 + ) + elif expected_source_ref is not None: + raise ValueError("no-op verification cannot bind a source") checked_at = _require_utc("now", now) try: claims = self._verify_signed(token) @@ -345,6 +379,7 @@ def verify( or claims.service_principal_id != expected_service_principal_id or claims.workload != expected_workload or claims.operation != expected_operation + or claims.source_ref != expected_source_ref or claims.worker_audience != expected_worker_audience or checked_at < claims.issued_at or checked_at >= claims.expires_at @@ -367,12 +402,15 @@ def _verify_signed(self, token: WorkerLeaseToken) -> WorkerLeaseClaims: header = _decode_document(encoded_header, _HEADER_FIELDS) if type(header["v"]) is not int: raise ValueError + token_version = header["v"] + if token_version not in {_TOKEN_VERSION, _FILE_IMPORT_TOKEN_VERSION}: + raise ValueError if header != { "alg": _ALGORITHM, "dom": _DOMAIN, "kid": header.get("kid"), "typ": _TOKEN_TYPE, - "v": _TOKEN_VERSION, + "v": token_version, }: raise ValueError key_version = _require_key_version(header["kid"]) @@ -386,7 +424,12 @@ def _verify_signed(self, token: WorkerLeaseToken) -> WorkerLeaseClaims: expected_signature = hmac.digest(key, signing_input, "sha256") if not hmac.compare_digest(supplied_signature, expected_signature): raise ValueError - document = _decode_document(encoded_claims, _CLAIM_FIELDS) + claim_fields = ( + _CLAIM_FIELDS + if token_version == _TOKEN_VERSION + else _FILE_IMPORT_CLAIM_FIELDS + ) + document = _decode_document(encoded_claims, claim_fields) if ( type(document["signing_key_version"]) is not int or document["signing_key_version"] != key_version @@ -394,7 +437,12 @@ def _verify_signed(self, token: WorkerLeaseToken) -> WorkerLeaseClaims: raise ValueError if document["actor_kind"] != WORKER_LEASE_ACTOR_KIND: raise ValueError - if document["operation"] != WORKER_LEASE_OPERATION: + expected_operation = ( + WORKER_LEASE_OPERATION + if token_version == _TOKEN_VERSION + else FILE_IMPORT_WORKER_LEASE_OPERATION + ) + if document["operation"] != expected_operation: raise ValueError return WorkerLeaseClaims( signing_key_version=key_version, @@ -406,6 +454,12 @@ def _verify_signed(self, token: WorkerLeaseToken) -> WorkerLeaseClaims: issued_at=_parse_timestamp(document["issued_at"]), expires_at=_parse_timestamp(document["expires_at"]), nonce=_base64url_decode(cast(str, document["nonce"])), + operation=cast(str, document["operation"]), + source_ref=( + cast(str, document["source_ref"]) + if token_version == _FILE_IMPORT_TOKEN_VERSION + else None + ), ) diff --git a/eval/catalogs/m0-security-evidence.yaml b/eval/catalogs/m0-security-evidence.yaml index 9d81d510..b29a4ab5 100644 --- a/eval/catalogs/m0-security-evidence.yaml +++ b/eval/catalogs/m0-security-evidence.yaml @@ -49,6 +49,7 @@ {"id": "PROP-TENANT-FK-002", "layer": "property", "selector": "tests/unit/test_schema_security_manifest.py::test_content_manifest_preserves_lineage_visibility_and_immutability"}, {"id": "PG-TENANT-FK-002", "layer": "postgres", "selector": "tests/integration/test_organization_isolation.py::test_composite_ownership_accepts_same_org_and_rejects_cross_org_parent"}, {"id": "PG-FILE-SOURCE-FK-021", "layer": "postgres", "selector": "tests/integration/test_file_source_registration.py::test_source_version_is_immutable_and_active_pointer_stays_in_organization"}, + {"id": "PG-FILE-IMPORT-023", "layer": "postgres", "selector": "tests/integration/test_file_import_tracer.py::test_registered_file_import_publishes_one_exact_authorized_http_package"}, {"id": "RUNTIME-TENANT-FK-002", "layer": "runtime", "selector": "tests/integration/test_runtime_authorized_evidence_integration.py::test_real_postgres_http_delivers_only_exact_authorized_evidence_bidirectionally"}, {"id": "PROP-RLS-FAIL-CLOSED-003", "layer": "property", "selector": "tests/unit/test_schema_security_manifest.py::test_tenant_owned_manifest_entry_preserves_every_security_property"}, {"id": "PG-RLS-FAIL-CLOSED-003", "layer": "postgres", "selector": "tests/integration/test_organization_isolation.py::test_missing_tenant_context_is_fail_closed_for_every_operation"}, @@ -104,13 +105,13 @@ {"id": "FIXTURE-ACCEPT-012", "layer": "runtime", "selector": "tests/unit/test_ticket_audience_separation.py::test_accept_012_context_read_ticket_cannot_create_an_action_effect"} ], "invariantMappings": [ - {"invariantRef": "TENANT-OWNERSHIP-001", "evidenceRefs": {"property": ["PROP-TENANT-OWNERSHIP-001"], "postgres": ["PG-FILE-SOURCE-RLS-021", "PG-RLS-ALL-TENANT-TABLES"], "runtime": ["RUNTIME-TENANT-OWNERSHIP-001"]}}, - {"invariantRef": "TENANT-FK-002", "evidenceRefs": {"property": ["PROP-TENANT-FK-002"], "postgres": ["PG-TENANT-FK-002", "PG-FILE-SOURCE-FK-021"], "runtime": ["RUNTIME-TENANT-FK-002"]}}, - {"invariantRef": "RLS-FAIL-CLOSED-003", "evidenceRefs": {"property": ["PROP-RLS-FAIL-CLOSED-003"], "postgres": ["PG-RLS-FAIL-CLOSED-003", "PG-RLS-ALL-TENANT-TABLES"], "runtime": ["RUNTIME-RLS-FAIL-CLOSED-003"]}}, + {"invariantRef": "TENANT-OWNERSHIP-001", "evidenceRefs": {"property": ["PROP-TENANT-OWNERSHIP-001"], "postgres": ["PG-FILE-SOURCE-RLS-021", "PG-RLS-ALL-TENANT-TABLES", "PG-FILE-IMPORT-023"], "runtime": ["RUNTIME-TENANT-OWNERSHIP-001"]}}, + {"invariantRef": "TENANT-FK-002", "evidenceRefs": {"property": ["PROP-TENANT-FK-002"], "postgres": ["PG-TENANT-FK-002", "PG-FILE-SOURCE-FK-021", "PG-FILE-IMPORT-023"], "runtime": ["RUNTIME-TENANT-FK-002"]}}, + {"invariantRef": "RLS-FAIL-CLOSED-003", "evidenceRefs": {"property": ["PROP-RLS-FAIL-CLOSED-003"], "postgres": ["PG-RLS-FAIL-CLOSED-003", "PG-RLS-ALL-TENANT-TABLES", "PG-FILE-IMPORT-023"], "runtime": ["RUNTIME-RLS-FAIL-CLOSED-003"]}}, {"invariantRef": "SCOPE-INTERSECTION-004", "evidenceRefs": {"property": ["PROP-SCOPE-INTERSECTION-004"], "postgres": ["PG-SCOPE-INTERSECTION-004", "PG-FIELD-PROJECTION-RLS-048"], "runtime": ["RUNTIME-SCOPE-INTERSECTION-004"]}}, - {"invariantRef": "INDEX-NOT-AUTHORITY-005", "evidenceRefs": {"property": ["PROP-INDEX-NOT-AUTHORITY-005"], "postgres": ["PG-INDEX-NOT-AUTHORITY-005"], "runtime": ["RUNTIME-INDEX-NOT-AUTHORITY-005"]}}, + {"invariantRef": "INDEX-NOT-AUTHORITY-005", "evidenceRefs": {"property": ["PROP-INDEX-NOT-AUTHORITY-005"], "postgres": ["PG-INDEX-NOT-AUTHORITY-005", "PG-FILE-IMPORT-023"], "runtime": ["RUNTIME-INDEX-NOT-AUTHORITY-005"]}}, {"invariantRef": "REVOCATION-006", "evidenceRefs": {"property": ["PROP-REVOCATION-006"], "postgres": ["PG-REVOCATION-006"], "runtime": ["RUNTIME-REVOCATION-006"]}}, - {"invariantRef": "WORKER-LEASE-007", "evidenceRefs": {"property": ["PROP-WORKER-LEASE-007"], "postgres": ["PG-WORKER-LEASE-007"], "runtime": ["RUNTIME-WORKER-LEASE-007"]}}, + {"invariantRef": "WORKER-LEASE-007", "evidenceRefs": {"property": ["PROP-WORKER-LEASE-007"], "postgres": ["PG-WORKER-LEASE-007", "PG-FILE-IMPORT-023"], "runtime": ["RUNTIME-WORKER-LEASE-007"]}}, {"invariantRef": "TRANSPORT-UNTRUSTED-008", "evidenceRefs": {"property": ["PROP-TRANSPORT-UNTRUSTED-008"], "postgres": ["PG-TRANSPORT-UNTRUSTED-008"], "runtime": ["RUNTIME-TRANSPORT-UNTRUSTED-008"]}}, {"invariantRef": "NON-ENUMERATION-009", "evidenceRefs": {"property": ["PROP-NON-ENUMERATION-009"], "postgres": ["PG-NON-ENUMERATION-009"], "runtime": ["RUNTIME-NON-ENUMERATION-009"]}}, {"invariantRef": "CITATION-AUTH-010", "evidenceRefs": {"property": ["PROP-CITATION-AUTH-010"], "postgres": ["PG-CITATION-AUTH-010"], "runtime": ["RUNTIME-CITATION-AUTH-010"]}}, diff --git a/migrations/versions/20260722_0011_file_import_tracer.py b/migrations/versions/20260722_0011_file_import_tracer.py new file mode 100644 index 00000000..a8525eb0 --- /dev/null +++ b/migrations/versions/20260722_0011_file_import_tracer.py @@ -0,0 +1,998 @@ +"""Publish one registered Markdown file through an exact WorkerLease. + +Revision ID: 20260722_0011 +Revises: 20260722_0010 +Create Date: 2026-07-22 +""" + +# ruff: noqa: E501 + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "20260722_0011" +down_revision: str | None = "20260722_0010" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_MIGRATOR = "context_engine_migrator" +_CONTROL = "context_engine_control" +_RUNTIME = "context_engine_runtime" +_WORKER = "context_engine_worker" +_DEFINER = "context_engine_worker_lease_definer" +_WORKLOAD = "supply.file-import" +_AUDIENCE = "context-engine-worker" +_OPERATION = "file.import" +_MAX_BIGINT = 2**63 - 1 +_MAX_TTL = 3600 + + +def _tenant_table(table: str) -> None: + for role in ("PUBLIC", _CONTROL, _RUNTIME, _WORKER, _DEFINER): + op.execute(f"REVOKE ALL ON TABLE {table} FROM {role}") + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute( + f"CREATE POLICY {table}_migrator_administration ON {table} " + f"FOR ALL TO {_MIGRATOR} USING (true) WITH CHECK (true)" + ) + + +def _immutable(table: str) -> None: + op.execute( + f"CREATE TRIGGER {table}_immutable BEFORE UPDATE OR DELETE ON {table} " + "FOR EACH ROW EXECUTE FUNCTION public.context_content_reject_mutation()" + ) + + +def upgrade() -> None: + """Create the narrow acquisition, job, publication, and exact index path.""" + + op.drop_constraint( + "ck_source_version_issue_21_capabilities", + "source_version", + type_="check", + ) + op.create_check_constraint( + "ck_source_version_file_capabilities", + "source_version", + "capability_manifest IN (" + "'{\"aclEvidenceMode\": \"mirrored\", \"authorizeAndProject\": " + "\"unavailable\", \"batchLimits\": \"unavailable\", \"checkpoint\": " + "\"unavailable\", \"checkpointSemantics\": \"unavailable\", " + "\"contentKinds\": [\"markdown\"], \"consistencyGuarantees\": " + "\"unavailable\", \"cursorSemantics\": \"unavailable\", " + "\"declarationVersion\": \"file-capabilities-v1\", \"deletion\": " + "\"unavailable\", \"describeCapabilities\": \"unavailable\", " + "\"discover\": \"unavailable\", \"fileSourceAccess\": \"unavailable\", " + "\"freshness\": \"unavailable\", \"ingestionJobs\": \"unavailable\", " + "\"projectionFields\": [], \"readChanges\": \"unavailable\", " + "\"resourceKinds\": [\"markdown_document\"], \"sourceMode\": " + "\"materialized\"}'::jsonb, " + "'{\"aclEvidenceMode\": \"mirrored\", \"authorizeAndProject\": " + "\"unavailable\", \"batchLimits\": \"unavailable\", \"checkpoint\": " + "\"unavailable\", \"checkpointSemantics\": \"unavailable\", " + "\"contentKinds\": [\"markdown\"], \"consistencyGuarantees\": " + "\"unavailable\", \"cursorSemantics\": \"unavailable\", " + "\"declarationVersion\": \"file-capabilities-v2\", \"deletion\": " + "\"unavailable\", \"describeCapabilities\": \"unavailable\", " + "\"discover\": \"unavailable\", \"fileSourceAccess\": \"available\", " + "\"freshness\": \"unavailable\", \"ingestionJobs\": \"available\", " + "\"projectionFields\": [], \"readChanges\": \"unavailable\", " + "\"resourceKinds\": [\"markdown_document\"], \"sourceMode\": " + "\"materialized\"}'::jsonb)", + ) + + op.drop_constraint( + "ck_service_principal_workload_issue17", + "service_principal", + type_="check", + ) + op.drop_constraint( + "ck_service_principal_operation_noop_complete", + "service_principal", + type_="check", + ) + op.create_check_constraint( + "ck_service_principal_workload_issue17", + "service_principal", + "workload IN ('supply.noop', 'supply.file-import')", + ) + op.create_check_constraint( + "ck_service_principal_operation_noop_complete", + "service_principal", + "operation IN ('noop.complete', 'file.import')", + ) + op.create_check_constraint( + "ck_service_principal_workload_operation_binding", + "service_principal", + "(workload = 'supply.noop' AND operation = 'noop.complete') OR " + "(workload = 'supply.file-import' AND operation = 'file.import')", + ) + + op.alter_column( + "context_resource", + "active_revision_id", + existing_type=postgresql.UUID(as_uuid=True), + nullable=True, + ) + + op.create_table( + "file_acquisition", + sa.Column("organization_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("acquisition_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("source_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("source_version_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("relative_path", sa.Text(), nullable=False), + sa.Column("audience_principal_ref", sa.Text(), nullable=False), + sa.Column("audience_membership_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("audience_membership_version", sa.BigInteger(), nullable=False), + sa.Column("idempotency_key", sa.Text(), nullable=False), + sa.Column("request_digest", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("organization_id", "acquisition_id", name="pk_file_acquisition"), + sa.UniqueConstraint( + "organization_id", "source_id", "idempotency_key", + name="uq_file_acquisition_source_idempotency", + ), + sa.ForeignKeyConstraint( + ["organization_id", "source_id", "source_version_id"], + ["source_version.organization_id", "source_version.source_id", "source_version.version_id"], + name="fk_file_acquisition_source_version_same_organization", + ), + sa.ForeignKeyConstraint( + ["organization_id", "audience_membership_id", "audience_membership_version"], + ["membership.organization_id", "membership.membership_id", "membership.membership_version"], + name="fk_file_acquisition_membership_version_same_organization", + ), + sa.CheckConstraint( + "relative_path ~ '^[^/\\\\]+\\.[mM][dD]$' AND relative_path NOT IN ('.', '..')", + name="ck_file_acquisition_one_markdown_filename", + ), + sa.CheckConstraint("btrim(audience_principal_ref) <> ''", name="ck_file_acquisition_principal_nonblank"), + sa.CheckConstraint("audience_membership_version > 0", name="ck_file_acquisition_membership_version_positive"), + sa.CheckConstraint("idempotency_key ~ '^[^[:space:]]{1,255}$'", name="ck_file_acquisition_idempotency_key"), + sa.CheckConstraint("request_digest ~ '^[0-9a-f]{64}$'", name="ck_file_acquisition_request_digest"), + ) + + op.create_table( + "file_import_job", + sa.Column("organization_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("job_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("acquisition_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("source_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("service_principal_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("workload", sa.Text(), nullable=False), + sa.Column("worker_audience", sa.Text(), nullable=False), + sa.Column("actor_kind", sa.Text(), nullable=False), + sa.Column("operation", sa.Text(), nullable=False), + sa.Column("state", sa.Text(), nullable=False), + sa.Column("signing_key_version", sa.BigInteger(), nullable=True), + sa.Column("lease_nonce_digest", postgresql.BYTEA(), nullable=True), + sa.Column("lease_issued_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("lease_redeemed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("failed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("resource_ref", sa.Text(), nullable=True), + sa.Column("revision_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("fragment_ref", sa.Text(), nullable=True), + sa.Column("effect_count", sa.SmallInteger(), nullable=False, server_default=sa.text("0")), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("organization_id", "job_id", name="pk_file_import_job"), + sa.UniqueConstraint("organization_id", "acquisition_id", name="uq_file_import_job_acquisition"), + sa.ForeignKeyConstraint( + ["organization_id", "acquisition_id"], + ["file_acquisition.organization_id", "file_acquisition.acquisition_id"], + name="fk_file_import_job_acquisition_same_organization", + ), + sa.ForeignKeyConstraint( + ["organization_id", "service_principal_id", "workload", "worker_audience", "operation"], + ["service_principal.organization_id", "service_principal.service_principal_id", "service_principal.workload", "service_principal.worker_audience", "service_principal.operation"], + name="fk_file_import_job_service_principal_binding", + ), + sa.CheckConstraint("workload = 'supply.file-import'", name="ck_file_import_job_workload"), + sa.CheckConstraint("worker_audience = 'context-engine-worker'", name="ck_file_import_job_worker_audience"), + sa.CheckConstraint("actor_kind = 'service'", name="ck_file_import_job_actor_kind"), + sa.CheckConstraint("operation = 'file.import'", name="ck_file_import_job_operation"), + sa.CheckConstraint("state IN ('available', 'leased', 'running', 'failed', 'completed')", name="ck_file_import_job_state"), + sa.CheckConstraint( + "(state = 'available' AND signing_key_version IS NULL AND lease_nonce_digest IS NULL AND lease_issued_at IS NULL AND lease_expires_at IS NULL AND lease_redeemed_at IS NULL AND failed_at IS NULL AND completed_at IS NULL AND resource_ref IS NULL AND revision_id IS NULL AND fragment_ref IS NULL AND effect_count = 0) OR " + "(state = 'leased' AND signing_key_version > 0 AND octet_length(lease_nonce_digest) = 32 AND lease_issued_at IS NOT NULL AND lease_expires_at > lease_issued_at AND lease_redeemed_at IS NULL AND failed_at IS NULL AND completed_at IS NULL AND resource_ref IS NULL AND revision_id IS NULL AND fragment_ref IS NULL AND effect_count = 0) OR " + "(state = 'running' AND signing_key_version > 0 AND octet_length(lease_nonce_digest) = 32 AND lease_issued_at IS NOT NULL AND lease_expires_at > lease_issued_at AND lease_redeemed_at >= lease_issued_at AND failed_at IS NULL AND completed_at IS NULL AND resource_ref IS NULL AND revision_id IS NULL AND fragment_ref IS NULL AND effect_count = 0) OR " + "(state = 'failed' AND signing_key_version > 0 AND octet_length(lease_nonce_digest) = 32 AND lease_issued_at IS NOT NULL AND lease_expires_at > lease_issued_at AND lease_redeemed_at >= lease_issued_at AND failed_at >= lease_redeemed_at AND completed_at IS NULL AND resource_ref IS NULL AND revision_id IS NULL AND fragment_ref IS NULL AND effect_count = 0) OR " + "(state = 'completed' AND signing_key_version > 0 AND octet_length(lease_nonce_digest) = 32 AND lease_issued_at IS NOT NULL AND lease_expires_at > lease_issued_at AND lease_redeemed_at >= lease_issued_at AND failed_at IS NULL AND completed_at >= lease_redeemed_at AND resource_ref IS NOT NULL AND revision_id IS NOT NULL AND fragment_ref IS NOT NULL AND effect_count = 1)", + name="ck_file_import_job_state_consistency", + ), + ) + + op.create_table( + "file_revision_snapshot", + sa.Column("organization_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("resource_ref", sa.Text(), nullable=False), + sa.Column("revision_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("acquisition_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("canonical_text", sa.Text(), nullable=False), + sa.Column("content_hash", sa.Text(), nullable=False), + sa.Column("compilation_digest", sa.Text(), nullable=False), + sa.Column("compiler_version", sa.Text(), nullable=False), + sa.Column("config_version", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("organization_id", "resource_ref", "revision_id", name="pk_file_revision_snapshot"), + sa.ForeignKeyConstraint( + ["organization_id", "resource_ref", "revision_id"], + ["context_revision.organization_id", "context_revision.resource_ref", "context_revision.revision_id"], + name="fk_file_revision_snapshot_revision_same_organization", + ), + sa.ForeignKeyConstraint( + ["organization_id", "acquisition_id"], + ["file_acquisition.organization_id", "file_acquisition.acquisition_id"], + name="fk_file_revision_snapshot_acquisition_same_organization", + ), + sa.CheckConstraint("content_hash ~ '^[0-9a-f]{64}$'", name="ck_file_revision_snapshot_content_hash"), + sa.CheckConstraint("compilation_digest ~ '^[0-9a-f]{64}$'", name="ck_file_revision_snapshot_compilation_digest"), + ) + + op.create_table( + "revision_publication_event", + sa.Column("organization_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("resource_ref", sa.Text(), nullable=False), + sa.Column("revision_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("ordinal", sa.SmallInteger(), nullable=False), + sa.Column("state", sa.Text(), nullable=False), + sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("organization_id", "resource_ref", "revision_id", "ordinal", name="pk_revision_publication_event"), + sa.UniqueConstraint("organization_id", "resource_ref", "revision_id", "state", name="uq_revision_publication_event_state"), + sa.ForeignKeyConstraint( + ["organization_id", "resource_ref", "revision_id"], + ["context_revision.organization_id", "context_revision.resource_ref", "context_revision.revision_id"], + name="fk_revision_publication_event_revision_same_organization", + ), + sa.CheckConstraint("(ordinal, state) IN ((0, 'prepared'), (1, 'indexed'), (2, 'active'))", name="ck_revision_publication_event_order"), + ) + + op.create_table( + "exact_phrase_candidate", + sa.Column("organization_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("phrase_digest", sa.Text(), nullable=False), + sa.Column("source_ref", sa.Text(), nullable=False), + sa.Column("resource_ref", sa.Text(), nullable=False), + sa.Column("revision_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("fragment_ref", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("organization_id", "phrase_digest", "resource_ref", "revision_id", "fragment_ref", name="pk_exact_phrase_candidate"), + sa.ForeignKeyConstraint( + ["organization_id", "resource_ref", "revision_id", "fragment_ref"], + ["context_fragment.organization_id", "context_fragment.resource_ref", "context_fragment.revision_id", "context_fragment.fragment_ref"], + name="fk_exact_phrase_candidate_fragment_same_organization", + ), + sa.CheckConstraint("phrase_digest ~ '^[0-9a-f]{64}$'", name="ck_exact_phrase_candidate_digest"), + ) + + for table in ( + "file_acquisition", "file_import_job", "file_revision_snapshot", + "revision_publication_event", "exact_phrase_candidate", + ): + _tenant_table(table) + + for table in ("file_acquisition", "file_revision_snapshot", "revision_publication_event", "exact_phrase_candidate"): + _immutable(table) + + tenant = ( + "organization_id = NULLIF(" + "current_setting('app.organization_id', true), ''" + ")::uuid" + ) + + actor = ( + f"{tenant} AND current_setting('app.actor_kind', true) = 'user' " + "AND NULLIF(current_setting('app.principal_ref', true), '') IS NOT NULL " + "AND NULLIF(current_setting('app.request_id', true), '') IS NOT NULL " + "AND NULLIF(current_setting('app.authentication_binding_ref', true), '') " + "IS NOT NULL AND NULLIF(current_setting('app.checked_at', true), '') " + "IS NOT NULL AND EXISTS (SELECT 1 FROM public.membership AS actor_membership " + "WHERE actor_membership.organization_id = " + "exact_phrase_candidate.organization_id " + "AND actor_membership.user_id = NULLIF(current_setting('app.user_id', true), '')::uuid " + "AND actor_membership.membership_id = NULLIF(current_setting('app.membership_id', true), '')::uuid " + "AND actor_membership.membership_version = NULLIF(current_setting('app.membership_version', true), '')::bigint " + "AND actor_membership.status = 'active' " + "AND actor_membership.valid_from <= NULLIF(current_setting('app.checked_at', true), '')::timestamptz " + "AND (actor_membership.valid_until IS NULL OR actor_membership.valid_until > " + "NULLIF(current_setting('app.checked_at', true), '')::timestamptz))" + ) + op.execute( + "CREATE POLICY exact_phrase_candidate_runtime ON exact_phrase_candidate " + f"FOR SELECT TO {_RUNTIME} USING ({actor})" + ) + op.execute("GRANT SELECT ON TABLE exact_phrase_candidate TO context_engine_runtime") + + job_binding = ( + f"organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid " + "AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid " + f"AND workload = '{_WORKLOAD}' AND worker_audience = '{_AUDIENCE}' " + f"AND operation = '{_OPERATION}'" + ) + for command in ("SELECT", "UPDATE"): + check = f" WITH CHECK ({job_binding})" if command == "UPDATE" else "" + op.execute( + f"CREATE POLICY file_import_job_definer_{command.lower()} ON file_import_job " + f"FOR {command} TO {_DEFINER} USING ({job_binding}){check}" + ) + op.execute(f"GRANT INSERT ON TABLE source_version TO {_DEFINER}") + op.execute( + "GRANT UPDATE (active_version_id) ON TABLE context_source TO " + f"{_DEFINER}" + ) + op.execute( + "CREATE POLICY file_import_job_definer_insert ON file_import_job " + f"FOR INSERT TO {_DEFINER} WITH CHECK ({job_binding})" + ) + + definer_commands = { + "context_source": ("SELECT", "UPDATE"), + "source_version": ("SELECT", "INSERT"), + "membership": ("SELECT",), + "service_principal": ("SELECT",), + "file_acquisition": ("SELECT", "INSERT"), + } + for table, commands in definer_commands.items(): + suffix = ( + " AND workload = 'supply.file-import' AND " + "worker_audience = 'context-engine-worker' AND " + "operation = 'file.import' AND enabled IS TRUE" + if table == "service_principal" + else "" + ) + for command in commands: + using = ( + f" USING ({tenant}{suffix})" + if command in {"SELECT", "UPDATE"} + else "" + ) + check = ( + f" WITH CHECK ({tenant}{suffix})" + if command in {"INSERT", "UPDATE"} + else "" + ) + op.execute( + f"CREATE POLICY {table}_file_import_definer_" + f"{command.lower()} ON {table} FOR {command} " + f"TO {_DEFINER}{using}{check}" + ) + op.execute( + "GRANT SELECT ON TABLE context_source, source_version, membership, " + "service_principal, file_acquisition, file_import_job " + f"TO {_DEFINER}" + ) + op.execute( + "GRANT INSERT ON TABLE file_acquisition, file_import_job " + f"TO {_DEFINER}" + ) + op.execute( + "GRANT UPDATE (state, signing_key_version, lease_nonce_digest, lease_issued_at, lease_expires_at, lease_redeemed_at, failed_at, completed_at, resource_ref, revision_id, fragment_ref, effect_count) ON file_import_job TO context_engine_worker_lease_definer" + ) + + definer_org = ( + "organization_id = NULLIF(" + "current_setting('app.organization_id', true), ''" + ")::uuid" + ) + publication_commands = { + "context_resource": ("SELECT", "INSERT", "UPDATE"), + "context_revision": ("INSERT",), + "context_fragment": ("INSERT",), + "resource_access_policy": ("INSERT",), + "membership_resource_field_right": ("INSERT",), + "file_revision_snapshot": ("INSERT",), + "revision_publication_event": ("INSERT",), + "exact_phrase_candidate": ("INSERT",), + } + for table, commands in publication_commands.items(): + for command in commands: + using = ( + f" USING ({definer_org})" + if command in {"SELECT", "UPDATE"} + else "" + ) + check = ( + f" WITH CHECK ({definer_org})" + if command in {"INSERT", "UPDATE"} + else "" + ) + op.execute( + f"CREATE POLICY {table}_file_import_definer_" + f"{command.lower()} ON {table} FOR {command} " + f"TO {_DEFINER}{using}{check}" + ) + op.execute( + f"GRANT SELECT, INSERT ON TABLE context_resource TO {_DEFINER}" + ) + op.execute( + "GRANT INSERT ON TABLE context_revision, context_fragment, " + "resource_access_policy, membership_resource_field_right, " + "file_revision_snapshot, revision_publication_event, " + f"exact_phrase_candidate TO {_DEFINER}" + ) + op.execute(f"GRANT UPDATE (active_revision_id) ON TABLE context_resource TO {_DEFINER}") + + publication_read = ( + f"{tenant} AND current_setting('app.actor_kind', true) = 'user' " + "AND EXISTS (SELECT 1 FROM public.membership AS actor_membership " + "WHERE actor_membership.organization_id = revision_publication_event.organization_id " + "AND actor_membership.user_id = NULLIF(current_setting('app.user_id', true), '')::uuid " + "AND actor_membership.membership_id = NULLIF(current_setting('app.membership_id', true), '')::uuid " + "AND actor_membership.membership_version = NULLIF(current_setting('app.membership_version', true), '')::bigint " + "AND actor_membership.status = 'active' " + "AND actor_membership.valid_from <= NULLIF(current_setting('app.checked_at', true), '')::timestamptz " + "AND (actor_membership.valid_until IS NULL OR actor_membership.valid_until > " + "NULLIF(current_setting('app.checked_at', true), '')::timestamptz)) " + "AND EXISTS (SELECT 1 FROM public.resource_access_policy AS access_policy " + "WHERE access_policy.organization_id = revision_publication_event.organization_id " + "AND access_policy.resource_ref = revision_publication_event.resource_ref " + "AND access_policy.principal_ref = current_setting('app.principal_ref', true) " + "AND access_policy.access_state = 'allowed')" + ) + op.execute( + "CREATE POLICY revision_publication_event_current_user_actor " + "ON revision_publication_event FOR SELECT TO context_engine_runtime " + f"USING ({publication_read})" + ) + op.execute( + "GRANT SELECT ON TABLE revision_publication_event TO context_engine_runtime" + ) + + _create_functions() + + +def _create_functions() -> None: + op.execute( + f""" + CREATE FUNCTION public.context_control_prepare_file_import( + requested_organization_id uuid, requested_acquisition_id uuid, + requested_job_id uuid, requested_activated_version_id uuid, + requested_source_id uuid, + requested_relative_path text, requested_audience_principal_ref text, + requested_audience_membership_id uuid, + requested_audience_membership_version bigint, + requested_idempotency_key text, requested_request_digest text, + requested_service_principal_id uuid + ) RETURNS TABLE (job_id uuid, service_principal_id uuid) + LANGUAGE plpgsql SECURITY DEFINER + SET search_path = pg_catalog, pg_temp + SET row_security = on + AS $function$ + DECLARE selected_version_id uuid; selected_acquisition_id uuid; + selected_root_ref text; selected_capabilities jsonb; + trusted_now timestamptz; + BEGIN + IF SESSION_USER <> '{_CONTROL}' THEN RETURN; END IF; + trusted_now := pg_catalog.statement_timestamp(); + PERFORM pg_catalog.set_config('app.organization_id', requested_organization_id::text, true); + PERFORM pg_catalog.set_config('app.worker_job_id', requested_job_id::text, true); + SELECT version.version_id, version.root_ref, + version.capability_manifest + INTO selected_version_id, selected_root_ref, + selected_capabilities + FROM public.context_source AS source + JOIN public.source_version AS version + ON version.organization_id = source.organization_id + AND version.source_id = source.source_id + AND version.version_id = source.active_version_id + WHERE source.organization_id = requested_organization_id + AND source.source_id = requested_source_id + AND version.capability_manifest->>'declarationVersion' + IN ('file-capabilities-v1', 'file-capabilities-v2'); + IF selected_version_id IS NULL OR NOT EXISTS ( + SELECT 1 FROM public.membership AS audience_membership + WHERE audience_membership.organization_id = requested_organization_id + AND audience_membership.membership_id = requested_audience_membership_id + AND audience_membership.membership_version = requested_audience_membership_version + AND audience_membership.status = 'active' + AND audience_membership.valid_from <= trusted_now + AND (audience_membership.valid_until IS NULL OR audience_membership.valid_until > trusted_now) + ) OR NOT EXISTS ( + SELECT 1 FROM public.service_principal AS receiver + WHERE receiver.organization_id = requested_organization_id + AND receiver.service_principal_id = requested_service_principal_id + AND receiver.workload = '{_WORKLOAD}' + AND receiver.worker_audience = '{_AUDIENCE}' + AND receiver.operation = '{_OPERATION}' + AND receiver.enabled IS TRUE + ) THEN RETURN; END IF; + + IF selected_capabilities->>'declarationVersion' + = 'file-capabilities-v1' THEN + INSERT INTO public.source_version ( + organization_id, source_id, version_id, source_kind, + root_ref, capability_manifest, created_at + ) VALUES ( + requested_organization_id, requested_source_id, + requested_activated_version_id, 'file', selected_root_ref, + jsonb_set( + jsonb_set( + jsonb_set( + selected_capabilities, + '{{declarationVersion}}', + '"file-capabilities-v2"'::jsonb + ), + '{{fileSourceAccess}}', '"available"'::jsonb + ), + '{{ingestionJobs}}', '"available"'::jsonb + ), + trusted_now + ); + UPDATE public.context_source + SET active_version_id = requested_activated_version_id + WHERE organization_id = requested_organization_id + AND source_id = requested_source_id + AND active_version_id = selected_version_id; + IF NOT FOUND THEN RETURN; END IF; + selected_version_id := requested_activated_version_id; + ELSIF selected_capabilities->>'fileSourceAccess' <> 'available' + OR selected_capabilities->>'ingestionJobs' <> 'available' THEN + RETURN; + END IF; + + INSERT INTO public.file_acquisition ( + organization_id, acquisition_id, source_id, source_version_id, + relative_path, audience_principal_ref, audience_membership_id, + audience_membership_version, idempotency_key, request_digest, created_at + ) VALUES ( + requested_organization_id, requested_acquisition_id, requested_source_id, + selected_version_id, requested_relative_path, requested_audience_principal_ref, + requested_audience_membership_id, requested_audience_membership_version, + requested_idempotency_key, requested_request_digest, trusted_now + ) ON CONFLICT (organization_id, source_id, idempotency_key) DO NOTHING; + SELECT acquisition_id INTO selected_acquisition_id + FROM public.file_acquisition + WHERE organization_id = requested_organization_id + AND source_id = requested_source_id + AND idempotency_key = requested_idempotency_key + AND request_digest = requested_request_digest; + IF selected_acquisition_id IS NULL THEN RETURN; END IF; + INSERT INTO public.file_import_job ( + organization_id, job_id, acquisition_id, source_id, + service_principal_id, workload, worker_audience, actor_kind, + operation, state, created_at + ) VALUES ( + requested_organization_id, requested_job_id, selected_acquisition_id, + requested_source_id, requested_service_principal_id, '{_WORKLOAD}', + '{_AUDIENCE}', 'service', '{_OPERATION}', 'available', trusted_now + ) ON CONFLICT (organization_id, acquisition_id) DO NOTHING; + RETURN QUERY SELECT job.job_id, job.service_principal_id + FROM public.file_import_job AS job + WHERE job.organization_id = requested_organization_id + AND job.acquisition_id = selected_acquisition_id + AND job.service_principal_id = requested_service_principal_id; + END; $function$ + """ + ) + + op.execute( + f""" + CREATE FUNCTION public.context_worker_issue_file_import_lease( + requested_organization_id uuid, requested_job_id uuid, + requested_service_principal_id uuid, requested_source_ref text, + requested_signing_key_version bigint, requested_nonce bytea, + requested_ttl_seconds integer + ) RETURNS TABLE (issued_at timestamptz, expires_at timestamptz) + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp SET row_security = on + AS $function$ + DECLARE now_at timestamptz; + BEGIN + IF SESSION_USER <> '{_CONTROL}' OR requested_signing_key_version NOT BETWEEN 1 AND {_MAX_BIGINT} + OR pg_catalog.octet_length(requested_nonce) <> 32 + OR requested_ttl_seconds NOT BETWEEN 1 AND {_MAX_TTL} THEN RETURN; END IF; + PERFORM pg_catalog.set_config('app.organization_id', requested_organization_id::text, true); + PERFORM pg_catalog.set_config('app.worker_job_id', requested_job_id::text, true); + now_at := pg_catalog.date_trunc('second', pg_catalog.transaction_timestamp()); + UPDATE public.file_import_job AS job SET state = 'leased', + signing_key_version = requested_signing_key_version, + lease_nonce_digest = public.digest(requested_nonce, 'sha256'), + lease_issued_at = now_at, + lease_expires_at = now_at + pg_catalog.make_interval(secs => requested_ttl_seconds) + WHERE job.organization_id = requested_organization_id AND job.job_id = requested_job_id + AND job.service_principal_id = requested_service_principal_id + AND job.source_id::text = requested_source_ref AND job.state = 'available' + AND EXISTS (SELECT 1 FROM public.service_principal AS principal + WHERE principal.organization_id = job.organization_id + AND principal.service_principal_id = job.service_principal_id + AND principal.workload = job.workload + AND principal.worker_audience = job.worker_audience + AND principal.operation = job.operation AND principal.enabled IS TRUE) + RETURNING job.lease_issued_at, job.lease_expires_at INTO issued_at, expires_at; + IF issued_at IS NOT NULL THEN RETURN NEXT; END IF; RETURN; + END; $function$ + """ + ) + + op.execute( + f""" + CREATE FUNCTION public.context_worker_redeem_file_import( + requested_organization_id uuid, requested_job_id uuid, + requested_service_principal_id uuid, requested_source_ref text, + requested_signing_key_version bigint, requested_nonce bytea, + requested_issued_at timestamptz, requested_expires_at timestamptz + ) RETURNS TABLE (source_ref text, root_ref text, relative_path text, + audience_principal_ref text, audience_membership_id uuid, + audience_membership_version bigint, acquisition_id uuid) + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp SET row_security = on + AS $function$ + DECLARE redeemed_at timestamptz; + BEGIN + IF SESSION_USER <> '{_WORKER}' THEN RETURN; END IF; + PERFORM pg_catalog.set_config('app.organization_id', requested_organization_id::text, true); + PERFORM pg_catalog.set_config('app.worker_job_id', requested_job_id::text, true); + redeemed_at := pg_catalog.statement_timestamp(); + UPDATE public.file_import_job AS job SET state = 'running', lease_redeemed_at = redeemed_at + WHERE job.organization_id = requested_organization_id AND job.job_id = requested_job_id + AND job.service_principal_id = requested_service_principal_id + AND job.source_id::text = requested_source_ref + AND job.state = 'leased' AND job.signing_key_version = requested_signing_key_version + AND job.lease_nonce_digest = public.digest(requested_nonce, 'sha256') + AND job.lease_issued_at = requested_issued_at AND job.lease_expires_at = requested_expires_at + AND redeemed_at >= job.lease_issued_at AND redeemed_at < job.lease_expires_at + AND EXISTS (SELECT 1 FROM public.service_principal AS principal + WHERE principal.organization_id = job.organization_id + AND principal.service_principal_id = job.service_principal_id + AND principal.workload = job.workload + AND principal.worker_audience = job.worker_audience + AND principal.operation = job.operation + AND principal.enabled IS TRUE); + IF NOT FOUND THEN RETURN; END IF; + RETURN QUERY SELECT job.source_id::text, version.root_ref, acquisition.relative_path, + acquisition.audience_principal_ref, acquisition.audience_membership_id, + acquisition.audience_membership_version, acquisition.acquisition_id + FROM public.file_import_job AS job + JOIN public.file_acquisition AS acquisition + ON acquisition.organization_id = job.organization_id AND acquisition.acquisition_id = job.acquisition_id + JOIN public.source_version AS version + ON version.organization_id = acquisition.organization_id + AND version.source_id = acquisition.source_id + AND version.version_id = acquisition.source_version_id + WHERE job.organization_id = requested_organization_id AND job.job_id = requested_job_id; + END; $function$ + """ + ) + + op.execute( + f""" + CREATE FUNCTION public.context_worker_fail_file_import( + requested_organization_id uuid, requested_job_id uuid, + requested_service_principal_id uuid, requested_source_ref text, + requested_signing_key_version bigint, requested_nonce bytea, + requested_issued_at timestamptz, requested_expires_at timestamptz + ) RETURNS boolean + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp SET row_security = on + AS $function$ + DECLARE changed boolean := false; failed_now timestamptz; + BEGIN + IF SESSION_USER <> '{_WORKER}' THEN RETURN false; END IF; + PERFORM pg_catalog.set_config('app.organization_id', requested_organization_id::text, true); + PERFORM pg_catalog.set_config('app.worker_job_id', requested_job_id::text, true); + failed_now := pg_catalog.statement_timestamp(); + UPDATE public.file_import_job AS job + SET state = 'failed', failed_at = failed_now + WHERE job.organization_id = requested_organization_id + AND job.job_id = requested_job_id AND job.state = 'running' + AND job.service_principal_id = requested_service_principal_id + AND job.source_id::text = requested_source_ref + AND job.signing_key_version = requested_signing_key_version + AND job.lease_nonce_digest = public.digest(requested_nonce, 'sha256') + AND job.lease_issued_at = requested_issued_at + AND job.lease_expires_at = requested_expires_at + AND failed_now >= job.lease_issued_at + AND failed_now < job.lease_expires_at + AND EXISTS (SELECT 1 FROM public.service_principal AS principal + WHERE principal.organization_id = job.organization_id + AND principal.service_principal_id = job.service_principal_id + AND principal.workload = job.workload + AND principal.worker_audience = job.worker_audience + AND principal.operation = job.operation + AND principal.enabled IS TRUE); + changed := FOUND; + RETURN changed; + END; $function$ + """ + ) + + op.execute( + f""" + CREATE FUNCTION public.context_worker_publish_file_import( + requested_organization_id uuid, requested_job_id uuid, + requested_service_principal_id uuid, requested_source_ref text, + requested_resource_ref text, requested_revision_id uuid, + requested_fragment_ref text, requested_canonical_text text, + requested_paragraph text, requested_content_hash text, + requested_compilation_digest text, requested_compiler_version text, + requested_config_version text, requested_phrase_digest text, + requested_signing_key_version bigint, requested_nonce bytea, + requested_issued_at timestamptz, requested_expires_at timestamptz + ) RETURNS TABLE (effect_count smallint) + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp SET row_security = on + AS $function$ + DECLARE job_row public.file_import_job%ROWTYPE; acquisition_row public.file_acquisition%ROWTYPE; now_at timestamptz; + BEGIN + IF SESSION_USER <> '{_WORKER}' OR requested_content_hash !~ '^[0-9a-f]{{64}}$' + OR requested_compilation_digest !~ '^[0-9a-f]{{64}}$' + OR requested_phrase_digest !~ '^[0-9a-f]{{64}}$' THEN RETURN; END IF; + PERFORM pg_catalog.set_config('app.organization_id', requested_organization_id::text, true); + PERFORM pg_catalog.set_config('app.worker_job_id', requested_job_id::text, true); + SELECT * INTO job_row FROM public.file_import_job + WHERE organization_id = requested_organization_id AND job_id = requested_job_id + AND service_principal_id = requested_service_principal_id + AND source_id::text = requested_source_ref + AND state = 'running' + AND signing_key_version = requested_signing_key_version + AND lease_nonce_digest = public.digest(requested_nonce, 'sha256') + AND lease_issued_at = requested_issued_at + AND lease_expires_at = requested_expires_at + AND pg_catalog.statement_timestamp() < lease_expires_at + AND EXISTS (SELECT 1 FROM public.service_principal AS principal + WHERE principal.organization_id = file_import_job.organization_id + AND principal.service_principal_id = file_import_job.service_principal_id + AND principal.workload = file_import_job.workload + AND principal.worker_audience = file_import_job.worker_audience + AND principal.operation = file_import_job.operation + AND principal.enabled IS TRUE) + FOR UPDATE; + IF job_row.job_id IS NULL THEN RETURN; END IF; + SELECT * INTO acquisition_row FROM public.file_acquisition + WHERE organization_id = job_row.organization_id AND acquisition_id = job_row.acquisition_id; + now_at := pg_catalog.statement_timestamp(); + SET CONSTRAINTS ALL DEFERRED; + INSERT INTO public.context_resource (organization_id, resource_ref, source_ref, active_revision_id, tombstoned) + VALUES (requested_organization_id, requested_resource_ref, job_row.source_id::text, NULL, false); + INSERT INTO public.context_revision (organization_id, resource_ref, revision_id) + VALUES (requested_organization_id, requested_resource_ref, requested_revision_id); + INSERT INTO public.file_revision_snapshot VALUES ( + requested_organization_id, requested_resource_ref, requested_revision_id, + job_row.acquisition_id, requested_canonical_text, requested_content_hash, + requested_compilation_digest, requested_compiler_version, requested_config_version + ); + INSERT INTO public.context_fragment ( + organization_id, resource_ref, revision_id, fragment_ref, ordinal, content, projection_kind + ) VALUES (requested_organization_id, requested_resource_ref, requested_revision_id, requested_fragment_ref, 0, requested_paragraph, 'body'); + INSERT INTO public.revision_publication_event VALUES + (requested_organization_id, requested_resource_ref, requested_revision_id, 0, 'prepared', now_at); + INSERT INTO public.exact_phrase_candidate VALUES ( + requested_organization_id, requested_phrase_digest, job_row.source_id::text, + requested_resource_ref, requested_revision_id, requested_fragment_ref + ); + INSERT INTO public.revision_publication_event VALUES + (requested_organization_id, requested_resource_ref, requested_revision_id, 1, 'indexed', now_at); + INSERT INTO public.resource_access_policy VALUES ( + requested_organization_id, requested_resource_ref, + acquisition_row.audience_principal_ref, 1, 'allowed', NULL + ); + INSERT INTO public.membership_resource_field_right VALUES ( + requested_organization_id, acquisition_row.audience_membership_id, + acquisition_row.audience_membership_version, requested_resource_ref, 'body' + ); + UPDATE public.context_resource SET active_revision_id = requested_revision_id + WHERE organization_id = requested_organization_id AND resource_ref = requested_resource_ref + AND active_revision_id IS NULL; + IF NOT FOUND THEN RETURN; END IF; + INSERT INTO public.revision_publication_event VALUES + (requested_organization_id, requested_resource_ref, requested_revision_id, 2, 'active', now_at); + UPDATE public.file_import_job SET state = 'completed', completed_at = now_at, + resource_ref = requested_resource_ref, revision_id = requested_revision_id, + fragment_ref = requested_fragment_ref, effect_count = 1 + WHERE organization_id = requested_organization_id AND job_id = requested_job_id AND state = 'running' + RETURNING file_import_job.effect_count INTO effect_count; + IF effect_count IS NOT NULL THEN RETURN NEXT; END IF; RETURN; + END; $function$ + """ + ) + + functions = ( + ("context_control_prepare_file_import", "(uuid, uuid, uuid, uuid, uuid, text, text, uuid, bigint, text, text, uuid)", _CONTROL), + ("context_worker_issue_file_import_lease", "(uuid, uuid, uuid, text, bigint, bytea, integer)", _CONTROL), + ("context_worker_redeem_file_import", "(uuid, uuid, uuid, text, bigint, bytea, timestamp with time zone, timestamp with time zone)", _WORKER), + ("context_worker_fail_file_import", "(uuid, uuid, uuid, text, bigint, bytea, timestamp with time zone, timestamp with time zone)", _WORKER), + ("context_worker_publish_file_import", "(uuid, uuid, uuid, text, text, uuid, text, text, text, text, text, text, text, text, bigint, bytea, timestamp with time zone, timestamp with time zone)", _WORKER), + ) + op.execute(f"GRANT CREATE ON SCHEMA public TO {_DEFINER}") + for name, signature, _grantee in functions: + op.execute(f"REVOKE ALL ON FUNCTION public.{name}{signature} FROM PUBLIC") + op.execute(f"ALTER FUNCTION public.{name}{signature} OWNER TO {_DEFINER}") + op.execute(f"SET LOCAL ROLE {_DEFINER}") + for name, signature, grantee in functions: + op.execute(f"GRANT EXECUTE ON FUNCTION public.{name}{signature} TO {grantee}") + op.execute("RESET ROLE") + op.execute(f"REVOKE CREATE ON SCHEMA public FROM {_DEFINER}") + + +def downgrade() -> None: + """Remove the Issue #23 tracer and restore the Issue #21/17 contracts.""" + + op.execute( + "LOCK TABLE context_source, source_version, context_resource, " + "context_revision, context_fragment, resource_access_policy, " + "membership_resource_field_right, file_acquisition, file_import_job, " + "file_revision_snapshot, revision_publication_event, " + "exact_phrase_candidate IN ACCESS EXCLUSIVE MODE" + ) + + functions = ( + "public.context_worker_publish_file_import(uuid, uuid, uuid, text, text, uuid, text, text, text, text, text, text, text, text, bigint, bytea, timestamp with time zone, timestamp with time zone)", + "public.context_worker_fail_file_import(uuid, uuid, uuid, text, bigint, bytea, timestamp with time zone, timestamp with time zone)", + "public.context_worker_redeem_file_import(uuid, uuid, uuid, text, bigint, bytea, timestamp with time zone, timestamp with time zone)", + "public.context_worker_issue_file_import_lease(uuid, uuid, uuid, text, bigint, bytea, integer)", + "public.context_control_prepare_file_import(uuid, uuid, uuid, uuid, uuid, text, text, uuid, bigint, text, text, uuid)", + ) + for function in functions: + op.execute(f"DROP FUNCTION {function}") + + op.execute( + "REVOKE SELECT ON TABLE revision_publication_event " + "FROM context_engine_runtime" + ) + op.execute( + "DROP POLICY revision_publication_event_current_user_actor " + "ON revision_publication_event" + ) + + for table in ( + "exact_phrase_candidate", + "revision_publication_event", + "file_revision_snapshot", + ): + op.drop_table(table) + + op.execute( + "DELETE FROM membership_resource_field_right AS field_right " + "USING file_import_job AS job WHERE job.state = 'completed' " + "AND field_right.organization_id = job.organization_id " + "AND field_right.resource_ref = job.resource_ref" + ) + op.execute( + "DELETE FROM resource_access_policy AS access_policy " + "USING file_import_job AS job WHERE job.state = 'completed' " + "AND access_policy.organization_id = job.organization_id " + "AND access_policy.resource_ref = job.resource_ref" + ) + op.execute( + "UPDATE context_resource AS resource SET active_revision_id = NULL " + "FROM file_import_job AS job WHERE job.state = 'completed' " + "AND resource.organization_id = job.organization_id " + "AND resource.resource_ref = job.resource_ref" + ) + op.execute("DROP TRIGGER context_fragment_reject_mutation ON context_fragment") + op.execute( + "DELETE FROM context_fragment AS fragment USING file_import_job AS job " + "WHERE job.state = 'completed' " + "AND fragment.organization_id = job.organization_id " + "AND fragment.resource_ref = job.resource_ref " + "AND fragment.revision_id = job.revision_id" + ) + op.execute( + "CREATE TRIGGER context_fragment_reject_mutation " + "BEFORE UPDATE OR DELETE ON context_fragment FOR EACH ROW " + "EXECUTE FUNCTION public.context_content_reject_mutation()" + ) + op.execute("DROP TRIGGER context_revision_reject_mutation ON context_revision") + op.execute( + "DELETE FROM context_revision AS revision USING file_import_job AS job " + "WHERE job.state = 'completed' " + "AND revision.organization_id = job.organization_id " + "AND revision.resource_ref = job.resource_ref " + "AND revision.revision_id = job.revision_id" + ) + op.execute( + "CREATE TRIGGER context_revision_reject_mutation " + "BEFORE UPDATE OR DELETE ON context_revision FOR EACH ROW " + "EXECUTE FUNCTION public.context_content_reject_mutation()" + ) + op.execute( + "DELETE FROM context_resource AS resource USING file_import_job AS job " + "WHERE job.state = 'completed' " + "AND resource.organization_id = job.organization_id " + "AND resource.resource_ref = job.resource_ref" + ) + + for table in ("file_import_job", "file_acquisition"): + op.drop_table(table) + + op.execute( + "WITH prior AS (SELECT DISTINCT ON (organization_id, source_id) " + "organization_id, source_id, version_id FROM source_version " + "WHERE capability_manifest->>'declarationVersion' = " + "'file-capabilities-v1' ORDER BY organization_id, source_id, " + "created_at, version_id) UPDATE context_source AS source " + "SET active_version_id = prior.version_id FROM prior " + "WHERE prior.organization_id = source.organization_id " + "AND prior.source_id = source.source_id AND EXISTS (SELECT 1 " + "FROM source_version AS active WHERE active.organization_id = " + "source.organization_id AND active.source_id = source.source_id " + "AND active.version_id = source.active_version_id AND " + "active.capability_manifest->>'declarationVersion' = " + "'file-capabilities-v2')" + ) + op.execute("DROP TRIGGER source_version_immutable ON source_version") + op.execute( + "DELETE FROM source_version WHERE " + "capability_manifest->>'declarationVersion' = 'file-capabilities-v2'" + ) + op.execute( + "CREATE TRIGGER source_version_immutable BEFORE UPDATE OR DELETE " + "ON source_version FOR EACH ROW EXECUTE FUNCTION " + "public.source_version_reject_mutation()" + ) + + for table, commands in { + "context_source": ("select", "update"), + "source_version": ("select", "insert"), + "membership": ("select",), + "service_principal": ("select",), + "context_resource": ("select", "insert", "update"), + "context_revision": ("insert",), + "context_fragment": ("insert",), + "resource_access_policy": ("insert",), + "membership_resource_field_right": ("insert",), + }.items(): + for command in commands: + op.execute( + f"DROP POLICY {table}_file_import_definer_{command} ON {table}" + ) + + op.execute(f"REVOKE SELECT, UPDATE ON TABLE context_source FROM {_DEFINER}") + op.execute(f"REVOKE SELECT, INSERT ON TABLE source_version FROM {_DEFINER}") + op.execute(f"REVOKE SELECT ON TABLE membership FROM {_DEFINER}") + op.execute( + f"REVOKE SELECT, INSERT, UPDATE ON TABLE context_resource FROM {_DEFINER}" + ) + op.execute( + "REVOKE INSERT ON TABLE context_revision, context_fragment, " + "resource_access_policy, membership_resource_field_right " + f"FROM {_DEFINER}" + ) + op.alter_column( + "context_resource", + "active_revision_id", + existing_type=postgresql.UUID(as_uuid=True), + nullable=False, + ) + op.drop_constraint( + "ck_source_version_file_capabilities", "source_version", type_="check" + ) + op.create_check_constraint( + "ck_source_version_issue_21_capabilities", + "source_version", + "capability_manifest = " + "'{\"aclEvidenceMode\": \"mirrored\", \"authorizeAndProject\": " + "\"unavailable\", \"batchLimits\": \"unavailable\", \"checkpoint\": " + "\"unavailable\", \"checkpointSemantics\": \"unavailable\", " + "\"contentKinds\": [\"markdown\"], \"consistencyGuarantees\": " + "\"unavailable\", \"cursorSemantics\": \"unavailable\", " + "\"declarationVersion\": \"file-capabilities-v1\", \"deletion\": " + "\"unavailable\", \"describeCapabilities\": \"unavailable\", " + "\"discover\": \"unavailable\", \"fileSourceAccess\": " + "\"unavailable\", \"freshness\": \"unavailable\", \"ingestionJobs\": " + "\"unavailable\", \"projectionFields\": [], \"readChanges\": " + "\"unavailable\", \"resourceKinds\": [\"markdown_document\"], " + "\"sourceMode\": \"materialized\"}'::jsonb", + ) + for constraint in ( + "ck_service_principal_workload_operation_binding", + "ck_service_principal_workload_issue17", + "ck_service_principal_operation_noop_complete", + ): + op.drop_constraint(constraint, "service_principal", type_="check") + op.execute( + "DELETE FROM service_principal WHERE workload = 'supply.file-import' " + "AND operation = 'file.import'" + ) + op.create_check_constraint( + "ck_service_principal_workload_issue17", + "service_principal", + "workload = 'supply.noop'", + ) + op.create_check_constraint( + "ck_service_principal_operation_noop_complete", + "service_principal", + "operation = 'noop.complete'", + ) + op.execute("SET CONSTRAINTS ALL IMMEDIATE") diff --git a/scripts/security_gate/rls.py b/scripts/security_gate/rls.py index 7767f8d7..78d8ad89 100644 --- a/scripts/security_gate/rls.py +++ b/scripts/security_gate/rls.py @@ -24,6 +24,10 @@ "context_resource": "PG-INDEX-NOT-AUTHORITY-005", "context_revision": "PG-INDEX-NOT-AUTHORITY-005", "context_fragment": "PG-INDEX-NOT-AUTHORITY-005", + "exact_phrase_candidate": "PG-FILE-IMPORT-023", + "file_acquisition": "PG-FILE-IMPORT-023", + "file_import_job": "PG-FILE-IMPORT-023", + "file_revision_snapshot": "PG-FILE-IMPORT-023", "organization_policy_epoch": "PG-REVOCATION-006", "resource_access_policy": "PG-REVOCATION-006", "context_run": "PG-TRACE-REDACTION-012", @@ -40,6 +44,7 @@ "release_operator_grant": "PG-RELEASE-OWNER-019", "active_release_manifest": "PG-RELEASE-OWNER-019", "release_promotion_audit": "PG-RELEASE-OWNER-019", + "revision_publication_event": "PG-FILE-IMPORT-023", } _SQL_TOKEN = re.compile( diff --git a/tests/integration/test_authorized_field_schema.py b/tests/integration/test_authorized_field_schema.py index 04eb13b2..c3f2e3f6 100644 --- a/tests/integration/test_authorized_field_schema.py +++ b/tests/integration/test_authorized_field_schema.py @@ -1258,6 +1258,11 @@ def test_field_authority_tables_have_force_rls_and_least_privilege_grants( assert grants == { (RUNTIME_ROLE, "context_fragment_field", "SELECT"), (RUNTIME_ROLE, "membership_resource_field_right", "SELECT"), + ( + "context_engine_worker_lease_definer", + "membership_resource_field_right", + "INSERT", + ), } assert { "pk_context_fragment_field", diff --git a/tests/integration/test_file_import_tracer.py b/tests/integration/test_file_import_tracer.py new file mode 100644 index 00000000..6cf252bb --- /dev/null +++ b/tests/integration/test_file_import_tracer.py @@ -0,0 +1,1554 @@ +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import UUID, uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import Engine, text +from sqlalchemy.engine import Connection +from sqlalchemy.exc import SQLAlchemyError + +from adapters.exact_phrase import PostgreSQLExactPhraseCandidateIndex +from adapters.file_source import FileReadLimits, FileRootRegistry +from adapters.http.app import create_app +from adapters.http.authentication import VerifiedAuthenticationContext +from adapters.http.organization_authority import OrganizationVerificationRejected +from adapters.http.scope_authority import ScopeAuthorityIdentity +from engine.control import ( + ContextControl, + ControlOperation, + ControlOperatorAuthority, + FileImportAudience, + FileImportPath, + FileImportReceiver, + FileRootRef, + PreparedFileImport, + PrepareFileImport, + RegisterFileSource, + SourceNotAvailable, + SourceRef, + VerifiedControlOperatorIdentity, +) +from engine.persistence import ( + DatabaseConfiguration, + FileImportLeaseRedemption, + FileImportUnavailable, + PostgreSQLControlStore, + PostgreSQLFileImportWorker, + PostgreSQLMembershipAuthority, + PostgreSQLWorkerLeaseIssuer, + create_database_engine, +) +from engine.persistence.membership_context import MembershipIdentity +from engine.runtime.construction import Runtime, required_kernel_dependencies +from engine.runtime.context_run import ContextRunOutcome +from engine.runtime.contracts import Acquire +from engine.runtime.evidence import CandidateRef +from engine.runtime.materialized import ( + MaterializedProjectionSession, + _observe_materialized_publication, +) +from engine.runtime.organization import ( + ExistingOrganizationVerification, + _construct_existing_http_organization_verification, +) +from engine.runtime.package_digest import QueryDigestKeyring +from engine.runtime.scope import ScopeSet, ScopeTarget +from engine.runtime.scope_authority import ( + TrustedScopeSnapshot, + _close_scope_authority_scope, + _construct_trusted_scope_snapshot, + _open_scope_authority_scope, +) +from engine.supply import ( + MarkdownCompilerConfig, + WorkerLeaseClaims, + WorkerLeaseCodec, + WorkerLeaseKeyring, + WorkerLeaseToken, +) +from tests.support.context_run_operator import exact_test_context_run_operator_read + +pytestmark = pytest.mark.integration +NOW = datetime.now(UTC).replace(microsecond=0) +SIGNING_KEY = bytes(range(32)) + + +def _publication_effect_counts( + connection: Connection, + organization_id: UUID, +) -> tuple[int, ...]: + row = connection.execute( + text( + """ + SELECT + (SELECT count(*) FROM file_acquisition + WHERE organization_id = :organization_id), + (SELECT count(*) FROM file_import_job + WHERE organization_id = :organization_id), + (SELECT count(*) FROM context_resource + WHERE organization_id = :organization_id), + (SELECT count(*) FROM context_revision + WHERE organization_id = :organization_id), + (SELECT count(*) FROM context_fragment + WHERE organization_id = :organization_id), + (SELECT count(*) FROM file_revision_snapshot + WHERE organization_id = :organization_id), + (SELECT count(*) FROM revision_publication_event + WHERE organization_id = :organization_id), + (SELECT count(*) FROM exact_phrase_candidate + WHERE organization_id = :organization_id), + (SELECT count(*) FROM resource_access_policy + WHERE organization_id = :organization_id), + (SELECT count(*) FROM membership_resource_field_right + WHERE organization_id = :organization_id) + """ + ), + {"organization_id": organization_id}, + ).one() + return tuple(row) + + +class _ControlAuthenticator: + def __init__(self, organization_id: UUID) -> None: + self.organization_id = organization_id + + def authenticate(self, opaque_credential: str) -> VerifiedControlOperatorIdentity: + if opaque_credential != "control-secret": + raise AssertionError("unexpected Control credential") + return VerifiedControlOperatorIdentity( + organization_id=self.organization_id, + operator_ref="operator:file-import", + authentication_binding_ref="binding:file-import", + authority_ref="authority:file-import", + allowed_operations=frozenset( + { + ControlOperation.REGISTER_SOURCE, + ControlOperation.READ_SOURCE, + ControlOperation.IMPORT_FILE, + } + ), + valid_from=NOW - timedelta(minutes=1), + expires_at=NOW + timedelta(hours=1), + ) + + +class _RuntimeAuthenticator: + def __init__( + self, + organization_id: UUID, + user_id: UUID, + membership_id: UUID, + *, + token: str = "runtime-secret", + ) -> None: + self.organization_id = organization_id + self.user_id = user_id + self.membership_id = membership_id + self.token = token + + def authenticate(self, opaque_credential: str) -> VerifiedAuthenticationContext: + assert opaque_credential == self.token + return VerifiedAuthenticationContext( + organization_ref=str(self.organization_id), + user_ref=str(self.user_id), + principal_ref="principal:file-reader", + membership_ref=str(self.membership_id), + membership_version=1, + agent_version_ref="agent:file-tracer", + authenticated_application_ref="application:file-tracer", + authentication_binding_ref="binding:file-tracer", + ) + + +class _MultiTenantRuntimeAuthenticator: + def __init__( + self, + identities: dict[str, tuple[UUID, UUID, UUID]], + ) -> None: + self.identities = identities + + def authenticate(self, opaque_credential: str) -> VerifiedAuthenticationContext: + organization_id, user_id, membership_id = self.identities[opaque_credential] + return _RuntimeAuthenticator( + organization_id, + user_id, + membership_id, + token=opaque_credential, + ).authenticate(opaque_credential) + + +class _OrganizationAuthority: + + def verify_existing( + self, + authentication: VerifiedAuthenticationContext, + *, + request_id: str, + verified_at: datetime, + ) -> ExistingOrganizationVerification: + try: + organization_id = UUID(authentication.organization_ref) + except ValueError: + raise OrganizationVerificationRejected from None + return _construct_existing_http_organization_verification( + organization_id=organization_id, + request_id=request_id, + authentication_binding_ref=authentication.authentication_binding_ref, + verified_at=verified_at, + ) + + +class _ExactScopeAuthority: + def __init__( + self, + source_ref: str, + resource_ref: str, + *, + allowed: bool = True, + ) -> None: + self.source_ref = source_ref + self.resource_ref = resource_ref + self.allowed = allowed + + @contextmanager + def current_scope( + self, identity: ScopeAuthorityIdentity + ) -> Iterator[TrustedScopeSnapshot]: + scope = _open_scope_authority_scope() + try: + target = ScopeSet( + frozenset( + { + ScopeTarget( + identity.organization_id, + self.source_ref, + self.resource_ref, + ) + } + if self.allowed + else set() + ) + ) + yield _construct_trusted_scope_snapshot( + authority_scope=scope, + organization_id=identity.organization_id, + user_id=identity.user_id, + membership_id=identity.membership_id, + membership_version=identity.membership_version, + policy_epoch=identity.policy_epoch, + principal_ref=identity.principal_ref, + agent_version_ref=identity.agent_version_ref, + purpose=identity.purpose, + request_id=identity.request_id, + authentication_binding_ref=identity.authentication_binding_ref, + checked_at=identity.checked_at, + organization_boundary=target, + membership_rights=target, + principal_grants=target, + agent_ceiling=target, + source_native_acl=target, + resource_acl=target, + purpose_policy=target, + ) + finally: + _close_scope_authority_scope(scope) + + +class _ExactThenReplayCandidateIndex: + def __init__(self, replay: CandidateRef) -> None: + self.exact = PostgreSQLExactPhraseCandidateIndex() + self.replay = replay + + def discover( + self, + request: Acquire, + projection_session: MaterializedProjectionSession, + ) -> tuple[CandidateRef, ...]: + exact = self.exact.discover(request, projection_session) + return exact or (self.replay,) + + +@dataclass(frozen=True, slots=True) +class _FileImportScenario: + organization_id: UUID + membership_id: UUID + receiver: FileImportReceiver + source_ref: SourceRef + prepared: PreparedFileImport + codec: WorkerLeaseCodec + token: WorkerLeaseToken | None + root_ref: FileRootRef + root: Path + + +def _prepare_file_import_scenario( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + *, + payload: bytes | None = b"# Handbook\n\nContextEngine delivers context.\n", + issue_lease: bool = True, + lease_ttl_seconds: int = 300, +) -> _FileImportScenario: + organization_id = uuid4() + user_id = uuid4() + membership_id = uuid4() + receiver = FileImportReceiver(uuid4()) + root_ref = FileRootRef(f"root-{organization_id.hex}") + root = tmp_path / root_ref.value + root.mkdir() + if payload is not None: + (root / "handbook.md").write_bytes(payload) + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text("INSERT INTO organization (organization_id) VALUES (:org)"), + {"org": organization_id}, + ) + connection.execute( + text("INSERT INTO user_account (user_id) VALUES (:user_id)"), + {"user_id": user_id}, + ) + connection.execute( + text( + """ + INSERT INTO membership ( + organization_id, membership_id, user_id, status, + membership_version, valid_from + ) VALUES (:org, :membership_id, :user_id, 'active', 1, :now) + """ + ), + { + "org": organization_id, + "membership_id": membership_id, + "user_id": user_id, + "now": NOW - timedelta(days=1), + }, + ) + connection.execute( + text( + """ + INSERT INTO service_principal ( + organization_id, service_principal_id, workload, + worker_audience, operation, enabled + ) VALUES (:org, :receiver, 'supply.file-import', + 'context-engine-worker', 'file.import', true) + """ + ), + { + "org": organization_id, + "receiver": receiver.service_principal_id, + }, + ) + finally: + migration_engine.dispose() + + authority = ControlOperatorAuthority( + _ControlAuthenticator(organization_id), + call_ttl=timedelta(minutes=5), + clock=lambda: NOW, + ) + control = ContextControl( + store=PostgreSQLControlStore( + guarded_control_engine, + clock=lambda: NOW, + file_import_receiver=receiver, + ), + authority=authority, + clock=lambda: NOW, + ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.REGISTER_SOURCE, + request_id="register-file-security-scenario", + ) as call: + source = control.register_source( + call, + RegisterFileSource("Handbook", root_ref, organization_id.hex), + ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.IMPORT_FILE, + request_id="prepare-file-security-scenario", + ) as call: + prepared = control.prepare_file_import( + call, + PrepareFileImport( + source_ref=source.source_ref, + path=FileImportPath("handbook.md"), + audience=FileImportAudience( + principal_ref="principal:file-reader", + membership_id=membership_id, + membership_version=1, + ), + idempotency_key="file-security-scenario", + ), + ) + codec = WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) + ) + token = ( + PostgreSQLWorkerLeaseIssuer( + guarded_control_engine, + codec, + lease_ttl_seconds=lease_ttl_seconds, + ).issue_file_import_lease(prepared) + if issue_lease + else None + ) + return _FileImportScenario( + organization_id=organization_id, + membership_id=membership_id, + receiver=receiver, + source_ref=source.source_ref, + prepared=prepared, + codec=codec, + token=token, + root_ref=root_ref, + root=root, + ) + + +def _scenario_claims(scenario: _FileImportScenario) -> WorkerLeaseClaims: + assert scenario.token is not None + return scenario.codec.verify( + scenario.token, + expected_organization_id=scenario.organization_id, + expected_job_id=scenario.prepared.job_id, + expected_service_principal_id=scenario.receiver.service_principal_id, + expected_workload=scenario.receiver.workload, + expected_operation=scenario.receiver.operation, + expected_worker_audience=scenario.receiver.worker_audience, + expected_source_ref=str(scenario.source_ref.value), + now=datetime.now(UTC).replace(microsecond=0), + ) + + +def _redeem_direct( + guarded_worker_engine: Engine, + claims: WorkerLeaseClaims, + *, + organization_id: UUID | None = None, + job_id: UUID | None = None, + service_principal_id: UUID | None = None, + source_ref: str | None = None, +) -> object | None: + with guarded_worker_engine.begin() as connection: + return connection.execute( + text( + """ + SELECT * FROM public.context_worker_redeem_file_import( + :organization_id, :job_id, :service_principal_id, + :source_ref, :signing_key_version, :nonce, + :issued_at, :expires_at + ) + """ + ), + { + "organization_id": organization_id or claims.organization_id, + "job_id": job_id or claims.job_id, + "service_principal_id": ( + service_principal_id or claims.service_principal_id + ), + "source_ref": source_ref or claims.source_ref, + "signing_key_version": claims.signing_key_version, + "nonce": claims.nonce, + "issued_at": claims.issued_at, + "expires_at": claims.expires_at, + }, + ).one_or_none() + + +def _publish_direct( + guarded_worker_engine: Engine, + claims: WorkerLeaseClaims, + *, + resource_ref: str, + revision_id: UUID, + organization_id: UUID | None = None, + job_id: UUID | None = None, + service_principal_id: UUID | None = None, + source_ref: str | None = None, +) -> int | None: + with guarded_worker_engine.begin() as connection: + return connection.execute( + text( + """ + SELECT effect_count + FROM public.context_worker_publish_file_import( + :organization_id, :job_id, :service_principal_id, + :source_ref, :resource_ref, + :revision_id, 'fragment:paragraph:1', + '# Handbook\n\nContextEngine delivers context.\n', + 'ContextEngine delivers context.', + :content_hash, :compilation_digest, + 'markdown-v1', 'markdown-config-v1', :phrase_digest, + :signing_key_version, :nonce, :issued_at, :expires_at + ) + """ + ), + { + "organization_id": organization_id or claims.organization_id, + "job_id": job_id or claims.job_id, + "service_principal_id": ( + service_principal_id or claims.service_principal_id + ), + "source_ref": source_ref or claims.source_ref, + "resource_ref": resource_ref, + "revision_id": revision_id, + "content_hash": "a" * 64, + "compilation_digest": "b" * 64, + "phrase_digest": "c" * 64, + "signing_key_version": claims.signing_key_version, + "nonce": claims.nonce, + "issued_at": claims.issued_at, + "expires_at": claims.expires_at, + }, + ).scalar_one_or_none() + + +def _fail_direct( + guarded_worker_engine: Engine, + claims: WorkerLeaseClaims, + *, + organization_id: UUID | None = None, + job_id: UUID | None = None, + service_principal_id: UUID | None = None, + source_ref: str | None = None, +) -> bool: + with guarded_worker_engine.begin() as connection: + return bool( + connection.execute( + text( + """ + SELECT public.context_worker_fail_file_import( + :organization_id, :job_id, :service_principal_id, + :source_ref, :signing_key_version, :nonce, + :issued_at, :expires_at + ) + """ + ), + { + "organization_id": organization_id or claims.organization_id, + "job_id": job_id or claims.job_id, + "service_principal_id": ( + service_principal_id or claims.service_principal_id + ), + "source_ref": source_ref or claims.source_ref, + "signing_key_version": claims.signing_key_version, + "nonce": claims.nonce, + "issued_at": claims.issued_at, + "expires_at": claims.expires_at, + }, + ).scalar_one() + ) + + +def _job_state( + migration_configuration: DatabaseConfiguration, + scenario: _FileImportScenario, +) -> tuple[str, int]: + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + row = connection.execute( + text( + """ + SELECT state, effect_count FROM file_import_job + WHERE organization_id = :org AND job_id = :job_id + """ + ), + { + "org": scenario.organization_id, + "job_id": scenario.prepared.job_id, + }, + ).one() + return row.state, row.effect_count + finally: + migration_engine.dispose() + + +def _scenario_effect_counts( + migration_configuration: DatabaseConfiguration, + scenario: _FileImportScenario, +) -> tuple[int, ...]: + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + return _publication_effect_counts(connection, scenario.organization_id) + finally: + migration_engine.dispose() + + +@pytest.mark.security_evidence(id="PG-FILE-IMPORT-023", layer="postgres") +def test_registered_file_import_publishes_one_exact_authorized_http_package( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, + guarded_runtime_engine: Engine, + guarded_operator_engine: Engine, + query_digest_keyring: QueryDigestKeyring, +) -> None: + organization_id = uuid4() + other_organization_id = uuid4() + user_id = uuid4() + membership_id = uuid4() + other_user_id = uuid4() + other_membership_id = uuid4() + receiver = FileImportReceiver(uuid4()) + migration_engine = create_database_engine(migration_configuration) + root = tmp_path / "handbook" + root.mkdir() + source_bytes = b"# Handbook\n\nContextEngine delivers context.\n" + (root / "handbook.md").write_bytes(source_bytes) + with migration_engine.begin() as connection: + connection.execute( + text("INSERT INTO organization (organization_id) VALUES (:a), (:b)"), + {"a": organization_id, "b": other_organization_id}, + ) + connection.execute( + text( + "INSERT INTO user_account (user_id) " + "VALUES (:user_id), (:other_user_id)" + ), + {"user_id": user_id, "other_user_id": other_user_id}, + ) + connection.execute( + text( + """ + INSERT INTO membership ( + organization_id, membership_id, user_id, status, + membership_version, valid_from + ) VALUES (:organization_id, :membership_id, :user_id, + 'active', 1, :valid_from) + """ + ), + { + "organization_id": organization_id, + "membership_id": membership_id, + "user_id": user_id, + "valid_from": NOW - timedelta(days=1), + }, + ) + connection.execute( + text( + """ + INSERT INTO membership ( + organization_id, membership_id, user_id, status, + membership_version, valid_from + ) VALUES (:organization_id, :membership_id, :user_id, + 'active', 1, :valid_from) + """ + ), + { + "organization_id": other_organization_id, + "membership_id": other_membership_id, + "user_id": other_user_id, + "valid_from": NOW - timedelta(days=1), + }, + ) + connection.execute( + text( + """ + INSERT INTO service_principal ( + organization_id, service_principal_id, workload, + worker_audience, operation, enabled + ) VALUES (:organization_id, :service_principal_id, + 'supply.file-import', 'context-engine-worker', + 'file.import', true) + """ + ), + { + "organization_id": organization_id, + "service_principal_id": receiver.service_principal_id, + }, + ) + + authority = ControlOperatorAuthority( + _ControlAuthenticator(organization_id), + call_ttl=timedelta(minutes=5), + clock=lambda: NOW, + ) + control = ContextControl( + store=PostgreSQLControlStore( + guarded_control_engine, + clock=lambda: NOW, + file_import_receiver=receiver, + ), + authority=authority, + clock=lambda: NOW, + ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.REGISTER_SOURCE, + request_id="register-file", + ) as call: + source = control.register_source( + call, + RegisterFileSource("Handbook", FileRootRef("handbook"), "handbook"), + ) + with migration_engine.connect() as connection: + before_invalid = _publication_effect_counts(connection, organization_id) + with pytest.raises(ValueError, match="Markdown filename"): + PrepareFileImport( + source_ref=source.source_ref, + path=FileImportPath("../outside.md"), + audience=FileImportAudience( + principal_ref="principal:file-reader", + membership_id=membership_id, + membership_version=1, + ), + idempotency_key="outside-import", + ) + with migration_engine.connect() as connection: + assert _publication_effect_counts( + connection, organization_id + ) == before_invalid == (0,) * 10 + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.IMPORT_FILE, + request_id="import-file", + ) as call: + prepared = control.prepare_file_import( + call, + PrepareFileImport( + source_ref=source.source_ref, + path=FileImportPath("handbook.md"), + audience=FileImportAudience( + principal_ref="principal:file-reader", + membership_id=membership_id, + membership_version=1, + ), + idempotency_key="handbook-import", + ), + ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.READ_SOURCE, + request_id="read-activated-file", + ) as call: + activated_source = control.read_source(call, source.source_ref) + assert activated_source.active_version.capabilities.file_source_access.value == ( + "available" + ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.REGISTER_SOURCE, + request_id="register-file-idempotent-after-activation", + ) as call: + registered_again = control.register_source( + call, + RegisterFileSource("Handbook", FileRootRef("handbook"), "handbook"), + ) + assert registered_again == activated_source + + expired_user_id = uuid4() + expired_membership_id = uuid4() + with migration_engine.begin() as connection: + connection.execute( + text("INSERT INTO user_account (user_id) VALUES (:user_id)"), + {"user_id": expired_user_id}, + ) + connection.execute( + text( + """ + INSERT INTO membership ( + organization_id, membership_id, user_id, status, + membership_version, valid_from, valid_until + ) VALUES (:organization_id, :membership_id, :user_id, + 'active', 1, :valid_from, :valid_until) + """ + ), + { + "organization_id": organization_id, + "membership_id": expired_membership_id, + "user_id": expired_user_id, + "valid_from": NOW - timedelta(days=2), + "valid_until": NOW - timedelta(days=1), + }, + ) + with ( + pytest.raises(SourceNotAvailable), + authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.IMPORT_FILE, + request_id="expired-membership-import", + ) as call, + ): + control.prepare_file_import( + call, + PrepareFileImport( + source_ref=source.source_ref, + path=FileImportPath("handbook.md"), + audience=FileImportAudience( + principal_ref="principal:expired-reader", + membership_id=expired_membership_id, + membership_version=1, + ), + idempotency_key="expired-membership-import", + ), + ) + with migration_engine.connect() as connection: + assert _publication_effect_counts(connection, organization_id)[:2] == (1, 1) + + codec = WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) + ) + token = PostgreSQLWorkerLeaseIssuer( + guarded_control_engine, + codec, + ).issue_file_import_lease(prepared) + published = PostgreSQLFileImportWorker( + guarded_worker_engine, + codec, + receiver, + FileRootRegistry( + {FileRootRef("handbook"): root}, + limits=FileReadLimits(max_file_bytes=1024), + ), + MarkdownCompilerConfig("markdown-config-v1"), + clock=lambda: datetime.now(UTC).replace(microsecond=0), + ).run( + FileImportLeaseRedemption( + token, + prepared.organization_id, + prepared.job_id, + prepared.source_ref, + ) + ) + + runtime = Runtime( + required_kernel_dependencies(), + candidate_index=_ExactThenReplayCandidateIndex(published.candidate_ref), + clock=lambda: NOW, + query_digest_keyring=query_digest_keyring, + ) + app = create_app( + authenticator=_MultiTenantRuntimeAuthenticator( + { + "runtime-secret": (organization_id, user_id, membership_id), + "other-runtime-secret": ( + other_organization_id, + other_user_id, + other_membership_id, + ), + } + ), + organization_authority=_OrganizationAuthority(), + membership_authority=PostgreSQLMembershipAuthority(guarded_runtime_engine), + scope_authority=_ExactScopeAuthority( + published.candidate_ref.source_ref, + published.candidate_ref.resource_ref, + ), + runtime=runtime, + clock=lambda: NOW, + request_id_factory=lambda: "file-import-http", + ) + client = TestClient(app) + response = client.post( + "/v1/context:resolve", + headers={"Authorization": "Bearer runtime-secret"}, + json={ + "kind": "acquire", + "need": {"query": "ContextEngine delivers context."}, + }, + ) + + assert response.status_code == 200 + package = response.json()["package"] + assert package["blocks"][0]["text"] == "ContextEngine delivers context." + assert package["evidence"][0]["sourceRef"] == str(source.source_ref.value) + assert package["evidence"][0]["resourceRef"] == published.candidate_ref.resource_ref + assert package["evidence"][0]["revisionRef"] == published.candidate_ref.revision_ref + assert package["evidence"][0]["fragmentRef"] == "fragment:paragraph:1" + assert package["blocks"][0]["evidenceRefs"] == [ + package["evidence"][0]["evidenceRef"] + ] + with exact_test_context_run_operator_read( + control_engine=guarded_control_engine, + operator_engine=guarded_operator_engine, + organization_id=organization_id, + decision_ref=package["decisionRef"], + request_id="file-import-context-run-read", + opaque_credential="file-import-operator-secret", + authorized_at=NOW, + ) as (reader, authorization): + run = reader.find_by_decision_ref(authorization, package["decisionRef"]) + assert run is not None + assert run.outcome is ContextRunOutcome.DELIVERED_AUTHORIZED + assert run.authorized_evidence_refs == ( + package["evidence"][0]["evidenceRef"], + ) + assert run.decision_audit_category is None + + with PostgreSQLMembershipAuthority( + guarded_runtime_engine + ).current_projection_session( + MembershipIdentity( + organization_id=organization_id, + user_id=user_id, + membership_id=membership_id, + membership_version=1, + principal_ref="principal:file-reader", + request_id="file-import-publication-read", + authentication_binding_ref="binding:file-tracer", + checked_at=NOW, + ) + ) as projection_session: + publication = _observe_materialized_publication( + projection_session, + published.candidate_ref, + ) + assert publication is not None + assert publication.states == ("prepared", "indexed", "active") + assert publication.active_revision_ref == published.candidate_ref.revision_ref + + with migration_engine.connect() as connection: + assert _publication_effect_counts(connection, organization_id) == ( + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + ) + assert _publication_effect_counts(connection, other_organization_id) == ( + 0, + ) * 10 + + unauthorized_runtime = Runtime( + required_kernel_dependencies(), + candidate_index=PostgreSQLExactPhraseCandidateIndex(), + clock=lambda: NOW, + query_digest_keyring=query_digest_keyring, + ) + unauthorized = TestClient( + create_app( + authenticator=_RuntimeAuthenticator( + organization_id, user_id, membership_id + ), + organization_authority=_OrganizationAuthority(), + membership_authority=PostgreSQLMembershipAuthority( + guarded_runtime_engine + ), + scope_authority=_ExactScopeAuthority( + published.candidate_ref.source_ref, + published.candidate_ref.resource_ref, + allowed=False, + ), + runtime=unauthorized_runtime, + clock=lambda: NOW, + request_id_factory=lambda: "file-import-unauthorized-http", + ) + ).post( + "/v1/context:resolve", + headers={"Authorization": "Bearer runtime-secret"}, + json={ + "kind": "acquire", + "need": {"query": "ContextEngine delivers context."}, + }, + ) + assert unauthorized.status_code == 200 + assert unauthorized.json()["package"]["blocks"] == [] + assert unauthorized.json()["package"]["evidence"] == [] + assert "ContextEngine delivers context." not in unauthorized.text + + cross_organization = client.post( + "/v1/context:resolve", + headers={"Authorization": "Bearer other-runtime-secret"}, + json={ + "kind": "acquire", + "need": {"query": "ContextEngine delivers context."}, + }, + ) + assert cross_organization.status_code == 200 + assert cross_organization.json()["package"]["blocks"] == [] + assert cross_organization.json()["package"]["evidence"] == [] + assert "ContextEngine delivers context." not in cross_organization.text + with migration_engine.connect() as connection: + assert _publication_effect_counts(connection, organization_id) == ( + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + ) + assert _publication_effect_counts(connection, other_organization_id) == ( + 0, + ) * 10 + + +def test_missing_file_after_redemption_records_terminal_zero_effect_failure( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + organization_id = uuid4() + user_id = uuid4() + membership_id = uuid4() + receiver = FileImportReceiver(uuid4()) + migration_engine = create_database_engine(migration_configuration) + with migration_engine.begin() as connection: + connection.execute( + text("INSERT INTO organization (organization_id) VALUES (:org)"), + {"org": organization_id}, + ) + connection.execute( + text("INSERT INTO user_account (user_id) VALUES (:user_id)"), + {"user_id": user_id}, + ) + connection.execute( + text( + """ + INSERT INTO membership ( + organization_id, membership_id, user_id, status, + membership_version, valid_from + ) VALUES (:org, :membership_id, :user_id, 'active', 1, :now) + """ + ), + { + "org": organization_id, + "membership_id": membership_id, + "user_id": user_id, + "now": NOW - timedelta(days=1), + }, + ) + connection.execute( + text( + """ + INSERT INTO service_principal ( + organization_id, service_principal_id, workload, + worker_audience, operation, enabled + ) VALUES (:org, :receiver, 'supply.file-import', + 'context-engine-worker', 'file.import', true) + """ + ), + {"org": organization_id, "receiver": receiver.service_principal_id}, + ) + authority = ControlOperatorAuthority( + _ControlAuthenticator(organization_id), + call_ttl=timedelta(minutes=5), + clock=lambda: NOW, + ) + control = ContextControl( + store=PostgreSQLControlStore( + guarded_control_engine, + clock=lambda: NOW, + file_import_receiver=receiver, + ), + authority=authority, + clock=lambda: NOW, + ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.REGISTER_SOURCE, + request_id="register-missing-file", + ) as call: + source = control.register_source( + call, + RegisterFileSource("Missing", FileRootRef("missing"), "missing"), + ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.IMPORT_FILE, + request_id="import-missing-file", + ) as call: + prepared = control.prepare_file_import( + call, + PrepareFileImport( + source_ref=source.source_ref, + path=FileImportPath("missing.md"), + audience=FileImportAudience( + principal_ref="principal:file-reader", + membership_id=membership_id, + membership_version=1, + ), + idempotency_key="missing-file", + ), + ) + codec = WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) + ) + token = PostgreSQLWorkerLeaseIssuer( + guarded_control_engine, + codec, + ).issue_file_import_lease(prepared) + root = tmp_path / "missing-root" + root.mkdir() + worker = PostgreSQLFileImportWorker( + guarded_worker_engine, + codec, + receiver, + FileRootRegistry( + {FileRootRef("missing"): root}, + limits=FileReadLimits(max_file_bytes=1024), + ), + MarkdownCompilerConfig("markdown-config-v1"), + clock=lambda: datetime.now(UTC).replace(microsecond=0), + ) + with pytest.raises(FileImportUnavailable): + worker.run( + FileImportLeaseRedemption( + token, + prepared.organization_id, + prepared.job_id, + prepared.source_ref, + ) + ) + with migration_engine.connect() as connection: + state, failed_at, effect_count = connection.execute( + text( + """ + SELECT state, failed_at, effect_count + FROM file_import_job + WHERE organization_id = :org AND job_id = :job_id + """ + ), + {"org": organization_id, "job_id": prepared.job_id}, + ).one() + assert state == "failed" + assert failed_at is not None + assert effect_count == 0 + assert _publication_effect_counts(connection, organization_id) == ( + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + + +@pytest.mark.parametrize( + "changed_binding", + ["organization", "job", "receiver", "source"], +) +def test_redeem_database_boundary_rejects_every_wrong_exact_binding( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, + changed_binding: str, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, migration_configuration, guarded_control_engine + ) + claims = _scenario_claims(scenario) + organization_id: UUID | None = None + job_id: UUID | None = None + service_principal_id: UUID | None = None + source_ref: str | None = None + if changed_binding == "organization": + organization_id = uuid4() + elif changed_binding == "job": + job_id = uuid4() + elif changed_binding == "receiver": + service_principal_id = uuid4() + else: + source_ref = str(uuid4()) + + assert ( + _redeem_direct( + guarded_worker_engine, + claims, + organization_id=organization_id, + job_id=job_id, + service_principal_id=service_principal_id, + source_ref=source_ref, + ) + is None + ) + assert _job_state(migration_configuration, scenario) == ("leased", 0) + assert _scenario_effect_counts(migration_configuration, scenario) == ( + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + + +def test_redeem_is_one_shot_before_any_content_effect( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, migration_configuration, guarded_control_engine + ) + claims = _scenario_claims(scenario) + + assert _redeem_direct(guarded_worker_engine, claims) is not None + assert _redeem_direct(guarded_worker_engine, claims) is None + assert _job_state(migration_configuration, scenario) == ("running", 0) + assert _scenario_effect_counts(migration_configuration, scenario)[2:] == ( + 0, + ) * 8 + + +@pytest.mark.parametrize("operation", ["publish", "fail"]) +@pytest.mark.parametrize( + "changed_binding", + ["organization", "job", "receiver", "source"], +) +def test_running_job_database_boundary_rejects_every_wrong_exact_binding( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, + operation: str, + changed_binding: str, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, migration_configuration, guarded_control_engine + ) + claims = _scenario_claims(scenario) + assert _redeem_direct(guarded_worker_engine, claims) is not None + organization_id: UUID | None = None + job_id: UUID | None = None + service_principal_id: UUID | None = None + source_ref: str | None = None + if changed_binding == "organization": + organization_id = uuid4() + elif changed_binding == "job": + job_id = uuid4() + elif changed_binding == "receiver": + service_principal_id = uuid4() + else: + source_ref = str(uuid4()) + + if operation == "publish": + result: int | bool | None = _publish_direct( + guarded_worker_engine, + claims, + resource_ref=f"resource:test:{uuid4()}", + revision_id=uuid4(), + organization_id=organization_id, + job_id=job_id, + service_principal_id=service_principal_id, + source_ref=source_ref, + ) + assert result is None + else: + result = _fail_direct( + guarded_worker_engine, + claims, + organization_id=organization_id, + job_id=job_id, + service_principal_id=service_principal_id, + source_ref=source_ref, + ) + assert result is False + assert _job_state(migration_configuration, scenario) == ("running", 0) + assert _scenario_effect_counts(migration_configuration, scenario)[2:] == ( + 0, + ) * 8 + + +def test_completed_file_import_rejects_redeem_publish_and_fail_replay( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, migration_configuration, guarded_control_engine + ) + claims = _scenario_claims(scenario) + resource_ref = f"resource:test:{uuid4()}" + assert _redeem_direct(guarded_worker_engine, claims) is not None + assert ( + _publish_direct( + guarded_worker_engine, + claims, + resource_ref=resource_ref, + revision_id=uuid4(), + ) + == 1 + ) + + before_replay = _scenario_effect_counts(migration_configuration, scenario) + assert _redeem_direct(guarded_worker_engine, claims) is None + assert ( + _publish_direct( + guarded_worker_engine, + claims, + resource_ref=f"resource:test:{uuid4()}", + revision_id=uuid4(), + ) + is None + ) + assert _fail_direct(guarded_worker_engine, claims) is False + assert _job_state(migration_configuration, scenario) == ("completed", 1) + assert _scenario_effect_counts( + migration_configuration, scenario + ) == before_replay == (1, 1, 1, 1, 1, 1, 3, 1, 1, 1) + + +def test_failed_file_import_rejects_fail_redeem_and_publish_replay( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, migration_configuration, guarded_control_engine + ) + claims = _scenario_claims(scenario) + assert _redeem_direct(guarded_worker_engine, claims) is not None + assert _fail_direct(guarded_worker_engine, claims) is True + + before_replay = _scenario_effect_counts(migration_configuration, scenario) + assert _fail_direct(guarded_worker_engine, claims) is False + assert _redeem_direct(guarded_worker_engine, claims) is None + assert ( + _publish_direct( + guarded_worker_engine, + claims, + resource_ref=f"resource:test:{uuid4()}", + revision_id=uuid4(), + ) + is None + ) + assert _job_state(migration_configuration, scenario) == ("failed", 0) + assert _scenario_effect_counts( + migration_configuration, scenario + ) == before_replay == (1, 1, 0, 0, 0, 0, 0, 0, 0, 0) + + +def test_disabled_receiver_after_redeem_cannot_publish_or_record_failure( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, migration_configuration, guarded_control_engine + ) + claims = _scenario_claims(scenario) + assert _redeem_direct(guarded_worker_engine, claims) is not None + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text( + """ + UPDATE service_principal SET enabled = false + WHERE organization_id = :org + AND service_principal_id = :receiver + """ + ), + { + "org": scenario.organization_id, + "receiver": scenario.receiver.service_principal_id, + }, + ) + finally: + migration_engine.dispose() + + assert ( + _publish_direct( + guarded_worker_engine, + claims, + resource_ref=f"resource:test:{uuid4()}", + revision_id=uuid4(), + ) + is None + ) + assert _fail_direct(guarded_worker_engine, claims) is False + assert _job_state(migration_configuration, scenario) == ("running", 0) + assert _scenario_effect_counts(migration_configuration, scenario)[2:] == ( + 0, + ) * 8 + + +def test_expired_redeemed_lease_cannot_publish_or_record_failure( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, + migration_configuration, + guarded_control_engine, + lease_ttl_seconds=1, + ) + claims = _scenario_claims(scenario) + assert _redeem_direct(guarded_worker_engine, claims) is not None + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + connection.execute(text("SELECT pg_sleep(1.1)")) + finally: + migration_engine.dispose() + + assert ( + _publish_direct( + guarded_worker_engine, + claims, + resource_ref=f"resource:test:{uuid4()}", + revision_id=uuid4(), + ) + is None + ) + assert _fail_direct(guarded_worker_engine, claims) is False + assert _job_state(migration_configuration, scenario) == ("running", 0) + assert _scenario_effect_counts(migration_configuration, scenario)[2:] == ( + 0, + ) * 8 + + +def test_invalid_markdown_records_terminal_failure_without_content_effects( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, + migration_configuration, + guarded_control_engine, + payload=b"## Unsupported heading\n", + ) + assert scenario.token is not None + worker = PostgreSQLFileImportWorker( + guarded_worker_engine, + scenario.codec, + scenario.receiver, + FileRootRegistry( + {scenario.root_ref: scenario.root}, + limits=FileReadLimits(max_file_bytes=1024), + ), + MarkdownCompilerConfig("markdown-config-v1"), + clock=lambda: datetime.now(UTC).replace(microsecond=0), + ) + + with pytest.raises(FileImportUnavailable): + worker.run( + FileImportLeaseRedemption( + scenario.token, + scenario.organization_id, + scenario.prepared.job_id, + scenario.source_ref, + ) + ) + + assert _job_state(migration_configuration, scenario) == ("failed", 0) + assert _scenario_effect_counts(migration_configuration, scenario) == ( + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + + +def test_late_publication_error_rolls_back_every_content_and_access_write( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, + guarded_control_engine: Engine, + guarded_worker_engine: Engine, +) -> None: + scenario = _prepare_file_import_scenario( + tmp_path, migration_configuration, guarded_control_engine + ) + claims = _scenario_claims(scenario) + assert _redeem_direct(guarded_worker_engine, claims) is not None + migration_engine = create_database_engine(migration_configuration) + trigger_installed = False + try: + with migration_engine.begin() as connection: + connection.execute( + text( + """ + CREATE FUNCTION public.context_test_reject_file_activation() + RETURNS trigger LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'injected late file publication failure'; + END; $function$ + """ + ) + ) + connection.execute( + text( + """ + CREATE TRIGGER reject_file_activation + BEFORE UPDATE OF active_revision_id ON context_resource + FOR EACH ROW EXECUTE FUNCTION + public.context_test_reject_file_activation() + """ + ) + ) + trigger_installed = True + with pytest.raises(SQLAlchemyError, match="injected late"): + _publish_direct( + guarded_worker_engine, + claims, + resource_ref=f"resource:test:{uuid4()}", + revision_id=uuid4(), + ) + finally: + if trigger_installed: + with migration_engine.begin() as connection: + connection.execute( + text( + "DROP TRIGGER IF EXISTS reject_file_activation " + "ON context_resource" + ) + ) + connection.execute( + text( + "DROP FUNCTION IF EXISTS " + "public.context_test_reject_file_activation()" + ) + ) + migration_engine.dispose() + + assert _job_state(migration_configuration, scenario) == ("running", 0) + assert _scenario_effect_counts(migration_configuration, scenario) == ( + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) diff --git a/tests/integration/test_m0_security_gate_rls.py b/tests/integration/test_m0_security_gate_rls.py index ebc032c6..21abcfb5 100644 --- a/tests/integration/test_m0_security_gate_rls.py +++ b/tests/integration/test_m0_security_gate_rls.py @@ -30,7 +30,7 @@ def _manifest() -> dict[str, object]: def test_all_manifest_tenant_tables_pass_live_non_owner_rls_audit( guarded_runtime_engine: Engine, ) -> None: - """PG-RLS-ALL-TENANT-TABLES: the live denominator is exactly 22/22.""" + """PG-RLS-ALL-TENANT-TABLES: the live denominator is exactly 27/27.""" with guarded_runtime_engine.connect() as connection: report = audit_live_rls( @@ -41,13 +41,13 @@ def test_all_manifest_tenant_tables_pass_live_non_owner_rls_audit( assert report["passed"] is True assert report["denominator"] == { - "allTables": 25, - "tenantOwned": 22, + "allTables": 30, + "tenantOwned": 27, "global": 3, } assert report["coverage"] == { - "numerator": 22, - "denominator": 22, + "numerator": 27, + "denominator": 27, "percent": 100.0, } assert report["failures"] == [] @@ -89,9 +89,9 @@ def test_no_force_row_level_security_mutation_fails_and_rolls_back( assert mutated["passed"] is False assert mutated["coverage"] == { - "numerator": 21, - "denominator": 22, - "percent": 95.45, + "numerator": 26, + "denominator": 27, + "percent": 96.3, } tenant_tables = cast( list[dict[str, Any]], mutated["tenantTables"] @@ -116,8 +116,8 @@ def test_no_force_row_level_security_mutation_fails_and_rolls_back( ) assert restored["passed"] is True assert restored["coverage"] == { - "numerator": 22, - "denominator": 22, + "numerator": 27, + "denominator": 27, "percent": 100.0, } diff --git a/tests/integration/test_membership_field_projection_integration.py b/tests/integration/test_membership_field_projection_integration.py index 9debf473..fc81954d 100644 --- a/tests/integration/test_membership_field_projection_integration.py +++ b/tests/integration/test_membership_field_projection_integration.py @@ -180,7 +180,10 @@ def __init__(self, candidate: CandidateRef) -> None: self.calls: list[Acquire] = [] self.returned_candidates: list[CandidateRef] = [] - def discover(self, request: Acquire) -> tuple[CandidateRef, ...]: + def discover( + self, request: Acquire, projection_session: object + ) -> tuple[CandidateRef, ...]: + del projection_session self.calls.append(request) self.returned_candidates.append(self.candidate) return (self.candidate,) diff --git a/tests/integration/test_membership_schema.py b/tests/integration/test_membership_schema.py index b35e640f..5a230262 100644 --- a/tests/integration/test_membership_schema.py +++ b/tests/integration/test_membership_schema.py @@ -510,6 +510,7 @@ def test_runtime_worker_and_public_grants_are_least_privilege( assert security == (True, True) assert set(policies) == { "membership_current_user_actor", + "membership_file_import_definer_select", "membership_migrator_administration", } runtime_policy = policies["membership_current_user_actor"] diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 2d88d511..4c0dd1d4 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -19,6 +19,38 @@ pytestmark = pytest.mark.integration ROOT = Path(__file__).parents[2] +HEAD_TABLES = [ + "active_release_manifest", + "alembic_version", + "context_fragment", + "context_fragment_field", + "context_resource", + "context_revision", + "context_run", + "context_run_operator_read_ticket", + "context_source", + "decision_audit", + "exact_phrase_candidate", + "file_acquisition", + "file_import_job", + "file_revision_snapshot", + "membership", + "membership_resource_field_right", + "organization", + "organization_policy_epoch", + "organization_record", + "release_candidate", + "release_evaluation", + "release_manifest", + "release_operator_grant", + "release_promotion_audit", + "resource_access_policy", + "revision_publication_event", + "service_principal", + "source_version", + "user_account", + "worker_noop_job", +] def _revision_rows(configuration: DatabaseConfiguration) -> list[str]: @@ -67,7 +99,7 @@ def test_empty_baseline_remains_a_reversible_historical_revision( assert _application_tables(migration_configuration) == ["alembic_version"] finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] + assert _revision_rows(migration_configuration) == ["20260722_0011"] def test_organization_isolation_revision_downgrades_and_reapplies_cleanly( @@ -82,34 +114,8 @@ def test_organization_isolation_revision_downgrades_and_reapplies_cleanly( finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] - assert _application_tables(migration_configuration) == [ - "active_release_manifest", - "alembic_version", - "context_fragment", - "context_fragment_field", - "context_resource", - "context_revision", - "context_run", - "context_run_operator_read_ticket", - "context_source", - "decision_audit", - "membership", - "membership_resource_field_right", - "organization", - "organization_policy_epoch", - "organization_record", - "release_candidate", - "release_evaluation", - "release_manifest", - "release_operator_grant", - "release_promotion_audit", - "resource_access_policy", - "service_principal", - "source_version", - "user_account", - "worker_noop_job", - ] + assert _revision_rows(migration_configuration) == ["20260722_0011"] + assert _application_tables(migration_configuration) == HEAD_TABLES def test_membership_revision_downgrades_to_issue_8_and_reapplies_cleanly( @@ -128,7 +134,7 @@ def test_membership_revision_downgrades_to_issue_8_and_reapplies_cleanly( finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] + assert _revision_rows(migration_configuration) == ["20260722_0011"] def test_content_schema_revision_downgrades_to_membership_and_reapplies_cleanly( @@ -149,34 +155,8 @@ def test_content_schema_revision_downgrades_to_membership_and_reapplies_cleanly( finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] - assert _application_tables(migration_configuration) == [ - "active_release_manifest", - "alembic_version", - "context_fragment", - "context_fragment_field", - "context_resource", - "context_revision", - "context_run", - "context_run_operator_read_ticket", - "context_source", - "decision_audit", - "membership", - "membership_resource_field_right", - "organization", - "organization_policy_epoch", - "organization_record", - "release_candidate", - "release_evaluation", - "release_manifest", - "release_operator_grant", - "release_promotion_audit", - "resource_access_policy", - "service_principal", - "source_version", - "user_account", - "worker_noop_job", - ] + assert _revision_rows(migration_configuration) == ["20260722_0011"] + assert _application_tables(migration_configuration) == HEAD_TABLES def test_policy_epoch_revision_downgrades_to_content_and_reapplies_cleanly( @@ -202,34 +182,8 @@ def test_policy_epoch_revision_downgrades_to_content_and_reapplies_cleanly( finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] - assert _application_tables(migration_configuration) == [ - "active_release_manifest", - "alembic_version", - "context_fragment", - "context_fragment_field", - "context_resource", - "context_revision", - "context_run", - "context_run_operator_read_ticket", - "context_source", - "decision_audit", - "membership", - "membership_resource_field_right", - "organization", - "organization_policy_epoch", - "organization_record", - "release_candidate", - "release_evaluation", - "release_manifest", - "release_operator_grant", - "release_promotion_audit", - "resource_access_policy", - "service_principal", - "source_version", - "user_account", - "worker_noop_job", - ] + assert _revision_rows(migration_configuration) == ["20260722_0011"] + assert _application_tables(migration_configuration) == HEAD_TABLES def test_worker_lease_revision_downgrades_to_policy_epoch_and_reapplies_cleanly( @@ -257,34 +211,8 @@ def test_worker_lease_revision_downgrades_to_policy_epoch_and_reapplies_cleanly( finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] - assert _application_tables(migration_configuration) == [ - "active_release_manifest", - "alembic_version", - "context_fragment", - "context_fragment_field", - "context_resource", - "context_revision", - "context_run", - "context_run_operator_read_ticket", - "context_source", - "decision_audit", - "membership", - "membership_resource_field_right", - "organization", - "organization_policy_epoch", - "organization_record", - "release_candidate", - "release_evaluation", - "release_manifest", - "release_operator_grant", - "release_promotion_audit", - "resource_access_policy", - "service_principal", - "source_version", - "user_account", - "worker_noop_job", - ] + assert _revision_rows(migration_configuration) == ["20260722_0011"] + assert _application_tables(migration_configuration) == HEAD_TABLES def test_decision_lineage_revision_downgrades_to_worker_lease_and_reapplies_cleanly( @@ -305,34 +233,8 @@ def test_decision_lineage_revision_downgrades_to_worker_lease_and_reapplies_clea finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] - assert _application_tables(migration_configuration) == [ - "active_release_manifest", - "alembic_version", - "context_fragment", - "context_fragment_field", - "context_resource", - "context_revision", - "context_run", - "context_run_operator_read_ticket", - "context_source", - "decision_audit", - "membership", - "membership_resource_field_right", - "organization", - "organization_policy_epoch", - "organization_record", - "release_candidate", - "release_evaluation", - "release_manifest", - "release_operator_grant", - "release_promotion_audit", - "resource_access_policy", - "service_principal", - "source_version", - "user_account", - "worker_noop_job", - ] + assert _revision_rows(migration_configuration) == ["20260722_0011"] + assert _application_tables(migration_configuration) == HEAD_TABLES def test_field_projection_revision_downgrades_to_decision_lineage_and_reapplies_cleanly( @@ -369,7 +271,7 @@ def test_field_projection_revision_downgrades_to_decision_lineage_and_reapplies_ finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] + assert _revision_rows(migration_configuration) == ["20260722_0011"] assert "context_fragment_field" in _application_tables(migration_configuration) assert "membership_resource_field_right" in _application_tables( migration_configuration @@ -392,7 +294,7 @@ def test_file_source_revision_downgrades_to_learning_release_and_reapplies_clean finally: command.upgrade(alembic_configuration, "head") - assert _revision_rows(migration_configuration) == ["20260722_0010"] + assert _revision_rows(migration_configuration) == ["20260722_0011"] tables = _application_tables(migration_configuration) assert "context_source" in tables assert "source_version" in tables @@ -630,7 +532,7 @@ def test_field_projection_downgrade_refuses_populated_content_atomically( ): command.downgrade(alembic_configuration, "20260722_0007") - assert _revision_rows(migration_configuration) == ["20260722_0010"] + assert _revision_rows(migration_configuration) == ["20260722_0011"] with engine.connect() as connection: assert connection.execute( text( @@ -692,7 +594,7 @@ def test_field_projection_downgrade_refuses_populated_content_atomically( ): connection.execute(text(statement), parameters) except SQLAlchemyError: - if _revision_rows(migration_configuration) != ["20260722_0010"]: + if _revision_rows(migration_configuration) != ["20260722_0011"]: command.upgrade(alembic_configuration, "head") raise finally: @@ -843,7 +745,7 @@ def test_field_projection_downgrade_serializes_with_concurrent_fragment_insert( parameters, ).scalar_one() == "concurrent-private-body" finally: - if _revision_rows(migration_configuration) != ["20260722_0010"]: + if _revision_rows(migration_configuration) != ["20260722_0011"]: command.upgrade(alembic_configuration, "head") with engine.begin() as connection: connection.execute( diff --git a/tests/integration/test_runtime_authorized_evidence_integration.py b/tests/integration/test_runtime_authorized_evidence_integration.py index 5d97e8cd..b014976b 100644 --- a/tests/integration/test_runtime_authorized_evidence_integration.py +++ b/tests/integration/test_runtime_authorized_evidence_integration.py @@ -190,7 +190,10 @@ def __init__( ) self.calls: list[Acquire] = [] - def discover(self, request: Acquire) -> tuple[CandidateRef, ...]: + def discover( + self, request: Acquire, projection_session: object + ) -> tuple[CandidateRef, ...]: + del projection_session self.calls.append(request) return self._ranked diff --git a/tests/integration/test_runtime_empty_package_integration.py b/tests/integration/test_runtime_empty_package_integration.py index 21f046e2..49aefd46 100644 --- a/tests/integration/test_runtime_empty_package_integration.py +++ b/tests/integration/test_runtime_empty_package_integration.py @@ -107,7 +107,8 @@ class ContentIoSpy: def __init__(self) -> None: self.calls = 0 - def discover(self, request: Acquire) -> tuple[()]: + def discover(self, request: Acquire, projection_session: object) -> tuple[()]: + del projection_session self.calls += 1 return () diff --git a/tests/integration/test_runtime_non_enumeration_integration.py b/tests/integration/test_runtime_non_enumeration_integration.py index e5dbb525..0a0d297c 100644 --- a/tests/integration/test_runtime_non_enumeration_integration.py +++ b/tests/integration/test_runtime_non_enumeration_integration.py @@ -123,7 +123,10 @@ def __init__( self.rankings = rankings self.calls: list[Acquire] = [] - def discover(self, request: Acquire) -> tuple[CandidateRef, ...]: + def discover( + self, request: Acquire, projection_session: object + ) -> tuple[CandidateRef, ...]: + del projection_session call_index = len(self.calls) self.calls.append(request) if call_index >= len(self.rankings): diff --git a/tests/unit/test_context_control.py b/tests/unit/test_context_control.py index 7b5d3afb..feeeb421 100644 --- a/tests/unit/test_context_control.py +++ b/tests/unit/test_context_control.py @@ -23,6 +23,12 @@ TrustedControlCall, VerifiedControlOperatorIdentity, ) +from engine.supply import ( + FileImportAudience, + FileImportPath, + PreparedFileImport, + PrepareFileImport, +) ORGANIZATION_ID = UUID("a6776454-3a24-4c1c-998c-3a69a1d3de23") NOW = datetime(2026, 7, 22, 18, 50, tzinfo=UTC) @@ -38,7 +44,11 @@ def authenticate(self, opaque_credential: str) -> VerifiedControlOperatorIdentit authentication_binding_ref="control-binding-a", authority_ref="source-admin-a", allowed_operations=frozenset( - {ControlOperation.REGISTER_SOURCE, ControlOperation.READ_SOURCE} + { + ControlOperation.IMPORT_FILE, + ControlOperation.REGISTER_SOURCE, + ControlOperation.READ_SOURCE, + } ), valid_from=NOW - timedelta(minutes=1), expires_at=NOW + timedelta(hours=1), @@ -74,6 +84,20 @@ def read_source( raise SourceNotAvailable return self.manifest + def prepare_file_import( + self, call: TrustedControlCall, command: PrepareFileImport + ) -> PreparedFileImport: + assert call.organization_id == ORGANIZATION_ID + assert call.operation is ControlOperation.IMPORT_FILE + assert self.manifest is not None + assert command.source_ref == self.manifest.source_ref + return PreparedFileImport( + organization_id=ORGANIZATION_ID, + job_id=UUID("9de5b515-540b-4c9c-b1d3-f9b691dfbb7a"), + source_ref=command.source_ref, + service_principal_id=UUID("0f7bc78d-a76a-477c-b097-ce557b7844b9"), + ) + def _authority() -> ControlOperatorAuthority: return ControlOperatorAuthority( @@ -182,6 +206,44 @@ def test_authorized_operator_registers_and_reads_one_honest_file_manifest() -> N assert control.read_source(call, registered.source_ref) == registered +def test_authorized_operator_prepares_one_credential_free_file_import() -> None: + store = _Store() + authority = _authority() + control = ContextControl(store=store, authority=authority, clock=lambda: NOW) + with authority.authorize( + opaque_credential="control-credential-a", + operation=ControlOperation.REGISTER_SOURCE, + request_id="register-for-import", + ) as call: + source = control.register_source( + call, + RegisterFileSource("Handbook", FileRootRef("handbook"), "handbook"), + ) + command = PrepareFileImport( + source_ref=source.source_ref, + path=FileImportPath("handbook.md"), + audience=FileImportAudience( + principal_ref="principal:handbook-reader", + membership_id=UUID("82a11990-7a87-4693-a3de-c3cab5fab7aa"), + membership_version=1, + ), + idempotency_key="handbook-import-v1", + ) + + with authority.authorize( + opaque_credential="control-credential-a", + operation=ControlOperation.IMPORT_FILE, + request_id="prepare-import", + ) as call: + prepared = control.prepare_file_import(call, command) + + assert prepared.organization_id == ORGANIZATION_ID + assert prepared.source_ref == source.source_ref + assert prepared.workload == "supply.file-import" + assert prepared.operation == "file.import" + assert "credential" not in repr(prepared).casefold() + + def test_source_ref_and_forged_or_wrong_operation_calls_never_authorize_control() -> ( None ): diff --git a/tests/unit/test_effective_scope_runtime.py b/tests/unit/test_effective_scope_runtime.py index cac501c7..d6bddded 100644 --- a/tests/unit/test_effective_scope_runtime.py +++ b/tests/unit/test_effective_scope_runtime.py @@ -63,8 +63,8 @@ class ContentIoSpy: def __init__(self) -> None: self.calls = 0 - def discover(self, request: Acquire) -> tuple[()]: - del request + def discover(self, request: Acquire, projection_session: object) -> tuple[()]: + del request, projection_session self.calls += 1 return () diff --git a/tests/unit/test_file_import.py b/tests/unit/test_file_import.py new file mode 100644 index 00000000..18840e34 --- /dev/null +++ b/tests/unit/test_file_import.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from dataclasses import fields +from datetime import UTC, datetime +from pathlib import Path +from uuid import UUID + +import pytest + +from adapters.file_source import FileReadLimits, FileRootRegistry +from engine.control import ( + FileImportAudience, + FileImportPath, + FileRootRef, + PrepareFileImport, + SourceRef, +) + +ORGANIZATION_ID = UUID("62f7e3b4-e7cf-44c5-afaf-2f58032801e0") +MEMBERSHIP_ID = UUID("48e1ab62-8f7f-44c2-9d38-4a918d315f07") +SOURCE_ID = UUID("1a4743a4-747e-423f-8dd9-7cccfb5c1d3c") +NOW = datetime(2026, 7, 22, 22, 0, tzinfo=UTC) + + +@pytest.mark.parametrize( + "value", + ( + "", + ".", + "..", + "../handbook.md", + "/tmp/handbook.md", + "folder/handbook.md", + "folder\\handbook.md", + "handbook.txt", + " handbook.md", + "handbook.md ", + ), +) +def test_manual_import_path_is_one_bounded_markdown_filename(value: str) -> None: + with pytest.raises(ValueError, match="Markdown filename"): + FileImportPath(value) + + +def test_manual_import_command_contains_no_host_path_or_credentials() -> None: + command = PrepareFileImport( + source_ref=SourceRef(SOURCE_ID), + path=FileImportPath("handbook.md"), + audience=FileImportAudience( + principal_ref="principal:handbook-reader", + membership_id=MEMBERSHIP_ID, + membership_version=3, + ), + idempotency_key="handbook-import-v1", + ) + + assert [field.name for field in fields(command)] == [ + "source_ref", + "path", + "audience", + "idempotency_key", + ] + rendered = repr(command).casefold() + for forbidden in ("credential", "password", "token", "/tmp", "file://"): + assert forbidden not in rendered + + +def test_root_registry_resolves_only_a_registered_logical_root( + tmp_path: Path, +) -> None: + root = tmp_path / "registered-root" + root.mkdir() + registry = FileRootRegistry( + {FileRootRef("handbook-root"): root}, + limits=FileReadLimits(max_file_bytes=1024), + ) + + target = registry.resolve( + FileRootRef("handbook-root"), + FileImportPath("handbook.md"), + ) + + assert target == root / "handbook.md" + with pytest.raises(LookupError, match="File root is not configured"): + registry.resolve( + FileRootRef("unknown-root"), + FileImportPath("handbook.md"), + ) + + +def test_root_registry_reads_one_regular_file_and_rejects_symlinks( + tmp_path: Path, +) -> None: + root = tmp_path / "registered-root" + root.mkdir() + expected = b"# Handbook\n\nContextEngine delivers context.\n" + (root / "handbook.md").write_bytes(expected) + outside = tmp_path / "outside.md" + outside.write_bytes(b"outside") + (root / "linked.md").symlink_to(outside) + registry = FileRootRegistry( + {FileRootRef("handbook-root"): root}, + limits=FileReadLimits(max_file_bytes=1024), + ) + + assert registry.read( + FileRootRef("handbook-root"), FileImportPath("handbook.md") + ) == expected + with pytest.raises(LookupError, match="regular configured-root file"): + registry.read( + FileRootRef("handbook-root"), FileImportPath("linked.md") + ) + + +def test_root_registry_anchors_directory_before_a_path_swap(tmp_path: Path) -> None: + root = tmp_path / "registered-root" + outside = tmp_path / "outside-root" + root.mkdir() + outside.mkdir() + (root / "handbook.md").write_bytes(b"inside") + (outside / "handbook.md").write_bytes(b"outside") + registry = FileRootRegistry( + {FileRootRef("handbook-root"): root}, + limits=FileReadLimits(max_file_bytes=1024), + ) + anchored = tmp_path / "anchored-root" + root.rename(anchored) + root.symlink_to(outside, target_is_directory=True) + + assert registry.read( + FileRootRef("handbook-root"), FileImportPath("handbook.md") + ) == b"inside" + + +def test_root_registry_rejects_symlinked_roots_and_oversized_files( + tmp_path: Path, +) -> None: + root = tmp_path / "registered-root" + root.mkdir() + linked_root = tmp_path / "linked-root" + linked_root.symlink_to(root, target_is_directory=True) + with pytest.raises(ValueError, match="non-symlink directory"): + FileRootRegistry( + {FileRootRef("handbook-root"): linked_root}, + limits=FileReadLimits(max_file_bytes=4), + ) + + (root / "handbook.md").write_bytes(b"12345") + registry = FileRootRegistry( + {FileRootRef("handbook-root"): root}, + limits=FileReadLimits(max_file_bytes=4), + ) + with pytest.raises(LookupError, match="regular configured-root file"): + registry.read( + FileRootRef("handbook-root"), FileImportPath("handbook.md") + ) diff --git a/tests/unit/test_http_trust_boundary.py b/tests/unit/test_http_trust_boundary.py index 69eff9d2..36adb6d4 100644 --- a/tests/unit/test_http_trust_boundary.py +++ b/tests/unit/test_http_trust_boundary.py @@ -217,8 +217,8 @@ def __init__(self) -> None: self.provider_calls = 0 self.source_content_calls = 0 - def discover(self, request: Acquire) -> tuple[()]: - del request + def discover(self, request: Acquire, projection_session: object) -> tuple[()]: + del request, projection_session self.index_calls += 1 return () diff --git a/tests/unit/test_m0_rls_inventory.py b/tests/unit/test_m0_rls_inventory.py index d3331ca3..6f7c9019 100644 --- a/tests/unit/test_m0_rls_inventory.py +++ b/tests/unit/test_m0_rls_inventory.py @@ -23,6 +23,10 @@ "context_run_operator_read_ticket", "context_source", "decision_audit", + "exact_phrase_candidate", + "file_acquisition", + "file_import_job", + "file_revision_snapshot", "membership", "membership_resource_field_right", "organization_policy_epoch", @@ -32,6 +36,7 @@ "release_manifest", "release_operator_grant", "release_promotion_audit", + "revision_publication_event", "resource_access_policy", "service_principal", "source_version", @@ -120,7 +125,7 @@ def test_manifest_declares_exact_live_table_denominator_and_rls_evidence() -> No assert global_tables == GLOBAL_TABLES assert tenant_tables == TENANT_TABLES - assert len(tables) == 25 + assert len(tables) == 30 for name in sorted(GLOBAL_TABLES): rationale = tables[name]["classificationRationale"] @@ -143,8 +148,8 @@ def test_rls_auditor_requires_every_live_control_and_non_owner_evidence() -> Non assert report["passed"] is True assert report["coverage"] == { - "numerator": 22, - "denominator": 22, + "numerator": 27, + "denominator": 27, "percent": 100.0, } inventory = cast(dict[str, object], report["inventory"]) @@ -169,7 +174,7 @@ def test_rls_auditor_does_not_count_force_rls_or_evidence_gaps() -> None: assert report["passed"] is False assert report["coverage"] == { "numerator": 0, - "denominator": 22, + "denominator": 27, "percent": 0.0, } tenant_reports = cast(list[dict[str, Any]], report["tenantTables"]) diff --git a/tests/unit/test_materialized_projection.py b/tests/unit/test_materialized_projection.py index c19ce0c0..1b8b519f 100644 --- a/tests/unit/test_materialized_projection.py +++ b/tests/unit/test_materialized_projection.py @@ -50,6 +50,13 @@ def __init__(self) -> None: self.locator_calls: list[CandidateRef] = [] self.projection_calls: list[MaterializedFragmentLocator] = [] + def discover_exact_phrase(self, phrase_digest: str) -> tuple[CandidateRef, ...]: + del phrase_digest + return () + + def observe_publication(self, candidate_ref: CandidateRef) -> None: + del candidate_ref + def locate( self, selected_candidate: CandidateRef, diff --git a/tests/unit/test_membership_field_projection.py b/tests/unit/test_membership_field_projection.py index 5cc36097..8897475c 100644 --- a/tests/unit/test_membership_field_projection.py +++ b/tests/unit/test_membership_field_projection.py @@ -39,6 +39,13 @@ def __init__(self, projection: MaterializedFragmentProjection | None) -> None: self.projection = projection self.calls: list[MaterializedFragmentLocator] = [] + def discover_exact_phrase(self, phrase_digest: str) -> tuple[()]: + del phrase_digest + return () + + def observe_publication(self, candidate_ref: object) -> None: + del candidate_ref + def locate(self, candidate_ref: object) -> MaterializedFragmentLocator: del candidate_ref return locator() diff --git a/tests/unit/test_runtime_authorized_evidence.py b/tests/unit/test_runtime_authorized_evidence.py index 4f32e423..9cc19711 100644 --- a/tests/unit/test_runtime_authorized_evidence.py +++ b/tests/unit/test_runtime_authorized_evidence.py @@ -138,8 +138,10 @@ def __init__(self, ranked: tuple[CandidateRef, ...]) -> None: self.ranked = ranked self.calls = 0 - def discover(self, request: Acquire) -> tuple[CandidateRef, ...]: - del request + def discover( + self, request: Acquire, projection_session: object + ) -> tuple[CandidateRef, ...]: + del request, projection_session self.calls += 1 return self.ranked @@ -155,6 +157,13 @@ def __init__(self) -> None: CROSS_ORGANIZATION: "CROSS-ORG-BODY-MUST-NEVER-BE-READ", } + def discover_exact_phrase(self, phrase_digest: str) -> tuple[()]: + del phrase_digest + return () + + def observe_publication(self, candidate_ref: CandidateRef) -> None: + del candidate_ref + def locate( self, candidate_ref: CandidateRef, diff --git a/tests/unit/test_runtime_empty_package.py b/tests/unit/test_runtime_empty_package.py index 5847cd16..b34184b5 100644 --- a/tests/unit/test_runtime_empty_package.py +++ b/tests/unit/test_runtime_empty_package.py @@ -3,6 +3,7 @@ from collections.abc import Iterator from contextlib import contextmanager from datetime import UTC, datetime +from typing import Any, cast from uuid import UUID import pytest @@ -68,7 +69,8 @@ def __init__(self) -> None: self.provider_calls = 0 self.source_content_calls = 0 - def discover(self, request: Acquire) -> tuple[()]: + def discover(self, request: Acquire, projection_session: object) -> tuple[()]: + del projection_session self.index_calls += 1 return () @@ -235,7 +237,7 @@ def test_content_io_spy_would_detect_every_runtime_dependency_call() -> None: candidate = runtime(spy) request = Acquire(need=ContextNeed(query="mutation control")) - candidate._content_io.index.discover(request) + candidate._content_io.index.discover(request, cast(Any, None)) candidate._content_io.provider.authorize_and_project() candidate._content_io.source_content.read_content() diff --git a/tests/unit/test_runtime_unavailable_capabilities.py b/tests/unit/test_runtime_unavailable_capabilities.py index 063e2542..b05674ca 100644 --- a/tests/unit/test_runtime_unavailable_capabilities.py +++ b/tests/unit/test_runtime_unavailable_capabilities.py @@ -86,8 +86,8 @@ def __init__(self) -> None: self.provider_calls = 0 self.source_calls = 0 - def discover(self, request: Acquire) -> tuple[()]: - del request + def discover(self, request: Acquire, projection_session: object) -> tuple[()]: + del request, projection_session self.index_calls += 1 return () @@ -319,7 +319,9 @@ def test_content_twins_are_observable_controls_for_every_prohibited_call() -> No twin = ContentIoTwin() content_io = RuntimeContentIo(index=twin, provider=twin, source_content=twin) - content_io.index.discover(Acquire(need=ContextNeed(query="control"))) + content_io.index.discover( + Acquire(need=ContextNeed(query="control")), cast(Any, None) + ) content_io.provider.authorize_and_project() content_io.source_content.read_content() diff --git a/tests/unit/test_schema_security_manifest.py b/tests/unit/test_schema_security_manifest.py index 6977e204..2ddc80b5 100644 --- a/tests/unit/test_schema_security_manifest.py +++ b/tests/unit/test_schema_security_manifest.py @@ -32,7 +32,7 @@ def test_manifest_classifies_the_exact_current_release_schema() -> None: document = manifest() tables = table_entries(document) - assert document["manifestVersion"] == "10.0.0" + assert document["manifestVersion"] == "11.0.0" assert set(tables) == { "active_release_manifest", "alembic_version", @@ -44,6 +44,10 @@ def test_manifest_classifies_the_exact_current_release_schema() -> None: "context_run_operator_read_ticket", "context_source", "decision_audit", + "exact_phrase_candidate", + "file_acquisition", + "file_import_job", + "file_revision_snapshot", "membership", "membership_resource_field_right", "organization", @@ -54,6 +58,7 @@ def test_manifest_classifies_the_exact_current_release_schema() -> None: "release_manifest", "release_operator_grant", "release_promotion_audit", + "revision_publication_event", "resource_access_policy", "service_principal", "source_version", @@ -83,6 +88,14 @@ def test_manifest_classifies_the_exact_current_release_schema() -> None: assert tables["worker_noop_job"]["classification"] == "tenant_owned" assert tables["context_source"]["classification"] == "tenant_owned" assert tables["source_version"]["classification"] == "tenant_owned" + for file_import_table in ( + "exact_phrase_candidate", + "file_acquisition", + "file_import_job", + "file_revision_snapshot", + "revision_publication_event", + ): + assert tables[file_import_table]["classification"] == "tenant_owned" for release_table in ( "active_release_manifest", "release_candidate", @@ -142,7 +155,7 @@ def test_issue_21_file_source_manifest_is_closed_and_role_separated() -> None: capability_constraint = next( constraint for constraint in version["checkConstraints"] - if constraint["name"] == "ck_source_version_issue_21_capabilities" + if constraint["name"] == "ck_source_version_file_capabilities" ) assert "materialized" in capability_constraint["expression"] assert "markdown" in capability_constraint["expression"] @@ -164,16 +177,36 @@ def test_issue_21_file_source_manifest_is_closed_and_role_separated() -> None: assert "\"describeCapabilities\": \"unavailable\"" in ( capability_constraint["expression"] ) - assert capability_constraint["expression"].count("unavailable") == 13 + assert '"declarationVersion": "file-capabilities-v1"' in ( + capability_constraint["expression"] + ) + assert '"declarationVersion": "file-capabilities-v2"' in ( + capability_constraint["expression"] + ) + assert '"fileSourceAccess": "available"' in ( + capability_constraint["expression"] + ) + assert '"ingestionJobs": "available"' in ( + capability_constraint["expression"] + ) + assert source["permittedOperations"] == { + "context_engine_control": ["SELECT", "INSERT"], + "context_engine_learning": [], + "context_engine_runtime": [], + "context_engine_security_operator": [], + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["SELECT", "UPDATE"], + } + assert version["permittedOperations"] == { + "context_engine_control": ["SELECT", "INSERT"], + "context_engine_learning": [], + "context_engine_runtime": [], + "context_engine_security_operator": [], + "context_engine_worker": [], + "context_engine_worker_lease_definer": ["SELECT", "INSERT"], + } for entry in (source, version): - assert entry["permittedOperations"] == { - "context_engine_control": ["SELECT", "INSERT"], - "context_engine_learning": [], - "context_engine_runtime": [], - "context_engine_security_operator": [], - "context_engine_worker": [], - } assert entry["rowLevelSecurity"]["enabled"] is True assert entry["rowLevelSecurity"]["forced"] is True @@ -525,6 +558,7 @@ def test_worker_lease_manifest_requires_exact_receiver_and_job() -> None: "ck_service_principal_worker_audience_bounds", "ck_service_principal_worker_audience_issue17", "ck_service_principal_operation_noop_complete", + "ck_service_principal_workload_operation_binding", } assert {constraint["name"] for constraint in job["checkConstraints"]} == { "ck_worker_noop_job_workload_bounds", @@ -689,6 +723,7 @@ def test_membership_manifest_requires_exact_user_actor_and_read_only_runtime() - assert entry["permittedOperations"] == { "context_engine_runtime": ["SELECT"], "context_engine_worker": [], + "context_engine_worker_lease_definer": ["SELECT"], } rls = entry["rowLevelSecurity"] @@ -917,11 +952,19 @@ def test_content_manifest_preserves_lineage_visibility_and_immutability() -> Non }, } + expected_definer_operations = { + "context_resource": ["SELECT", "INSERT", "UPDATE"], + "context_revision": ["INSERT"], + "context_fragment": ["INSERT"], + } for entry in (resource, revision, fragment): assert entry["organizationColumn"] == "organization_id" assert entry["permittedOperations"] == { "context_engine_runtime": ["SELECT"], "context_engine_worker": [], + "context_engine_worker_lease_definer": expected_definer_operations[ + entry["name"] + ], } rls = entry["rowLevelSecurity"] assert rls["enabled"] is True @@ -1089,10 +1132,13 @@ def test_content_manifest_preserves_lineage_visibility_and_immutability() -> Non for entry in (field, right): assert entry["organizationColumn"] == "organization_id" - assert entry["permittedOperations"] == { + expected_operations = { "context_engine_runtime": ["SELECT"], "context_engine_worker": [], } + if entry["name"] == "membership_resource_field_right": + expected_operations["context_engine_worker_lease_definer"] = ["INSERT"] + assert entry["permittedOperations"] == expected_operations rls = entry["rowLevelSecurity"] assert rls["enabled"] is True assert rls["forced"] is True @@ -1204,12 +1250,15 @@ def test_policy_epoch_manifest_seals_runtime_reads_and_control_mutation() -> Non for entry in (epoch, access): assert entry["organizationColumn"] == "organization_id" - assert entry["permittedOperations"] == { + expected_operations = { "context_engine_access_policy_definer": ["SELECT", "UPDATE"], "context_engine_control": ["EXECUTE change_resource_access"], "context_engine_runtime": ["SELECT"], "context_engine_worker": [], } + if entry["name"] == "resource_access_policy": + expected_operations["context_engine_worker_lease_definer"] = ["INSERT"] + assert entry["permittedOperations"] == expected_operations rls = entry["rowLevelSecurity"] assert rls["enabled"] is True assert rls["forced"] is True diff --git a/tests/unit/test_worker_lease.py b/tests/unit/test_worker_lease.py index 6cc7ba37..b838848f 100644 --- a/tests/unit/test_worker_lease.py +++ b/tests/unit/test_worker_lease.py @@ -11,6 +11,7 @@ import engine.supply as supply from engine.supply import ( + FILE_IMPORT_WORKER_LEASE_OPERATION, WORKER_LEASE_OPERATION, WorkerLeaseClaims, WorkerLeaseCodec, @@ -228,6 +229,52 @@ def test_token_uses_one_exact_protected_header_and_fixed_claim_set() -> None: } +def test_file_import_lease_uses_a_distinct_version_and_exact_source_binding() -> None: + codec = _codec() + token = codec.mint( + _claims( + workload="supply.file-import", + operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + source_ref="source:handbook", + ) + ) + + header, payload = _decoded_token(token) + + assert header["v"] == 2 + assert payload["operation"] == "file.import" + assert payload["source_ref"] == "source:handbook" + assert codec.verify( + token, + **_verification_arguments( + expected_workload="supply.file-import", + expected_operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + expected_source_ref="source:handbook", + ), # type: ignore[arg-type] + ).source_ref == "source:handbook" + + +def test_file_import_lease_rejects_a_wrong_source_generically() -> None: + codec = _codec() + token = codec.mint( + _claims( + workload="supply.file-import", + operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + source_ref="source:handbook", + ) + ) + + with pytest.raises(WorkNotAvailable, match="^work not available$"): + codec.verify( + token, + **_verification_arguments( + expected_workload="supply.file-import", + expected_operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + expected_source_ref="source:other", + ), # type: ignore[arg-type] + ) + + @pytest.mark.parametrize( "opaque_value", [ From 79ef7063b179b0b5d26a5554348c06bc847948ac Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 23 Jul 2026 01:11:51 +0800 Subject: [PATCH 2/3] fix: preserve authorized exact-file retries --- ...h-first-file-through-exact-worker-lease.md | 4 +- engine/persistence/membership_context.py | 1 - .../persistence/schema_security_manifest.yaml | 2 +- .../20260722_0011_file_import_tracer.py | 22 ++- tests/integration/test_file_import_tracer.py | 130 +++++++++++++++++- 5 files changed, 153 insertions(+), 6 deletions(-) diff --git a/docs/decisions/0037-publish-first-file-through-exact-worker-lease.md b/docs/decisions/0037-publish-first-file-through-exact-worker-lease.md index 95856645..7210cc9e 100644 --- a/docs/decisions/0037-publish-first-file-through-exact-worker-lease.md +++ b/docs/decisions/0037-publish-first-file-through-exact-worker-lease.md @@ -59,7 +59,9 @@ before restoring the Issue #21 and Issue #17 constraints. The exact-phrase index stores a SHA-256 query digest and lineage references, not content. Its Runtime SELECT requires the complete current UserActor transaction context and runs inside the same retained projection transaction used by the -Kernel; returned `CandidateRef` values remain untrusted discovery output. +Kernel. It does not truncate digest matches before authorization because doing +so could hide a later authorized candidate behind earlier denied candidates; +returned `CandidateRef` values remain untrusted discovery output. Every candidate still crosses the sealed AuthorizationKernel and `AuthorizedProjection` gates before content-bearing assembly. Cross-Organization or scope-denied resolution returns the canonical empty package. diff --git a/engine/persistence/membership_context.py b/engine/persistence/membership_context.py index 36ced7ff..fd34e84e 100644 --- a/engine/persistence/membership_context.py +++ b/engine/persistence/membership_context.py @@ -156,7 +156,6 @@ def discover_exact_phrase( FROM exact_phrase_candidate WHERE phrase_digest = :phrase_digest ORDER BY resource_ref, revision_id, fragment_ref - LIMIT 64 """ ), {"phrase_digest": phrase_digest}, diff --git a/engine/persistence/schema_security_manifest.yaml b/engine/persistence/schema_security_manifest.yaml index 94bfe865..004b106d 100644 --- a/engine/persistence/schema_security_manifest.yaml +++ b/engine/persistence/schema_security_manifest.yaml @@ -2454,7 +2454,7 @@ "forced": true, "policies": [ {"name": "file_import_job_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, - {"name": "file_import_job_definer_select", "command": "SELECT", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'"}, + {"name": "file_import_job_definer_select", "command": "SELECT", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND (job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid OR acquisition_id = NULLIF(current_setting('app.file_acquisition_id', true), '')::uuid) AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'"}, {"name": "file_import_job_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'"}, {"name": "file_import_job_definer_update", "command": "UPDATE", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'", "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import'"} ] diff --git a/migrations/versions/20260722_0011_file_import_tracer.py b/migrations/versions/20260722_0011_file_import_tracer.py index a8525eb0..47f27301 100644 --- a/migrations/versions/20260722_0011_file_import_tracer.py +++ b/migrations/versions/20260722_0011_file_import_tracer.py @@ -314,11 +314,19 @@ def upgrade() -> None: f"AND workload = '{_WORKLOAD}' AND worker_audience = '{_AUDIENCE}' " f"AND operation = '{_OPERATION}'" ) + job_select_binding = ( + f"organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid " + "AND (job_id = NULLIF(current_setting('app.worker_job_id', true), '')::uuid " + "OR acquisition_id = NULLIF(current_setting('app.file_acquisition_id', true), '')::uuid) " + f"AND workload = '{_WORKLOAD}' AND worker_audience = '{_AUDIENCE}' " + f"AND operation = '{_OPERATION}'" + ) for command in ("SELECT", "UPDATE"): check = f" WITH CHECK ({job_binding})" if command == "UPDATE" else "" + using = job_select_binding if command == "SELECT" else job_binding op.execute( f"CREATE POLICY file_import_job_definer_{command.lower()} ON file_import_job " - f"FOR {command} TO {_DEFINER} USING ({job_binding}){check}" + f"FOR {command} TO {_DEFINER} USING ({using}){check}" ) op.execute(f"GRANT INSERT ON TABLE source_version TO {_DEFINER}") op.execute( @@ -552,6 +560,9 @@ def _create_functions() -> None: AND idempotency_key = requested_idempotency_key AND request_digest = requested_request_digest; IF selected_acquisition_id IS NULL THEN RETURN; END IF; + PERFORM pg_catalog.set_config( + 'app.file_acquisition_id', selected_acquisition_id::text, true + ); INSERT INTO public.file_import_job ( organization_id, job_id, acquisition_id, source_id, service_principal_id, workload, worker_audience, actor_kind, @@ -561,6 +572,15 @@ def _create_functions() -> None: requested_source_id, requested_service_principal_id, '{_WORKLOAD}', '{_AUDIENCE}', 'service', '{_OPERATION}', 'available', trusted_now ) ON CONFLICT (organization_id, acquisition_id) DO NOTHING; + SELECT job.job_id INTO requested_job_id + FROM public.file_import_job AS job + WHERE job.organization_id = requested_organization_id + AND job.acquisition_id = selected_acquisition_id + AND job.service_principal_id = requested_service_principal_id; + IF requested_job_id IS NULL THEN RETURN; END IF; + PERFORM pg_catalog.set_config( + 'app.worker_job_id', requested_job_id::text, true + ); RETURN QUERY SELECT job.job_id, job.service_principal_id FROM public.file_import_job AS job WHERE job.organization_id = requested_organization_id diff --git a/tests/integration/test_file_import_tracer.py b/tests/integration/test_file_import_tracer.py index 6cf252bb..6d3c5d73 100644 --- a/tests/integration/test_file_import_tracer.py +++ b/tests/integration/test_file_import_tracer.py @@ -44,14 +44,21 @@ PostgreSQLWorkerLeaseIssuer, create_database_engine, ) -from engine.persistence.membership_context import MembershipIdentity +from engine.persistence.membership_context import ( + MembershipIdentity, + _PostgreSQLMaterializedProjectionPort, +) from engine.runtime.construction import Runtime, required_kernel_dependencies +from engine.runtime.content_io import exact_phrase_digest from engine.runtime.context_run import ContextRunOutcome -from engine.runtime.contracts import Acquire +from engine.runtime.contracts import Acquire, ContextNeed from engine.runtime.evidence import CandidateRef from engine.runtime.materialized import ( MaterializedProjectionSession, + _close_materialized_projection_scope, + _construct_materialized_projection_session, _observe_materialized_publication, + _open_materialized_projection_scope, ) from engine.runtime.organization import ( ExistingOrganizationVerification, @@ -392,6 +399,25 @@ def _prepare_file_import_scenario( idempotency_key="file-security-scenario", ), ) + with authority.authorize( + opaque_credential="control-secret", + operation=ControlOperation.IMPORT_FILE, + request_id="retry-import-after-lost-response", + ) as call: + prepared_retry = control.prepare_file_import( + call, + PrepareFileImport( + source_ref=source.source_ref, + path=FileImportPath("handbook.md"), + audience=FileImportAudience( + principal_ref="principal:file-reader", + membership_id=membership_id, + membership_version=1, + ), + idempotency_key="file-security-scenario", + ), + ) + assert prepared_retry == prepared codec = WorkerLeaseCodec( WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) ) @@ -1142,6 +1168,106 @@ def test_missing_file_after_redemption_records_terminal_zero_effect_failure( ) +def test_exact_phrase_discovery_does_not_hide_a_match_after_sixty_four_rows( + migration_configuration: DatabaseConfiguration, +) -> None: + organization_id = uuid4() + candidate_rows: list[dict[str, object]] = [] + for index in range(65): + resource_ref = f"resource:exact-limit:{index:03d}" + candidate_rows.append( + { + "organization_id": organization_id, + "source_ref": "source:exact-limit", + "resource_ref": resource_ref, + "revision_id": uuid4(), + "fragment_ref": "fragment:paragraph:1", + "ordinal": index, + } + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + transaction = connection.begin() + connection.execute( + text("INSERT INTO organization (organization_id) VALUES (:org)"), + {"org": organization_id}, + ) + connection.execute(text("SET CONSTRAINTS ALL DEFERRED")) + connection.execute( + text( + """ + INSERT INTO context_resource ( + organization_id, resource_ref, source_ref, + active_revision_id, tombstoned + ) VALUES (:organization_id, :resource_ref, :source_ref, + :revision_id, false) + """ + ), + candidate_rows, + ) + connection.execute( + text( + """ + INSERT INTO context_revision ( + organization_id, resource_ref, revision_id + ) VALUES (:organization_id, :resource_ref, :revision_id) + """ + ), + candidate_rows, + ) + connection.execute( + text( + """ + INSERT INTO context_fragment ( + organization_id, resource_ref, revision_id, + fragment_ref, ordinal, content, projection_kind + ) VALUES (:organization_id, :resource_ref, :revision_id, + :fragment_ref, 0, 'same exact paragraph', 'body') + """ + ), + candidate_rows, + ) + connection.execute( + text( + """ + INSERT INTO exact_phrase_candidate ( + organization_id, phrase_digest, source_ref, + resource_ref, revision_id, fragment_ref + ) VALUES (:organization_id, :phrase_digest, :source_ref, + :resource_ref, :revision_id, :fragment_ref) + """ + ), + [ + { + **row, + "phrase_digest": exact_phrase_digest( + "same exact paragraph" + ), + } + for row in candidate_rows + ], + ) + projection_scope = _open_materialized_projection_scope() + try: + projection_session = _construct_materialized_projection_session( + authority_scope=projection_scope, + port=_PostgreSQLMaterializedProjectionPort(connection), + ) + discovered = PostgreSQLExactPhraseCandidateIndex().discover( + Acquire(need=ContextNeed(query="same exact paragraph")), + projection_session, + ) + finally: + _close_materialized_projection_scope(projection_scope) + transaction.rollback() + finally: + migration_engine.dispose() + + assert len(discovered) == 65 + assert discovered[-1].resource_ref == "resource:exact-limit:064" + + @pytest.mark.parametrize( "changed_binding", ["organization", "job", "receiver", "source"], From 1ba64067a1f0a4130f8fd0341c0fbf5dbd8eead4 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 23 Jul 2026 01:14:06 +0800 Subject: [PATCH 3/3] test: strengthen file import policy assertions --- tests/integration/test_membership_schema.py | 16 +++++++++++++++- tests/unit/test_worker_lease.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_membership_schema.py b/tests/integration/test_membership_schema.py index 5a230262..30419fa5 100644 --- a/tests/integration/test_membership_schema.py +++ b/tests/integration/test_membership_schema.py @@ -12,7 +12,11 @@ from sqlalchemy.exc import DBAPIError, IntegrityError from engine.persistence import DatabaseConfiguration, create_database_engine -from engine.persistence.configuration import RUNTIME_ROLE, WORKER_ROLE +from engine.persistence.configuration import ( + RUNTIME_ROLE, + WORKER_LEASE_DEFINER_ROLE, + WORKER_ROLE, +) pytestmark = pytest.mark.integration CHECKED_AT = datetime(2026, 7, 21, 8, 0, tzinfo=UTC) @@ -538,6 +542,16 @@ def test_runtime_worker_and_public_grants_are_least_privilege( ): assert required_fragment in normalized_policy + file_import_policy = policies["membership_file_import_definer_select"] + assert file_import_policy[:3] == ( + "PERMISSIVE", + (WORKER_LEASE_DEFINER_ROLE,), + "SELECT", + ) + assert file_import_policy[3] is not None + assert file_import_policy[4] is None + assert "app.organization_id" in str(file_import_policy[3]).lower() + migrator_policy = policies["membership_migrator_administration"] assert migrator_policy[:3] == ( "PERMISSIVE", diff --git a/tests/unit/test_worker_lease.py b/tests/unit/test_worker_lease.py index b838848f..5418b20c 100644 --- a/tests/unit/test_worker_lease.py +++ b/tests/unit/test_worker_lease.py @@ -176,7 +176,7 @@ def test_wrong_binding_not_yet_valid_and_expired_are_generic( def test_unknown_signing_key_version_is_generic() -> None: token = _codec(version=8).mint(_claims(signing_key_version=8)) - with pytest.raises(WorkNotAvailable, match="^work not available$"): + with pytest.raises(WorkNotAvailable, match=r"^work not available$"): _codec().verify(token, **_verification_arguments()) # type: ignore[arg-type]