diff --git a/README.md b/README.md index 8ffac8f2..1694fa10 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,21 @@ ContextEngine 的安全协议依据自身需求与威胁模型独立设计,零 - [D0 Baseline Candidate](./DESIGN-BASELINE.md):当前候选状态与尚未关闭的 evidence gates。 +独立 Supply worker 的确定性单周期 File dispatch 使用 +`context-engine-worker --dispatch-file-once`。生产长运行入口是 +`context-engine-worker --dispatch-files`;它以服务端固定的一秒间隔轮询无工作结果, +并在 `SIGTERM` / `SIGINT` 时结束。两种入口都只读取 role-specific scheduler、 +worker URL、WorkerLease signing key 和服务端 JSON root registry +(`CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON`);调用方不得提供 +Organization、Source、job 或 token。输出仅包含 `dispatched` / `no_work` / `refused`;Provider +polling、过期 lease reclaim、retry/dead-letter 与 delete execution 仍未激活。Worker +基础设施不可用会终止 dispatch,不会继续 claim 并滞留后续 job。 +文件/内容失败仅在该 job 已持久化为 terminal failed 或当前 authority 拒绝该精确 +failure transition 后返回 `refused` 并继续调度;failure recording 基础设施不可用仍会 +终止 dispatch。 +Lease 的立即验证使用 worker PostgreSQL 时钟,与数据库签发时间保持同一时间域, +不依赖 worker host clock 对齐。 + 当前除固定 commit 的四仓静态证据与仓库内设计拆解外,已有 [`compose.yaml`](./compose.yaml) 固定的真实 PostgreSQL + pgvector 基础 harness, 以及首个 Organization-owned 代表表的 RLS 动态证据。 diff --git a/applications/worker.py b/applications/worker.py index 9eab0f49..861c6c37 100644 --- a/applications/worker.py +++ b/applications/worker.py @@ -3,23 +3,34 @@ import argparse import json import os +import signal import threading -from collections.abc import Sequence +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path from typing import Protocol from uuid import UUID +from sqlalchemy import Engine, text +from sqlalchemy.exc import SQLAlchemyError + from adapters.file_source import FileReadLimits, FileRootRegistry from engine import BUILD_IDENTIFIER from engine.control import FileImportReceiver, FileRootRef, SourceRef from engine.persistence import ( DatabasePurpose, + FileDispatchLease, + FileDispatchNoWork, FileImportLeaseRedemption, + FileImportRefused, + FileImportUnavailable, + PostgreSQLFileDispatchAuthority, PostgreSQLFileImportWorker, create_database_engine, load_database_configuration, ) +from engine.persistence.role_guard import assert_worker_role from engine.persistence.worker_jobs import ( WorkerLeaseRedemption, WorkerNoOpCompletion, @@ -31,8 +42,11 @@ WorkerLeaseCodec, WorkerLeaseKeyring, WorkerLeaseToken, + WorkNotAvailable, ) +_FILE_DISPATCH_POLL_SECONDS = 1.0 + class WorkerNoOpCompletionAuthority(Protocol): """Application port for one verified persistent no-op completion.""" @@ -42,6 +56,70 @@ def complete_noop( ) -> WorkerNoOpCompletion: ... +class FileDispatchAuthority(Protocol): + """Application port for database-selected first-attempt File work.""" + + def claim(self) -> FileDispatchLease | FileDispatchNoWork: ... + + +class FileDispatchWorker(Protocol): + """Existing exact File import execution seam.""" + + def run(self, redemption: FileImportLeaseRedemption) -> object: ... + + +class FileDispatchWorkerFactory(Protocol): + def __call__(self, receiver: FileImportReceiver) -> FileDispatchWorker: ... + + +@dataclass(frozen=True, slots=True) +class FileDispatchCycleResult: + """Content-free process result for one autonomous dispatch cycle.""" + + outcome: str + status: str = field(default="complete", init=False) + + def __post_init__(self) -> None: + if self.outcome not in {"dispatched", "no_work", "refused"}: + raise ValueError("File dispatch cycle outcome must remain closed") + + +def dispatch_one_file_import( + authority: FileDispatchAuthority, + worker_factory: FileDispatchWorkerFactory, +) -> FileDispatchCycleResult: + """Claim and run at most one exact job without caller routing input.""" + + claim = authority.claim() + if type(claim) is FileDispatchNoWork: + return FileDispatchCycleResult("no_work") + if type(claim) is not FileDispatchLease: + raise TypeError("File dispatch authority returned an invalid result") + try: + worker_factory(FileImportReceiver(claim.service_principal_id)).run( + claim.redemption + ) + except (FileImportRefused, WorkNotAvailable): + return FileDispatchCycleResult("refused") + return FileDispatchCycleResult("dispatched") + + +def dispatch_file_imports_until_stopped( + authority: FileDispatchAuthority, + worker_factory: FileDispatchWorkerFactory, + stop_event: threading.Event, + outcome_observer: Callable[[FileDispatchCycleResult], None] | None = None, +) -> None: + """Run bounded single-job cycles until process shutdown is requested.""" + + while not stop_event.is_set(): + result = dispatch_one_file_import(authority, worker_factory) + if outcome_observer is not None: + outcome_observer(result) + if result.outcome == "no_work": + stop_event.wait(_FILE_DISPATCH_POLL_SECONDS) + + def complete_persistent_noop_job( authority: WorkerNoOpCompletionAuthority, redemption: WorkerLeaseRedemption, @@ -63,17 +141,7 @@ def _required_environment(name: str) -> str: def _run_one_file_import() -> int: """Consume one exact, signed File job in the independent Supply process.""" - signing_key_hex = _required_environment( - "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX" - ) - if len(signing_key_hex) != 64: - raise ValueError("Supply worker configuration is not available") - try: - signing_key = bytes.fromhex(signing_key_hex) - except ValueError: - raise ValueError("Supply worker configuration is not available") from None - if len(signing_key) != 32: - raise ValueError("Supply worker configuration is not available") + signing_key = _worker_signing_key() configuration = load_database_configuration(DatabasePurpose.SUPPLY_WORKER) engine = create_database_engine(configuration) roots = FileRootRegistry( @@ -143,7 +211,175 @@ def _run_one_file_import() -> int: engine.dispose() -def run(*, test_mode: bool, run_file_job: bool = False) -> int: +def _worker_signing_key() -> bytes: + signing_key_hex = _required_environment( + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX" + ) + if len(signing_key_hex) != 64: + raise ValueError("Supply worker configuration is not available") + try: + signing_key = bytes.fromhex(signing_key_hex) + except ValueError: + raise ValueError("Supply worker configuration is not available") from None + if len(signing_key) != 32: + raise ValueError("Supply worker configuration is not available") + return signing_key + + +def _file_dispatch_root_bindings() -> dict[FileRootRef, Path]: + """Load the server-owned registry for every root this dispatcher serves.""" + + raw_registry = _required_environment("CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON") + try: + document = json.loads(raw_registry) + except json.JSONDecodeError: + raise ValueError("Supply worker configuration is not available") from None + if type(document) is not dict or not document: + raise ValueError("Supply worker configuration is not available") + bindings: dict[FileRootRef, Path] = {} + for raw_ref, raw_path in document.items(): + if ( + type(raw_ref) is not str + or type(raw_path) is not str + or not raw_path + or raw_path != raw_path.strip() + ): + raise ValueError("Supply worker configuration is not available") + bindings[FileRootRef(raw_ref)] = Path(raw_path) + return bindings + + +def _file_dispatch_roots() -> FileRootRegistry: + return FileRootRegistry( + _file_dispatch_root_bindings(), + limits=FileReadLimits(max_file_bytes=4_096), + ) + + +def _worker_database_time(engine: Engine) -> datetime: + """Read the worker authority's clock for immediate lease verification.""" + + try: + with engine.connect() as connection: + assert_worker_role(connection) + checked_at = connection.execute( + text("SELECT pg_catalog.date_trunc('second', clock_timestamp())") + ).scalar_one() + except (SQLAlchemyError, AssertionError, ValueError): + raise FileImportUnavailable("File import clock is unavailable") from None + if type(checked_at) is not datetime or checked_at.tzinfo is None: + raise FileImportUnavailable("File import clock is unavailable") + return checked_at.astimezone(UTC) + + +def _run_file_dispatch(*, single_cycle: bool) -> int: + """Run configured autonomous File dispatch without caller routing facts.""" + + codec = WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: _worker_signing_key()}) + ) + scheduler_engine = create_database_engine( + load_database_configuration(DatabasePurpose.SUPPLY_SCHEDULER) + ) + worker_engine = create_database_engine( + load_database_configuration(DatabasePurpose.SUPPLY_WORKER) + ) + root_bindings = _file_dispatch_root_bindings() + roots = FileRootRegistry( + root_bindings, + limits=FileReadLimits(max_file_bytes=4_096), + ) + try: + authority = PostgreSQLFileDispatchAuthority( + scheduler_engine, + codec, + configured_root_refs=tuple( + root_ref.value for root_ref in root_bindings + ), + ) + + def worker_factory(receiver: FileImportReceiver) -> PostgreSQLFileImportWorker: + return PostgreSQLFileImportWorker( + worker_engine, + codec, + receiver, + roots, + MarkdownCompilerConfig("markdown-config-v1"), + clock=lambda: _worker_database_time(worker_engine), + ) + + if single_cycle: + result = dispatch_one_file_import(authority, worker_factory) + print( + json.dumps( + { + "dispatch": "file.import", + "outcome": result.outcome, + "service": "context-engine-worker", + "status": result.status, + }, + sort_keys=True, + ), + flush=True, + ) + else: + stop_event = threading.Event() + + def request_stop(_signum: int, _frame: object) -> None: + stop_event.set() + + previous_sigterm = signal.signal(signal.SIGTERM, request_stop) + previous_sigint = signal.signal(signal.SIGINT, request_stop) + try: + print( + json.dumps( + { + "dispatch": "file.import", + "service": "context-engine-worker", + "status": "ready", + }, + sort_keys=True, + ), + flush=True, + ) + dispatch_file_imports_until_stopped( + authority, + worker_factory, + stop_event, + outcome_observer=lambda result: print( + json.dumps( + { + "dispatch": "file.import", + "outcome": result.outcome, + "service": "context-engine-worker", + "status": result.status, + }, + sort_keys=True, + ), + flush=True, + ), + ) + finally: + signal.signal(signal.SIGINT, previous_sigint) + signal.signal(signal.SIGTERM, previous_sigterm) + return 0 + finally: + roots.close() + worker_engine.dispose() + scheduler_engine.dispose() + + +def run( + *, + test_mode: bool, + run_file_job: bool = False, + dispatch_file_once: bool = False, + dispatch_files: bool = False, +) -> int: + if dispatch_file_once: + return _run_file_dispatch(single_cycle=True) + if dispatch_files: + return _run_file_dispatch(single_cycle=False) if run_file_job: return _run_one_file_import() Runtime(required_kernel_dependencies()) @@ -177,10 +413,33 @@ def main(argv: Sequence[str] | None = None) -> int: action="store_true", help="consume one exact configured FileImport WorkerLease and exit", ) + parser.add_argument( + "--dispatch-file-once", + action="store_true", + help="claim and execute at most one eligible scheduled File import", + ) + parser.add_argument( + "--dispatch-files", + action="store_true", + help="continuously claim eligible scheduled File imports until shutdown", + ) args = parser.parse_args(argv) - if args.test_mode and args.run_file_job: - parser.error("--test-mode and --run-file-job are mutually exclusive") - return run(test_mode=args.test_mode, run_file_job=args.run_file_job) + selected_modes = sum( + ( + args.test_mode, + args.run_file_job, + args.dispatch_file_once, + args.dispatch_files, + ) + ) + if selected_modes > 1: + parser.error("worker execution modes are mutually exclusive") + return run( + test_mode=args.test_mode, + run_file_job=args.run_file_job, + dispatch_file_once=args.dispatch_file_once, + dispatch_files=args.dispatch_files, + ) if __name__ == "__main__": diff --git a/compose.yaml b/compose.yaml index e02f5193..f537e43a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,6 +20,8 @@ services: CONTEXT_ENGINE_RUNTIME_PASSWORD: ${CONTEXT_ENGINE_RUNTIME_PASSWORD:?CONTEXT_ENGINE_RUNTIME_PASSWORD is required} CONTEXT_ENGINE_WORKER_ROLE: ${CONTEXT_ENGINE_WORKER_ROLE:?CONTEXT_ENGINE_WORKER_ROLE is required} CONTEXT_ENGINE_WORKER_PASSWORD: ${CONTEXT_ENGINE_WORKER_PASSWORD:?CONTEXT_ENGINE_WORKER_PASSWORD is required} + CONTEXT_ENGINE_SCHEDULER_ROLE: ${CONTEXT_ENGINE_SCHEDULER_ROLE:?CONTEXT_ENGINE_SCHEDULER_ROLE is required} + CONTEXT_ENGINE_SCHEDULER_PASSWORD: ${CONTEXT_ENGINE_SCHEDULER_PASSWORD:?CONTEXT_ENGINE_SCHEDULER_PASSWORD is required} CONTEXT_ENGINE_LEARNING_ROLE: ${CONTEXT_ENGINE_LEARNING_ROLE:?CONTEXT_ENGINE_LEARNING_ROLE is required} CONTEXT_ENGINE_LEARNING_PASSWORD: ${CONTEXT_ENGINE_LEARNING_PASSWORD:?CONTEXT_ENGINE_LEARNING_PASSWORD is required} CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE: ${CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE:?CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE is required} diff --git a/docs/decisions/0059-dispatch-scheduled-file-imports-through-exact-leases.md b/docs/decisions/0059-dispatch-scheduled-file-imports-through-exact-leases.md new file mode 100644 index 00000000..2ff4fd99 --- /dev/null +++ b/docs/decisions/0059-dispatch-scheduled-file-imports-through-exact-leases.md @@ -0,0 +1,89 @@ +--- +name: adr-0059-dispatch-scheduled-file-imports-through-exact-leases +version: "1.0.0" +description: > + Let the independent Supply worker select first-attempt scheduled File imports + without caller tenant routing or broader Control authority. +--- + +# 0059. Dispatch scheduled File imports through exact leases + +- Status: accepted +- Date: 2026-07-25 +- Refines: ADR-0029, ADR-0037, ADR-0043, ADR-0055, ADR-0058 + +## Context + +Accepted File pages can create exact `file_import_job` rows, but the Supply +process previously required an operator to provide Organization, Source, job, +receiver, and a pre-minted WorkerLease. Those values are an evidence seam, not +a safe autonomous selector. Giving the process the Control login would also +combine tenant choice, scheduling, and lease issuance authority. + +## Decision + +The existing Supply process may open two independently guarded pools: the +function-only `context_engine_scheduler` login and the existing worker login. +The scheduler login can execute only `context_scheduler_claim_file_import`; it +cannot read application tables or mutate Control/Runtime state. The function is +owned by a dedicated NOLOGIN File-dispatch definer so its cross-Organization +selection policies do not enlarge any existing definer function. + +The function accepts only a fresh 32-byte nonce, current signing-key version, +and the distinct set of server-configured logical root references. It returns +typed no-work without leasing anything if the globally oldest eligible +candidate's exact root is absent from that capability set. The set is an +all-or-nothing capability assertion applied after global selection, never a +routing filter; host paths never enter PostgreSQL. +It selects the oldest current v3/v4 page-scheduled upsert by database acceptance +time, then checkpoint/page/change and stable Organization/Source/job identity. +It first locks the selected job, then holds the same per-Source progress advisory +lock as page acceptance, refreshes its database statement snapshot, and +refreshes database wall-clock time before revalidating the latest scan epoch. +It also requires the selected SourceVersion root to remain in the configured +registry and locks mutable Source, Membership, and +receiver authority rows with `FOR UPDATE SKIP LOCKED`, and changes only that job +from available generation zero to a database-timed leased generation one. +Current SourceVersion, Membership version, and exact enabled File import receiver +are thus revalidated and fenced in the claim transaction. Manual jobs, delete +observations, disabled sources, stale pages, and every later job state remain +invisible to the scheduler. + +Python mints the existing versioned WorkerLease solely from returned claims and +fresh nonce, constructs the existing `FileImportLeaseRedemption`, and invokes +`PostgreSQLFileImportWorker`. Immediate verification reads the worker database +clock at whole-second protocol precision, matching the database-issued lease +timestamps and the redemption function's authoritative expiry check rather than +depending on host-clock alignment. PostgreSQL `timestamptz` results are normalized +from their session-zone representation to UTC before the strict lease contract is +constructed or verified. Dispatch loads every served logical root from one +server-owned JSON registry, so cross-Organization selection cannot consume an +eligible job merely because another configured root was omitted. A claim is +marked internally for downgrade +fencing; process output contains only `dispatched`, `no_work`, or the closed +job-level `refused` outcome for exact lease rejection and never raw +claims, token, nonce, tenant identity, source bytes, or host path. +A file/content failure becomes job-level `refused` only after the existing +failure transaction durably seals that exact job or current authority rejects +that exact failure transition. Worker infrastructure or failure-recording +unavailability terminates dispatch after the already claimed lease instead of +claiming and stranding additional jobs; automatic retry/backoff remains inactive. + +## Consequences + +- Concurrent scheduler transactions claim different rows or typed no-work. +- A stop after claim retains one ordinary expiring generation-one lease and no + publication effect; reclaim and retry remain inactive. +- Delete execution stays exclusively in the trusted #87 Control carrier. +- Stopping dispatch requires only stopping the loop or revoking function + execution; retained jobs and publication lineage remain untouched. +- Downgrade is refused after this scheduler has claimed a job, because the prior + schema cannot safely preserve the capability's provenance. + +## Revisit trigger + +Revisit before expired-lease reclaim, retry/backoff, dead-letter policy, +provider polling, automatic page acceptance, implicit audience selection, or +automatic upsert/delete ordering. Any revision must retain separate scheduler +and worker roles, database-selected tenant routing, exact WorkerLease fencing, +and content-free no-work/output contracts. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f9298275..4734db5e 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -48,6 +48,7 @@ kernel, capability separation, and publication visibility model. | File change-page acknowledgement | [0054 — Acknowledge File change pages before cursor advance](0054-acknowledge-file-change-pages-before-cursor-advance.md) | Provider pages remain pending until one whole-page Control transaction appends the page, changes, and existing acquisition checkpoint, after which Control issues the continuation cursor | Provider-observation cursor advance, partial page acceptance, implicit publication audience, a second checkpoint protocol, or Runtime authority from Supply progress | | File delete observation execution | [0057 — Execute current File deletes through tombstone authority](0057-execute-current-file-deletes-through-tombstone-authority.md) | One trusted exact locator is revalidated against the current complete v4 scan, derives effect identity server-side, calls the sole tombstone authority, and records one immutable exact binding | Provider deletion authority, caller-authored path/effect identity, stale scan deletion, a second visibility authority, or Runtime trust in Supply metadata | | Mixed File page upsert scheduling | [0058 — Schedule only upserts from mixed File pages](0058-schedule-only-upserts-from-mixed-file-pages.md) | Validate one complete current v4 page, then atomically schedule and replay its nonempty ordered upsert projection through the existing import path while preserving original ordinals | Scheduler delete/tombstone authority, implicit audience, automatic upsert/delete ordering, a second queue, or Runtime trust in Supply metadata | +| Autonomous File dispatch | [0059 — Dispatch scheduled File imports through exact leases](0059-dispatch-scheduled-file-imports-through-exact-leases.md) | A function-only scheduler login atomically claims the oldest current page-scheduled upsert and mints the existing exact first-attempt WorkerLease | Caller tenant/job routing, broad Control credentials, direct scheduler table access, retry/reclaim, delete execution, or a second queue/process | | Private delivery ingress | [0045 — Redeem private delivery evidence at ingress](0045-redeem-private-delivery-evidence-at-ingress.md) | One digest-only service/request/asker/audience/epoch-bound DeliveryEvidenceRef constructs private TrustedDeliveryContext inside the current UserActor transaction before content work | Raw trusted delivery facts on the wire, bearer persistence, application-role minting/table reads, alternate Runtime paths, or claiming later M2 carriers | | Exact Package egress | [0046 — Bind egress to one exact Package hop](0046-bind-egress-to-one-exact-package-hop.md) | One digest-only grant binds one exact audience-bound Package to one model or channel preflight hop and redeems atomically | Treating Package construction as disclosure authority, arbitrary content at egress, cross-hop reuse, or bypassing final policy | | Public OpenAPI v0 | [0047 — Freeze OpenAPI v0 through one Runtime path](0047-freeze-openapi-v0-through-one-runtime-path.md) | One public `/v0/resolve` schema and a hidden provisional v1 bridge share the same sealed Runtime; Package release lineage is read-only from the Learning-published active manifest | Two authorization compositions, caller-authored release facts, Runtime publication/fallback, or in-place mutation of historical snapshots | @@ -152,6 +153,7 @@ touched: - [0045 — Redeem private delivery evidence at ingress](0045-redeem-private-delivery-evidence-at-ingress.md) - [0046 — Bind egress to one exact Package hop](0046-bind-egress-to-one-exact-package-hop.md) - [0054 — Acknowledge File change pages before cursor advance](0054-acknowledge-file-change-pages-before-cursor-advance.md) +- [0055 — Schedule accepted File observations explicitly](0055-schedule-accepted-file-observations-explicitly.md) - [0056 — Detect File deletions without tombstone authority](0056-detect-file-deletions-without-tombstone-authority.md) - [0057 — Execute current File deletes through tombstone authority](0057-execute-current-file-deletes-through-tombstone-authority.md) - [0058 — Schedule only upserts from mixed File pages](0058-schedule-only-upserts-from-mixed-file-pages.md) diff --git a/engine/persistence/__init__.py b/engine/persistence/__init__.py index 28c33a31..ac9a7178 100644 --- a/engine/persistence/__init__.py +++ b/engine/persistence/__init__.py @@ -16,6 +16,7 @@ DELIVERY_EVIDENCE_DEFINER_ROLE, EGRESS_GRANT_DEFINER_ROLE, EGRESS_ROLE, + FILE_DISPATCH_DEFINER_ROLE, IDENTITY_ROLE, LEARNING_ROLE, OPERATOR_ROLE, @@ -50,6 +51,7 @@ from engine.persistence.file_imports import ( FileImportInterrupted, FileImportLeaseRedemption, + FileImportRefused, FileImportUnavailable, FilePublicationBoundary, PostgreSQLFileImportWorker, @@ -69,6 +71,7 @@ assert_identity_role, assert_learning_role, assert_runtime_role, + assert_scheduler_role, assert_security_operator_role, assert_worker_role, ) @@ -79,6 +82,9 @@ from engine.persistence.worker_jobs import ( DEFAULT_WORKER_LEASE_TTL_SECONDS, MAX_WORKER_LEASE_TTL_SECONDS, + FileDispatchLease, + FileDispatchNoWork, + PostgreSQLFileDispatchAuthority, PostgreSQLWorkerLeaseAuthority, PostgreSQLWorkerLeaseIssuer, WorkerExecutionIdentity, @@ -106,6 +112,7 @@ "DELIVERY_EVIDENCE_DEFINER_ROLE", "EGRESS_GRANT_DEFINER_ROLE", "EGRESS_ROLE", + "FILE_DISPATCH_DEFINER_ROLE", "LEARNING_ROLE", "OPERATOR_ROLE", "RELEASE_DEFINER_ROLE", @@ -130,6 +137,10 @@ "PostgreSQLDeliveryEvidenceRetentionPort", "PostgreSQLEgressGrantRedemptionAuthority", "FileImportLeaseRedemption", + "FileImportRefused", + "FileDispatchLease", + "FileDispatchNoWork", + "PostgreSQLFileDispatchAuthority", "FileImportInterrupted", "FileImportUnavailable", "FilePublicationBoundary", @@ -158,6 +169,7 @@ "assert_identity_role", "assert_egress_role", "assert_security_operator_role", + "assert_scheduler_role", "assert_control_role", "assert_worker_role", "create_database_engine", diff --git a/engine/persistence/configuration.py b/engine/persistence/configuration.py index a1bf78e3..a926585d 100644 --- a/engine/persistence/configuration.py +++ b/engine/persistence/configuration.py @@ -22,9 +22,11 @@ CITATION_DEFINER_ROLE = "context_engine_citation_definer" ACCESS_POLICY_DEFINER_ROLE = "context_engine_access_policy_definer" WORKER_LEASE_DEFINER_ROLE = "context_engine_worker_lease_definer" +FILE_DISPATCH_DEFINER_ROLE = "context_engine_file_dispatch_definer" CONTEXT_RUN_READER_DEFINER_ROLE = "context_engine_context_run_reader_definer" RUNTIME_ROLE = "context_engine_runtime" WORKER_ROLE = "context_engine_worker" +SCHEDULER_ROLE = "context_engine_scheduler" LEARNING_ROLE = "context_engine_learning" OPERATOR_ROLE = "context_engine_security_operator" RELEASE_DEFINER_ROLE = "context_engine_release_definer" @@ -40,6 +42,10 @@ class DatabasePurpose(Enum): TRUSTED_ACTION = ("CONTEXT_ENGINE_ACTION_DATABASE_URL", ACTION_ROLE) API_RUNTIME = ("CONTEXT_ENGINE_RUNTIME_DATABASE_URL", RUNTIME_ROLE) SUPPLY_WORKER = ("CONTEXT_ENGINE_WORKER_DATABASE_URL", WORKER_ROLE) + SUPPLY_SCHEDULER = ( + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL", + SCHEDULER_ROLE, + ) LEARNING = ("CONTEXT_ENGINE_LEARNING_DATABASE_URL", LEARNING_ROLE) SECURITY_OPERATOR = ( "CONTEXT_ENGINE_SECURITY_OPERATOR_DATABASE_URL", @@ -64,6 +70,7 @@ def expected_role(self) -> str: DatabasePurpose.TRUSTED_ACTION: "CONTEXT_ENGINE_ACTION_ROLE", DatabasePurpose.API_RUNTIME: "CONTEXT_ENGINE_RUNTIME_ROLE", DatabasePurpose.SUPPLY_WORKER: "CONTEXT_ENGINE_WORKER_ROLE", + DatabasePurpose.SUPPLY_SCHEDULER: "CONTEXT_ENGINE_SCHEDULER_ROLE", DatabasePurpose.LEARNING: "CONTEXT_ENGINE_LEARNING_ROLE", DatabasePurpose.SECURITY_OPERATOR: "CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE", DatabasePurpose.SECURITY_TEST: "CONTEXT_ENGINE_RUNTIME_ROLE", @@ -128,6 +135,7 @@ class HarnessDatabaseConfigurations: action: DatabaseConfiguration runtime: DatabaseConfiguration worker: DatabaseConfiguration + scheduler: DatabaseConfiguration learning: DatabaseConfiguration operator: DatabaseConfiguration security_test: DatabaseConfiguration @@ -204,6 +212,9 @@ def load_harness_database_configurations( action=load_database_configuration(DatabasePurpose.TRUSTED_ACTION, source), runtime=load_database_configuration(DatabasePurpose.API_RUNTIME, source), worker=load_database_configuration(DatabasePurpose.SUPPLY_WORKER, source), + scheduler=load_database_configuration( + DatabasePurpose.SUPPLY_SCHEDULER, source + ), learning=load_database_configuration(DatabasePurpose.LEARNING, source), operator=load_database_configuration(DatabasePurpose.SECURITY_OPERATOR, source), security_test=load_database_configuration( @@ -218,13 +229,14 @@ def load_harness_database_configurations( configurations.action.expected_role, configurations.runtime.expected_role, configurations.worker.expected_role, + configurations.scheduler.expected_role, configurations.learning.expected_role, configurations.operator.expected_role, } - if len(distinct_roles) != 9: + if len(distinct_roles) != 10: raise DatabaseConfigurationError( "migration, control, identity, egress, action, runtime, worker, " - "learning, and security-operator " + "scheduler, learning, and security-operator " "database roles must be distinct" ) if configurations.security_test.url != configurations.runtime.url: diff --git a/engine/persistence/file_imports.py b/engine/persistence/file_imports.py index 114e810e..d35047c6 100644 --- a/engine/persistence/file_imports.py +++ b/engine/persistence/file_imports.py @@ -144,6 +144,10 @@ class FileImportUnavailable(RuntimeError): """Generic failure after a valid lease reaches acquisition/publication.""" +class FileImportRefused(FileImportUnavailable): + """Content-free refusal after the exact job was durably sealed as failed.""" + + class FilePublicationBoundary(StrEnum): """The three explicit post-commit fault-injection boundaries.""" @@ -265,26 +269,17 @@ def run(self, redemption: FileImportLeaseRedemption) -> PublishedFileImport: raise LookupError("accepted File observation changed") outcome = compile_markdown(source, self._config) except LookupError: - if redeemed.expected_content_sha256 is None: + with suppress(WorkNotAvailable): self._fail(redemption.token, claims) - else: - with suppress(FileImportUnavailable, WorkNotAvailable): - self._fail(redemption.token, claims) - raise FileImportUnavailable("File import is unavailable") from None + raise FileImportRefused("File import is unavailable") from None if type(outcome) is CompilationFailure: - if redeemed.expected_content_sha256 is None: + with suppress(WorkNotAvailable): self._fail(redemption.token, claims) - else: - with suppress(FileImportUnavailable, WorkNotAvailable): - self._fail(redemption.token, claims) - raise FileImportUnavailable("File import is unavailable") + raise FileImportRefused("File import is unavailable") if type(outcome) is not ParsedDocument: # pragma: no cover - closed union - if redeemed.expected_content_sha256 is None: + with suppress(WorkNotAvailable): self._fail(redemption.token, claims) - else: - with suppress(FileImportUnavailable, WorkNotAvailable): - self._fail(redemption.token, claims) - raise FileImportUnavailable("File import is unavailable") + raise FileImportRefused("File import is unavailable") try: return self._publish(redemption.token, claims, redeemed, outcome) except FileImportInterrupted: diff --git a/engine/persistence/role_guard.py b/engine/persistence/role_guard.py index f9bd06fe..f03a1bc9 100644 --- a/engine/persistence/role_guard.py +++ b/engine/persistence/role_guard.py @@ -13,6 +13,7 @@ MIGRATOR_ROLE, OPERATOR_ROLE, RUNTIME_ROLE, + SCHEDULER_ROLE, WORKER_ROLE, ) @@ -138,6 +139,13 @@ def assert_worker_role(connection: Connection) -> None: _assert_non_owner_role(connection, WORKER_ROLE) +def assert_scheduler_role(connection: Connection) -> None: + """Require the dedicated content-free File scheduler login.""" + + _assert_non_owner_role(connection, SCHEDULER_ROLE) + _assert_no_owned_objects_or_role_members(connection) + + def _assert_no_owned_objects_or_role_members(connection: Connection) -> None: """Reject object ownership and incoming memberships for sensitive roles.""" diff --git a/engine/persistence/schema_security_manifest.yaml b/engine/persistence/schema_security_manifest.yaml index 42088acc..d2d86abf 100644 --- a/engine/persistence/schema_security_manifest.yaml +++ b/engine/persistence/schema_security_manifest.yaml @@ -1,5 +1,5 @@ { - "manifestVersion": "30.0.0", + "manifestVersion": "31.0.0", "controlOperations": [ { "name": "register_file_source", @@ -39,7 +39,7 @@ "definerRole": "context_engine_worker_lease_definer", "directTableMutationAllowed": false, "trustedOrganizationSource": "TrustedControlCall", - "databaseOwnedTime": true, + "databaseOwnedTime": "clock_timestamp refreshed after every progress-lock wait", "idempotencyBinding": [ "organization_id", "source_id" @@ -304,6 +304,27 @@ "file_import_job_event" ] }, + { + "name": "claim_scheduled_file_import", + "databaseFunction": "context_scheduler_claim_file_import", + "role": "context_engine_scheduler", + "definerRole": "context_engine_file_dispatch_definer", + "directTableMutationAllowed": false, + "callerSelectedRoutingDimensions": [], + "serverCapabilityInput": "all-or-nothing configured logical File root assertion applied only after globally oldest eligible selection and again during final post-lock revalidation; an omitted root yields no-work and cannot redirect selection", + "databaseOwnedTime": "full-precision clock_timestamp authority check refreshed after every progress-lock wait; protocol lease timestamps and immediate worker verification use the PostgreSQL clock normalized to whole seconds", + "fixedTtlSeconds": 300, + "selection": "oldest current v3/v4 page-scheduled upsert ordered by database acceptance time, checkpoint sequence, page ordinal, change ordinal, Organization, Source, and job id; the claim holds the per-Source progress advisory lock, refreshes its statement snapshot, revalidates latest accepted scan epoch, and uses FOR UPDATE SKIP LOCKED authority-row fencing before leasing", + "currentAuthority": [ + "active exact SourceVersion", + "latest accepted scan epoch", + "active exact Membership/version", + "enabled exact File import ServicePrincipal" + ], + "atomicWrites": [ + "file_import_job" + ] + }, { "name": "redeem_file_import_lease", "databaseFunction": "context_worker_redeem_file_import", @@ -728,6 +749,19 @@ "context_engine_citation_definer" ], "using": "true" + }, + { + "name": "membership_file_dispatch_definer_select", + "command": "SELECT", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true" + }, + { + "name": "membership_file_dispatch_definer_update", + "command": "UPDATE", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true", + "withCheck": "true" } ] }, @@ -753,6 +787,10 @@ ], "context_engine_citation_definer": [ "SELECT" + ], + "context_engine_file_dispatch_definer": [ + "SELECT", + "UPDATE status, valid_from, valid_until" ] }, "partitions": [], @@ -1476,6 +1514,19 @@ "context_engine_action_execute_definer" ], "using": "true" + }, + { + "name": "context_source_file_dispatch_definer_select", + "command": "SELECT", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true" + }, + { + "name": "context_source_file_dispatch_definer_update", + "command": "UPDATE", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true", + "withCheck": "true" } ] }, @@ -1504,6 +1555,10 @@ ], "context_engine_action_execute_definer": [ "SELECT" + ], + "context_engine_file_dispatch_definer": [ + "SELECT", + "UPDATE lifecycle_state, active_version_id" ] }, "partitions": [], @@ -1635,6 +1690,12 @@ "context_engine_action_execute_definer" ], "using": "true" + }, + { + "name": "source_version_file_dispatch_definer_select", + "command": "SELECT", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true" } ] }, @@ -1667,6 +1728,9 @@ ], "context_engine_action_execute_definer": [ "SELECT" + ], + "context_engine_file_dispatch_definer": [ + "SELECT" ] }, "partitions": [], @@ -3642,6 +3706,19 @@ "context_engine_worker_lease_definer" ], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid AND workload = 'supply.file-import' AND worker_audience = 'context-engine-worker' AND operation = 'file.import' AND enabled IS TRUE" + }, + { + "name": "service_principal_file_dispatch_definer_select", + "command": "SELECT", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true" + }, + { + "name": "service_principal_file_dispatch_definer_update", + "command": "UPDATE", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true", + "withCheck": "true" } ] }, @@ -3650,6 +3727,10 @@ "context_engine_worker": [], "context_engine_worker_lease_definer": [ "SELECT" + ], + "context_engine_file_dispatch_definer": [ + "SELECT", + "UPDATE enabled" ] }, "partitions": [], @@ -5179,6 +5260,12 @@ "context_engine_worker_lease_definer" ], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "file_acquisition_file_dispatch_definer_select", + "command": "SELECT", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true" } ] }, @@ -5714,6 +5801,23 @@ ], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid", "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "file_import_job_file_dispatch_definer_select", + "command": "SELECT", + "roles": [ + "context_engine_file_dispatch_definer" + ], + "using": "true" + }, + { + "name": "file_import_job_file_dispatch_definer_update", + "command": "UPDATE", + "roles": [ + "context_engine_file_dispatch_definer" + ], + "using": "true", + "withCheck": "true" } ] }, @@ -5735,10 +5839,12 @@ "context_worker_index_file_publication", "context_worker_activate_recoverable_file_publication", "context_worker_record_file_import_interruption" + ,"context_scheduler_claim_file_import" ], "definerRoles": [ "context_engine_worker_lease_definer", "context_engine_access_policy_definer" + ,"context_engine_file_dispatch_definer" ], "directTableMutationAllowed": false }, @@ -5754,6 +5860,9 @@ "EXECUTE context_worker_issue_file_import_lease" ], "context_engine_runtime": [], + "context_engine_scheduler": [ + "EXECUTE context_scheduler_claim_file_import" + ], "context_engine_worker": [ "EXECUTE context_worker_redeem_file_import", "EXECUTE context_worker_fail_file_import", @@ -5772,6 +5881,10 @@ "SELECT", "INSERT", "UPDATE" + ], + "context_engine_file_dispatch_definer": [ + "SELECT", + "UPDATE state, signing_key_version, lease_nonce_digest, lease_issued_at, lease_expires_at, lease_generation, dispatch_claimed" ] }, "partitions": [], @@ -5790,6 +5903,8 @@ "WORKER-LEASE-007", "PG-FILE-SOURCE-OFFBOARD-030", "PG-FILE-CHANGE-SUPERSESSION-083" + ,"PG-FILE-DISPATCH-091" + ,"PG-FILE-DISPATCH-CONCURRENCY-091" ] }, { @@ -7368,7 +7483,8 @@ "rowLevelSecurity": {"enabled": true, "forced": true, "policies": [ {"name": "file_source_change_page_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, {"name": "file_source_change_page_file_change_definer_select", "command": "SELECT", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"}, - {"name": "file_source_change_page_file_change_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"} + {"name": "file_source_change_page_file_change_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"}, + {"name": "file_source_change_page_file_dispatch_definer_select", "command": "SELECT", "roles": ["context_engine_file_dispatch_definer"], "using": "true"} ]}, "immutableRows": {"trigger": "file_source_change_page_immutable", "function": "context_content_reject_mutation", "events": ["UPDATE", "DELETE"], "sqlstate": "55000"}, "functionOnlyMutation": {"databaseFunctions": ["context_control_accept_file_change_page", "context_control_accept_file_delete_observation_page"], "definerRole": "context_engine_worker_lease_definer", "directTableMutationAllowed": false}, @@ -7401,7 +7517,8 @@ "rowLevelSecurity": {"enabled": true, "forced": true, "policies": [ {"name": "file_source_change_migrator_administration", "command": "ALL", "roles": ["context_engine_migrator"], "using": "true", "withCheck": "true"}, {"name": "file_source_change_file_change_definer_select", "command": "SELECT", "roles": ["context_engine_worker_lease_definer"], "using": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"}, - {"name": "file_source_change_file_change_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"} + {"name": "file_source_change_file_change_definer_insert", "command": "INSERT", "roles": ["context_engine_worker_lease_definer"], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid"}, + {"name": "file_source_change_file_dispatch_definer_select", "command": "SELECT", "roles": ["context_engine_file_dispatch_definer"], "using": "true"} ]}, "immutableRows": {"trigger": "file_source_change_immutable", "function": "context_content_reject_mutation", "events": ["UPDATE", "DELETE"], "sqlstate": "55000"}, "functionOnlyMutation": {"databaseFunctions": ["context_control_accept_file_change_page", "context_control_accept_file_delete_observation_page"], "definerRole": "context_engine_worker_lease_definer", "directTableMutationAllowed": false}, @@ -7589,6 +7706,12 @@ "context_engine_worker_lease_definer" ], "withCheck": "organization_id = NULLIF(current_setting('app.organization_id', true), '')::uuid" + }, + { + "name": "file_source_acquisition_checkpoint_file_dispatch_definer_select", + "command": "SELECT", + "roles": ["context_engine_file_dispatch_definer"], + "using": "true" } ] }, diff --git a/engine/persistence/worker_jobs.py b/engine/persistence/worker_jobs.py index 120ae98a..979d49d4 100644 --- a/engine/persistence/worker_jobs.py +++ b/engine/persistence/worker_jobs.py @@ -6,14 +6,18 @@ from dataclasses import dataclass, field from datetime import UTC, datetime from enum import StrEnum -from typing import Final, Literal +from typing import TYPE_CHECKING, Final, Literal from uuid import UUID from sqlalchemy import Connection, Engine, text from sqlalchemy.exc import SQLAlchemyError -from engine.control import PreparedFileImport -from engine.persistence.role_guard import assert_control_role, assert_worker_role +from engine.control import PreparedFileImport, SourceRef +from engine.persistence.role_guard import ( + assert_control_role, + assert_scheduler_role, + assert_worker_role, +) from engine.supply.jobs import ( FILE_IMPORT_WORKER_LEASE_OPERATION, WORKER_LEASE_ACTOR_KIND, @@ -30,12 +34,25 @@ worker_lease_digest, ) +if TYPE_CHECKING: + from engine.persistence.file_imports import FileImportLeaseRedemption + DEFAULT_WORKER_LEASE_TTL_SECONDS: Final = 300 MAX_WORKER_LEASE_TTL_SECONDS: Final = 3600 + + def _utc_now() -> datetime: return datetime.now(UTC).replace(microsecond=0) +def _database_timestamp_utc(field_name: str, value: object) -> datetime: + """Normalize one PostgreSQL timestamptz representation to protocol UTC.""" + + if type(value) is not datetime or value.tzinfo is None: + raise ValueError(f"{field_name} must be timezone-aware") + return _require_utc(field_name, value.astimezone(UTC)) + + @dataclass(frozen=True, slots=True) class WorkerLeaseIssueRequest: """Trusted durable-row locator; the issuer owns lease time and entropy.""" @@ -146,6 +163,167 @@ class WorkerLeaseAuthorityUnavailable(RuntimeError): """A trusted lease database authority could not complete safely.""" +@dataclass(frozen=True, slots=True) +class FileDispatchNoWork: + """Closed, content-free result when no first-attempt File job is eligible.""" + + status: Literal["no_work"] = field(default="no_work", init=False) + + +@dataclass(frozen=True, slots=True, repr=False) +class FileDispatchLease: + """Scheduler-minted exact first-attempt lease and internal routing facts.""" + + token: WorkerLeaseToken = field(repr=False) + organization_id: UUID = field(repr=False) + job_id: UUID = field(repr=False) + source_ref: SourceRef = field(repr=False) + service_principal_id: UUID = field(repr=False) + lease_generation: int + issued_at: datetime = field(repr=False) + expires_at: datetime = field(repr=False) + + def __post_init__(self) -> None: + if type(self.token) is not WorkerLeaseToken: + raise TypeError("File dispatch requires a WorkerLeaseToken") + _require_uuid("organization_id", self.organization_id) + _require_uuid("job_id", self.job_id) + if type(self.source_ref) is not SourceRef: + raise TypeError("File dispatch source must be SourceRef") + _require_uuid("service_principal_id", self.service_principal_id) + if self.lease_generation != 1: + raise ValueError("first-attempt File dispatch requires generation one") + issued_at = _require_utc("issued_at", self.issued_at) + expires_at = _require_utc("expires_at", self.expires_at) + if expires_at <= issued_at: + raise ValueError("File dispatch expiry must follow issuance") + + @property + def redemption(self) -> FileImportLeaseRedemption: + """Construct the existing untrusted worker carrier only on demand.""" + + from engine.persistence.file_imports import FileImportLeaseRedemption + + return FileImportLeaseRedemption( + token=self.token, + expected_organization_id=self.organization_id, + expected_job_id=self.job_id, + expected_source_ref=self.source_ref, + ) + + def __repr__(self) -> str: + return f"FileDispatchLease(lease_generation={self.lease_generation})" + + +FileDispatchClaim = FileDispatchLease | FileDispatchNoWork + + +class PostgreSQLFileDispatchAuthority: + """Claim the oldest eligible scheduled File import without tenant input.""" + + __slots__ = ("_codec", "_configured_root_refs", "_scheduler_engine") + + def __init__( + self, + scheduler_engine: Engine, + codec: WorkerLeaseCodec, + *, + configured_root_refs: tuple[str, ...], + ) -> None: + if type(codec) is not WorkerLeaseCodec: + raise TypeError("File dispatch authority requires WorkerLeaseCodec") + self._scheduler_engine = scheduler_engine + self._codec = codec + if ( + type(configured_root_refs) is not tuple + or not configured_root_refs + or len(set(configured_root_refs)) != len(configured_root_refs) + ): + raise ValueError("File dispatch requires distinct configured roots") + for root_ref in configured_root_refs: + _require_identifier("configured_root_ref", root_ref, maximum_length=128) + self._configured_root_refs = configured_root_refs + + def claim(self) -> FileDispatchClaim: + """Atomically fence one first attempt, then mint its exact existing token.""" + + nonce = generate_worker_lease_nonce() + try: + with self._scheduler_engine.begin() as connection: + self._require_scheduler_role(connection) + row = connection.execute( + text( + """ + SELECT organization_id, job_id, source_id, + service_principal_id, lease_generation, + issued_at, expires_at + FROM public.context_scheduler_claim_file_import( + :signing_key_version, :nonce, :configured_root_refs + ) + """ + ), + { + "signing_key_version": ( + self._codec.active_signing_key_version + ), + "nonce": nonce, + "configured_root_refs": list(self._configured_root_refs), + }, + ).one_or_none() + if row is None: + return FileDispatchNoWork() + issued_at = _database_timestamp_utc("issued_at", row.issued_at) + expires_at = _database_timestamp_utc("expires_at", row.expires_at) + organization_id = _require_uuid( + "organization_id", row.organization_id + ) + job_id = _require_uuid("job_id", row.job_id) + service_principal_id = _require_uuid( + "service_principal_id", row.service_principal_id + ) + source_ref = SourceRef(_require_uuid("source_id", row.source_id)) + lease_generation = row.lease_generation + claims = WorkerLeaseClaims( + signing_key_version=self._codec.active_signing_key_version, + organization_id=organization_id, + job_id=job_id, + service_principal_id=service_principal_id, + workload="supply.file-import", + worker_audience="context-engine-worker", + issued_at=issued_at, + expires_at=expires_at, + nonce=nonce, + operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + source_ref=str(source_ref.value), + lease_generation=lease_generation, + ) + return FileDispatchLease( + token=self._codec.mint(claims), + organization_id=organization_id, + job_id=job_id, + source_ref=source_ref, + service_principal_id=service_principal_id, + lease_generation=lease_generation, + issued_at=issued_at, + expires_at=expires_at, + ) + except WorkerLeaseAuthorityUnavailable: + raise + except SQLAlchemyError: + raise WorkerLeaseAuthorityUnavailable( + "File dispatch claim database work failed" + ) from None + + @staticmethod + def _require_scheduler_role(connection: Connection) -> None: + try: + assert_scheduler_role(connection) + except AssertionError as error: + raise WorkerLeaseAuthorityUnavailable( + "File dispatch authority is not the dedicated scheduler role" + ) from error + + def _rejection(token: WorkerLeaseToken) -> WorkNotAvailable: return WorkNotAvailable( WorkerLeaseRejectionAuditReceipt(lease_digest=worker_lease_digest(token)) diff --git a/eval/catalogs/m0-security-evidence.schema.json b/eval/catalogs/m0-security-evidence.schema.json index f4919040..05d61e38 100644 --- a/eval/catalogs/m0-security-evidence.schema.json +++ b/eval/catalogs/m0-security-evidence.schema.json @@ -189,7 +189,7 @@ "selector": { "type": "string", "minLength": 1, - "pattern": "^tests/(?:unit|integration)/test_[A-Za-z0-9_]+\\.py::test_[A-Za-z0-9_]+$" + "pattern": "^tests/(?:unit|integration|process)/test_[A-Za-z0-9_]+\\.py::test_[A-Za-z0-9_]+$" } } }, diff --git a/eval/catalogs/m0-security-evidence.yaml b/eval/catalogs/m0-security-evidence.yaml index 9c3f1001..c2662699 100644 --- a/eval/catalogs/m0-security-evidence.yaml +++ b/eval/catalogs/m0-security-evidence.yaml @@ -525,6 +525,21 @@ "layer": "postgres", "selector": "tests/integration/test_file_change_pages.py::test_control_schedules_only_the_upserts_from_a_current_mixed_file_page" }, + { + "id": "PG-FILE-DISPATCH-091", + "layer": "postgres", + "selector": "tests/integration/test_file_dispatch.py::test_scheduler_claims_only_current_page_scheduled_upsert" + }, + { + "id": "PG-FILE-DISPATCH-CONCURRENCY-091", + "layer": "postgres", + "selector": "tests/integration/test_file_dispatch.py::test_concurrent_dispatchers_never_claim_the_same_job" + }, + { + "id": "PROC-FILE-DISPATCH-091", + "layer": "runtime", + "selector": "tests/integration/test_file_dispatch.py::test_independent_worker_process_dispatches_and_publishes_one_job" + }, { "id": "HTTP-FILE-MIXED-UPSERT-NO-DELETE-089", "layer": "runtime", @@ -669,11 +684,14 @@ "PG-FILE-DELETE-PAGE-085", "PG-FILE-DELETE-NO-EFFECT-085", "PG-FILE-MIXED-UPSERT-SCHEDULE-089", - "PG-FILE-MIXED-UPSERT-REPLAY-089" + "PG-FILE-MIXED-UPSERT-REPLAY-089", + "PG-FILE-DISPATCH-091", + "PG-FILE-DISPATCH-CONCURRENCY-091" ], "runtime": [ "RUNTIME-WORKER-LEASE-007", - "HTTP-FILE-MIXED-UPSERT-NO-DELETE-089" + "HTTP-FILE-MIXED-UPSERT-NO-DELETE-089", + "PROC-FILE-DISPATCH-091" ] } }, diff --git a/eval/catalogs/security-catalog.schema.json b/eval/catalogs/security-catalog.schema.json index 3b05e968..991f7adb 100644 --- a/eval/catalogs/security-catalog.schema.json +++ b/eval/catalogs/security-catalog.schema.json @@ -95,8 +95,8 @@ }, "activations": { "type": "array", - "minItems": 20, - "maxItems": 20, + "minItems": 21, + "maxItems": 21, "uniqueItems": true, "prefixItems": [ { @@ -989,6 +989,48 @@ "Runtime authorization from File page or checkpoint metadata" ] } + }, + { + "const": { + "issueRef": "#91", + "invariantRef": "WORKER-LEASE-007", + "carrier": "autonomous first-attempt dispatch of explicit scheduled File upserts", + "status": "active_fail_closed", + "policyEpochScope": "not-runtime-authority", + "controlBoundary": "function-only scheduler login -> current page/acquisition/audience/receiver eligibility -> deterministic SKIP LOCKED selector -> database-timed generation-one lease -> existing WorkerLease and File worker", + "testEvidence": [ + { + "id": "PG-FILE-DISPATCH-091", + "surface": "tests/integration/test_file_dispatch.py::test_scheduler_claims_only_current_page_scheduled_upsert", + "oracle": "The non-owner scheduler cannot read job tables and can claim only one current page-scheduled upsert through the exact function; the next claim is typed content-free no-work and revoked audience authority remains unclaimed." + }, + { + "id": "PG-FILE-DISPATCH-CONCURRENCY-091", + "surface": "tests/integration/test_file_dispatch.py::test_concurrent_dispatchers_never_claim_the_same_job", + "oracle": "Two concurrent scheduler-role authorities claim the two oldest eligible jobs exactly once through FOR UPDATE SKIP LOCKED; no job identity is returned twice." + }, + { + "id": "PROC-FILE-DISPATCH-091", + "surface": "tests/integration/test_file_dispatch.py::test_independent_worker_process_dispatches_and_publishes_one_job", + "oracle": "The independent worker runs a configured dispatch cycle with no caller Organization, Source, job, or token, completes the existing publication path, and emits only the closed content-free dispatched result." + } + ], + "deferredEvidence": [ + "expired-lease reclaim, automatic retry/backoff, dead-letter handling, and operator remediation", + "provider polling, automatic page acceptance, and automatic delete ordering" + ], + "futureCarriers": [ + "retry and dead-letter owner", + "provider polling and full resync", + "explicit mixed-change ordering policy" + ], + "notActive": [ + "scheduler tenant, Source, job, audience, path, lease-time, or generation choice", + "manual import or delete execution", + "automatic retry or reclaim", + "Runtime authorization from Supply scheduling or lease state" + ] + } } ], "items": false @@ -1290,7 +1332,10 @@ "HTTP-FILE-DELETE-INVISIBLE-087", "PG-FILE-MIXED-UPSERT-SCHEDULE-089", "PG-FILE-MIXED-UPSERT-REPLAY-089", - "HTTP-FILE-MIXED-UPSERT-NO-DELETE-089" + "HTTP-FILE-MIXED-UPSERT-NO-DELETE-089", + "PG-FILE-DISPATCH-091", + "PG-FILE-DISPATCH-CONCURRENCY-091", + "PROC-FILE-DISPATCH-091" ] }, "surface": { @@ -1338,7 +1383,8 @@ "#83", "#85", "#87", - "#89" + "#89", + "#91" ] }, "invariantRef": { @@ -1375,7 +1421,8 @@ "explicit accepted File page scheduling through existing import jobs", "bounded durable File deletion observations without execution", "explicit current File delete execution through sole tombstone authority", - "explicit current mixed File page upsert-projection scheduling" + "explicit current mixed File page upsert-projection scheduling", + "autonomous first-attempt dispatch of explicit scheduled File upserts" ] }, "status": { @@ -1410,7 +1457,8 @@ "accepted File page -> trusted ContextControl explicit FileImportAudience -> exact file_acquisition/file_import_job lineage -> existing WorkerLease -> current accepted scan-epoch redemption fence -> pre-compiler raw observation verification -> atomic current-epoch publication fence", "latest complete same-SourceVersion v4 baseline -> stable shallow File snapshot -> provider-authenticated upsert/delete ChangePage -> exact baseline-bound ContextControl PostgreSQL acceptance", "exact trusted Source/SourceVersion/page/ordinal locator -> current complete v4 scan revalidation -> server-derived Resource/event lineage -> existing tombstone authority -> immutable exact execution binding", - "complete current v4 upsert/delete page validation -> trusted ContextControl explicit FileImportAudience -> nonempty original-ordinal upsert projection -> exact existing acquisition/import-job lineage -> existing WorkerLease and current-scan publication fences" + "complete current v4 upsert/delete page validation -> trusted ContextControl explicit FileImportAudience -> nonempty original-ordinal upsert projection -> exact existing acquisition/import-job lineage -> existing WorkerLease and current-scan publication fences", + "function-only scheduler login -> current page/acquisition/audience/receiver eligibility -> deterministic SKIP LOCKED selector -> database-timed generation-one lease -> existing WorkerLease and File worker" ] }, "testEvidence": { diff --git a/eval/catalogs/security-invariants.yaml b/eval/catalogs/security-invariants.yaml index 5322a939..8582bfb0 100644 --- a/eval/catalogs/security-invariants.yaml +++ b/eval/catalogs/security-invariants.yaml @@ -23,7 +23,8 @@ "#83", "#85", "#87", - "#89" + "#89", + "#91" ], "documentRefs": [ "README.md", @@ -51,9 +52,10 @@ "docs/decisions/0055-schedule-accepted-file-observations-explicitly.md", "docs/decisions/0056-detect-file-deletions-without-tombstone-authority.md", "docs/decisions/0057-execute-current-file-deletes-through-tombstone-authority.md", - "docs/decisions/0058-schedule-only-upserts-from-mixed-file-pages.md" + "docs/decisions/0058-schedule-only-upserts-from-mixed-file-pages.md", + "docs/decisions/0059-dispatch-scheduled-file-imports-through-exact-leases.md" ], - "reconciliation": "Issue #2 fixes the product and testing decisions, issue #5 requires exactly fifteen release invariants and twelve canonical acceptance fixtures, and ADR-0019 resolves the later nineteen-label prose expansion without weakening any safeguard. Issue #15 activates only Organization-level next-request resolve(Acquire) revocation evidence under REVOCATION-006: at that activation, PG-REVOCATION-006, RUN-006, and CACHE-002 are active while BLOB-002 and Continue, citation, Policy-Epoch-bound WorkerLease, production ContextAccessTicket/ActionTicket, audit, outbox, cleanup, finer-epoch, UI, and external-admin carriers remain future or NOT_ACTIVE; later issue records are authoritative for subsequently activated carriers. Issue #16 activates only the M0 refusal gate for unavailable Continue, profile-disabled OpenCitation, and server-owned unavailable Acquire plans: at that activation its real continuation, profile-enabled citation, federated/source-native, and File carriers remain future, while its Runtime and HTTP refusal surfaces prove generic outcomes before content I/O; Issue #69 later activates the private/direct File profile-enabled citation carrier. Issue #17 activates only the signed one-shot persistent no-op durable-job WorkerLease subcarrier under WORKER-LEASE-007. It binds one exact worker audience but no end-user delivery audience or Policy Epoch, and proves only LEASE-SIGNING-017, PG-WORKER-LEASE-NOOP-017, and WORKER-LEASE-REPLAY-007; Source, Resource, Revision, Policy Epoch, end-user delivery audience, idempotency, generation, business mutation, outbox, File publication, and the full ACCEPT-008 matrix remain deferred or NOT_ACTIVE. Issue #18 activates only distinct signed synthetic ContextAccessTicket Provider-read and ActionTicket no-op channel-action subcarriers under ACTION-SEPARATION-014, with current Organization-v0 Policy Epoch validation. TICKET-AUDIENCE-018 and PG-TICKET-EPOCH-018 do not activate production ContextProvider integration, ContextRuntime ticket integration, BotDelivery, full M2 ActionPlane.prepare/perform, a real Sender or external effect, payload/destination/approval/idempotency binding, durable one-shot/replay/reconciliation, or full ACCEPT-012 PASS; those remain future or NOT_ACTIVE. Issue #19 activates only the current Acquire authorized-only ContextRun and restricted delivered-empty DecisionAudit subcarrier under TRACE-REDACTION-012. DIGEST-019, RUN-LINEAGE-019, AUTHORIZED-RUN-019, and PG-TRACE-REDACTION-012 prove deterministic Package and Organization-bound query digests, retained-UserActor-transaction persistence, decisionRef resolution, redaction, and short-lived exact-Organization operator ticket reads with no application-role table access; the supported reader commits deletion before returning, while a direct caller rollback is not claimed as durable exactly-once redemption. Raw query retention, full ContextPackage body retention, unauthenticated transport failures as ContextRuns, cross-Organization analytics, and general observability redaction remain NOT_ACTIVE. Issue #48 activates only the current ACCEPT-002 authenticated HTTP Acquire Membership field-projection carrier under SCOPE-INTERSECTION-004, INDEX-NOT-AUTHORITY-005, and TRACE-REDACTION-012. PROP-FIELD-PROJECTION-048, PG-FIELD-PROJECTION-048, and HTTP-ACCEPT-002-048 bind one current Membership/version field ceiling to same-transaction FORCE-RLS reduction, the sealed AuthorizationKernel, AuthorizedProjection and Evidence integrity, and authorized-only ContextRun/audit persistence. General permission DSLs, caller-authored projection lists, CandidateRef or index field authority, production Provider/source-native ACL negotiation, Supply publication, File/Base field ACL, typed fields, Continue, and Issue #20 runner substitution remain future or NOT_ACTIVE; Issue #69 later activates private/direct File OpenCitation through the same field-projection gates. Issue #63 activates only the digest-only private authenticated HTTP Acquire DeliveryEvidenceRef carrier under TRANSPORT-UNTRUSTED-008. PROP-DELIVERY-EVIDENCE-063, PG-DELIVERY-EVIDENCE-063, HTTP-DELIVERY-EVIDENCE-063, and FILE-DELIVERY-EVIDENCE-063 prove exact service/request/Organization/asker/Membership-version/destination/consumer/purpose/audience/epoch/lifetime binding, stable identical retry identity, role isolation, expiry cleanup, pre-content generic rejection, and one File-backed sealed Runtime delivery. Group AudienceSnapshot, group/public DeliveryEvidenceRef, production ModelGateway, ActionPlane, and the BotDelivery application remain future or NOT_ACTIVE; Issues #64, #66, #69, and #70 later activate the frozen OpenAPI, generated TypeScript SDK, private/direct OpenCitation, and deterministic private model-egress carriers. Issue #65 activates only one opaque digest-only model or channel EgressGrant after final Package policy, exact atomic PostgreSQL redemption and restricted audit, nominal BotDelivery inputs, and deterministic network-free ModelGateway or Sender-preflight spies under EGRESS-011. PROP-EGRESS-011, PG-EGRESS-011, and RUNTIME-EGRESS-011 prove exact Package/Organization/purpose/audience/epoch/hop/profile/lifetime binding and zero additional bytes on replay. Real model/provider calls, a real Sender or channel write, ActionTicket effects, group AudienceSnapshot revalidation, and the BotDelivery application process remain future or NOT_ACTIVE; Issue #64 later activates the generated SDK consumer and Issue #70 later activates the deterministic private TypeScript ModelGateway. Issue #66 activates the frozen public POST /v0/resolve OpenAPI carrier under TRANSPORT-UNTRUSTED-008. OPENAPI-CONTRACT-066, OPENAPI-BREAKING-066, HTTP-V0-066, and PG-RUNTIME-RELEASE-066 prove one public closed operation, deterministic immutable snapshot and breaking-change refusal, a hidden v1 bridge through the same handler and sealed Runtime path, and exact read-only observation of the active Learning-promoted release with fail-closed missing-release behavior before content work. A production BotDelivery caller, Continue redemption, MCP, group AudienceSnapshot, and external effects remain future or NOT_ACTIVE; Issues #64 and #69 later activate the generated TypeScript SDK and private/direct OpenCitation redemption through this frozen operation. Issue #64 activates only the packaged generated TypeScript POST /v0/resolve client under TRANSPORT-UNTRUSTED-008. SDK-CONTRACT-064 and SDK-LIVE-FILE-064 prove deterministic pinned generation, strict closed types, a narrow export map and metadata-only facade, installable tarball consumption, and one real PostgreSQL/File-backed Acquire through CandidateRef, AuthorizationKernel, AuthorizedProjection, ContextPackage, and opaque model egress grant. Issue #69 later extends SDK-LIVE-FILE-064 with a successful private/direct File OpenCitation through a second request-bound DeliveryEvidenceRef; Issue #70 extends the installed SDK fixture into deterministic Package-bound model generation; generated Continue remains generic unavailable. External package publication, production provider access, MCP, group AudienceSnapshot, real Continue redemption, and external effects remain future or NOT_ACTIVE. Issue #67 activates only private ActionPlane.prepare for create-placeholder, finalize-reply, and private-follow-up operation-specific tickets under ACTION-SEPARATION-014. PG-ACTION-PREPARE-067 proves exact current delivery, Organization, destination, audience, source, payload, approval, epoch, lifetime, and idempotency binding under a dedicated non-owner PostgreSQL role with digest-only FORCE-RLS persistence and zero effects. Issue #68 activates private ActionPlane.perform only through a deterministic Sender twin. PG-ACTION-PERFORM-068 proves one pre-Sender current-authority validation, one provider-attempt identity, immutable receipt replay, zero-effect ticket/payload mutation and stale-audience refusal, same-label cross-Organization isolation, and monotonic applied/rejected reconciliation including crash interleavings. Real provider or channel network effects, group AudienceSnapshot, compensation/delete, production BotDelivery orchestration, and the full ACCEPT-012 pass remain future or NOT_ACTIVE. The canonical set is IDs 001 through 012, 014, 015, and 019: CACHE-SCOPE-013 remains a preregistered conditional extension; AUDIENCE-016 is absorbed by SCOPE-INTERSECTION-004 and EGRESS-011; ACL-PROOF-017 is absorbed by INDEX-NOT-AUTHORITY-005 and REVOCATION-006; DELIVERY-EVIDENCE-018 is absorbed by TRANSPORT-UNTRUSTED-008. ACCEPT-001 through ACCEPT-012 follow ADR-0019's category order. Protected-asset references A-01 through A-08 refer, in order, to the eight bullets in the threat model's Protected assets section. Every expectedEvidence value below is a stable planned case identifier, not a claim that the case ran or passed; only an exact activation record upgrades named evidence, while fixture carrier status and the explicit M0 oracle preserve every other accepted-versus-active distinction. Issue #69 activates private/direct File CitationOpenRef issuance and OpenCitation under CITATION-AUTH-010: digest-only multi-use locators reveal only prior Package/Evidence and Fragment location lineage, every open obtains a current UserActor and trusted delivery context then traverses CandidateRef, AuthorizationKernel, AuthorizedProjection, a replacement ContextPackage, EgressGrant, ContextRun, and restricted DecisionAudit. PG-CITATION-AUTH-010, RUNTIME-CITATION-AUTH-010, and SDK-LIVE-FILE-064 prove A/B/A reauthorization, non-consumption on denial, database-clock expiry, cross-kind and cross-Organization opacity, and the generated SDK carrier. Group/public AudienceSnapshot, non-File providers, raw source URL locators, and Continue remain future or NOT_ACTIVE. Issue #70 activates only the private deterministic TypeScript ModelGateway under EGRESS-011. TS-MODEL-EGRESS-070, SDK-MODEL-EGRESS-070, and PG-MODEL-EGRESS-070 prove one current Package, exact grant redemption, closed provider input, bounded Package-subset citations, replay zero bytes, and digest-only retained audit. Real providers, streaming, group AudienceSnapshot, model-authored ActionPlane authority, and external effects remain future or NOT_ACTIVE. Issue #71 activates the complete private File-backed deterministic-twin BotDelivery carrier under TRANSPORT-UNTRUSTED-008, CITATION-AUTH-010, EGRESS-011, and ACTION-SEPARATION-014. TS-PRIVATE-BOT-FLOW-071, SDK-PRIVATE-BOT-FLOW-071, and PG-PRIVATE-BOT-FLOW-071 prove the independent TypeScript process and import boundary, exact verified private event binding, opaque DeliveryEvidenceRef transport, installed generated-SDK HTTP resolve, sealed File Package path, controlled model generation, distinct placeholder/final ActionPlane effects, digest-only DeliveryReceipt audit, citation reopening, and the composed wrong-binding oracles. The historical FIXTURE-ACCEPT-012 continues to prove the M0 unavailable cross-capability baseline separately. Live Feishu/model/Sender network carriers, group/public delivery, compensation/delete, Continue, and MCP remain NOT_ACTIVE. Issue #81 activates deterministic shallow File readChanges and whole-page acknowledgement under WORKER-LEASE-007. PG-FILE-CHANGE-ACTIVATE-081, PG-FILE-CHANGE-PAGE-081, and PG-FILE-CHANGE-DENY-081 prove immutable v3 activation, provider-authenticated content-free paging, exact replay, predecessor ordering, post-commit cursor issuance, FORCE-RLS tenant isolation, source/version invalidation, and zero implicit job or publication effect. Automatic scheduling, deletion execution, recursive discovery, full resync, and Runtime authority from cursor or checkpoint metadata remain NOT_ACTIVE. Issue #83 activates only explicit accepted File page scheduling through the existing file.import acquisition, WorkerLease, and publication path under WORKER-LEASE-007. PG-FILE-CHANGE-SCHEDULE-083, PG-FILE-CHANGE-SCHEDULE-DENY-083, and PG-FILE-CHANGE-SUPERSESSION-083 prove whole-page atomicity, exact job replay, explicit current FileImportAudience and receiver validation, immutable raw-byte observation lineage, pre-compiler drift refusal, and scan-epoch fences before content read and visible publication with zero stale publication effect. Autonomous polling, implicit audience inheritance, deletion execution, automatic retry/reclaim, dead-letter handling, full resync, and Runtime authority from provider checkpoints remain NOT_ACTIVE. Issue #85 activates bounded File delete observations only under WORKER-LEASE-007. PG-FILE-DELETE-DETECT-085, PG-FILE-DELETE-PAGE-085, and PG-FILE-DELETE-NO-EFFECT-085 prove exact latest-complete same-SourceVersion baseline binding, stable shallow diff, immutable canonical persistence/replay, forged/incomplete/stale/cross-Organization refusal, mixed-page scheduling refusal, and zero tombstone, Policy Epoch, cleanup, watermark, or generated-SDK Runtime visibility effect. At that activation, deletion execution remains NOT_ACTIVE; autonomous polling, retry/reclaim/dead-letter, full resync, recursive scan, and Runtime authority from baseline or delete metadata remain NOT_ACTIVE. Issue #87 later activates only exact trusted current File delete execution through the existing #28 tombstone authority under REVOCATION-006. PG-FILE-DELETE-EXECUTE-087, PG-FILE-DELETE-REPLAY-087, and HTTP-FILE-DELETE-INVISIBLE-087 prove current complete-scan revalidation, server-derived effect identity, atomic tombstone/epoch/cleanup/binding, exact replay, mismatch rollback, and immediate sealed-Runtime invisibility. Issue #89 later activates only the exact upsert projection of one current mixed v4 page under WORKER-LEASE-007. PG-FILE-MIXED-UPSERT-SCHEDULE-089, PG-FILE-MIXED-UPSERT-REPLAY-089, and HTTP-FILE-MIXED-UPSERT-NO-DELETE-089 prove complete-page validation, gapped original-ordinal job binding, exact replay, partial-lineage refusal, and zero delete/tombstone authority through the generated SDK. Autonomous scheduling, automatic upsert/delete ordering, batch deletion execution, Provider deletion authority, physical cleanup, restore/recreate, retry/reclaim/dead-letter, full resync, and Runtime authority from observation, page, checkpoint, or execution metadata remain NOT_ACTIVE." + "reconciliation": "Issue #2 fixes the product and testing decisions, issue #5 requires exactly fifteen release invariants and twelve canonical acceptance fixtures, and ADR-0019 resolves the later nineteen-label prose expansion without weakening any safeguard. Issue #15 activates only Organization-level next-request resolve(Acquire) revocation evidence under REVOCATION-006: at that activation, PG-REVOCATION-006, RUN-006, and CACHE-002 are active while BLOB-002 and Continue, citation, Policy-Epoch-bound WorkerLease, production ContextAccessTicket/ActionTicket, audit, outbox, cleanup, finer-epoch, UI, and external-admin carriers remain future or NOT_ACTIVE; later issue records are authoritative for subsequently activated carriers. Issue #16 activates only the M0 refusal gate for unavailable Continue, profile-disabled OpenCitation, and server-owned unavailable Acquire plans: at that activation its real continuation, profile-enabled citation, federated/source-native, and File carriers remain future, while its Runtime and HTTP refusal surfaces prove generic outcomes before content I/O; Issue #69 later activates the private/direct File profile-enabled citation carrier. Issue #17 activates only the signed one-shot persistent no-op durable-job WorkerLease subcarrier under WORKER-LEASE-007. It binds one exact worker audience but no end-user delivery audience or Policy Epoch, and proves only LEASE-SIGNING-017, PG-WORKER-LEASE-NOOP-017, and WORKER-LEASE-REPLAY-007; Source, Resource, Revision, Policy Epoch, end-user delivery audience, idempotency, generation, business mutation, outbox, File publication, and the full ACCEPT-008 matrix remain deferred or NOT_ACTIVE. Issue #18 activates only distinct signed synthetic ContextAccessTicket Provider-read and ActionTicket no-op channel-action subcarriers under ACTION-SEPARATION-014, with current Organization-v0 Policy Epoch validation. TICKET-AUDIENCE-018 and PG-TICKET-EPOCH-018 do not activate production ContextProvider integration, ContextRuntime ticket integration, BotDelivery, full M2 ActionPlane.prepare/perform, a real Sender or external effect, payload/destination/approval/idempotency binding, durable one-shot/replay/reconciliation, or full ACCEPT-012 PASS; those remain future or NOT_ACTIVE. Issue #19 activates only the current Acquire authorized-only ContextRun and restricted delivered-empty DecisionAudit subcarrier under TRACE-REDACTION-012. DIGEST-019, RUN-LINEAGE-019, AUTHORIZED-RUN-019, and PG-TRACE-REDACTION-012 prove deterministic Package and Organization-bound query digests, retained-UserActor-transaction persistence, decisionRef resolution, redaction, and short-lived exact-Organization operator ticket reads with no application-role table access; the supported reader commits deletion before returning, while a direct caller rollback is not claimed as durable exactly-once redemption. Raw query retention, full ContextPackage body retention, unauthenticated transport failures as ContextRuns, cross-Organization analytics, and general observability redaction remain NOT_ACTIVE. Issue #48 activates only the current ACCEPT-002 authenticated HTTP Acquire Membership field-projection carrier under SCOPE-INTERSECTION-004, INDEX-NOT-AUTHORITY-005, and TRACE-REDACTION-012. PROP-FIELD-PROJECTION-048, PG-FIELD-PROJECTION-048, and HTTP-ACCEPT-002-048 bind one current Membership/version field ceiling to same-transaction FORCE-RLS reduction, the sealed AuthorizationKernel, AuthorizedProjection and Evidence integrity, and authorized-only ContextRun/audit persistence. General permission DSLs, caller-authored projection lists, CandidateRef or index field authority, production Provider/source-native ACL negotiation, Supply publication, File/Base field ACL, typed fields, Continue, and Issue #20 runner substitution remain future or NOT_ACTIVE; Issue #69 later activates private/direct File OpenCitation through the same field-projection gates. Issue #63 activates only the digest-only private authenticated HTTP Acquire DeliveryEvidenceRef carrier under TRANSPORT-UNTRUSTED-008. PROP-DELIVERY-EVIDENCE-063, PG-DELIVERY-EVIDENCE-063, HTTP-DELIVERY-EVIDENCE-063, and FILE-DELIVERY-EVIDENCE-063 prove exact service/request/Organization/asker/Membership-version/destination/consumer/purpose/audience/epoch/lifetime binding, stable identical retry identity, role isolation, expiry cleanup, pre-content generic rejection, and one File-backed sealed Runtime delivery. Group AudienceSnapshot, group/public DeliveryEvidenceRef, production ModelGateway, ActionPlane, and the BotDelivery application remain future or NOT_ACTIVE; Issues #64, #66, #69, and #70 later activate the frozen OpenAPI, generated TypeScript SDK, private/direct OpenCitation, and deterministic private model-egress carriers. Issue #65 activates only one opaque digest-only model or channel EgressGrant after final Package policy, exact atomic PostgreSQL redemption and restricted audit, nominal BotDelivery inputs, and deterministic network-free ModelGateway or Sender-preflight spies under EGRESS-011. PROP-EGRESS-011, PG-EGRESS-011, and RUNTIME-EGRESS-011 prove exact Package/Organization/purpose/audience/epoch/hop/profile/lifetime binding and zero additional bytes on replay. Real model/provider calls, a real Sender or channel write, ActionTicket effects, group AudienceSnapshot revalidation, and the BotDelivery application process remain future or NOT_ACTIVE; Issue #64 later activates the generated SDK consumer and Issue #70 later activates the deterministic private TypeScript ModelGateway. Issue #66 activates the frozen public POST /v0/resolve OpenAPI carrier under TRANSPORT-UNTRUSTED-008. OPENAPI-CONTRACT-066, OPENAPI-BREAKING-066, HTTP-V0-066, and PG-RUNTIME-RELEASE-066 prove one public closed operation, deterministic immutable snapshot and breaking-change refusal, a hidden v1 bridge through the same handler and sealed Runtime path, and exact read-only observation of the active Learning-promoted release with fail-closed missing-release behavior before content work. A production BotDelivery caller, Continue redemption, MCP, group AudienceSnapshot, and external effects remain future or NOT_ACTIVE; Issues #64 and #69 later activate the generated TypeScript SDK and private/direct OpenCitation redemption through this frozen operation. Issue #64 activates only the packaged generated TypeScript POST /v0/resolve client under TRANSPORT-UNTRUSTED-008. SDK-CONTRACT-064 and SDK-LIVE-FILE-064 prove deterministic pinned generation, strict closed types, a narrow export map and metadata-only facade, installable tarball consumption, and one real PostgreSQL/File-backed Acquire through CandidateRef, AuthorizationKernel, AuthorizedProjection, ContextPackage, and opaque model egress grant. Issue #69 later extends SDK-LIVE-FILE-064 with a successful private/direct File OpenCitation through a second request-bound DeliveryEvidenceRef; Issue #70 extends the installed SDK fixture into deterministic Package-bound model generation; generated Continue remains generic unavailable. External package publication, production provider access, MCP, group AudienceSnapshot, real Continue redemption, and external effects remain future or NOT_ACTIVE. Issue #67 activates only private ActionPlane.prepare for create-placeholder, finalize-reply, and private-follow-up operation-specific tickets under ACTION-SEPARATION-014. PG-ACTION-PREPARE-067 proves exact current delivery, Organization, destination, audience, source, payload, approval, epoch, lifetime, and idempotency binding under a dedicated non-owner PostgreSQL role with digest-only FORCE-RLS persistence and zero effects. Issue #68 activates private ActionPlane.perform only through a deterministic Sender twin. PG-ACTION-PERFORM-068 proves one pre-Sender current-authority validation, one provider-attempt identity, immutable receipt replay, zero-effect ticket/payload mutation and stale-audience refusal, same-label cross-Organization isolation, and monotonic applied/rejected reconciliation including crash interleavings. Real provider or channel network effects, group AudienceSnapshot, compensation/delete, production BotDelivery orchestration, and the full ACCEPT-012 pass remain future or NOT_ACTIVE. The canonical set is IDs 001 through 012, 014, 015, and 019: CACHE-SCOPE-013 remains a preregistered conditional extension; AUDIENCE-016 is absorbed by SCOPE-INTERSECTION-004 and EGRESS-011; ACL-PROOF-017 is absorbed by INDEX-NOT-AUTHORITY-005 and REVOCATION-006; DELIVERY-EVIDENCE-018 is absorbed by TRANSPORT-UNTRUSTED-008. ACCEPT-001 through ACCEPT-012 follow ADR-0019's category order. Protected-asset references A-01 through A-08 refer, in order, to the eight bullets in the threat model's Protected assets section. Every expectedEvidence value below is a stable planned case identifier, not a claim that the case ran or passed; only an exact activation record upgrades named evidence, while fixture carrier status and the explicit M0 oracle preserve every other accepted-versus-active distinction. Issue #69 activates private/direct File CitationOpenRef issuance and OpenCitation under CITATION-AUTH-010: digest-only multi-use locators reveal only prior Package/Evidence and Fragment location lineage, every open obtains a current UserActor and trusted delivery context then traverses CandidateRef, AuthorizationKernel, AuthorizedProjection, a replacement ContextPackage, EgressGrant, ContextRun, and restricted DecisionAudit. PG-CITATION-AUTH-010, RUNTIME-CITATION-AUTH-010, and SDK-LIVE-FILE-064 prove A/B/A reauthorization, non-consumption on denial, database-clock expiry, cross-kind and cross-Organization opacity, and the generated SDK carrier. Group/public AudienceSnapshot, non-File providers, raw source URL locators, and Continue remain future or NOT_ACTIVE. Issue #70 activates only the private deterministic TypeScript ModelGateway under EGRESS-011. TS-MODEL-EGRESS-070, SDK-MODEL-EGRESS-070, and PG-MODEL-EGRESS-070 prove one current Package, exact grant redemption, closed provider input, bounded Package-subset citations, replay zero bytes, and digest-only retained audit. Real providers, streaming, group AudienceSnapshot, model-authored ActionPlane authority, and external effects remain future or NOT_ACTIVE. Issue #71 activates the complete private File-backed deterministic-twin BotDelivery carrier under TRANSPORT-UNTRUSTED-008, CITATION-AUTH-010, EGRESS-011, and ACTION-SEPARATION-014. TS-PRIVATE-BOT-FLOW-071, SDK-PRIVATE-BOT-FLOW-071, and PG-PRIVATE-BOT-FLOW-071 prove the independent TypeScript process and import boundary, exact verified private event binding, opaque DeliveryEvidenceRef transport, installed generated-SDK HTTP resolve, sealed File Package path, controlled model generation, distinct placeholder/final ActionPlane effects, digest-only DeliveryReceipt audit, citation reopening, and the composed wrong-binding oracles. The historical FIXTURE-ACCEPT-012 continues to prove the M0 unavailable cross-capability baseline separately. Live Feishu/model/Sender network carriers, group/public delivery, compensation/delete, Continue, and MCP remain NOT_ACTIVE. Issue #81 activates deterministic shallow File readChanges and whole-page acknowledgement under WORKER-LEASE-007. PG-FILE-CHANGE-ACTIVATE-081, PG-FILE-CHANGE-PAGE-081, and PG-FILE-CHANGE-DENY-081 prove immutable v3 activation, provider-authenticated content-free paging, exact replay, predecessor ordering, post-commit cursor issuance, FORCE-RLS tenant isolation, source/version invalidation, and zero implicit job or publication effect. Automatic scheduling, deletion execution, recursive discovery, full resync, and Runtime authority from cursor or checkpoint metadata remain NOT_ACTIVE. Issue #83 activates only explicit accepted File page scheduling through the existing file.import acquisition, WorkerLease, and publication path under WORKER-LEASE-007. PG-FILE-CHANGE-SCHEDULE-083, PG-FILE-CHANGE-SCHEDULE-DENY-083, and PG-FILE-CHANGE-SUPERSESSION-083 prove whole-page atomicity, exact job replay, explicit current FileImportAudience and receiver validation, immutable raw-byte observation lineage, pre-compiler drift refusal, and scan-epoch fences before content read and visible publication with zero stale publication effect. Autonomous polling, implicit audience inheritance, deletion execution, automatic retry/reclaim, dead-letter handling, full resync, and Runtime authority from provider checkpoints remain NOT_ACTIVE. Issue #85 activates bounded File delete observations only under WORKER-LEASE-007. PG-FILE-DELETE-DETECT-085, PG-FILE-DELETE-PAGE-085, and PG-FILE-DELETE-NO-EFFECT-085 prove exact latest-complete same-SourceVersion baseline binding, stable shallow diff, immutable canonical persistence/replay, forged/incomplete/stale/cross-Organization refusal, mixed-page scheduling refusal, and zero tombstone, Policy Epoch, cleanup, watermark, or generated-SDK Runtime visibility effect. At that activation, deletion execution remains NOT_ACTIVE; autonomous polling, retry/reclaim/dead-letter, full resync, recursive scan, and Runtime authority from baseline or delete metadata remain NOT_ACTIVE. Issue #87 later activates only exact trusted current File delete execution through the existing #28 tombstone authority under REVOCATION-006. PG-FILE-DELETE-EXECUTE-087, PG-FILE-DELETE-REPLAY-087, and HTTP-FILE-DELETE-INVISIBLE-087 prove current complete-scan revalidation, server-derived effect identity, atomic tombstone/epoch/cleanup/binding, exact replay, mismatch rollback, and immediate sealed-Runtime invisibility. Issue #89 later activates only the exact upsert projection of one current mixed v4 page under WORKER-LEASE-007. PG-FILE-MIXED-UPSERT-SCHEDULE-089, PG-FILE-MIXED-UPSERT-REPLAY-089, and HTTP-FILE-MIXED-UPSERT-NO-DELETE-089 prove complete-page validation, gapped original-ordinal job binding, exact replay, partial-lineage refusal, and zero delete/tombstone authority through the generated SDK. Autonomous scheduling, automatic upsert/delete ordering, batch deletion execution, Provider deletion authority, physical cleanup, restore/recreate, retry/reclaim/dead-letter, full resync, and Runtime authority from observation, page, checkpoint, or execution metadata remain NOT_ACTIVE. Issue #91 activates only scheduler-owned first-attempt File dispatch through the existing exact WorkerLease. Automatic reclaim, retry/backoff, dead-letter handling, provider polling, and delete ordering remain NOT_ACTIVE." }, "hardOracles": [ { @@ -923,6 +925,46 @@ "filesystem watcher", "Runtime authorization from File page or checkpoint metadata" ] + }, + { + "issueRef": "#91", + "invariantRef": "WORKER-LEASE-007", + "carrier": "autonomous first-attempt dispatch of explicit scheduled File upserts", + "status": "active_fail_closed", + "policyEpochScope": "not-runtime-authority", + "controlBoundary": "function-only scheduler login -> current page/acquisition/audience/receiver eligibility -> deterministic SKIP LOCKED selector -> database-timed generation-one lease -> existing WorkerLease and File worker", + "testEvidence": [ + { + "id": "PG-FILE-DISPATCH-091", + "surface": "tests/integration/test_file_dispatch.py::test_scheduler_claims_only_current_page_scheduled_upsert", + "oracle": "The non-owner scheduler cannot read job tables and can claim only one current page-scheduled upsert through the exact function; the next claim is typed content-free no-work and revoked audience authority remains unclaimed." + }, + { + "id": "PG-FILE-DISPATCH-CONCURRENCY-091", + "surface": "tests/integration/test_file_dispatch.py::test_concurrent_dispatchers_never_claim_the_same_job", + "oracle": "Two concurrent scheduler-role authorities claim the two oldest eligible jobs exactly once through FOR UPDATE SKIP LOCKED; no job identity is returned twice." + }, + { + "id": "PROC-FILE-DISPATCH-091", + "surface": "tests/integration/test_file_dispatch.py::test_independent_worker_process_dispatches_and_publishes_one_job", + "oracle": "The independent worker runs a configured dispatch cycle with no caller Organization, Source, job, or token, completes the existing publication path, and emits only the closed content-free dispatched result." + } + ], + "deferredEvidence": [ + "expired-lease reclaim, automatic retry/backoff, dead-letter handling, and operator remediation", + "provider polling, automatic page acceptance, and automatic delete ordering" + ], + "futureCarriers": [ + "retry and dead-letter owner", + "provider polling and full resync", + "explicit mixed-change ordering policy" + ], + "notActive": [ + "scheduler tenant, Source, job, audience, path, lease-time, or generation choice", + "manual import or delete execution", + "automatic retry or reclaim", + "Runtime authorization from Supply scheduling or lease state" + ] } ], "invariants": [ diff --git a/infra/postgres/init/10-security-roles.sh b/infra/postgres/init/10-security-roles.sh index e1d12bea..0e1db4a1 100755 --- a/infra/postgres/init/10-security-roles.sh +++ b/infra/postgres/init/10-security-roles.sh @@ -18,6 +18,8 @@ required_environment=( CONTEXT_ENGINE_RUNTIME_PASSWORD CONTEXT_ENGINE_WORKER_ROLE CONTEXT_ENGINE_WORKER_PASSWORD + CONTEXT_ENGINE_SCHEDULER_ROLE + CONTEXT_ENGINE_SCHEDULER_PASSWORD CONTEXT_ENGINE_LEARNING_ROLE CONTEXT_ENGINE_LEARNING_PASSWORD CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE @@ -51,6 +53,8 @@ psql \ \getenv runtime_password CONTEXT_ENGINE_RUNTIME_PASSWORD \getenv worker_role CONTEXT_ENGINE_WORKER_ROLE \getenv worker_password CONTEXT_ENGINE_WORKER_PASSWORD +\getenv scheduler_role CONTEXT_ENGINE_SCHEDULER_ROLE +\getenv scheduler_password CONTEXT_ENGINE_SCHEDULER_PASSWORD \getenv learning_role CONTEXT_ENGINE_LEARNING_ROLE \getenv learning_password CONTEXT_ENGINE_LEARNING_PASSWORD \getenv security_operator_role CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE @@ -96,6 +100,16 @@ CREATE ROLE :"worker_role" NOREPLICATION NOBYPASSRLS; +CREATE ROLE :"scheduler_role" + LOGIN + PASSWORD :'scheduler_password' + NOSUPERUSER + NOCREATEDB + NOCREATEROLE + NOINHERIT + NOREPLICATION + NOBYPASSRLS; + CREATE ROLE :"identity_role" LOGIN PASSWORD :'identity_password' @@ -148,14 +162,14 @@ CREATE ROLE :"security_operator_role" REVOKE ALL ON DATABASE :"database_name" FROM PUBLIC; GRANT CONNECT ON DATABASE :"database_name" - TO :"migrator_role", :"control_role", :"runtime_role", :"worker_role", + TO :"migrator_role", :"control_role", :"runtime_role", :"worker_role", :"scheduler_role", :"identity_role", :"egress_role", :"action_role", :"learning_role", :"security_operator_role"; ALTER DATABASE :"database_name" OWNER TO :"migrator_role"; REVOKE ALL ON SCHEMA public FROM PUBLIC; ALTER SCHEMA public OWNER TO :"migrator_role"; GRANT USAGE ON SCHEMA public - TO :"control_role", :"runtime_role", :"worker_role", + TO :"control_role", :"runtime_role", :"worker_role", :"scheduler_role", :"identity_role", :"egress_role", :"action_role", :"learning_role", :"security_operator_role"; -- pgvector is an untrusted extension, so only the disposable bootstrap diff --git a/migrations/versions/20260725_0033_autonomous_file_dispatch.py b/migrations/versions/20260725_0033_autonomous_file_dispatch.py new file mode 100644 index 00000000..e9742cdd --- /dev/null +++ b/migrations/versions/20260725_0033_autonomous_file_dispatch.py @@ -0,0 +1,469 @@ +"""Claim current scheduled File imports through exact first-attempt leases. + +Revision ID: 20260725_0033 +Revises: 20260725_0032 +Create Date: 2026-07-25 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "20260725_0033" +down_revision: str | None = "20260725_0032" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_SCHEDULER = "context_engine_scheduler" +_DEFINER = "context_engine_file_dispatch_definer" +_FUNCTION = "context_scheduler_claim_file_import" +_SIGNATURE = "(bigint, bytea, text[])" +_TTL_SECONDS = 300 +_MIGRATION_FENCE = "context-engine.file-dispatch-migration-fence" +_SCHEDULING_MIGRATION_FENCE = "context-engine.file-change-scheduling-migration-fence" +_MAX_BIGINT = 9_223_372_036_854_775_807 +_V3 = "file-capabilities-v3" +_V4 = "file-capabilities-v4" + + +def upgrade() -> None: + """Install one cross-tenant selector available only through the scheduler call.""" + + op.add_column( + "file_import_job", + sa.Column( + "dispatch_claimed", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + op.create_index( + "ix_file_import_job_dispatch_available", + "file_import_job", + ["organization_id", "source_id", "job_id"], + postgresql_where=sa.text("state = 'available' AND lease_generation = 0"), + ) + op.create_index( + "ix_file_source_checkpoint_dispatch_order", + "file_source_acquisition_checkpoint", + [ + "accepted_at", + "sequence", + "organization_id", + "source_id", + "acquisition_id", + "job_id", + ], + postgresql_where=sa.text("change_kind = 'file_import'"), + ) + for table in ( + "context_source", + "source_version", + "membership", + "service_principal", + "file_acquisition", + "file_import_job", + "file_source_change_page", + "file_source_change", + "file_source_acquisition_checkpoint", + ): + op.execute( + f"CREATE POLICY {table}_file_dispatch_definer_select ON {table} " + f"FOR SELECT TO {_DEFINER} USING (true)" + ) + op.execute(f"GRANT SELECT ON TABLE {table} TO {_DEFINER}") + op.execute( + "CREATE POLICY file_import_job_file_dispatch_definer_update " + f"ON file_import_job FOR UPDATE TO {_DEFINER} USING (true) " + "WITH CHECK (true)" + ) + for table in ("context_source", "membership", "service_principal"): + op.execute( + f"CREATE POLICY {table}_file_dispatch_definer_update ON {table} " + f"FOR UPDATE TO {_DEFINER} USING (true) WITH CHECK (true)" + ) + op.execute( + f"GRANT UPDATE (lifecycle_state, active_version_id) " + f"ON context_source TO {_DEFINER}" + ) + op.execute( + f"GRANT UPDATE (status, valid_from, valid_until) ON membership TO {_DEFINER}" + ) + op.execute(f"GRANT UPDATE (enabled) ON service_principal TO {_DEFINER}") + op.execute( + "GRANT UPDATE (state, signing_key_version, lease_nonce_digest, " + "lease_issued_at, lease_expires_at, lease_generation, dispatch_claimed) " + f"ON file_import_job TO {_DEFINER}" + ) + op.execute( + f""" + CREATE FUNCTION public.{_FUNCTION}( + requested_signing_key_version bigint, + requested_nonce bytea, + configured_root_refs text[] + ) RETURNS TABLE ( + organization_id uuid, + job_id uuid, + source_id uuid, + service_principal_id uuid, + lease_generation bigint, + issued_at timestamptz, + expires_at timestamptz + ) + LANGUAGE plpgsql SECURITY DEFINER + SET search_path = pg_catalog, pg_temp + SET row_security = on + AS $function$ + DECLARE + authority_checked_at timestamptz; + minted_at timestamptz; + selected_organization_id uuid; + selected_job_id uuid; + selected_source_id uuid; + BEGIN + IF requested_signing_key_version IS NULL + OR requested_nonce IS NULL + OR configured_root_refs IS NULL + OR SESSION_USER <> '{_SCHEDULER}' + OR requested_signing_key_version NOT BETWEEN 1 AND {_MAX_BIGINT} + OR pg_catalog.octet_length(requested_nonce) <> 32 + OR pg_catalog.cardinality(configured_root_refs) < 1 + OR EXISTS ( + SELECT 1 FROM pg_catalog.unnest(configured_root_refs) AS root_ref + WHERE root_ref !~ '^[A-Za-z0-9][A-Za-z0-9._-]{{0,127}}$' + OR root_ref IN ('.', '..') + ) + OR pg_catalog.cardinality(configured_root_refs) <> ( + SELECT count(DISTINCT root_ref) + FROM pg_catalog.unnest(configured_root_refs) AS root_ref + ) + THEN RETURN; END IF; + PERFORM pg_catalog.pg_advisory_xact_lock_shared( + pg_catalog.hashtextextended('{_MIGRATION_FENCE}', 0) + ); + authority_checked_at := pg_catalog.clock_timestamp(); + SELECT job.organization_id, job.job_id, job.source_id + INTO selected_organization_id, selected_job_id, selected_source_id + FROM public.file_import_job AS job + JOIN public.file_acquisition AS acquisition + ON acquisition.organization_id = job.organization_id + AND acquisition.acquisition_id = job.acquisition_id + AND acquisition.source_id = job.source_id + JOIN public.file_source_change_page AS accepted_page + ON accepted_page.organization_id = acquisition.organization_id + AND accepted_page.source_id = acquisition.source_id + AND accepted_page.source_version_id = acquisition.source_version_id + AND accepted_page.page_ref = acquisition.change_page_ref + JOIN public.file_source_change AS accepted_change + ON accepted_change.organization_id = acquisition.organization_id + AND accepted_change.source_id = acquisition.source_id + AND accepted_change.source_version_id = acquisition.source_version_id + AND accepted_change.page_ref = acquisition.change_page_ref + AND accepted_change.change_ordinal = acquisition.change_ordinal + AND accepted_change.relative_path = acquisition.relative_path + AND accepted_change.content_sha256 = acquisition.expected_content_sha256 + AND accepted_change.content_length = acquisition.expected_content_length + JOIN public.file_source_acquisition_checkpoint AS scheduled + ON scheduled.organization_id = job.organization_id + AND scheduled.source_id = job.source_id + AND scheduled.acquisition_id = job.acquisition_id + AND scheduled.job_id = job.job_id + AND scheduled.change_kind = 'file_import' + JOIN public.context_source AS source + ON source.organization_id = acquisition.organization_id + AND source.source_id = acquisition.source_id + AND source.active_version_id = acquisition.source_version_id + JOIN public.source_version AS version + ON version.organization_id = source.organization_id + AND version.source_id = source.source_id + AND version.version_id = source.active_version_id + JOIN public.membership AS audience + ON audience.organization_id = acquisition.organization_id + AND audience.membership_id = acquisition.audience_membership_id + AND audience.membership_version = acquisition.audience_membership_version + JOIN public.service_principal AS receiver + ON receiver.organization_id = job.organization_id + AND receiver.service_principal_id = job.service_principal_id + AND receiver.workload = job.workload + AND receiver.worker_audience = job.worker_audience + AND receiver.operation = job.operation + WHERE job.state = 'available' + AND job.lease_generation = 0 + AND job.workload = 'supply.file-import' + AND job.worker_audience = 'context-engine-worker' + AND job.actor_kind = 'service' + AND job.operation = 'file.import' + AND acquisition.change_page_ref IS NOT NULL + AND accepted_change.change_kind = 'upsert' + AND source.source_kind = 'file' + AND source.lifecycle_state = 'active' + AND version.capability_manifest->>'declarationVersion' + IN ('{_V3}', '{_V4}') + AND audience.status = 'active' + AND audience.valid_from <= authority_checked_at + AND (audience.valid_until IS NULL + OR audience.valid_until > authority_checked_at) + AND receiver.enabled IS TRUE + AND accepted_page.scan_epoch = ( + SELECT current_page.scan_epoch + FROM public.file_source_acquisition_checkpoint AS current_checkpoint + JOIN public.file_source_change_page AS current_page + ON current_page.organization_id = current_checkpoint.organization_id + AND current_page.source_id = current_checkpoint.source_id + AND current_page.source_version_id = + current_checkpoint.source_version_id + AND current_page.page_ref = current_checkpoint.change_page_ref + WHERE current_checkpoint.organization_id = acquisition.organization_id + AND current_checkpoint.source_id = acquisition.source_id + AND current_checkpoint.change_kind = 'file_change_page' + ORDER BY current_checkpoint.sequence DESC + LIMIT 1 + ) + ORDER BY scheduled.accepted_at, scheduled.sequence, + accepted_page.page_ordinal, accepted_change.change_ordinal, + job.organization_id, job.source_id, job.job_id + FOR UPDATE OF job SKIP LOCKED + LIMIT 1; + IF NOT FOUND THEN RETURN; END IF; + -- The configured capability set is an all-or-nothing assertion, + -- never a routing filter. Selection above always chooses the + -- globally oldest eligible row first; an omitted root returns + -- content-free no-work and cannot redirect the claim elsewhere. + IF NOT EXISTS ( + SELECT 1 + FROM public.source_version AS selected_version + WHERE selected_version.organization_id = + selected_organization_id + AND selected_version.source_id = selected_source_id + AND selected_version.root_ref = ANY(configured_root_refs) + ) THEN RETURN; END IF; + + -- Page acceptance owns this same exclusive Organization/Source + -- progress lock. Take it before mutable authority row locks, then + -- use a fresh READ COMMITTED statement snapshot below so a scan + -- accepted while this selector waited makes the old job ineligible. + PERFORM pg_catalog.pg_advisory_xact_lock( + pg_catalog.hashtextextended( + 'context-engine.file-source-progress:' + || selected_organization_id::text || ':' + || selected_source_id::text, 0 + ) + ); + authority_checked_at := pg_catalog.clock_timestamp(); + minted_at := pg_catalog.date_trunc( + 'second', pg_catalog.clock_timestamp() + ); + + RETURN QUERY + WITH candidate AS ( + SELECT job.organization_id, job.job_id + FROM public.file_import_job AS job + JOIN public.file_acquisition AS acquisition + ON acquisition.organization_id = job.organization_id + AND acquisition.acquisition_id = job.acquisition_id + AND acquisition.source_id = job.source_id + JOIN public.file_source_change_page AS accepted_page + ON accepted_page.organization_id = acquisition.organization_id + AND accepted_page.source_id = acquisition.source_id + AND accepted_page.source_version_id = + acquisition.source_version_id + AND accepted_page.page_ref = acquisition.change_page_ref + JOIN public.file_source_change AS accepted_change + ON accepted_change.organization_id = acquisition.organization_id + AND accepted_change.source_id = acquisition.source_id + AND accepted_change.source_version_id = + acquisition.source_version_id + AND accepted_change.page_ref = acquisition.change_page_ref + AND accepted_change.change_ordinal = acquisition.change_ordinal + AND accepted_change.relative_path = acquisition.relative_path + AND accepted_change.content_sha256 = + acquisition.expected_content_sha256 + AND accepted_change.content_length = + acquisition.expected_content_length + JOIN public.file_source_acquisition_checkpoint AS scheduled + ON scheduled.organization_id = job.organization_id + AND scheduled.source_id = job.source_id + AND scheduled.acquisition_id = job.acquisition_id + AND scheduled.job_id = job.job_id + AND scheduled.change_kind = 'file_import' + JOIN public.context_source AS source + ON source.organization_id = acquisition.organization_id + AND source.source_id = acquisition.source_id + AND source.active_version_id = acquisition.source_version_id + JOIN public.source_version AS version + ON version.organization_id = source.organization_id + AND version.source_id = source.source_id + AND version.version_id = source.active_version_id + JOIN public.membership AS audience + ON audience.organization_id = acquisition.organization_id + AND audience.membership_id = + acquisition.audience_membership_id + AND audience.membership_version = + acquisition.audience_membership_version + JOIN public.service_principal AS receiver + ON receiver.organization_id = job.organization_id + AND receiver.service_principal_id = job.service_principal_id + AND receiver.workload = job.workload + AND receiver.worker_audience = job.worker_audience + AND receiver.operation = job.operation + WHERE job.state = 'available' + AND job.lease_generation = 0 + AND job.organization_id = selected_organization_id + AND job.job_id = selected_job_id + AND job.source_id = selected_source_id + AND job.workload = 'supply.file-import' + AND job.worker_audience = 'context-engine-worker' + AND job.actor_kind = 'service' + AND job.operation = 'file.import' + AND acquisition.change_page_ref IS NOT NULL + AND accepted_change.change_kind = 'upsert' + AND source.source_kind = 'file' + AND source.lifecycle_state = 'active' + AND version.capability_manifest->>'declarationVersion' + IN ('{_V3}', '{_V4}') + AND version.root_ref = ANY(configured_root_refs) + AND audience.status = 'active' + AND audience.valid_from <= authority_checked_at + AND (audience.valid_until IS NULL + OR audience.valid_until > authority_checked_at) + AND receiver.enabled IS TRUE + AND accepted_page.scan_epoch = ( + SELECT current_page.scan_epoch + FROM public.file_source_acquisition_checkpoint + AS current_checkpoint + JOIN public.file_source_change_page AS current_page + ON current_page.organization_id = + current_checkpoint.organization_id + AND current_page.source_id = current_checkpoint.source_id + AND current_page.source_version_id = + current_checkpoint.source_version_id + AND current_page.page_ref = + current_checkpoint.change_page_ref + WHERE current_checkpoint.organization_id = + acquisition.organization_id + AND current_checkpoint.source_id = acquisition.source_id + AND current_checkpoint.change_kind = 'file_change_page' + ORDER BY current_checkpoint.sequence DESC + LIMIT 1 + ) + FOR UPDATE OF source, audience, receiver SKIP LOCKED + ), claimed AS ( + UPDATE public.file_import_job AS job + SET state = 'leased', + signing_key_version = requested_signing_key_version, + lease_nonce_digest = public.digest(requested_nonce, 'sha256'), + lease_issued_at = minted_at, + lease_expires_at = minted_at + + pg_catalog.make_interval(secs => {_TTL_SECONDS}), + lease_generation = 1, + dispatch_claimed = true + FROM candidate + WHERE job.organization_id = candidate.organization_id + AND job.job_id = candidate.job_id + AND job.state = 'available' + AND job.lease_generation = 0 + RETURNING job.organization_id, job.job_id, job.source_id, + job.service_principal_id, job.lease_generation, + job.lease_issued_at, job.lease_expires_at + ) + SELECT claimed.organization_id, claimed.job_id, claimed.source_id, + claimed.service_principal_id, claimed.lease_generation, + claimed.lease_issued_at, claimed.lease_expires_at + FROM claimed; + END; + $function$ + """ + ) + op.execute(f"REVOKE ALL ON FUNCTION public.{_FUNCTION}{_SIGNATURE} FROM PUBLIC") + op.execute(f"GRANT CREATE ON SCHEMA public TO {_DEFINER}") + op.execute(f"ALTER FUNCTION public.{_FUNCTION}{_SIGNATURE} OWNER TO {_DEFINER}") + op.execute(f"REVOKE CREATE ON SCHEMA public FROM {_DEFINER}") + op.execute(f"SET LOCAL ROLE {_DEFINER}") + op.execute( + f"GRANT EXECUTE ON FUNCTION public.{_FUNCTION}{_SIGNATURE} TO {_SCHEDULER}" + ) + op.execute("RESET ROLE") + + +def downgrade() -> None: + """Remove dispatch only when it has never issued a generation-one lease.""" + + connection = op.get_bind() + # Acquire the upstream scheduling fence before DROP POLICY takes relation + # locks. This preserves the established migration lock order for an + # in-flight page scheduler and every older downgrade in the same Alembic + # transaction. + connection.execute( + sa.text( + "SELECT pg_catalog.pg_advisory_xact_lock(" + "pg_catalog.hashtextextended(:migration_fence, 0))" + ), + {"migration_fence": _SCHEDULING_MIGRATION_FENCE}, + ) + connection.execute( + sa.text( + "SELECT pg_catalog.pg_advisory_xact_lock(" + "pg_catalog.hashtextextended(:migration_fence, 0))" + ), + {"migration_fence": _MIGRATION_FENCE}, + ) + claimed = connection.execute( + sa.text( + "SELECT EXISTS (SELECT 1 FROM public.file_import_job " + "WHERE dispatch_claimed IS TRUE)" + ) + ).scalar_one() + if claimed: + raise RuntimeError( + "autonomous File dispatch downgrade requires no retained " + "generation-one lease; use a forward fix" + ) + # Preserve the established scheduling migration relation-lock order. An + # in-flight manual acquisition must be observed before authority-policy + # relations such as Membership are touched by this downgrade. + op.execute("LOCK TABLE file_acquisition IN ACCESS EXCLUSIVE MODE") + op.execute(f"DROP FUNCTION public.{_FUNCTION}{_SIGNATURE}") + op.execute( + "REVOKE UPDATE (state, signing_key_version, lease_nonce_digest, " + "lease_issued_at, lease_expires_at, lease_generation, dispatch_claimed) " + f"ON file_import_job FROM {_DEFINER}" + ) + op.execute( + f"REVOKE UPDATE (lifecycle_state, active_version_id) " + f"ON context_source FROM {_DEFINER}" + ) + op.execute( + f"REVOKE UPDATE (status, valid_from, valid_until) ON membership FROM {_DEFINER}" + ) + op.execute(f"REVOKE UPDATE (enabled) ON service_principal FROM {_DEFINER}") + op.execute( + "DROP POLICY file_import_job_file_dispatch_definer_update ON file_import_job" + ) + for table in ("context_source", "membership", "service_principal"): + op.execute(f"DROP POLICY {table}_file_dispatch_definer_update ON {table}") + # Request this relation lock first, before touching other policy tables, so + # the established scheduling downgrade remains first in the lock order. + for table in ( + "file_acquisition", + "context_source", + "source_version", + "membership", + "service_principal", + "file_import_job", + "file_source_change_page", + "file_source_change", + "file_source_acquisition_checkpoint", + ): + op.execute(f"DROP POLICY {table}_file_dispatch_definer_select ON {table}") + op.execute(f"REVOKE SELECT ON TABLE {table} FROM {_DEFINER}") + op.drop_index( + "ix_file_source_checkpoint_dispatch_order", + table_name="file_source_acquisition_checkpoint", + ) + op.drop_index( + "ix_file_import_job_dispatch_available", + table_name="file_import_job", + ) + op.drop_column("file_import_job", "dispatch_claimed") diff --git a/scripts/database_harness.sh b/scripts/database_harness.sh index fad3b320..bd701636 100755 --- a/scripts/database_harness.sh +++ b/scripts/database_harness.sh @@ -46,6 +46,7 @@ generate_environment() { local action_password local runtime_password local worker_password + local scheduler_password local learning_password local security_operator_password local postgres_port @@ -58,6 +59,7 @@ generate_environment() { action_password="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" runtime_password="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" worker_password="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" + scheduler_password="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" learning_password="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" security_operator_password="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" postgres_port="$(python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')" @@ -89,6 +91,8 @@ generate_environment() { printf 'CONTEXT_ENGINE_RUNTIME_PASSWORD=%s\n' "$runtime_password" printf 'CONTEXT_ENGINE_WORKER_ROLE=context_engine_worker\n' printf 'CONTEXT_ENGINE_WORKER_PASSWORD=%s\n' "$worker_password" + printf 'CONTEXT_ENGINE_SCHEDULER_ROLE=context_engine_scheduler\n' + printf 'CONTEXT_ENGINE_SCHEDULER_PASSWORD=%s\n' "$scheduler_password" printf 'CONTEXT_ENGINE_LEARNING_ROLE=context_engine_learning\n' printf 'CONTEXT_ENGINE_LEARNING_PASSWORD=%s\n' "$learning_password" printf 'CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE=context_engine_security_operator\n' @@ -108,6 +112,8 @@ generate_environment() { "$runtime_password" "$postgres_port" printf 'CONTEXT_ENGINE_WORKER_DATABASE_URL=postgresql+psycopg://context_engine_worker:%s@127.0.0.1:%s/context_engine\n' \ "$worker_password" "$postgres_port" + printf 'CONTEXT_ENGINE_SCHEDULER_DATABASE_URL=postgresql+psycopg://context_engine_scheduler:%s@127.0.0.1:%s/context_engine\n' \ + "$scheduler_password" "$postgres_port" printf 'CONTEXT_ENGINE_LEARNING_DATABASE_URL=postgresql+psycopg://context_engine_learning:%s@127.0.0.1:%s/context_engine\n' \ "$learning_password" "$postgres_port" printf 'CONTEXT_ENGINE_SECURITY_OPERATOR_DATABASE_URL=postgresql+psycopg://context_engine_security_operator:%s@127.0.0.1:%s/context_engine\n' \ @@ -154,12 +160,43 @@ migrate_legacy_environment() { (migrate_legacy_identity_identity) (migrate_legacy_egress_identity) (migrate_legacy_action_identity) + (migrate_legacy_scheduler_identity) (migrate_legacy_learning_identity) (migrate_legacy_security_operator_identity) rmdir "$ENV_MIGRATION_LOCK" trap - EXIT } +migrate_legacy_scheduler_identity() { + if grep -q '^CONTEXT_ENGINE_SCHEDULER_ROLE=' "$ENV_FILE"; then + return + fi + local postgres_port + postgres_port="$(sed -n 's/^CONTEXT_ENGINE_POSTGRES_PORT=//p' "$ENV_FILE")" + if [[ ! "$postgres_port" =~ ^[0-9]+$ ]]; then + printf 'legacy database environment has no valid PostgreSQL port\n' >&2 + exit 1 + fi + local scheduler_password + scheduler_password="$(python3 -c 'import secrets; print(secrets.token_hex(32))')" + local migration_file + migration_file="$(mktemp "$STATE_DIR/database.env.scheduler.XXXXXX")" + trap 'rm -f "$migration_file"' EXIT + ( + umask 077 + while IFS= read -r environment_line || [[ -n "$environment_line" ]]; do + printf '%s\n' "$environment_line" + done <"$ENV_FILE" + printf 'CONTEXT_ENGINE_SCHEDULER_ROLE=context_engine_scheduler\n' + printf 'CONTEXT_ENGINE_SCHEDULER_PASSWORD=%s\n' "$scheduler_password" + printf 'CONTEXT_ENGINE_SCHEDULER_DATABASE_URL=postgresql+psycopg://context_engine_scheduler:%s@127.0.0.1:%s/context_engine\n' \ + "$scheduler_password" "$postgres_port" + ) >"$migration_file" + chmod 600 "$migration_file" + mv "$migration_file" "$ENV_FILE" + trap - EXIT +} + read_embedded_project_identity() { local variable_name local variable_value @@ -444,7 +481,7 @@ load_environment() { local variable_name local variable_value local loaded_variable_names=' ' - local allowed_variables=' POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD CONTEXT_ENGINE_POSTGRES_PORT CONTEXT_ENGINE_COMPOSE_PROJECT CONTEXT_ENGINE_MIGRATOR_ROLE CONTEXT_ENGINE_MIGRATOR_PASSWORD CONTEXT_ENGINE_CONTROL_ROLE CONTEXT_ENGINE_CONTROL_PASSWORD CONTEXT_ENGINE_IDENTITY_ROLE CONTEXT_ENGINE_IDENTITY_PASSWORD CONTEXT_ENGINE_EGRESS_ROLE CONTEXT_ENGINE_EGRESS_PASSWORD CONTEXT_ENGINE_ACTION_ROLE CONTEXT_ENGINE_ACTION_PASSWORD CONTEXT_ENGINE_RUNTIME_ROLE CONTEXT_ENGINE_RUNTIME_PASSWORD CONTEXT_ENGINE_WORKER_ROLE CONTEXT_ENGINE_WORKER_PASSWORD CONTEXT_ENGINE_LEARNING_ROLE CONTEXT_ENGINE_LEARNING_PASSWORD CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE CONTEXT_ENGINE_SECURITY_OPERATOR_PASSWORD CONTEXT_ENGINE_MIGRATION_DATABASE_URL CONTEXT_ENGINE_CONTROL_DATABASE_URL CONTEXT_ENGINE_IDENTITY_DATABASE_URL CONTEXT_ENGINE_EGRESS_DATABASE_URL CONTEXT_ENGINE_ACTION_DATABASE_URL CONTEXT_ENGINE_RUNTIME_DATABASE_URL CONTEXT_ENGINE_WORKER_DATABASE_URL CONTEXT_ENGINE_LEARNING_DATABASE_URL CONTEXT_ENGINE_SECURITY_OPERATOR_DATABASE_URL CONTEXT_ENGINE_TEST_DATABASE_URL ' + local allowed_variables=' POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD CONTEXT_ENGINE_POSTGRES_PORT CONTEXT_ENGINE_COMPOSE_PROJECT CONTEXT_ENGINE_MIGRATOR_ROLE CONTEXT_ENGINE_MIGRATOR_PASSWORD CONTEXT_ENGINE_CONTROL_ROLE CONTEXT_ENGINE_CONTROL_PASSWORD CONTEXT_ENGINE_IDENTITY_ROLE CONTEXT_ENGINE_IDENTITY_PASSWORD CONTEXT_ENGINE_EGRESS_ROLE CONTEXT_ENGINE_EGRESS_PASSWORD CONTEXT_ENGINE_ACTION_ROLE CONTEXT_ENGINE_ACTION_PASSWORD CONTEXT_ENGINE_RUNTIME_ROLE CONTEXT_ENGINE_RUNTIME_PASSWORD CONTEXT_ENGINE_WORKER_ROLE CONTEXT_ENGINE_WORKER_PASSWORD CONTEXT_ENGINE_SCHEDULER_ROLE CONTEXT_ENGINE_SCHEDULER_PASSWORD CONTEXT_ENGINE_LEARNING_ROLE CONTEXT_ENGINE_LEARNING_PASSWORD CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE CONTEXT_ENGINE_SECURITY_OPERATOR_PASSWORD CONTEXT_ENGINE_MIGRATION_DATABASE_URL CONTEXT_ENGINE_CONTROL_DATABASE_URL CONTEXT_ENGINE_IDENTITY_DATABASE_URL CONTEXT_ENGINE_EGRESS_DATABASE_URL CONTEXT_ENGINE_ACTION_DATABASE_URL CONTEXT_ENGINE_RUNTIME_DATABASE_URL CONTEXT_ENGINE_WORKER_DATABASE_URL CONTEXT_ENGINE_SCHEDULER_DATABASE_URL CONTEXT_ENGINE_LEARNING_DATABASE_URL CONTEXT_ENGINE_SECURITY_OPERATOR_DATABASE_URL CONTEXT_ENGINE_TEST_DATABASE_URL ' while IFS='=' read -r variable_name variable_value; do if [[ -z "$variable_name" || "$allowed_variables" != *" $variable_name "* ]]; then @@ -479,6 +516,7 @@ load_environment() { "$CONTEXT_ENGINE_ACTION_ROLE" != 'context_engine_action' || \ "$CONTEXT_ENGINE_RUNTIME_ROLE" != 'context_engine_runtime' || \ "$CONTEXT_ENGINE_WORKER_ROLE" != 'context_engine_worker' || \ + "$CONTEXT_ENGINE_SCHEDULER_ROLE" != 'context_engine_scheduler' || \ "$CONTEXT_ENGINE_LEARNING_ROLE" != 'context_engine_learning' || \ "$CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE" != \ 'context_engine_security_operator' || \ @@ -492,6 +530,7 @@ load_environment() { ! "$CONTEXT_ENGINE_ACTION_PASSWORD" =~ ^[0-9a-f]{64}$ || \ ! "$CONTEXT_ENGINE_RUNTIME_PASSWORD" =~ ^[0-9a-f]{64}$ || \ ! "$CONTEXT_ENGINE_WORKER_PASSWORD" =~ ^[0-9a-f]{64}$ || \ + ! "$CONTEXT_ENGINE_SCHEDULER_PASSWORD" =~ ^[0-9a-f]{64}$ || \ ! "$CONTEXT_ENGINE_LEARNING_PASSWORD" =~ ^[0-9a-f]{64}$ || \ ! "$CONTEXT_ENGINE_SECURITY_OPERATOR_PASSWORD" =~ ^[0-9a-f]{64}$ ]]; then printf 'database environment failed its generated-value contract\n' >&2 @@ -514,6 +553,8 @@ load_environment() { "postgresql+psycopg://context_engine_runtime:$CONTEXT_ENGINE_RUNTIME_PASSWORD@$database_endpoint" || \ "$CONTEXT_ENGINE_WORKER_DATABASE_URL" != \ "postgresql+psycopg://context_engine_worker:$CONTEXT_ENGINE_WORKER_PASSWORD@$database_endpoint" || \ + "$CONTEXT_ENGINE_SCHEDULER_DATABASE_URL" != \ + "postgresql+psycopg://context_engine_scheduler:$CONTEXT_ENGINE_SCHEDULER_PASSWORD@$database_endpoint" || \ "$CONTEXT_ENGINE_LEARNING_DATABASE_URL" != \ "postgresql+psycopg://context_engine_learning:$CONTEXT_ENGINE_LEARNING_PASSWORD@$database_endpoint" || \ "$CONTEXT_ENGINE_SECURITY_OPERATOR_DATABASE_URL" != \ diff --git a/scripts/provision_database_roles.py b/scripts/provision_database_roles.py index e0500f68..16e57942 100644 --- a/scripts/provision_database_roles.py +++ b/scripts/provision_database_roles.py @@ -24,11 +24,13 @@ DELIVERY_EVIDENCE_DEFINER_ROLE, EGRESS_GRANT_DEFINER_ROLE, EGRESS_ROLE, + FILE_DISPATCH_DEFINER_ROLE, IDENTITY_ROLE, LEARNING_ROLE, MIGRATOR_ROLE, OPERATOR_ROLE, RELEASE_DEFINER_ROLE, + SCHEDULER_ROLE, WORKER_LEASE_DEFINER_ROLE, ) @@ -52,12 +54,15 @@ class RoleProvisioningContract: egress_password: str action_role: str action_password: str + scheduler_role: str + scheduler_password: str learning_role: str learning_password: str security_operator_role: str security_operator_password: str definer_role: str worker_lease_definer_role: str + file_dispatch_definer_role: str context_run_reader_definer_role: str release_definer_role: str delivery_evidence_definer_role: str @@ -75,10 +80,12 @@ def __post_init__(self) -> None: "identity_role", "egress_role", "action_role", + "scheduler_role", "learning_role", "security_operator_role", "definer_role", "worker_lease_definer_role", + "file_dispatch_definer_role", "context_run_reader_definer_role", "release_definer_role", "delivery_evidence_definer_role", @@ -96,10 +103,12 @@ def __post_init__(self) -> None: self.identity_role, self.egress_role, self.action_role, + self.scheduler_role, self.learning_role, self.security_operator_role, self.definer_role, self.worker_lease_definer_role, + self.file_dispatch_definer_role, self.context_run_reader_definer_role, self.release_definer_role, self.delivery_evidence_definer_role, @@ -108,7 +117,7 @@ def __post_init__(self) -> None: self.action_prepare_definer_role, self.action_execute_definer_role, } - if len(security_roles) != 16: + if len(security_roles) != 18: raise ValueError("provisioned database roles must be distinct") if type(self.postgres_port) is not int or not 1 <= self.postgres_port <= 65535: raise ValueError("postgres_port must be a valid TCP port") @@ -118,6 +127,7 @@ def __post_init__(self) -> None: "identity_password", "egress_password", "action_password", + "scheduler_password", "learning_password", "security_operator_password", ): @@ -143,6 +153,8 @@ def _contract_from_environment( "CONTEXT_ENGINE_EGRESS_PASSWORD", "CONTEXT_ENGINE_ACTION_ROLE", "CONTEXT_ENGINE_ACTION_PASSWORD", + "CONTEXT_ENGINE_SCHEDULER_ROLE", + "CONTEXT_ENGINE_SCHEDULER_PASSWORD", "CONTEXT_ENGINE_LEARNING_ROLE", "CONTEXT_ENGINE_LEARNING_PASSWORD", "CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE", @@ -168,6 +180,8 @@ def _contract_from_environment( raise ValueError("database role provisioning has an invalid egress role") if environment["CONTEXT_ENGINE_ACTION_ROLE"] != ACTION_ROLE: raise ValueError("database role provisioning has an invalid action role") + if environment["CONTEXT_ENGINE_SCHEDULER_ROLE"] != SCHEDULER_ROLE: + raise ValueError("database role provisioning has an invalid scheduler role") if environment["CONTEXT_ENGINE_LEARNING_ROLE"] != LEARNING_ROLE: raise ValueError("database role provisioning has an invalid learning role") if environment["CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE"] != OPERATOR_ROLE: @@ -192,6 +206,8 @@ def _contract_from_environment( egress_password=environment["CONTEXT_ENGINE_EGRESS_PASSWORD"], action_role=environment["CONTEXT_ENGINE_ACTION_ROLE"], action_password=environment["CONTEXT_ENGINE_ACTION_PASSWORD"], + scheduler_role=environment["CONTEXT_ENGINE_SCHEDULER_ROLE"], + scheduler_password=environment["CONTEXT_ENGINE_SCHEDULER_PASSWORD"], learning_role=environment["CONTEXT_ENGINE_LEARNING_ROLE"], learning_password=environment["CONTEXT_ENGINE_LEARNING_PASSWORD"], security_operator_role=environment["CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE"], @@ -200,6 +216,7 @@ def _contract_from_environment( ], definer_role=ACCESS_POLICY_DEFINER_ROLE, worker_lease_definer_role=WORKER_LEASE_DEFINER_ROLE, + file_dispatch_definer_role=FILE_DISPATCH_DEFINER_ROLE, context_run_reader_definer_role=CONTEXT_RUN_READER_DEFINER_ROLE, release_definer_role=RELEASE_DEFINER_ROLE, delivery_evidence_definer_role=DELIVERY_EVIDENCE_DEFINER_ROLE, @@ -316,10 +333,12 @@ def provision_security_roles( _create_role_if_missing(connection, contract.identity_role) _create_role_if_missing(connection, contract.egress_role) _create_role_if_missing(connection, contract.action_role) + _create_role_if_missing(connection, contract.scheduler_role) _create_role_if_missing(connection, contract.learning_role) _create_role_if_missing(connection, contract.security_operator_role) _create_role_if_missing(connection, contract.definer_role) _create_role_if_missing(connection, contract.worker_lease_definer_role) + _create_role_if_missing(connection, contract.file_dispatch_definer_role) _create_role_if_missing(connection, contract.context_run_reader_definer_role) _create_role_if_missing(connection, contract.release_definer_role) _create_role_if_missing(connection, contract.delivery_evidence_definer_role) @@ -352,6 +371,15 @@ def provision_security_roles( sql.Literal(contract.identity_password), ) ) + connection.execute( + sql.SQL( + "ALTER ROLE {} WITH LOGIN PASSWORD {} NOSUPERUSER NOCREATEDB " + "NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS" + ).format( + sql.Identifier(contract.scheduler_role), + sql.Literal(contract.scheduler_password), + ) + ) connection.execute( sql.SQL( "ALTER ROLE {} WITH LOGIN PASSWORD {} NOSUPERUSER NOCREATEDB " @@ -439,6 +467,12 @@ def provision_security_roles( "NOINHERIT NOREPLICATION NOBYPASSRLS" ).format(sql.Identifier(contract.worker_lease_definer_role)) ) + connection.execute( + sql.SQL( + "ALTER ROLE {} WITH NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE " + "NOINHERIT NOREPLICATION NOBYPASSRLS" + ).format(sql.Identifier(contract.file_dispatch_definer_role)) + ) connection.execute( sql.SQL( "ALTER ROLE {} WITH NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE " @@ -456,10 +490,12 @@ def provision_security_roles( _revoke_roles_granted_to(connection, contract.identity_role) _revoke_roles_granted_to(connection, contract.egress_role) _revoke_roles_granted_to(connection, contract.action_role) + _revoke_roles_granted_to(connection, contract.scheduler_role) _revoke_roles_granted_to(connection, contract.learning_role) _revoke_roles_granted_to(connection, contract.security_operator_role) _revoke_roles_granted_to(connection, contract.definer_role) _revoke_roles_granted_to(connection, contract.worker_lease_definer_role) + _revoke_roles_granted_to(connection, contract.file_dispatch_definer_role) _revoke_roles_granted_to(connection, contract.context_run_reader_definer_role) _revoke_roles_granted_to(connection, contract.release_definer_role) _revoke_roles_granted_to(connection, contract.delivery_evidence_definer_role) @@ -471,10 +507,12 @@ def provision_security_roles( _revoke_members_of(connection, contract.identity_role) _revoke_members_of(connection, contract.egress_role) _revoke_members_of(connection, contract.action_role) + _revoke_members_of(connection, contract.scheduler_role) _revoke_members_of(connection, contract.learning_role) _revoke_members_of(connection, contract.security_operator_role) _revoke_members_of(connection, contract.definer_role) _revoke_members_of(connection, contract.worker_lease_definer_role) + _revoke_members_of(connection, contract.file_dispatch_definer_role) _revoke_members_of(connection, contract.context_run_reader_definer_role) _revoke_members_of(connection, contract.release_definer_role) _revoke_members_of(connection, contract.delivery_evidence_definer_role) @@ -518,6 +556,12 @@ def provision_security_roles( sql.Identifier(contract.migrator_role), ) ) + connection.execute( + sql.SQL("GRANT {} TO {} WITH ADMIN FALSE, INHERIT FALSE, SET TRUE").format( + sql.Identifier(contract.file_dispatch_definer_role), + sql.Identifier(contract.migrator_role), + ) + ) connection.execute( sql.SQL("GRANT {} TO {} WITH ADMIN FALSE, INHERIT FALSE, SET TRUE").format( sql.Identifier(contract.delivery_evidence_definer_role), @@ -542,10 +586,12 @@ def provision_security_roles( contract.identity_role, contract.egress_role, contract.action_role, + contract.scheduler_role, contract.learning_role, contract.security_operator_role, contract.definer_role, contract.worker_lease_definer_role, + contract.file_dispatch_definer_role, contract.context_run_reader_definer_role, contract.release_definer_role, contract.delivery_evidence_definer_role, @@ -588,6 +634,12 @@ def provision_security_roles( sql.Identifier(contract.action_role), ) ) + connection.execute( + sql.SQL("GRANT CONNECT ON DATABASE {} TO {}").format( + sql.Identifier(contract.database_name), + sql.Identifier(contract.scheduler_role), + ) + ) connection.execute( sql.SQL("GRANT CONNECT ON DATABASE {} TO {}").format( sql.Identifier(contract.database_name), diff --git a/scripts/security_gate/runner.py b/scripts/security_gate/runner.py index 64a95236..98b22852 100644 --- a/scripts/security_gate/runner.py +++ b/scripts/security_gate/runner.py @@ -80,6 +80,8 @@ "CONTEXT_ENGINE_RUNTIME_PASSWORD", "CONTEXT_ENGINE_WORKER_ROLE", "CONTEXT_ENGINE_WORKER_PASSWORD", + "CONTEXT_ENGINE_SCHEDULER_ROLE", + "CONTEXT_ENGINE_SCHEDULER_PASSWORD", "CONTEXT_ENGINE_LEARNING_ROLE", "CONTEXT_ENGINE_LEARNING_PASSWORD", "CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE", @@ -91,6 +93,7 @@ "CONTEXT_ENGINE_ACTION_DATABASE_URL", "CONTEXT_ENGINE_RUNTIME_DATABASE_URL", "CONTEXT_ENGINE_WORKER_DATABASE_URL", + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL", "CONTEXT_ENGINE_LEARNING_DATABASE_URL", "CONTEXT_ENGINE_SECURITY_OPERATOR_DATABASE_URL", "CONTEXT_ENGINE_TEST_DATABASE_URL", diff --git a/scripts/validate_security_catalog.py b/scripts/validate_security_catalog.py index 872e0ba6..d997e62c 100644 --- a/scripts/validate_security_catalog.py +++ b/scripts/validate_security_catalog.py @@ -2011,6 +2011,76 @@ ], } +CANONICAL_FILE_DISPATCH_ACTIVATION: dict[str, object] = { + "issueRef": "#91", + "invariantRef": "WORKER-LEASE-007", + "carrier": "autonomous first-attempt dispatch of explicit scheduled File upserts", + "status": "active_fail_closed", + "policyEpochScope": "not-runtime-authority", + "controlBoundary": ( + "function-only scheduler login -> current page/acquisition/audience/receiver " + "eligibility -> deterministic SKIP LOCKED selector -> database-timed " + "generation-one lease -> existing WorkerLease and File worker" + ), + "testEvidence": [ + { + "id": "PG-FILE-DISPATCH-091", + "surface": ( + "tests/integration/test_file_dispatch.py::" + "test_scheduler_claims_only_current_page_scheduled_upsert" + ), + "oracle": ( + "The non-owner scheduler cannot read job tables and can claim only " + "one current page-scheduled upsert through the exact function; the " + "next claim is typed content-free no-work and revoked audience " + "authority remains unclaimed." + ), + }, + { + "id": "PG-FILE-DISPATCH-CONCURRENCY-091", + "surface": ( + "tests/integration/test_file_dispatch.py::" + "test_concurrent_dispatchers_never_claim_the_same_job" + ), + "oracle": ( + "Two concurrent scheduler-role authorities claim the two oldest " + "eligible jobs exactly once through FOR UPDATE SKIP LOCKED; no job " + "identity is returned twice." + ), + }, + { + "id": "PROC-FILE-DISPATCH-091", + "surface": ( + "tests/integration/test_file_dispatch.py::" + "test_independent_worker_process_dispatches_and_publishes_one_job" + ), + "oracle": ( + "The independent worker runs a configured dispatch cycle with no " + "caller Organization, Source, job, or token, completes the existing " + "publication path, and emits only the closed content-free " + "dispatched result." + ), + }, + ], + "deferredEvidence": [ + "expired-lease reclaim, automatic retry/backoff, dead-letter handling, " + "and operator remediation", + "provider polling, automatic page acceptance, and automatic delete ordering", + ], + "futureCarriers": [ + "retry and dead-letter owner", + "provider polling and full resync", + "explicit mixed-change ordering policy", + ], + "notActive": [ + "scheduler tenant, Source, job, audience, path, lease-time, or " + "generation choice", + "manual import or delete execution", + "automatic retry or reclaim", + "Runtime authorization from Supply scheduling or lease state", + ], +} + CANONICAL_ACTIVATIONS: list[dict[str, object]] = [ CANONICAL_REVOCATION_ACTIVATION, CANONICAL_UNAVAILABLE_CAPABILITY_ACTIVATION, @@ -2032,6 +2102,7 @@ CANONICAL_FILE_DELETE_OBSERVATION_ACTIVATION, CANONICAL_FILE_DELETE_EXECUTION_ACTIVATION, CANONICAL_FILE_MIXED_UPSERT_SCHEDULING_ACTIVATION, + CANONICAL_FILE_DISPATCH_ACTIVATION, ] CANONICAL_ACTIVATION_ISSUE_LIST = ", ".join( f"Issue {activation['issueRef']}" for activation in CANONICAL_ACTIVATIONS diff --git a/scripts/wait_for_database.py b/scripts/wait_for_database.py index 12f94598..3aea049f 100644 --- a/scripts/wait_for_database.py +++ b/scripts/wait_for_database.py @@ -12,6 +12,7 @@ from engine.persistence import ( DatabasePurpose, + assert_scheduler_role, assert_security_operator_role, create_database_engine, load_harness_database_configurations, @@ -41,6 +42,7 @@ def wait_for_database(timeout_seconds: float) -> None: configurations.action, configurations.runtime, configurations.worker, + configurations.scheduler, configurations.learning, configurations.operator, configurations.security_test, @@ -57,6 +59,8 @@ def wait_for_database(timeout_seconds: float) -> None: ) if configuration.purpose is DatabasePurpose.SECURITY_OPERATOR: assert_security_operator_role(connection) + if configuration.purpose is DatabasePurpose.SUPPLY_SCHEDULER: + assert_scheduler_role(connection) if configuration.purpose is DatabasePurpose.LEARNING: assert_learning_role(connection) if configuration.purpose is DatabasePurpose.TRUSTED_IDENTITY: @@ -85,7 +89,7 @@ def main(argv: Sequence[str] | None = None) -> int: purpose_names = ( "migration, control, identity, egress, action, runtime, worker, learning, " "security-operator, " - "security-test" + "scheduler, security-test" ) print("PostgreSQL harness ready: " + purpose_names) return 0 diff --git a/tests/catalog/test_m0_security_gate.py b/tests/catalog/test_m0_security_gate.py index fe2ebc04..a2b5ab46 100644 --- a/tests/catalog/test_m0_security_gate.py +++ b/tests/catalog/test_m0_security_gate.py @@ -59,6 +59,14 @@ def test_security_gate_database_contract_includes_the_action_role() -> None: } <= _ALLOWED_DATABASE_ENVIRONMENT_KEYS +def test_security_gate_database_contract_includes_the_scheduler_role() -> None: + assert { + "CONTEXT_ENGINE_SCHEDULER_ROLE", + "CONTEXT_ENGINE_SCHEDULER_PASSWORD", + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL", + } <= _ALLOWED_DATABASE_ENVIRONMENT_KEYS + + def complete_provenance(*, commit: str = "a" * 40) -> dict[str, object]: digest = "b" * 64 return { diff --git a/tests/catalog/test_validate_security_catalog.py b/tests/catalog/test_validate_security_catalog.py index d8ad4139..a0f7c120 100644 --- a/tests/catalog/test_validate_security_catalog.py +++ b/tests/catalog/test_validate_security_catalog.py @@ -34,6 +34,7 @@ CANONICAL_FILE_CHANGE_SCHEDULING_ACTIVATION, CANONICAL_FILE_DELETE_EXECUTION_ACTIVATION, CANONICAL_FILE_DELETE_OBSERVATION_ACTIVATION, + CANONICAL_FILE_DISPATCH_ACTIVATION, CANONICAL_FILE_MIXED_UPSERT_SCHEDULING_ACTIVATION, CANONICAL_INVARIANT_IDS, CANONICAL_MODEL_EGRESS_ACTIVATION, @@ -587,6 +588,7 @@ def make_catalog() -> dict[str, object]: copy.deepcopy(CANONICAL_FILE_DELETE_OBSERVATION_ACTIVATION), copy.deepcopy(CANONICAL_FILE_DELETE_EXECUTION_ACTIVATION), copy.deepcopy(CANONICAL_FILE_MIXED_UPSERT_SCHEDULING_ACTIVATION), + copy.deepcopy(CANONICAL_FILE_DISPATCH_ACTIVATION), ], "invariants": invariants, "fixtures": fixtures, @@ -705,6 +707,7 @@ def make_schema() -> dict[str, object]: CANONICAL_FILE_MIXED_UPSERT_SCHEDULING_ACTIVATION ) }, + {"const": copy.deepcopy(CANONICAL_FILE_DISPATCH_ACTIVATION)}, ], "items": False, }, @@ -1106,7 +1109,7 @@ def test_issue_71_activates_private_accept_012_without_rewriting_m0_history( assert isinstance(upgrade_trigger, str) self.assertIn("Issue #71 activates", upgrade_trigger) self.assertEqual( - object_list_at(catalog, "activations")[-6], + object_list_at(catalog, "activations")[-7], CANONICAL_PRIVATE_BOT_DELIVERY_ACTIVATION, ) @@ -1458,7 +1461,7 @@ def test_issue_70_model_egress_activation_stops_before_real_provider(self) -> No def test_issue_71_private_bot_activation_stops_before_live_providers(self) -> None: catalog = make_catalog() - activation = object_list_at(catalog, "activations")[-6] + activation = object_list_at(catalog, "activations")[-7] self.assertEqual(activation, CANONICAL_PRIVATE_BOT_DELIVERY_ACTIVATION) self.assertEqual(activation["invariantRef"], "ACTION-SEPARATION-014") @@ -1482,7 +1485,7 @@ def test_issue_71_private_bot_activation_stops_before_live_providers(self) -> No def test_issue_81_file_change_activation_stops_before_scheduling(self) -> None: catalog = make_catalog() - activation = object_list_at(catalog, "activations")[-5] + activation = object_list_at(catalog, "activations")[-6] self.assertEqual(activation, CANONICAL_FILE_CHANGE_FEED_ACTIVATION) self.assertEqual(activation["invariantRef"], "WORKER-LEASE-007") @@ -1503,7 +1506,7 @@ def test_issue_81_file_change_activation_stops_before_scheduling(self) -> None: def test_issue_83_file_change_scheduling_stays_explicit(self) -> None: catalog = make_catalog() - activation = object_list_at(catalog, "activations")[-4] + activation = object_list_at(catalog, "activations")[-5] self.assertEqual( activation, @@ -1525,7 +1528,7 @@ def test_issue_83_file_change_scheduling_stays_explicit(self) -> None: def test_issue_85_file_delete_observation_has_no_execution_authority(self) -> None: catalog = make_catalog() - activation = object_list_at(catalog, "activations")[-3] + activation = object_list_at(catalog, "activations")[-4] self.assertEqual( activation, @@ -1549,7 +1552,7 @@ def test_issue_85_file_delete_observation_has_no_execution_authority(self) -> No def test_issue_87_executes_only_current_exact_file_deletes(self) -> None: catalog = make_catalog() - activation = object_list_at(catalog, "activations")[-2] + activation = object_list_at(catalog, "activations")[-3] self.assertEqual( activation, @@ -1570,7 +1573,7 @@ def test_issue_87_executes_only_current_exact_file_deletes(self) -> None: def test_issue_89_schedules_only_the_mixed_page_upsert_projection(self) -> None: catalog = make_catalog() - activation = object_list_at(catalog, "activations")[-1] + activation = object_list_at(catalog, "activations")[-2] self.assertEqual( activation, @@ -2011,7 +2014,7 @@ def test_tracked_catalog_freezes_issue_19_authority_and_bounded_scope( self.assertEqual(catalog["catalogVersion"], "1.3.0") self.assertEqual( - issue_refs[-20:], + issue_refs[-21:], [ "#15", "#16", @@ -2033,6 +2036,7 @@ def test_tracked_catalog_freezes_issue_19_authority_and_bounded_scope( "#85", "#87", "#89", + "#91", ], ) self.assertIn( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 43cfefeb..b1c3b083 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -14,6 +14,7 @@ assert_action_role, assert_control_role, assert_runtime_role, + assert_scheduler_role, assert_security_operator_role, assert_worker_role, create_database_engine, @@ -106,6 +107,13 @@ def worker_configuration( return database_configurations.worker +@pytest.fixture(scope="session") +def scheduler_configuration( + database_configurations: HarnessDatabaseConfigurations, +) -> DatabaseConfiguration: + return database_configurations.scheduler + + @pytest.fixture(scope="session") def learning_configuration( database_configurations: HarnessDatabaseConfigurations, @@ -157,6 +165,21 @@ def guarded_worker_engine( engine.dispose() +@pytest.fixture(scope="session") +def guarded_scheduler_engine( + scheduler_configuration: DatabaseConfiguration, +) -> Iterator[Engine]: + """Expose only the function-only non-owner File scheduler engine.""" + + engine = create_database_engine(scheduler_configuration) + try: + with engine.connect() as connection: + assert_scheduler_role(connection) + yield engine + finally: + engine.dispose() + + @pytest.fixture(scope="session") def guarded_learning_engine( learning_configuration: DatabaseConfiguration, diff --git a/tests/integration/test_file_dispatch.py b/tests/integration/test_file_dispatch.py new file mode 100644 index 00000000..56734ed0 --- /dev/null +++ b/tests/integration/test_file_dispatch.py @@ -0,0 +1,1618 @@ +from __future__ import annotations + +import json +import os +import queue +import subprocess +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC +from hashlib import sha256 +from pathlib import Path +from threading import Thread +from time import monotonic, sleep +from typing import TextIO, cast +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import Engine, create_engine, text +from sqlalchemy.exc import DBAPIError +from uvicorn import Config, Server + +from adapters.exact_phrase import PostgreSQLExactPhraseCandidateIndex +from adapters.file_source import FileChangeProvider, FileReadLimits, FileRootRegistry +from adapters.http.app import create_app +from applications.worker import _worker_database_time +from engine.control import ( + ChangeLimit, + ControlOperation, + FileChangeSource, + FileImportAudience, + FileImportReceiver, + FileRootRef, + InitialScan, + ProviderOk, + ScheduleFileChangePage, +) +from engine.persistence import ( + DatabaseConfiguration, + FileDispatchLease, + FileDispatchNoWork, + PostgreSQLFileDispatchAuthority, + PostgreSQLMembershipAuthority, + create_database_engine, +) +from engine.runtime.construction import Runtime, required_kernel_dependencies +from engine.runtime.package_digest import QueryDigestKeyring +from engine.supply import WorkerLeaseCodec, WorkerLeaseKeyring +from engine.supply.jobs import FILE_IMPORT_WORKER_LEASE_OPERATION +from tests.integration.test_file_change_pages import ( + _SCENARIOS, + _activate_delete_observations, + _authorize, + _delete_observation_effect_snapshot, + _delete_scenarios, + _proofs, + _seed_file_change_source, +) +from tests.integration.test_file_import_tracer import ( + _ExactScopeAuthority, + _OrganizationAuthority, + _RuntimeAuthenticator, +) +from tests.integration.test_z_egress_grant_file import ( + _pack_and_install_resolve_sdk, + _run_installed_empty_consumer, + _unused_port, + _wait_for_tcp, +) +from tests.support.releases import ( + clear_test_runtime_release, + ensure_test_runtime_release, +) + +pytestmark = pytest.mark.integration +SIGNING_KEY = b"issue-91-file-dispatch-key-00001" +ALL_TEST_ROOTS = ("dispatch-root",) + + +def _drain_text_stream(stream: TextIO, lines: queue.Queue[str]) -> None: + for line in stream: + lines.put(line) + + +def _dispatch_authority( + engine: Engine, + codec: WorkerLeaseCodec | None = None, +) -> PostgreSQLFileDispatchAuthority: + return PostgreSQLFileDispatchAuthority( + engine, + codec + or WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) + ), + configured_root_refs=ALL_TEST_ROOTS, + ) + + +@pytest.fixture(autouse=True) +def _enable_shared_scenario_tracking( + migration_configuration: DatabaseConfiguration, +) -> object: + scenarios: list[tuple[UUID, UUID]] = [] + _SCENARIOS.append(scenarios) + try: + yield + finally: + _SCENARIOS.remove(scenarios) + _delete_scenarios(migration_configuration, scenarios) + + +def _schedule_one( + *, + root: Path, + guarded_control_engine: Engine, + migration_configuration: DatabaseConfiguration, + root_ref: FileRootRef | None = None, +) -> tuple[UUID, UUID, FileRootRef]: + root_ref = root_ref or FileRootRef("dispatch-root") + provider_proofs, control_proofs = _proofs() + organization_id = uuid4() + receiver = FileImportReceiver(uuid4()) + control, authority, source = _seed_file_change_source( + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + organization_id=organization_id, + receiver=receiver, + root_ref=root_ref, + control_proofs=control_proofs, + ) + source = _activate_delete_observations(control, authority, organization_id, source) + provider = FileChangeProvider( + FileRootRegistry( + {source.source_version.root_ref: root}, + limits=FileReadLimits(max_file_bytes=4_096), + ), + proofs=provider_proofs, + ) + page = provider.read_changes( + FileChangeSource(organization_id, source.source_version), + InitialScan(), + ChangeLimit(10), + ) + assert type(page) is ProviderOk + with _authorize( + authority, + organization_id, + ControlOperation.ACCEPT_FILE_CHANGE_PAGE, + f"accept-dispatch-{organization_id}", + ) as call: + accepted = control.accept_file_change_page(call, page.value) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + membership_id = connection.execute( + text( + "SELECT membership_id FROM membership " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ).scalar_one() + finally: + migration_engine.dispose() + with _authorize( + authority, + organization_id, + ControlOperation.SCHEDULE_FILE_CHANGE_PAGE, + f"schedule-dispatch-{organization_id}", + ) as call: + scheduled = control.schedule_file_change_page( + call, + ScheduleFileChangePage( + accepted.source_ref, + accepted.source_version_ref, + accepted.page_ref, + FileImportAudience("principal:file-reader", membership_id, 1), + ), + ) + return organization_id, scheduled.changes[0].prepared_import.job_id, root_ref + + +@pytest.mark.security_evidence(id="PG-FILE-DISPATCH-091", layer="postgres") +def test_scheduler_claims_only_current_page_scheduled_upsert( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "root" + root.mkdir() + (root / "handbook.md").write_text("# Current\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + codec = WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) + ) + authority = _dispatch_authority(guarded_scheduler_engine, codec) + + claim = authority.claim() + assert type(claim) is FileDispatchLease + assert claim.organization_id == organization_id + assert claim.job_id == job_id + assert claim.lease_generation == 1 + claims = codec.verify( + claim.token, + expected_organization_id=claim.organization_id, + expected_job_id=claim.job_id, + expected_service_principal_id=claim.service_principal_id, + expected_workload="supply.file-import", + expected_operation=FILE_IMPORT_WORKER_LEASE_OPERATION, + expected_worker_audience="context-engine-worker", + expected_source_ref=str(claim.source_ref.value), + now=claim.issued_at, + ) + assert claims.issued_at == claim.issued_at + assert claims.expires_at == claim.expires_at + assert claims.lease_generation == claim.lease_generation + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, lease_generation, dispatch_claimed, " + "(SELECT count(*) FROM context_revision " + "WHERE organization_id = :organization_id) AS revisions " + "FROM file_import_job WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("leased", 1, True, 0) + assert type(authority.claim()) is FileDispatchNoWork + + with pytest.raises(DBAPIError), guarded_scheduler_engine.connect() as connection: + connection.execute(text("SELECT count(*) FROM file_import_job")) + + +def test_dispatch_normalizes_non_utc_database_sessions( + tmp_path: Path, + guarded_control_engine: Engine, + scheduler_configuration: DatabaseConfiguration, + worker_configuration: DatabaseConfiguration, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "root" + root.mkdir() + (root / "offset.md").write_text("# Offset\n", encoding="utf-8") + _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + session_options = {"options": "-c timezone=Asia/Shanghai"} + scheduler_engine = create_engine( + scheduler_configuration.url, + connect_args=session_options, + ) + worker_engine = create_engine( + worker_configuration.url, + connect_args=session_options, + ) + try: + claim = _dispatch_authority(scheduler_engine).claim() + assert type(claim) is FileDispatchLease + assert claim.issued_at.utcoffset() == UTC.utcoffset(claim.issued_at) + assert claim.expires_at.utcoffset() == UTC.utcoffset(claim.expires_at) + checked_at = _worker_database_time(worker_engine) + assert checked_at.utcoffset() == UTC.utcoffset(checked_at) + finally: + scheduler_engine.dispose() + worker_engine.dispose() + + +def test_revoked_audience_returns_content_free_no_work( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "root" + root.mkdir() + (root / "revoked.md").write_text("# Revoked\n", encoding="utf-8") + organization_id, _job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text( + "UPDATE membership SET status = 'revoked' " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ) + finally: + migration_engine.dispose() + authority = _dispatch_authority(guarded_scheduler_engine) + + assert authority.claim() == FileDispatchNoWork() + + +def test_missing_server_root_capability_leases_nothing( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "missing-registry-root" + root.mkdir() + (root / "missing.md").write_text("# Missing\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + authority = PostgreSQLFileDispatchAuthority( + guarded_scheduler_engine, + WorkerLeaseCodec(WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY})), + configured_root_refs=("another-root",), + ) + + assert authority.claim() == FileDispatchNoWork() + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, lease_generation, dispatch_claimed " + "FROM file_import_job WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("available", 0, False) + + +def test_scheduler_null_inputs_lease_nothing( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "null-registry-root" + root.mkdir() + (root / "null.md").write_text("# Null\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + + with guarded_scheduler_engine.begin() as connection: + statements = ( + "SELECT * FROM public.context_scheduler_claim_file_import(" + "NULL::bigint, :nonce, ARRAY['dispatch-root']::text[])", + "SELECT * FROM public.context_scheduler_claim_file_import(" + ":key_version, NULL::bytea, ARRAY['dispatch-root']::text[])", + "SELECT * FROM public.context_scheduler_claim_file_import(" + ":key_version, :nonce, NULL::text[])", + ) + for statement in statements: + assert ( + connection.execute( + text(statement), + {"key_version": 1, "nonce": b"n" * 32}, + ).all() + == [] + ) + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, lease_generation, dispatch_claimed " + "FROM file_import_job WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("available", 0, False) + + +def test_scheduler_root_subset_cannot_redirect_global_oldest_selection( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + scheduled: list[tuple[UUID, UUID, FileRootRef]] = [] + for name in ("oldest-root", "newer-root"): + root = tmp_path / name + root.mkdir() + (root / f"{name}.md").write_text(f"# {name}\n", encoding="utf-8") + scheduled.append( + _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + root_ref=FileRootRef(name), + ) + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text( + "ALTER TABLE file_source_acquisition_checkpoint DISABLE TRIGGER " + "file_source_acquisition_checkpoint_immutable" + ) + ) + for offset, (_organization_id, job_id, _root_ref) in enumerate(scheduled): + connection.execute( + text( + "UPDATE file_source_acquisition_checkpoint SET accepted_at = " + "TIMESTAMPTZ '2026-07-25 12:00:00+00' + " + ":offset * interval '1 second' WHERE job_id = :job_id" + ), + {"offset": offset, "job_id": job_id}, + ) + connection.execute( + text( + "ALTER TABLE file_source_acquisition_checkpoint ENABLE TRIGGER " + "file_source_acquisition_checkpoint_immutable" + ) + ) + finally: + migration_engine.dispose() + subset = PostgreSQLFileDispatchAuthority( + guarded_scheduler_engine, + WorkerLeaseCodec(WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY})), + configured_root_refs=("newer-root",), + ) + assert subset.claim() == FileDispatchNoWork() + + complete = PostgreSQLFileDispatchAuthority( + guarded_scheduler_engine, + WorkerLeaseCodec(WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY})), + configured_root_refs=("oldest-root", "newer-root"), + ) + claim = complete.claim() + assert type(claim) is FileDispatchLease + assert claim.job_id == scheduled[0][1] + + +@pytest.mark.parametrize( + ("mutation", "parameters"), + [ + ( + "UPDATE context_source SET lifecycle_state = 'disabled', " + "disabled_version_id = active_version_id, disabled_at = now() " + "WHERE organization_id = :organization_id", + {}, + ), + ( + "UPDATE service_principal SET enabled = false " + "WHERE organization_id = :organization_id", + {}, + ), + ( + "UPDATE membership SET valid_until = now() - interval '1 second' " + "WHERE organization_id = :organization_id", + {}, + ), + ], +) +def test_current_authority_filter_leaves_ineligible_job_untouched( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, + mutation: str, + parameters: dict[str, object], +) -> None: + root = tmp_path / "filtered-root" + root.mkdir() + (root / "filtered.md").write_text("# Filtered\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text(mutation), + {"organization_id": organization_id, **parameters}, + ) + finally: + migration_engine.dispose() + + authority = _dispatch_authority(guarded_scheduler_engine) + assert authority.claim() == FileDispatchNoWork() + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, lease_generation, dispatch_claimed " + "FROM file_import_job WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("available", 0, False) + + +def test_non_available_and_later_generation_jobs_are_not_claimed( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "leased-root" + root.mkdir() + (root / "leased.md").write_text("# Leased\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text( + "UPDATE file_import_job SET state = 'leased', " + "signing_key_version = 1, " + "lease_nonce_digest = digest('x', 'sha256'), " + "lease_issued_at = now(), " + "lease_expires_at = now() + interval '5 min', " + "lease_generation = 2 WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ) + finally: + migration_engine.dispose() + authority = _dispatch_authority(guarded_scheduler_engine) + + assert authority.claim() == FileDispatchNoWork() + + +def test_superseded_scan_page_job_is_not_claimed( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + first_root = tmp_path / "first-scan" + first_root.mkdir() + path = first_root / "scan.md" + path.write_text("# First\n", encoding="utf-8") + organization_id, first_job_id, root_ref = _schedule_one( + root=first_root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text( + "SELECT set_config('app.organization_id', " + "CAST(:organization_id AS text), true)" + ), + {"organization_id": organization_id}, + ) + source_id, source_version_id = connection.execute( + text( + "SELECT source_id, active_version_id FROM context_source " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ).one() + latest_page = connection.execute( + text( + "SELECT page_ref, scan_epoch, accepted_at " + "FROM file_source_change_page " + "WHERE organization_id = :organization_id " + "AND source_id = :source_id ORDER BY accepted_at DESC LIMIT 1" + ), + {"organization_id": organization_id, "source_id": source_id}, + ).one() + new_page_ref = sha256( + organization_id.bytes + b"issue-91-new-scan-page" + ).hexdigest() + connection.execute( + text( + "SET CONSTRAINTS " + "fk_file_source_delete_observation_page_exact DEFERRED" + ) + ) + connection.execute( + text( + "INSERT INTO file_source_delete_observation_page (" + "organization_id, source_id, source_version_id, page_ref) VALUES (" + ":organization_id, :source_id, :version_id, :page_ref)" + ), + { + "organization_id": organization_id, + "source_id": source_id, + "version_id": source_version_id, + "page_ref": new_page_ref, + }, + ) + connection.execute( + text( + "INSERT INTO file_source_change_page (organization_id, source_id, " + "source_version_id, page_ref, scan_ref, scan_epoch, page_limit, " + "page_ordinal, change_count, complete, accepted_at) VALUES (" + ":organization_id, :source_id, :version_id, :page_ref, :scan_ref, " + ":scan_epoch, 1, 1, 0, true, :accepted_at + interval '1 second')" + ), + { + "organization_id": organization_id, + "source_id": source_id, + "version_id": source_version_id, + "page_ref": new_page_ref, + "scan_ref": sha256( + organization_id.bytes + b"issue-91-new-scan" + ).hexdigest(), + "scan_epoch": uuid4(), + "accepted_at": latest_page.accepted_at, + }, + ) + max_sequence = connection.execute( + text( + "SELECT max(sequence) FROM file_source_acquisition_checkpoint " + "WHERE organization_id = :organization_id " + "AND source_id = :source_id" + ), + {"organization_id": organization_id, "source_id": source_id}, + ).scalar_one() + connection.execute( + text( + "INSERT INTO file_source_acquisition_checkpoint (" + "organization_id, source_id, sequence, checkpoint_ref, " + "change_kind, " + "accepted_at, source_version_id, change_page_ref) VALUES (" + ":organization_id, :source_id, :sequence, :checkpoint_ref, " + "'file_change_page', :accepted_at + interval '1 second', " + ":version_id, :page_ref)" + ), + { + "organization_id": organization_id, + "source_id": source_id, + "sequence": max_sequence + 1, + "checkpoint_ref": "facp_" + + sha256( + organization_id.bytes + b"issue-91-checkpoint" + ).hexdigest(), + "accepted_at": latest_page.accepted_at, + "version_id": source_version_id, + "page_ref": new_page_ref, + }, + ) + finally: + migration_engine.dispose() + assert root_ref.value + authority = _dispatch_authority(guarded_scheduler_engine) + + assert authority.claim() == FileDispatchNoWork() + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, dispatch_claimed FROM file_import_job " + "WHERE organization_id = :organization_id AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": first_job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("available", False) + + +@pytest.mark.security_evidence(id="PG-FILE-DISPATCH-CONCURRENCY-091", layer="postgres") +def test_concurrent_dispatchers_never_claim_the_same_job( + tmp_path: Path, + guarded_control_engine: Engine, + scheduler_configuration: DatabaseConfiguration, + migration_configuration: DatabaseConfiguration, +) -> None: + roots = [] + expected_jobs = set() + for name in ("first", "second"): + root = tmp_path / name + root.mkdir() + (root / f"{name}.md").write_text(f"# {name}\n", encoding="utf-8") + roots.append(root) + _organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + expected_jobs.add(job_id) + + def claim() -> FileDispatchLease | FileDispatchNoWork: + engine = create_database_engine(scheduler_configuration) + try: + return _dispatch_authority( + engine, + WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) + ), + ).claim() + finally: + engine.dispose() + + with ThreadPoolExecutor(max_workers=2) as executor: + results = tuple(executor.map(lambda _index: claim(), range(2))) + + claims = tuple(result for result in results if type(result) is FileDispatchLease) + assert {claim.job_id for claim in claims} == expected_jobs + assert len({claim.job_id for claim in claims}) == len(claims) == 2 + + +def test_dispatch_order_is_global_and_deterministic_across_organizations( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + scheduled_jobs: list[tuple[UUID, UUID]] = [] + for name in ( + "sequence-first", + "page-first", + "change-first", + "stable-first", + "stable-last", + ): + root = tmp_path / name + root.mkdir() + (root / f"{name}.md").write_text(f"# {name}\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + scheduled_jobs.append((organization_id, job_id)) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.begin() as connection: + connection.execute( + text( + "ALTER TABLE file_acquisition DROP CONSTRAINT " + "fk_file_acquisition_change_observation_exact" + ) + ) + for table in ( + "file_source_acquisition_checkpoint", + "file_source_change_page", + "file_source_change", + "file_acquisition", + ): + connection.execute( + text(f"ALTER TABLE {table} DISABLE TRIGGER {table}_immutable") + ) + positions = { + scheduled_jobs[0][1]: (10, 9, 9), + scheduled_jobs[1][1]: (20, 1, 9), + scheduled_jobs[2][1]: (20, 2, 1), + scheduled_jobs[3][1]: (20, 2, 9), + scheduled_jobs[4][1]: (20, 2, 9), + } + for job_id, (sequence, page_ordinal, change_ordinal) in positions.items(): + lineage = connection.execute( + text( + "SELECT acquisition.organization_id, acquisition.source_id, " + "acquisition.change_page_ref, acquisition.change_ordinal " + "FROM file_import_job AS job JOIN file_acquisition " + "AS acquisition " + "ON acquisition.organization_id = job.organization_id " + "AND acquisition.acquisition_id = job.acquisition_id " + "WHERE job.job_id = :job_id" + ), + {"job_id": job_id}, + ).one() + connection.execute( + text( + "UPDATE file_source_acquisition_checkpoint SET " + "accepted_at = TIMESTAMPTZ '2026-07-25 12:00:00+00', " + "sequence = :sequence WHERE job_id = :job_id" + ), + {"job_id": job_id, "sequence": sequence}, + ) + connection.execute( + text( + "UPDATE file_source_change_page SET page_ordinal = :ordinal " + "WHERE organization_id = :organization_id " + "AND source_id = :source_id AND page_ref = :page_ref" + ), + { + "organization_id": lineage.organization_id, + "source_id": lineage.source_id, + "page_ref": lineage.change_page_ref, + "ordinal": page_ordinal, + }, + ) + connection.execute( + text( + "UPDATE file_source_change SET change_ordinal = :new_ordinal " + "WHERE organization_id = :organization_id " + "AND source_id = :source_id AND page_ref = :page_ref " + "AND change_ordinal = :old_ordinal" + ), + { + "organization_id": lineage.organization_id, + "source_id": lineage.source_id, + "page_ref": lineage.change_page_ref, + "old_ordinal": lineage.change_ordinal, + "new_ordinal": change_ordinal, + }, + ) + connection.execute( + text( + "UPDATE file_acquisition SET change_ordinal = :new_ordinal " + "WHERE organization_id = :organization_id " + "AND source_id = :source_id AND change_page_ref = :page_ref" + ), + { + "organization_id": lineage.organization_id, + "source_id": lineage.source_id, + "page_ref": lineage.change_page_ref, + "new_ordinal": change_ordinal, + }, + ) + connection.execute( + text( + "ALTER TABLE file_acquisition ADD CONSTRAINT " + "fk_file_acquisition_change_observation_exact FOREIGN KEY (" + "organization_id, source_id, source_version_id, change_page_ref, " + "change_ordinal, relative_path, expected_content_sha256, " + "expected_content_length) REFERENCES file_source_change (" + "organization_id, source_id, source_version_id, page_ref, " + "change_ordinal, relative_path, content_sha256, content_length)" + ) + ) + for table in ( + "file_source_acquisition_checkpoint", + "file_source_change_page", + "file_source_change", + "file_acquisition", + ): + connection.execute( + text(f"ALTER TABLE {table} ENABLE TRIGGER {table}_immutable") + ) + finally: + migration_engine.dispose() + authority = _dispatch_authority(guarded_scheduler_engine) + + claims = tuple(authority.claim() for _index in range(5)) + + assert all(type(claim) is FileDispatchLease for claim in claims) + stable_tie = sorted(scheduled_jobs[3:]) + expected = [ + scheduled_jobs[0][1], + scheduled_jobs[1][1], + scheduled_jobs[2][1], + *(job_id for _organization_id, job_id in stable_tie), + ] + assert [ + claim.job_id for claim in claims if type(claim) is FileDispatchLease + ] == expected + + +def test_post_claim_mint_failure_leaves_one_expiring_lease_and_zero_effect( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "crash-root" + root.mkdir() + (root / "crash.md").write_text("# Crash\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + codec = WorkerLeaseCodec( + WorkerLeaseKeyring(active_version=1, keys={1: SIGNING_KEY}) + ) + monkeypatch.setattr( + WorkerLeaseCodec, + "mint", + lambda _self, _claims: (_ for _ in ()).throw(RuntimeError("signing failed")), + ) + authority = _dispatch_authority(guarded_scheduler_engine, codec) + + with pytest.raises(RuntimeError, match="signing failed"): + authority.claim() + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + snapshot = connection.execute( + text( + "SELECT job.state, job.lease_generation, job.dispatch_claimed, " + "job.lease_expires_at > job.lease_issued_at, " + "(SELECT count(*) FROM context_revision " + " WHERE organization_id = :organization_id), " + "(SELECT count(*) FROM exact_phrase_candidate " + " WHERE organization_id = :organization_id), " + "(SELECT count(*) FROM file_source_publish_watermark " + " WHERE organization_id = :organization_id) " + "FROM file_import_job AS job " + "WHERE job.organization_id = :organization_id " + "AND job.job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(snapshot) == ("leased", 1, True, True, 0, 0, 0) + + monkeypatch.undo() + assert ( + _dispatch_authority( + guarded_scheduler_engine, + codec, + ).claim() + == FileDispatchNoWork() + ) + + +def test_claim_skips_authority_row_while_revocation_is_in_flight( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "revocation-race-root" + root.mkdir() + (root / "race.md").write_text("# Race\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as revoker: + transaction = revoker.begin() + revoker.execute( + text( + "UPDATE membership SET status = 'revoked' " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ) + authority = _dispatch_authority(guarded_scheduler_engine) + assert authority.claim() == FileDispatchNoWork() + transaction.commit() + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, lease_generation, dispatch_claimed " + "FROM file_import_job WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("available", 0, False) + + +def test_claim_refreshes_latest_scan_after_waiting_for_source_progress( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "scan-race-root" + root.mkdir() + (root / "race.md").write_text("# Old scan\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as accepter: + transaction = accepter.begin() + source_id, version_id = accepter.execute( + text( + "SELECT source_id, active_version_id FROM context_source " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ).one() + accepter.execute( + text( + "SELECT pg_catalog.pg_advisory_xact_lock(" + "pg_catalog.hashtextextended(" + "'context-engine.file-source-progress:' || " + "CAST(:organization_id AS text) || ':' || " + "CAST(:source_id AS text), 0))" + ), + {"organization_id": organization_id, "source_id": source_id}, + ) + accepter.execute( + text( + "SELECT pg_catalog.set_config('app.organization_id', " + "CAST(:organization_id AS text), true)" + ), + {"organization_id": organization_id}, + ) + + with ThreadPoolExecutor(max_workers=1) as executor: + pending_claim = executor.submit( + _dispatch_authority(guarded_scheduler_engine).claim + ) + deadline = monotonic() + 10 + waiting = False + while monotonic() < deadline: + with migration_engine.connect() as observer: + waiting = bool( + observer.execute( + text( + "SELECT EXISTS (SELECT 1 FROM " + "pg_catalog.pg_stat_activity AS activity JOIN " + "pg_catalog.pg_locks AS held_lock ON " + "held_lock.pid = activity.pid WHERE " + "activity.usename = " + "'context_engine_scheduler' AND " + "held_lock.locktype = " + "'advisory' AND held_lock.granted IS FALSE)" + ) + ).scalar_one() + ) + if waiting: + break + sleep(0.01) + if not waiting: + pytest.fail("File dispatch did not wait for Source progress") + + latest = accepter.execute( + text( + "SELECT page.accepted_at, (SELECT max(sequence) FROM " + "file_source_acquisition_checkpoint WHERE " + "organization_id = :organization_id AND " + "source_id = :source_id) AS sequence FROM " + "file_source_change_page AS page JOIN " + "file_source_acquisition_checkpoint AS checkpoint ON " + "checkpoint.organization_id = page.organization_id AND " + "checkpoint.source_id = page.source_id AND " + "checkpoint.change_page_ref = page.page_ref WHERE " + "page.organization_id = :organization_id AND " + "page.source_id = :source_id ORDER BY " + "checkpoint.sequence DESC LIMIT 1" + ), + {"organization_id": organization_id, "source_id": source_id}, + ).one() + page_ref = sha256(organization_id.bytes + b"scan-race-page").hexdigest() + scan_ref = sha256(organization_id.bytes + b"scan-race-scan").hexdigest() + accepter.execute( + text( + "INSERT INTO file_source_delete_observation_page (" + "organization_id, source_id, source_version_id, page_ref) " + "VALUES (:organization_id, :source_id, :version_id, :page_ref)" + ), + { + "organization_id": organization_id, + "source_id": source_id, + "version_id": version_id, + "page_ref": page_ref, + }, + ) + accepter.execute( + text( + "INSERT INTO file_source_change_page (organization_id, " + "source_id, source_version_id, page_ref, scan_ref, scan_epoch, " + "page_limit, " + "page_ordinal, change_count, complete, accepted_at) VALUES (" + ":organization_id, :source_id, :version_id, :page_ref, " + ":scan_ref, :scan_epoch, 1, 1, 0, true, " + ":accepted_at + interval '1 second')" + ), + { + "organization_id": organization_id, + "source_id": source_id, + "version_id": version_id, + "page_ref": page_ref, + "scan_ref": scan_ref, + "scan_epoch": uuid4(), + "accepted_at": latest.accepted_at, + }, + ) + accepter.execute( + text( + "INSERT INTO file_source_acquisition_checkpoint (" + "organization_id, source_id, sequence, checkpoint_ref, " + "change_kind, accepted_at, source_version_id, change_page_ref) " + "VALUES (:organization_id, :source_id, :sequence, " + ":checkpoint_ref, 'file_change_page', " + ":accepted_at + interval '1 second', :version_id, :page_ref)" + ), + { + "organization_id": organization_id, + "source_id": source_id, + "sequence": latest.sequence + 1, + "checkpoint_ref": "facp_" + + sha256( + organization_id.bytes + b"scan-race-checkpoint" + ).hexdigest(), + "accepted_at": latest.accepted_at, + "version_id": version_id, + "page_ref": page_ref, + }, + ) + transaction.commit() + assert pending_claim.result(timeout=5) == FileDispatchNoWork() + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, lease_generation, dispatch_claimed " + "FROM file_import_job WHERE organization_id = " + ":organization_id AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("available", 0, False) + + +def test_claim_refreshes_membership_expiry_after_waiting_for_source_progress( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "expiry-race-root" + root.mkdir() + (root / "expiry.md").write_text("# Expiry\n", encoding="utf-8") + organization_id, job_id, _root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as blocker: + transaction = blocker.begin() + source_id = blocker.execute( + text( + "SELECT source_id FROM context_source " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ).scalar_one() + blocker.execute( + text( + "UPDATE membership SET valid_until = " + "pg_catalog.clock_timestamp() + interval '250 milliseconds' " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ) + blocker.execute( + text( + "SELECT pg_catalog.pg_advisory_xact_lock(" + "pg_catalog.hashtextextended(" + "'context-engine.file-source-progress:' || " + "CAST(:organization_id AS text) || ':' || " + "CAST(:source_id AS text), 0))" + ), + {"organization_id": organization_id, "source_id": source_id}, + ) + transaction.commit() + + with migration_engine.connect() as blocker: + transaction = blocker.begin() + blocker.execute( + text( + "SELECT pg_catalog.pg_advisory_xact_lock(" + "pg_catalog.hashtextextended(" + "'context-engine.file-source-progress:' || " + "CAST(:organization_id AS text) || ':' || " + "CAST(:source_id AS text), 0))" + ), + {"organization_id": organization_id, "source_id": source_id}, + ) + with ThreadPoolExecutor(max_workers=1) as executor: + pending_claim = executor.submit( + _dispatch_authority(guarded_scheduler_engine).claim + ) + deadline = monotonic() + 5 + waiting = False + while monotonic() < deadline: + with migration_engine.connect() as observer: + waiting = bool( + observer.execute( + text( + "SELECT EXISTS (SELECT 1 FROM " + "pg_catalog.pg_stat_activity AS activity JOIN " + "pg_catalog.pg_locks AS held_lock ON " + "held_lock.pid = activity.pid WHERE " + "activity.usename = " + "'context_engine_scheduler' AND " + "held_lock.locktype = " + "'advisory' AND held_lock.granted IS FALSE)" + ) + ).scalar_one() + ) + if waiting: + break + sleep(0.01) + if not waiting: + pytest.fail("File dispatch did not wait for Source progress") + sleep(0.4) + transaction.commit() + assert pending_claim.result(timeout=5) == FileDispatchNoWork() + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, lease_generation, dispatch_claimed " + "FROM file_import_job WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("available", 0, False) + + +def test_autonomous_dispatch_projects_mixed_upserts_but_never_delete_effects( + tmp_path: Path, + guarded_control_engine: Engine, + guarded_scheduler_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + root = tmp_path / "mixed-dispatch-root" + root.mkdir() + for path in ("a.md", "b.md", "c.md"): + (root / path).write_text(f"# {path}\n", encoding="utf-8") + provider_proofs, control_proofs = _proofs() + organization_id = uuid4() + receiver = FileImportReceiver(uuid4()) + control, authority, source = _seed_file_change_source( + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + organization_id=organization_id, + receiver=receiver, + root_ref=FileRootRef("dispatch-root"), + control_proofs=control_proofs, + ) + source = _activate_delete_observations(control, authority, organization_id, source) + provider = FileChangeProvider( + FileRootRegistry( + {source.source_version.root_ref: root}, + limits=FileReadLimits(max_file_bytes=4_096), + ), + proofs=provider_proofs, + ) + baseline = provider.read_changes(source, InitialScan(), ChangeLimit(3)) + assert type(baseline) is ProviderOk + with _authorize( + authority, + organization_id, + ControlOperation.ACCEPT_FILE_CHANGE_PAGE, + "accept-dispatch-mixed-baseline", + ) as call: + accepted = control.accept_file_change_page(call, baseline.value) + with _authorize( + authority, + organization_id, + ControlOperation.READ_SOURCE_PROGRESS, + "read-dispatch-mixed-baseline", + ) as call: + progress = control.read_file_source_progress(call, accepted.source_ref) + assert progress.complete_change_baseline is not None + (root / "a.md").write_text("# changed a\n", encoding="utf-8") + (root / "b.md").unlink() + (root / "c.md").write_text("# changed c\n", encoding="utf-8") + mixed = provider.read_changes( + FileChangeSource( + organization_id, + source.source_version, + scan_head=progress.change_scan_head, + complete_baseline=progress.complete_change_baseline, + ), + InitialScan(), + ChangeLimit(3), + ) + assert type(mixed) is ProviderOk + with _authorize( + authority, + organization_id, + ControlOperation.ACCEPT_FILE_CHANGE_PAGE, + "accept-dispatch-current-mixed", + ) as call: + accepted_mixed = control.accept_file_change_page(call, mixed.value) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + membership_id = connection.execute( + text( + "SELECT membership_id FROM membership " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ).scalar_one() + finally: + migration_engine.dispose() + with _authorize( + authority, + organization_id, + ControlOperation.SCHEDULE_FILE_CHANGE_PAGE, + "schedule-dispatch-current-mixed", + ) as call: + scheduled = control.schedule_file_change_page( + call, + ScheduleFileChangePage( + accepted_mixed.source_ref, + accepted_mixed.source_version_ref, + accepted_mixed.page_ref, + FileImportAudience("principal:file-reader", membership_id, 1), + ), + ) + assert [change.ordinal for change in scheduled.changes] == [1, 3] + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + before = _delete_observation_effect_snapshot(connection, organization_id) + dispatch = _dispatch_authority(guarded_scheduler_engine) + assert type(dispatch.claim()) is FileDispatchLease + assert type(dispatch.claim()) is FileDispatchLease + assert dispatch.claim() == FileDispatchNoWork() + with migration_engine.connect() as connection: + after = _delete_observation_effect_snapshot(connection, organization_id) + states = connection.execute( + text( + "SELECT job.state, job.lease_generation FROM file_import_job " + "AS job JOIN file_acquisition AS acquisition ON " + "acquisition.organization_id = job.organization_id AND " + "acquisition.acquisition_id = job.acquisition_id WHERE " + "job.organization_id = :organization_id AND " + "acquisition.change_page_ref = :page_ref ORDER BY job.job_id" + ), + { + "organization_id": organization_id, + "page_ref": accepted_mixed.page_ref, + }, + ).all() + finally: + migration_engine.dispose() + assert after == before + assert [tuple(state) for state in states] == [("leased", 1), ("leased", 1)] + + +@pytest.mark.security_evidence(id="PROC-FILE-DISPATCH-091", layer="runtime") +def test_independent_worker_process_dispatches_and_publishes_one_job( + tmp_path: Path, + guarded_control_engine: Engine, + migration_configuration: DatabaseConfiguration, + guarded_runtime_engine: Engine, + query_digest_keyring: QueryDigestKeyring, +) -> None: + root = tmp_path / "process-root" + root.mkdir() + (root / "process.md").write_text( + "# Process\n\nContextEngine delivers context.\n", encoding="utf-8" + ) + organization_id, job_id, root_ref = _schedule_one( + root=root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + + completed = subprocess.run( + ["context-engine-worker", "--dispatch-file-once"], + env={ + **os.environ, + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX": SIGNING_KEY.hex(), + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON": json.dumps( + {root_ref.value: str(root)} + ), + }, + check=True, + capture_output=True, + text=True, + ) + + assert json.loads(completed.stdout) == { + "dispatch": "file.import", + "outcome": "dispatched", + "service": "context-engine-worker", + "status": "complete", + } + forbidden = (str(root), str(organization_id), str(job_id), SIGNING_KEY.hex()) + assert all(value not in completed.stdout for value in forbidden) + no_work = subprocess.run( + ["context-engine-worker", "--dispatch-file-once"], + env={ + **os.environ, + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX": SIGNING_KEY.hex(), + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON": json.dumps( + {root_ref.value: str(root)} + ), + }, + check=True, + capture_output=True, + text=True, + ) + assert json.loads(no_work.stdout) == { + "dispatch": "file.import", + "outcome": "no_work", + "service": "context-engine-worker", + "status": "complete", + } + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + state = connection.execute( + text( + "SELECT state, effect_count, dispatch_claimed " + "FROM file_import_job WHERE organization_id = :organization_id " + "AND job_id = :job_id" + ), + {"organization_id": organization_id, "job_id": job_id}, + ).one() + candidate = connection.execute( + text( + "SELECT source_ref, resource_ref, " + "CAST(revision_id AS text) AS revision_ref " + "FROM exact_phrase_candidate " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ).one() + membership_id, user_id = connection.execute( + text( + "SELECT membership_id, user_id FROM membership " + "WHERE organization_id = :organization_id" + ), + {"organization_id": organization_id}, + ).one() + finally: + migration_engine.dispose() + assert tuple(state) == ("completed", 1, True) + + ensure_test_runtime_release( + organization_id, + active_revision_refs=(candidate.revision_ref,), + ) + consumer_root = tmp_path / "dispatch-sdk-consumer" + consumer_root.mkdir() + _pack_and_install_resolve_sdk(consumer_root) + + def resolve_for( + requested_organization_id: UUID, + requested_user_id: UUID, + requested_membership_id: UUID, + ) -> dict[str, object]: + application = create_app( + authenticator=_RuntimeAuthenticator( + requested_organization_id, + requested_user_id, + requested_membership_id, + ), + organization_authority=_OrganizationAuthority(), + membership_authority=PostgreSQLMembershipAuthority(guarded_runtime_engine), + scope_authority=_ExactScopeAuthority( + candidate.source_ref, + candidate.resource_ref, + ), + runtime=Runtime( + required_kernel_dependencies(), + candidate_index=PostgreSQLExactPhraseCandidateIndex(), + query_digest_keyring=query_digest_keyring, + ), + ) + port = _unused_port() + server = Server( + Config( + application, + host="127.0.0.1", + port=port, + log_level="warning", + lifespan="off", + ) + ) + thread = Thread(target=server.run, daemon=True) + thread.start() + try: + _wait_for_tcp(port) + return _run_installed_empty_consumer( + consumer_root, + base_url=f"http://127.0.0.1:{port}", + ) + finally: + server.should_exit = True + thread.join(timeout=10) + assert not thread.is_alive() + + try: + authorized = resolve_for(organization_id, user_id, membership_id) + authorized_package = cast(dict[str, object], authorized["package"]) + authorized_blocks = cast(list[dict[str, object]], authorized_package["blocks"]) + assert authorized_blocks[0]["text"] == ("ContextEngine delivers context.") + + other_root = tmp_path / "other-root" + other_root.mkdir() + (other_root / "other.md").write_text("# Other\n", encoding="utf-8") + other_organization_id, _other_job_id, _other_root_ref = _schedule_one( + root=other_root, + guarded_control_engine=guarded_control_engine, + migration_configuration=migration_configuration, + ) + other_engine = create_database_engine(migration_configuration) + try: + with other_engine.connect() as connection: + other_membership_id, other_user_id = connection.execute( + text( + "SELECT membership_id, user_id FROM membership " + "WHERE organization_id = :organization_id" + ), + {"organization_id": other_organization_id}, + ).one() + finally: + other_engine.dispose() + ensure_test_runtime_release(other_organization_id) + try: + denied = resolve_for( + other_organization_id, + other_user_id, + other_membership_id, + ) + denied_package = cast(dict[str, object], denied["package"]) + assert denied_package["blocks"] == [] + assert denied_package["evidence"] == [] + finally: + clear_test_runtime_release(other_organization_id) + finally: + clear_test_runtime_release(organization_id) + + +def test_long_running_dispatch_process_exits_cleanly_on_sigterm( + tmp_path: Path, +) -> None: + root = tmp_path / "empty-process-root" + root.mkdir() + process = subprocess.Popen( + ["context-engine-worker", "--dispatch-files"], + env={ + **os.environ, + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX": SIGNING_KEY.hex(), + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON": json.dumps( + {"empty-process-root": str(root)} + ), + }, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdout is not None + assert process.stderr is not None + stdout_lines: queue.Queue[str] = queue.Queue() + stderr_lines: queue.Queue[str] = queue.Queue() + stdout_reader = Thread( + target=_drain_text_stream, + args=(process.stdout, stdout_lines), + daemon=True, + ) + stderr_reader = Thread( + target=_drain_text_stream, + args=(process.stderr, stderr_lines), + daemon=True, + ) + stdout_reader.start() + stderr_reader.start() + try: + assert json.loads(stdout_lines.get(timeout=10)) == { + "dispatch": "file.import", + "service": "context-engine-worker", + "status": "ready", + } + assert json.loads(stdout_lines.get(timeout=10)) == { + "dispatch": "file.import", + "outcome": "no_work", + "service": "context-engine-worker", + "status": "complete", + } + process.terminate() + process.wait(timeout=3) + stdout_reader.join(timeout=1) + stderr_reader.join(timeout=1) + assert process.returncode == 0 + assert not stdout_reader.is_alive() + assert not stderr_reader.is_alive() + assert stderr_lines.empty() + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) diff --git a/tests/integration/test_membership_schema.py b/tests/integration/test_membership_schema.py index 0966cc09..f407497a 100644 --- a/tests/integration/test_membership_schema.py +++ b/tests/integration/test_membership_schema.py @@ -18,6 +18,7 @@ CITATION_DEFINER_ROLE, DELIVERY_EVIDENCE_DEFINER_ROLE, EGRESS_GRANT_DEFINER_ROLE, + FILE_DISPATCH_DEFINER_ROLE, RUNTIME_ROLE, WORKER_LEASE_DEFINER_ROLE, WORKER_ROLE, @@ -473,7 +474,8 @@ def test_runtime_worker_and_public_grants_are_least_privilege( :egress_grant_definer_role, :action_prepare_definer_role, :action_execute_definer_role, - :citation_definer_role + :citation_definer_role, + :file_dispatch_definer_role ) """ ), @@ -487,9 +489,21 @@ def test_runtime_worker_and_public_grants_are_least_privilege( "action_prepare_definer_role": ACTION_PREPARE_DEFINER_ROLE, "action_execute_definer_role": ACTION_EXECUTE_DEFINER_ROLE, "citation_definer_role": CITATION_DEFINER_ROLE, + "file_dispatch_definer_role": FILE_DISPATCH_DEFINER_ROLE, }, ) } + dispatch_update_columns = set( + connection.execute( + text( + "SELECT column_name FROM information_schema.column_privileges " + "WHERE table_schema = 'public' AND table_name = 'membership' " + "AND grantee = :file_dispatch_definer_role " + "AND privilege_type = 'UPDATE'" + ), + {"file_dispatch_definer_role": FILE_DISPATCH_DEFINER_ROLE}, + ).scalars() + ) security = tuple( connection.execute( text( @@ -539,7 +553,9 @@ def test_runtime_worker_and_public_grants_are_least_privilege( (ACTION_PREPARE_DEFINER_ROLE, "membership", "SELECT"), (ACTION_EXECUTE_DEFINER_ROLE, "membership", "SELECT"), (CITATION_DEFINER_ROLE, "membership", "SELECT"), + (FILE_DISPATCH_DEFINER_ROLE, "membership", "SELECT"), } + assert dispatch_update_columns == {"status", "valid_from", "valid_until"} assert security == (True, True) assert set(policies) == { "membership_current_user_actor", @@ -548,6 +564,8 @@ def test_runtime_worker_and_public_grants_are_least_privilege( "membership_action_prepare_definer_select", "membership_action_execute_definer_select", "membership_file_import_definer_select", + "membership_file_dispatch_definer_select", + "membership_file_dispatch_definer_update", "membership_citation_definer_select", "membership_migrator_administration", } @@ -586,6 +604,21 @@ def test_runtime_worker_and_public_grants_are_least_privilege( assert file_import_policy[4] is None assert "app.organization_id" in str(file_import_policy[3]).lower() + assert policies["membership_file_dispatch_definer_select"] == ( + "PERMISSIVE", + (FILE_DISPATCH_DEFINER_ROLE,), + "SELECT", + "true", + None, + ) + assert policies["membership_file_dispatch_definer_update"] == ( + "PERMISSIVE", + (FILE_DISPATCH_DEFINER_ROLE,), + "UPDATE", + "true", + "true", + ) + delivery_evidence_policy = policies[ "membership_delivery_evidence_definer_select" ] diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index d5f6ee71..3ecce9fa 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -1139,6 +1139,63 @@ def test_mixed_file_upsert_scheduling_revision_downgrades_and_reapplies_empty( engine.dispose() +def test_file_dispatch_revision_downgrades_and_reapplies_empty( + migration_configuration: DatabaseConfiguration, +) -> None: + """Issue #91 removes only its unclaimed scheduler capability cleanly.""" + + alembic_configuration = Config(ROOT / "alembic.ini") + try: + command.downgrade(alembic_configuration, "20260725_0032") + assert _revision_rows(migration_configuration) == ["20260725_0032"] + engine = create_database_engine(migration_configuration) + try: + with engine.connect() as connection: + assert ( + connection.execute( + text( + "SELECT to_regprocedure(" + "'public.context_scheduler_claim_file_import(" + "bigint,bytea,text[])'" + ") IS NULL" + ) + ).scalar_one() + is True + ) + privileges = connection.execute( + text( + "SELECT table_name, privilege_type, NULL::text AS column_name " + "FROM information_schema.table_privileges " + "WHERE table_schema = 'public' AND grantee = " + "'context_engine_file_dispatch_definer' " + "UNION ALL " + "SELECT table_name, privilege_type, column_name " + "FROM information_schema.column_privileges " + "WHERE table_schema = 'public' AND grantee = " + "'context_engine_file_dispatch_definer'" + ) + ).all() + assert privileges == [] + assert ( + connection.execute( + text( + "SELECT NOT EXISTS (" + "SELECT 1 FROM information_schema.columns " + "WHERE table_schema = 'public' " + "AND table_name = 'file_import_job' " + "AND column_name = 'dispatch_claimed')" + ) + ).scalar_one() + is True + ) + finally: + engine.dispose() + finally: + command.upgrade(alembic_configuration, "head") + + assert _revision_rows(migration_configuration) == [HEAD_REVISION] + + def test_mixed_file_upsert_downgrade_waits_for_in_flight_scheduler( tmp_path: Path, migration_configuration: DatabaseConfiguration, diff --git a/tests/integration/test_postgres_harness.py b/tests/integration/test_postgres_harness.py index 72db5aa3..d0f7e517 100644 --- a/tests/integration/test_postgres_harness.py +++ b/tests/integration/test_postgres_harness.py @@ -27,12 +27,14 @@ DELIVERY_EVIDENCE_DEFINER_ROLE, EGRESS_GRANT_DEFINER_ROLE, EGRESS_ROLE, + FILE_DISPATCH_DEFINER_ROLE, IDENTITY_ROLE, LEARNING_ROLE, MIGRATOR_ROLE, OPERATOR_ROLE, RELEASE_DEFINER_ROLE, RUNTIME_ROLE, + SCHEDULER_ROLE, WORKER_LEASE_DEFINER_ROLE, WORKER_ROLE, ) @@ -102,6 +104,7 @@ def test_all_login_roles_have_reviewed_capabilities( action_configuration: DatabaseConfiguration, runtime_configuration: DatabaseConfiguration, worker_configuration: DatabaseConfiguration, + scheduler_configuration: DatabaseConfiguration, learning_configuration: DatabaseConfiguration, operator_configuration: DatabaseConfiguration, ) -> None: @@ -113,6 +116,7 @@ def test_all_login_roles_have_reviewed_capabilities( action_configuration, runtime_configuration, worker_configuration, + scheduler_configuration, learning_configuration, operator_configuration, ) @@ -132,6 +136,7 @@ def test_all_login_roles_have_reviewed_capabilities( ACTION_ROLE, RUNTIME_ROLE, WORKER_ROLE, + SCHEDULER_ROLE, LEARNING_ROLE, OPERATOR_ROLE, } @@ -172,6 +177,8 @@ def test_post_init_role_provisioning_repairs_a_legacy_volume_idempotently( egress_password=os.environ["CONTEXT_ENGINE_EGRESS_PASSWORD"], action_role=ACTION_ROLE, action_password=os.environ["CONTEXT_ENGINE_ACTION_PASSWORD"], + scheduler_role=SCHEDULER_ROLE, + scheduler_password=os.environ["CONTEXT_ENGINE_SCHEDULER_PASSWORD"], learning_role=LEARNING_ROLE, learning_password=os.environ["CONTEXT_ENGINE_LEARNING_PASSWORD"], security_operator_role=OPERATOR_ROLE, @@ -180,6 +187,7 @@ def test_post_init_role_provisioning_repairs_a_legacy_volume_idempotently( ], definer_role=ACCESS_POLICY_DEFINER_ROLE, worker_lease_definer_role=WORKER_LEASE_DEFINER_ROLE, + file_dispatch_definer_role=FILE_DISPATCH_DEFINER_ROLE, context_run_reader_definer_role=CONTEXT_RUN_READER_DEFINER_ROLE, release_definer_role=RELEASE_DEFINER_ROLE, delivery_evidence_definer_role=DELIVERY_EVIDENCE_DEFINER_ROLE, @@ -215,6 +223,7 @@ def test_post_init_role_provisioning_repairs_a_legacy_volume_idempotently( for role_name in ( ACCESS_POLICY_DEFINER_ROLE, WORKER_LEASE_DEFINER_ROLE, + FILE_DISPATCH_DEFINER_ROLE, CONTEXT_RUN_READER_DEFINER_ROLE, DELIVERY_EVIDENCE_DEFINER_ROLE, CITATION_DEFINER_ROLE, @@ -238,13 +247,14 @@ def test_post_init_role_provisioning_repairs_a_legacy_volume_idempotently( FROM pg_roles WHERE rolname IN ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s + %s, %s ) """, ( CONTROL_ROLE, ACCESS_POLICY_DEFINER_ROLE, WORKER_LEASE_DEFINER_ROLE, + FILE_DISPATCH_DEFINER_ROLE, CONTEXT_RUN_READER_DEFINER_ROLE, DELIVERY_EVIDENCE_DEFINER_ROLE, RELEASE_DEFINER_ROLE, @@ -460,6 +470,55 @@ def test_post_init_role_provisioning_repairs_a_legacy_volume_idempotently( True, 1, ) + dispatch_facts = bootstrap_connection.execute( + """ + SELECT + dispatch.rolcanlogin, + dispatch.rolsuper, + dispatch.rolcreaterole, + dispatch.rolcreatedb, + dispatch.rolinherit, + dispatch.rolreplication, + dispatch.rolbypassrls, + has_database_privilege(dispatch.oid, current_database(), 'CONNECT'), + NOT EXISTS ( + SELECT 1 + FROM pg_auth_members AS granted_to_dispatch + WHERE granted_to_dispatch.member = dispatch.oid + ), + ( + SELECT count(*) + FROM pg_auth_members AS dispatch_members + WHERE dispatch_members.roleid = dispatch.oid + ), + dispatch_membership.admin_option, + dispatch_membership.inherit_option, + dispatch_membership.set_option + FROM pg_roles AS dispatch + JOIN pg_auth_members AS dispatch_membership + ON dispatch_membership.roleid = dispatch.oid + JOIN pg_roles AS migrator + ON migrator.oid = dispatch_membership.member + WHERE dispatch.rolname = %s + AND migrator.rolname = %s + """, + (FILE_DISPATCH_DEFINER_ROLE, MIGRATOR_ROLE), + ).fetchone() + assert dispatch_facts == ( + False, + False, + False, + False, + False, + False, + False, + False, + True, + 1, + False, + False, + True, + ) learning_facts = bootstrap_connection.execute( """ SELECT diff --git a/tests/support/migrations.py b/tests/support/migrations.py index ef3a95cf..76c2bf1b 100644 --- a/tests/support/migrations.py +++ b/tests/support/migrations.py @@ -1,3 +1,3 @@ """Shared migration assertions for tests that require the current schema head.""" -HEAD_REVISION = "20260725_0032" +HEAD_REVISION = "20260725_0033" diff --git a/tests/unit/test_database_configuration.py b/tests/unit/test_database_configuration.py index a88f1c1c..707fe3fe 100644 --- a/tests/unit/test_database_configuration.py +++ b/tests/unit/test_database_configuration.py @@ -16,6 +16,7 @@ MIGRATOR_ROLE, OPERATOR_ROLE, RUNTIME_ROLE, + SCHEDULER_ROLE, WORKER_ROLE, DatabaseConfigurationError, DatabasePurpose, @@ -55,6 +56,10 @@ def database_environment() -> dict[str, str]: "postgresql+psycopg://context_engine_worker:worker-secret@" "127.0.0.1:5432/context_engine" ), + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL": ( + "postgresql+psycopg://context_engine_scheduler:scheduler-secret@" + "127.0.0.1:5432/context_engine" + ), "CONTEXT_ENGINE_LEARNING_DATABASE_URL": ( "postgresql+psycopg://context_engine_learning:learning-secret@" "127.0.0.1:5432/context_engine" @@ -74,6 +79,7 @@ def database_environment() -> dict[str, str]: "CONTEXT_ENGINE_EGRESS_ROLE": EGRESS_ROLE, "CONTEXT_ENGINE_ACTION_ROLE": ACTION_ROLE, "CONTEXT_ENGINE_WORKER_ROLE": WORKER_ROLE, + "CONTEXT_ENGINE_SCHEDULER_ROLE": SCHEDULER_ROLE, "CONTEXT_ENGINE_LEARNING_ROLE": LEARNING_ROLE, "CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE": OPERATOR_ROLE, } @@ -89,6 +95,10 @@ def database_environment() -> dict[str, str]: (DatabasePurpose.TRUSTED_ACTION, "CONTEXT_ENGINE_ACTION_DATABASE_URL"), (DatabasePurpose.API_RUNTIME, "CONTEXT_ENGINE_RUNTIME_DATABASE_URL"), (DatabasePurpose.SUPPLY_WORKER, "CONTEXT_ENGINE_WORKER_DATABASE_URL"), + ( + DatabasePurpose.SUPPLY_SCHEDULER, + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL", + ), (DatabasePurpose.LEARNING, "CONTEXT_ENGINE_LEARNING_DATABASE_URL"), ( DatabasePurpose.SECURITY_OPERATOR, diff --git a/tests/unit/test_database_harness_contract.py b/tests/unit/test_database_harness_contract.py index 27020167..4dd6f560 100644 --- a/tests/unit/test_database_harness_contract.py +++ b/tests/unit/test_database_harness_contract.py @@ -52,6 +52,7 @@ def test_compose_project_identity_is_generated_per_checkout() -> None: "action_role", "runtime_role", "worker_role", + "scheduler_role", "learning_role", "security_operator_role", ], @@ -100,6 +101,8 @@ def test_compose_passes_dedicated_operator_credentials_to_bootstrap() -> None: assert "CONTEXT_ENGINE_CONTROL_ROLE" in compose assert "CONTEXT_ENGINE_CONTROL_PASSWORD" in compose + assert "CONTEXT_ENGINE_SCHEDULER_ROLE" in compose + assert "CONTEXT_ENGINE_SCHEDULER_PASSWORD" in compose assert "CONTEXT_ENGINE_IDENTITY_ROLE" in compose assert "CONTEXT_ENGINE_IDENTITY_PASSWORD" in compose assert "CONTEXT_ENGINE_EGRESS_ROLE" in compose @@ -116,6 +119,7 @@ def test_readiness_probe_includes_dedicated_operator_configuration() -> None: wait_script = repository_text("scripts/wait_for_database.py") assert "configurations.control" in wait_script + assert "configurations.scheduler" in wait_script assert "configurations.identity" in wait_script assert "configurations.egress" in wait_script assert "configurations.action" in wait_script diff --git a/tests/unit/test_file_dispatch.py b/tests/unit/test_file_dispatch.py new file mode 100644 index 00000000..cd1c5a80 --- /dev/null +++ b/tests/unit/test_file_dispatch.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +import json +from dataclasses import asdict +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path +from threading import Event +from typing import cast +from uuid import uuid4 + +import pytest +from sqlalchemy import Engine +from sqlalchemy.exc import SQLAlchemyError + +from applications.worker import ( + FileDispatchCycleResult, + _file_dispatch_roots, + _worker_database_time, + dispatch_file_imports_until_stopped, + dispatch_one_file_import, +) +from engine.control import FileImportPath, FileRootRef, SourceRef +from engine.persistence.file_imports import FileImportRefused, FileImportUnavailable +from engine.persistence.worker_jobs import ( + FileDispatchLease, + FileDispatchNoWork, + PostgreSQLFileDispatchAuthority, + _database_timestamp_utc, +) +from engine.supply import ( + WorkerLeaseCodec, + WorkerLeaseKeyring, + WorkerLeaseRejectionAuditReceipt, + WorkerLeaseToken, + WorkNotAvailable, +) + + +def test_dispatch_no_work_is_a_closed_content_free_outcome() -> None: + outcome = FileDispatchNoWork() + + assert asdict(outcome) == {"status": "no_work"} + assert repr(outcome) == "FileDispatchNoWork(status='no_work')" + + +def test_dispatch_lease_redacts_every_routing_and_capability_value() -> None: + claimed_at = datetime(2026, 7, 25, 10, tzinfo=UTC) + claim = FileDispatchLease( + token=WorkerLeaseToken("lease-token"), + organization_id=uuid4(), + job_id=uuid4(), + source_ref=SourceRef(uuid4()), + service_principal_id=uuid4(), + lease_generation=1, + issued_at=claimed_at, + expires_at=claimed_at + timedelta(minutes=5), + ) + + rendered = repr(claim) + assert rendered == "FileDispatchLease(lease_generation=1)" + assert "lease-token" not in rendered + assert "organization_id" not in rendered + assert "source_ref" not in rendered + assert claim.redemption.expected_job_id == claim.job_id + assert claim.redemption.expected_source_ref == claim.source_ref + + +@pytest.mark.parametrize("generation", [0, 2]) +def test_first_attempt_dispatch_rejects_any_other_generation(generation: int) -> None: + claimed_at = datetime(2026, 7, 25, 10, tzinfo=UTC) + + with pytest.raises(ValueError, match="generation one"): + FileDispatchLease( + token=WorkerLeaseToken("lease-token"), + organization_id=uuid4(), + job_id=uuid4(), + source_ref=SourceRef(uuid4()), + service_principal_id=uuid4(), + lease_generation=generation, + issued_at=claimed_at, + expires_at=claimed_at + timedelta(minutes=5), + ) + + +class _NoWorkAuthority: + def claim(self) -> FileDispatchNoWork: + return FileDispatchNoWork() + + +class _ForbiddenWorkerFactory: + def __call__(self, _receiver: object) -> object: + raise AssertionError("no-work must not construct a worker") + + +def test_dispatch_cycle_stops_on_content_free_no_work() -> None: + result = dispatch_one_file_import( + _NoWorkAuthority(), + _ForbiddenWorkerFactory(), # type: ignore[arg-type] + ) + + assert asdict(result) == {"outcome": "no_work", "status": "complete"} + + +class _OneClaimAuthority: + def __init__(self, claim: FileDispatchLease) -> None: + self._claim = claim + + def claim(self) -> FileDispatchLease: + return self._claim + + +class _RefusingWorker: + def run(self, _redemption: object) -> object: + raise WorkNotAvailable(WorkerLeaseRejectionAuditReceipt(lease_digest="a" * 64)) + + +class _RefusingWorkerFactory: + def __call__(self, _receiver: object) -> _RefusingWorker: + return _RefusingWorker() + + +def test_dispatch_cycle_reports_job_refusal_without_routing_content() -> None: + claimed_at = datetime(2026, 7, 25, 10, tzinfo=UTC) + claim = FileDispatchLease( + token=WorkerLeaseToken("lease-token"), + organization_id=uuid4(), + job_id=uuid4(), + source_ref=SourceRef(uuid4()), + service_principal_id=uuid4(), + lease_generation=1, + issued_at=claimed_at, + expires_at=claimed_at + timedelta(minutes=5), + ) + + result = dispatch_one_file_import( + _OneClaimAuthority(claim), + _RefusingWorkerFactory(), # type: ignore[arg-type] + ) + + assert asdict(result) == {"outcome": "refused", "status": "complete"} + + +class _UnavailableWorker: + def run(self, _redemption: object) -> object: + raise FileImportUnavailable("File publication is unavailable") + + +class _UnavailableWorkerFactory: + def __call__(self, _receiver: object) -> _UnavailableWorker: + return _UnavailableWorker() + + +class _TerminallyFailedWorker: + def run(self, _redemption: object) -> object: + raise FileImportRefused("File import is unavailable") + + +class _TerminallyFailedWorkerFactory: + def __call__(self, _receiver: object) -> _TerminallyFailedWorker: + return _TerminallyFailedWorker() + + +def test_dispatch_stops_after_worker_infrastructure_failure() -> None: + claimed_at = datetime(2026, 7, 25, 10, tzinfo=UTC) + claim = FileDispatchLease( + token=WorkerLeaseToken("lease-token"), + organization_id=uuid4(), + job_id=uuid4(), + source_ref=SourceRef(uuid4()), + service_principal_id=uuid4(), + lease_generation=1, + issued_at=claimed_at, + expires_at=claimed_at + timedelta(minutes=5), + ) + + with pytest.raises(FileImportUnavailable, match="publication is unavailable"): + dispatch_one_file_import( + _OneClaimAuthority(claim), + _UnavailableWorkerFactory(), # type: ignore[arg-type] + ) + + +def test_dispatch_continues_after_durably_recorded_job_failure() -> None: + claimed_at = datetime(2026, 7, 25, 10, tzinfo=UTC) + claim = FileDispatchLease( + token=WorkerLeaseToken("lease-token"), + organization_id=uuid4(), + job_id=uuid4(), + source_ref=SourceRef(uuid4()), + service_principal_id=uuid4(), + lease_generation=1, + issued_at=claimed_at, + expires_at=claimed_at + timedelta(minutes=5), + ) + + result = dispatch_one_file_import( + _OneClaimAuthority(claim), + _TerminallyFailedWorkerFactory(), # type: ignore[arg-type] + ) + + assert asdict(result) == {"outcome": "refused", "status": "complete"} + + +class _StoppingNoWorkAuthority: + def __init__(self, stop_event: Event) -> None: + self.stop_event = stop_event + self.claim_count = 0 + + def claim(self) -> FileDispatchNoWork: + self.claim_count += 1 + self.stop_event.set() + return FileDispatchNoWork() + + +def test_long_running_dispatch_loop_honors_shutdown_after_no_work() -> None: + stop_event = Event() + authority = _StoppingNoWorkAuthority(stop_event) + observed: list[FileDispatchCycleResult] = [] + + dispatch_file_imports_until_stopped( + authority, + _ForbiddenWorkerFactory(), # type: ignore[arg-type] + stop_event, + observed.append, + ) + + assert authority.claim_count == 1 + assert [asdict(result) for result in observed] == [ + {"outcome": "no_work", "status": "complete"} + ] + + +def test_dispatch_loads_every_server_owned_file_root( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + first, second = tmp_path / "first", tmp_path / "second" + first.mkdir() + second.mkdir() + (first / "first.md").write_bytes(b"first") + (second / "second.md").write_bytes(b"second") + monkeypatch.setenv( + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON", + json.dumps({"first": str(first), "second": str(second)}), + ) + + with _file_dispatch_roots() as roots: + assert roots.read(FileRootRef("first"), FileImportPath("first.md")) == b"first" + assert ( + roots.read(FileRootRef("second"), FileImportPath("second.md")) == b"second" + ) + + +@pytest.mark.parametrize("document", ["[]", "{}", '{"root": 1}', "not-json"]) +def test_dispatch_rejects_invalid_server_root_registry( + monkeypatch: pytest.MonkeyPatch, + document: str, +) -> None: + monkeypatch.setenv("CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON", document) + + with pytest.raises(ValueError, match="configuration is not available"): + _file_dispatch_roots() + + +class _FailingEngine: + def begin(self) -> None: + raise SQLAlchemyError("nonce=" + (b"n" * 32).hex()) + + +class _FailingClockEngine: + def connect(self) -> None: + raise SQLAlchemyError("database clock failed") + + +def test_dispatch_database_clock_failure_is_generic() -> None: + with pytest.raises(FileImportUnavailable, match="clock is unavailable"): + _worker_database_time(cast(Engine, _FailingClockEngine())) + + +class _OffsetClockResult: + def scalar_one(self) -> datetime: + return datetime( + 2026, + 7, + 25, + 18, + tzinfo=timezone(timedelta(hours=8)), + ) + + +class _OffsetClockConnection: + def __enter__(self) -> _OffsetClockConnection: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, _statement: object) -> _OffsetClockResult: + return _OffsetClockResult() + + +class _OffsetClockEngine: + def connect(self) -> _OffsetClockConnection: + return _OffsetClockConnection() + + +def test_dispatch_database_clock_normalizes_session_offset_to_utc( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "applications.worker.assert_worker_role", lambda _connection: None + ) + + assert _worker_database_time(cast(Engine, _OffsetClockEngine())) == datetime( + 2026, 7, 25, 10, tzinfo=UTC + ) + + +def test_dispatch_claim_normalizes_database_timestamp_offset_to_utc() -> None: + session_timestamp = datetime( + 2026, + 7, + 25, + 18, + tzinfo=timezone(timedelta(hours=8)), + ) + + assert _database_timestamp_utc("issued_at", session_timestamp) == datetime( + 2026, 7, 25, 10, tzinfo=UTC + ) + + +def test_dispatch_database_failure_does_not_retain_generated_nonce( + monkeypatch: pytest.MonkeyPatch, +) -> None: + nonce = b"n" * 32 + monkeypatch.setattr( + "engine.persistence.worker_jobs.generate_worker_lease_nonce", + lambda: nonce, + ) + authority = PostgreSQLFileDispatchAuthority( + cast(Engine, _FailingEngine()), + WorkerLeaseCodec(WorkerLeaseKeyring(active_version=1, keys={1: b"k" * 32})), + configured_root_refs=("configured-root",), + ) + + with pytest.raises(RuntimeError) as failed: + authority.claim() + + rendered = (str(failed.value), repr(failed.value)) + assert all(nonce.hex() not in value for value in rendered) diff --git a/tests/unit/test_learning_database_configuration.py b/tests/unit/test_learning_database_configuration.py index 3f3c87b7..98b12716 100644 --- a/tests/unit/test_learning_database_configuration.py +++ b/tests/unit/test_learning_database_configuration.py @@ -11,6 +11,7 @@ MIGRATOR_ROLE, OPERATOR_ROLE, RUNTIME_ROLE, + SCHEDULER_ROLE, WORKER_ROLE, DatabaseConfigurationError, DatabasePurpose, @@ -49,6 +50,10 @@ def _database_environment() -> dict[str, str]: "postgresql+psycopg://context_engine_worker:worker-secret@" "127.0.0.1:5432/context_engine" ), + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL": ( + "postgresql+psycopg://context_engine_scheduler:scheduler-secret@" + "127.0.0.1:5432/context_engine" + ), "CONTEXT_ENGINE_LEARNING_DATABASE_URL": ( "postgresql+psycopg://context_engine_learning:learning-secret@" "127.0.0.1:5432/context_engine" @@ -68,6 +73,7 @@ def _database_environment() -> dict[str, str]: "CONTEXT_ENGINE_ACTION_ROLE": ACTION_ROLE, "CONTEXT_ENGINE_RUNTIME_ROLE": RUNTIME_ROLE, "CONTEXT_ENGINE_WORKER_ROLE": WORKER_ROLE, + "CONTEXT_ENGINE_SCHEDULER_ROLE": SCHEDULER_ROLE, "CONTEXT_ENGINE_LEARNING_ROLE": LEARNING_ROLE, "CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE": OPERATOR_ROLE, } @@ -100,6 +106,7 @@ def test_harness_exposes_learning_as_a_distinct_login() -> None: configurations.action.expected_role, configurations.runtime.expected_role, configurations.worker.expected_role, + configurations.scheduler.expected_role, configurations.learning.expected_role, configurations.operator.expected_role, } == { @@ -110,6 +117,7 @@ def test_harness_exposes_learning_as_a_distinct_login() -> None: ACTION_ROLE, RUNTIME_ROLE, WORKER_ROLE, + SCHEDULER_ROLE, LEARNING_ROLE, OPERATOR_ROLE, } diff --git a/tests/unit/test_learning_database_role_provisioning.py b/tests/unit/test_learning_database_role_provisioning.py index d506b4aa..e6835f95 100644 --- a/tests/unit/test_learning_database_role_provisioning.py +++ b/tests/unit/test_learning_database_role_provisioning.py @@ -21,6 +21,8 @@ def _provisioning_environment() -> dict[str, str]: "CONTEXT_ENGINE_EGRESS_PASSWORD": "8" * 64, "CONTEXT_ENGINE_ACTION_ROLE": "context_engine_action", "CONTEXT_ENGINE_ACTION_PASSWORD": "9" * 64, + "CONTEXT_ENGINE_SCHEDULER_ROLE": "context_engine_scheduler", + "CONTEXT_ENGINE_SCHEDULER_PASSWORD": "7" * 64, "CONTEXT_ENGINE_LEARNING_ROLE": LEARNING_ROLE, "CONTEXT_ENGINE_LEARNING_PASSWORD": "c" * 64, "CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE": "context_engine_security_operator", diff --git a/tests/unit/test_schema_security_manifest.py b/tests/unit/test_schema_security_manifest.py index f6fef4ce..5162a593 100644 --- a/tests/unit/test_schema_security_manifest.py +++ b/tests/unit/test_schema_security_manifest.py @@ -32,7 +32,7 @@ def test_manifest_classifies_the_exact_current_release_schema() -> None: document = manifest() tables = table_entries(document) - assert document["manifestVersion"] == "30.0.0" + assert document["manifestVersion"] == "31.0.0" assert set(tables) == { "active_release_manifest", "action_delivery_attempt", @@ -911,8 +911,12 @@ def test_issue_21_file_source_manifest_is_closed_and_role_separated() -> None: "context_engine_security_operator": [], "context_engine_worker": [], "context_engine_worker_lease_definer": ["SELECT", "UPDATE"], - "context_engine_action_prepare_definer": ["SELECT"], - "context_engine_action_execute_definer": ["SELECT"], + "context_engine_action_prepare_definer": ["SELECT"], + "context_engine_action_execute_definer": ["SELECT"], + "context_engine_file_dispatch_definer": [ + "SELECT", + "UPDATE lifecycle_state, active_version_id", + ], } assert version["permittedOperations"] == { "context_engine_control": [ @@ -928,6 +932,7 @@ def test_issue_21_file_source_manifest_is_closed_and_role_separated() -> None: "context_engine_worker_lease_definer": ["SELECT", "INSERT"], "context_engine_action_prepare_definer": ["SELECT"], "context_engine_action_execute_definer": ["SELECT"], + "context_engine_file_dispatch_definer": ["SELECT"], } for entry in (source, version): assert entry["rowLevelSecurity"]["enabled"] is True @@ -1318,6 +1323,7 @@ def test_worker_lease_manifest_requires_exact_receiver_and_job() -> None: "context_engine_runtime": [], "context_engine_worker": [], "context_engine_worker_lease_definer": ["SELECT"], + "context_engine_file_dispatch_definer": ["SELECT", "UPDATE enabled"], } assert job["permittedOperations"] == { "context_engine_control": ["EXECUTE issue_noop_worker_lease"], @@ -1452,6 +1458,10 @@ def test_membership_manifest_requires_exact_user_actor_and_read_only_runtime() - "context_engine_action_prepare_definer": ["SELECT"], "context_engine_action_execute_definer": ["SELECT"], "context_engine_citation_definer": ["SELECT"], + "context_engine_file_dispatch_definer": [ + "SELECT", + "UPDATE status, valid_from, valid_until", + ], } rls = entry["rowLevelSecurity"]