diff --git a/applications/control.py b/applications/control.py index 452551bd..dcf63a58 100644 --- a/applications/control.py +++ b/applications/control.py @@ -17,7 +17,7 @@ from applications.file_scan import FileScanReport, scan_file_source from applications.operator_authentication import ( CONTROL_OPERATOR_SECRET_ENV, - LocalOperatorAuthorities, + LocalControlOperatorConfiguration, LocalOperatorConfiguration, ) from applications.release_promotion import promote_release, release_report_json @@ -26,6 +26,7 @@ ActivateFileDeleteObservations, ContextControl, ControlOperation, + ControlOperatorAuthority, FileRootRef, FileSourceProgress, RegisterFileSource, @@ -161,13 +162,13 @@ def main(argv: Sequence[str] | None = None) -> None: print(rendered, flush=True) -def local_operator_authorities() -> LocalOperatorAuthorities | None: - """Construct local operator authority only after complete explicit opt-in.""" +def local_control_operator_authority() -> ControlOperatorAuthority | None: + """Construct routine Control authority without loading release credentials.""" - configuration = LocalOperatorConfiguration.load(os.environ) + configuration = LocalControlOperatorConfiguration.load(os.environ) if configuration is None: return None - return configuration.authorities() + return configuration.authority() def _run_operator_subcommand( @@ -179,8 +180,8 @@ def _run_operator_subcommand( | MultiSourceScanReport | MultiSourceStatusReport ): - authorities = local_operator_authorities() - if authorities is None: + authority = local_control_operator_authority() + if authority is None: raise SourceNotAvailable organization_id = UUID(arguments.organization_id) opaque_credential = os.environ[CONTROL_OPERATOR_SECRET_ENV] @@ -196,7 +197,7 @@ def clock() -> datetime: if arguments.subcommand == "scan-all": manifests = _list_sources( organization_id=organization_id, - authorities=authorities, + authority=authority, opaque_credential=opaque_credential, engine=engine, clock=clock, @@ -206,7 +207,7 @@ def clock() -> datetime: scan_file_source( organization_id=organization_id, source_ref=manifest.source_ref, - authority=authorities.control, + authority=authority, opaque_credential=opaque_credential, engine=engine, clock=clock, @@ -218,7 +219,7 @@ def clock() -> datetime: return scan_file_source( organization_id=organization_id, source_ref=SourceRef(UUID(arguments.source_ref)), - authority=authorities.control, + authority=authority, opaque_credential=opaque_credential, engine=engine, clock=clock, @@ -227,14 +228,14 @@ def clock() -> datetime: operation = _operation(arguments.subcommand) control = ContextControl( store=PostgreSQLControlStore(engine, clock=clock), - authority=authorities.control, + authority=authority, clock=clock, ) if arguments.subcommand == "status" and arguments.source_ref is None: manifests = _list_sources_with_control( control=control, organization_id=organization_id, - authorities=authorities, + authority=authority, opaque_credential=opaque_credential, ) progress = tuple( @@ -242,13 +243,13 @@ def clock() -> datetime: control=control, organization_id=organization_id, source_ref=manifest.source_ref, - authorities=authorities, + authority=authority, opaque_credential=opaque_credential, ) for manifest in manifests ) return MultiSourceStatusReport(progress) - with authorities.control.authorize( + with authority.authorize( opaque_credential=opaque_credential, operation=operation, request_id=f"local-{arguments.subcommand}-{uuid4().hex}", @@ -319,20 +320,20 @@ def __post_init__(self) -> None: def _list_sources( *, organization_id: UUID, - authorities: LocalOperatorAuthorities, + authority: ControlOperatorAuthority, opaque_credential: str, engine: Engine, clock: Callable[[], datetime], ) -> tuple[SourceManifest, ...]: control = ContextControl( store=PostgreSQLControlStore(engine, clock=clock), - authority=authorities.control, + authority=authority, clock=clock, ) return _list_sources_with_control( control=control, organization_id=organization_id, - authorities=authorities, + authority=authority, opaque_credential=opaque_credential, ) @@ -341,10 +342,10 @@ def _list_sources_with_control( *, control: ContextControl, organization_id: UUID, - authorities: LocalOperatorAuthorities, + authority: ControlOperatorAuthority, opaque_credential: str, ) -> tuple[SourceManifest, ...]: - with authorities.control.authorize( + with authority.authorize( opaque_credential=opaque_credential, operation=ControlOperation.READ_SOURCE, request_id=f"local-list-sources-{uuid4().hex}", @@ -359,10 +360,10 @@ def _read_status( control: ContextControl, organization_id: UUID, source_ref: SourceRef, - authorities: LocalOperatorAuthorities, + authority: ControlOperatorAuthority, opaque_credential: str, ) -> FileSourceProgress: - with authorities.control.authorize( + with authority.authorize( opaque_credential=opaque_credential, operation=ControlOperation.READ_SOURCE_PROGRESS, request_id=f"local-status-{uuid4().hex}", diff --git a/applications/file_scan.py b/applications/file_scan.py index 8ce909e4..80c56380 100644 --- a/applications/file_scan.py +++ b/applications/file_scan.py @@ -4,6 +4,7 @@ import hashlib import hmac +import os from collections.abc import Callable from dataclasses import dataclass, replace from datetime import datetime @@ -18,8 +19,11 @@ from applications.operator_authentication import ( CONTROL_OPERATOR_SECRET_ENV, DOGFOOD_SECRET_ENV, + DOGFOOD_SECRET_FINGERPRINT_ENV, RELEASE_OPERATOR_SECRET_ENV, + RELEASE_OPERATOR_SECRET_FINGERPRINT_ENV, WORKER_SECRET_ENV, + local_secret_fingerprint, ) from engine.control import ( FILE_DELETE_OBSERVATION_CAPABILITY_MANIFEST, @@ -355,14 +359,27 @@ def _proof_keys() -> tuple[Ed25519PrivateKey, Ed25519PrivateKey]: checkpoint_material = _private_key_material(CHECKPOINT_SIGNING_KEY_ENV) operator_secret_values = ( required_environment(CONTROL_OPERATOR_SECRET_ENV), - required_environment(RELEASE_OPERATOR_SECRET_ENV), - required_environment(DOGFOOD_SECRET_ENV), + required_environment(WORKER_SECRET_ENV), ) encoded_proof_values = ( provider_material.hex(), checkpoint_material.hex(), ) + external_fingerprints = tuple( + _external_secret_fingerprint(secret_name, fingerprint_name) + for secret_name, fingerprint_name in ( + (RELEASE_OPERATOR_SECRET_ENV, RELEASE_OPERATOR_SECRET_FINGERPRINT_ENV), + (DOGFOOD_SECRET_ENV, DOGFOOD_SECRET_FINGERPRINT_ENV), + ) + ) if any( + hmac.compare_digest( + local_secret_fingerprint(proof_value), + external_fingerprint, + ) + for proof_value in encoded_proof_values + for external_fingerprint in external_fingerprints + ) or any( hmac.compare_digest(proof_value, operator_secret.lower()) for proof_value in encoded_proof_values for operator_secret in operator_secret_values @@ -386,6 +403,22 @@ def _proof_keys() -> tuple[Ed25519PrivateKey, Ed25519PrivateKey]: ) +def _external_secret_fingerprint(secret_name: str, fingerprint_name: str) -> str: + raw = os.environ.get(secret_name) + if raw is not None: + return local_secret_fingerprint(raw) + fingerprint = required_environment(fingerprint_name) + if len(fingerprint) != 64: + raise SourceNotAvailable + try: + decoded = bytes.fromhex(fingerprint) + except ValueError: + raise SourceNotAvailable from None + if len(decoded) != 32: + raise SourceNotAvailable + return fingerprint.lower() + + def _positive_bigint(value: str) -> int: if not value.isascii() or not value.isdecimal(): raise SourceNotAvailable diff --git a/applications/operator_authentication.py b/applications/operator_authentication.py index 52175c21..2a274f16 100644 --- a/applications/operator_authentication.py +++ b/applications/operator_authentication.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import hmac from collections.abc import Callable, Mapping from dataclasses import dataclass, field @@ -27,6 +28,10 @@ CONTROL_OPERATOR_OPERATIONS_ENV = "CONTEXT_ENGINE_CONTROL_OPERATOR_OPERATIONS" DOGFOOD_SECRET_ENV = "CONTEXT_ENGINE_DOGFOOD_SECRET" WORKER_SECRET_ENV = "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX" +RELEASE_OPERATOR_SECRET_FINGERPRINT_ENV = ( + "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET_SHA256" +) +DOGFOOD_SECRET_FINGERPRINT_ENV = "CONTEXT_ENGINE_DOGFOOD_SECRET_SHA256" OPERATOR_ENVIRONMENT_VARIABLES = frozenset( { CONTROL_OPERATOR_SECRET_ENV, @@ -54,6 +59,70 @@ def __init__(self) -> None: super().__init__("operator authentication rejected") +def local_secret_fingerprint(value: str) -> str: + """Fingerprint one local secret for collision checks without delegating it.""" + + if type(value) is not str or not value: + raise LocalOperatorConfigurationUnavailable + return hashlib.sha256(value.lower().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True) +class LocalControlOperatorConfiguration: + """The routine Control identity without any release publication credential.""" + + organization_id: UUID + control_secret: bytes = field(repr=False) + control_operations: frozenset[ControlOperation] = field(repr=False) + + @classmethod + def load( + cls, + environment: Mapping[str, str], + ) -> LocalControlOperatorConfiguration | None: + names = frozenset( + { + CONTROL_OPERATOR_SECRET_ENV, + OPERATOR_ORGANIZATION_ENV, + CONTROL_OPERATOR_OPERATIONS_ENV, + } + ) + configured = names.intersection(environment) + if not configured: + return None + if configured != names: + raise LocalOperatorConfigurationUnavailable + try: + raw_operations = environment[CONTROL_OPERATOR_OPERATIONS_ENV].split(",") + if any(not value or value != value.strip() for value in raw_operations): + raise ValueError + operations = frozenset(ControlOperation(value) for value in raw_operations) + if len(operations) != len(raw_operations): + raise ValueError + return cls( + organization_id=UUID(environment[OPERATOR_ORGANIZATION_ENV]), + control_secret=_secret(environment[CONTROL_OPERATOR_SECRET_ENV]), + control_operations=operations, + ) + except (KeyError, TypeError, ValueError, UnicodeError): + raise LocalOperatorConfigurationUnavailable from None + + def authority( + self, + *, + clock: Callable[[], datetime] | None = None, + ) -> ControlOperatorAuthority: + active_clock = clock or (lambda: datetime.now(UTC)) + return ControlOperatorAuthority( + LocalControlOperatorAuthenticator(self, clock=active_clock), + call_ttl=LOCAL_OPERATOR_TTL, + clock=active_clock, + ) + + def __repr__(self) -> str: + return "LocalControlOperatorConfiguration()" + + def _secret(value: object) -> bytes: if ( type(value) is not str @@ -181,11 +250,14 @@ class LocalControlOperatorAuthenticator: def __init__( self, - configuration: LocalOperatorConfiguration, + configuration: LocalOperatorConfiguration | LocalControlOperatorConfiguration, *, clock: Callable[[], datetime], ) -> None: - if type(configuration) is not LocalOperatorConfiguration: + if type(configuration) not in { + LocalOperatorConfiguration, + LocalControlOperatorConfiguration, + }: raise TypeError("operator authentication rejected") if not callable(clock): raise TypeError("operator authentication rejected") diff --git a/deploy/daily-driver/api.plist.template b/deploy/daily-driver/api.plist.template new file mode 100644 index 00000000..ca6f1939 --- /dev/null +++ b/deploy/daily-driver/api.plist.template @@ -0,0 +1,40 @@ + + + + + Label + ${label_prefix}.api + ProgramArguments + + ${python} + -m + scripts.daily_driver.jobs + daemon + --service + api + --checkout + ${checkout} + --database-environment + ${database_environment} + --operator-environment + ${operator_environment} + --api-port + ${api_port} + + WorkingDirectory + ${checkout} + RunAtLoad + + KeepAlive + + SuccessfulExit + + + ProcessType + Background + StandardOutPath + ${log_root}/api.log + StandardErrorPath + ${log_root}/api.error.log + + diff --git a/deploy/daily-driver/backup.plist.template b/deploy/daily-driver/backup.plist.template new file mode 100644 index 00000000..1c174c36 --- /dev/null +++ b/deploy/daily-driver/backup.plist.template @@ -0,0 +1,40 @@ + + + + + Label + ${label_prefix}.backup + ProgramArguments + + ${python} + -m + scripts.daily_driver.jobs + run + --job + backup + --checkout + ${checkout} + --database-environment + ${database_environment} + --failure-root + ${failure_root} + --backup-root + ${backup_root} + --docker-executable + ${docker_executable} + + WorkingDirectory + ${checkout} + StartCalendarInterval + + Hour + ${backup_hour} + Minute + 0 + + StandardOutPath + ${log_root}/backup.log + StandardErrorPath + ${log_root}/backup.error.log + + diff --git a/deploy/daily-driver/database.plist.template b/deploy/daily-driver/database.plist.template new file mode 100644 index 00000000..4876f529 --- /dev/null +++ b/deploy/daily-driver/database.plist.template @@ -0,0 +1,40 @@ + + + + + Label + ${label_prefix}.database + ProgramArguments + + ${python} + -m + scripts.daily_driver.jobs + bootstrap + --service + database + --checkout + ${checkout} + --docker-executable + ${docker_executable} + --uv-executable + ${uv_executable} + + WorkingDirectory + ${checkout} + RunAtLoad + + KeepAlive + + SuccessfulExit + + + ThrottleInterval + 30 + ProcessType + Background + StandardOutPath + ${log_root}/database.log + StandardErrorPath + ${log_root}/database.error.log + + diff --git a/deploy/daily-driver/health.plist.template b/deploy/daily-driver/health.plist.template new file mode 100644 index 00000000..fa4a30e5 --- /dev/null +++ b/deploy/daily-driver/health.plist.template @@ -0,0 +1,33 @@ + + + + + Label + ${label_prefix}.health + ProgramArguments + + ${python} + -m + scripts.daily_driver.jobs + run + --job + health + --checkout + ${checkout} + --failure-root + ${failure_root} + --health-url + ${health_url} + + WorkingDirectory + ${checkout} + StartInterval + ${health_interval_seconds} + RunAtLoad + + StandardOutPath + ${log_root}/health.log + StandardErrorPath + ${log_root}/health.error.log + + diff --git a/deploy/daily-driver/scan.plist.template b/deploy/daily-driver/scan.plist.template new file mode 100644 index 00000000..1f87014b --- /dev/null +++ b/deploy/daily-driver/scan.plist.template @@ -0,0 +1,38 @@ + + + + + Label + ${label_prefix}.scan + ProgramArguments + + ${python} + -m + scripts.daily_driver.jobs + run + --job + scan + --checkout + ${checkout} + --database-environment + ${database_environment} + --operator-environment + ${operator_environment} + --failure-root + ${failure_root} + + WorkingDirectory + ${checkout} + StartCalendarInterval + + Hour + ${scan_hour} + Minute + 0 + + StandardOutPath + ${log_root}/scan.log + StandardErrorPath + ${log_root}/scan.error.log + + diff --git a/deploy/daily-driver/scheduled-jobs.json b/deploy/daily-driver/scheduled-jobs.json new file mode 100644 index 00000000..dcd2a5bb --- /dev/null +++ b/deploy/daily-driver/scheduled-jobs.json @@ -0,0 +1,27 @@ +{ + "allowedOperations": [ + "backup", + "drain", + "health", + "refresh", + "scan" + ], + "jobs": [ + { + "operation": "backup", + "publicationAuthority": "NONE", + "schedule": "StartCalendarInterval" + }, + { + "operation": "scan", + "publicationAuthority": "NONE", + "schedule": "StartCalendarInterval" + }, + { + "operation": "health", + "publicationAuthority": "NONE", + "schedule": "StartInterval" + } + ], + "schemaVersion": 1 +} diff --git a/deploy/daily-driver/worker.plist.template b/deploy/daily-driver/worker.plist.template new file mode 100644 index 00000000..081b6109 --- /dev/null +++ b/deploy/daily-driver/worker.plist.template @@ -0,0 +1,38 @@ + + + + + Label + ${label_prefix}.worker + ProgramArguments + + ${python} + -m + scripts.daily_driver.jobs + daemon + --service + worker + --checkout + ${checkout} + --database-environment + ${database_environment} + --operator-environment + ${operator_environment} + + WorkingDirectory + ${checkout} + RunAtLoad + + KeepAlive + + SuccessfulExit + + + ProcessType + Background + StandardOutPath + ${log_root}/worker.log + StandardErrorPath + ${log_root}/worker.error.log + + diff --git a/docs/operations/daily-driver-deployment.md b/docs/operations/daily-driver-deployment.md new file mode 100644 index 00000000..40c77c8d --- /dev/null +++ b/docs/operations/daily-driver-deployment.md @@ -0,0 +1,354 @@ +# Durable daily-driver deployment + +This runbook installs one local, loopback-only ContextEngine daily driver from +a dedicated plain Git checkout. The tracked setup and launchd files are +templates only: they do not install or load a service, handle a real vault, or +write to a password manager. The maintainer performs every machine-changing +step below. + +The deployment keeps the existing topology: one pinned compose PostgreSQL, one +API process, and one independent Supply worker. Scheduled work is limited by +[`scheduled-jobs.json`](../../deploy/daily-driver/scheduled-jobs.json) to scan, +refresh, drain, health, and backup categories. Release promotion, profile +activation, and rollback remain explicit operator actions under ADR-0073. + +## 1. Choose durable locations + +Set these values in the shell that performs installation. Both roots must be +absolute existing directories, must not be symlinks, must live outside every +Git worktree, and must never be below a `.context-engine` directory. The setup +script imports this convention from `engine.learning.golden_storage`, which is +the same durable-root contract used by the private golden corpus. + +```bash +export CONTEXT_ENGINE_DEPLOY_CHECKOUT='' +export CONTEXT_ENGINE_DATABASE_BACKUP_ROOT='' +export CONTEXT_ENGINE_DOCKER_EXECUTABLE="$(command -v docker)" +export CONTEXT_ENGINE_UV_EXECUTABLE="$(command -v uv)" +export CONTEXT_ENGINE_ORIGIN="$(git -C '' remote get-url origin)" +export CONTEXT_ENGINE_DEPLOY_BRANCH='main' +export CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX='' +export CONTEXT_ENGINE_API_PORT='' +export CONTEXT_ENGINE_BACKUP_HOUR='' +export CONTEXT_ENGINE_SCAN_HOUR='' +export CONTEXT_ENGINE_HEALTH_INTERVAL_SECONDS='' +``` + +Create the two parent-owned roots with restrictive permissions. Run these from +outside every Git worktree; the setup script refuses otherwise. + +```bash +install -d -m 700 "$(dirname "$CONTEXT_ENGINE_DEPLOY_CHECKOUT")" +install -d -m 700 "$CONTEXT_ENGINE_DATABASE_BACKUP_ROOT" +cd /private/tmp +python3 '/scripts/daily_driver_setup.py' \ + --checkout "$CONTEXT_ENGINE_DEPLOY_CHECKOUT" \ + --origin "$CONTEXT_ENGINE_ORIGIN" \ + --branch "$CONTEXT_ENGINE_DEPLOY_BRANCH" \ + --backup-root "$CONTEXT_ENGINE_DATABASE_BACKUP_ROOT" \ + --docker-executable "$CONTEXT_ENGINE_DOCKER_EXECUTABLE" \ + --uv-executable "$CONTEXT_ENGINE_UV_EXECUTABLE" \ + --label-prefix "$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX" \ + --api-port "$CONTEXT_ENGINE_API_PORT" \ + --backup-hour "$CONTEXT_ENGINE_BACKUP_HOUR" \ + --scan-hour "$CONTEXT_ENGINE_SCAN_HOUR" \ + --health-interval-seconds "$CONTEXT_ENGINE_HEALTH_INTERVAL_SECONDS" +``` + +Re-running that command is the update path: it requires a matching origin, +clean checkout, and fast-forward-only branch update; then it re-runs the locked +install, brings up the same compose project, and atomically re-renders identical +plists under `.context-engine/launchd/`. Never run `make db-reset` in this +checkout: it is the one command that deletes the durable compose volume. +Keep `CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX` stable across reruns. The owner-only +render manifest refuses a prefix change and refuses unknown plists instead of +deleting state it cannot prove it owns. +The setup also writes an owner-only durable-deployment marker; the tracked +database harness refuses `make db-reset` whenever that marker exists, before it +invokes Docker. `make db-down` remains the non-destructive stop command. + +## 2. Preserve the single live connection contract + +`make db-up` generated +`$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/database.env`. It is the only +database connection source and must remain a current-user-owned mode-`0600` +file. Do not copy individual credentials into a plist, another env file, this +runbook, an issue, or a pull request. + +The password-manager operation is deliberately manual. The following uses the +installed 1Password CLI; select the private vault explicitly. Record only the +returned item ID in the maintainer's private inventory. The repository never +runs these commands: + +```bash +export CONTEXT_ENGINE_PASSWORD_VAULT='' +op document create \ + "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/database.env" \ + --vault "$CONTEXT_ENGINE_PASSWORD_VAULT" \ + --title 'ContextEngine durable database.env' \ + --file-name 'database.env' +export CONTEXT_ENGINE_DATABASE_ENV_ITEM='' +export CONTEXT_ENGINE_DATABASE_ENV_RETRIEVAL="$(mktemp /private/tmp/context-engine-database-env.XXXXXX)" +op document get "$CONTEXT_ENGINE_DATABASE_ENV_ITEM" \ + --vault "$CONTEXT_ENGINE_PASSWORD_VAULT" \ + --out-file "$CONTEXT_ENGINE_DATABASE_ENV_RETRIEVAL" \ + --file-mode 0600 +test "$(stat -f '%Lp' "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/database.env")" = 600 +cmp -s \ + "$CONTEXT_ENGINE_DATABASE_ENV_RETRIEVAL" \ + "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/database.env" +rm -f "$CONTEXT_ENGINE_DATABASE_ENV_RETRIEVAL" +unset CONTEXT_ENGINE_DATABASE_ENV_RETRIEVAL +``` + +No repository script writes to the password manager. If the comparison fails, +delete the password-manager item and repeat before continuing. + +## 3. Configure and bootstrap the bounded local composition + +The setup command created an empty ignored `operators.env` without overwriting +any existing values. Populate it from the complete variables documented in the +repository README sections “Run the bounded dogfood API”, “Run the worker”, and +“Scan a local File source”. Add +`CONTEXT_ENGINE_OPERATOR_SOURCE_REF` after source registration; the scheduled +scan wrapper uses that identifier. Keep the file at mode `0600`: + +```bash +test "$(stat -f '%Lp' "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/operators.env")" = 600 +${EDITOR:?set EDITOR} \ + "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/operators.env" +``` + +Use plain `KEY=value` records. Shell-sensitive values such as the root-registry +JSON must be enclosed in single quotes so the same file is both sourceable for +interactive commands and parsed without evaluation by the launchd wrapper: + +```text +CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON='{"":""}' +``` + +The value of `CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON` must bind a curated, +bounded subtree rather than a disposable mirror or worktree. The current source +baseline bound and active Markdown contract are documented in the end-to-end +dogfood walkthrough; do not silently widen them. + +Load both ignored sources for interactive bootstrap only: + +```bash +cd "$CONTEXT_ENGINE_DEPLOY_CHECKOUT" +set -a +source .context-engine/database.env +source .context-engine/operators.env +set +a +uv run context-engine-control migrate +uv run context-engine-dogfood-seed \ + --organization-id "$CONTEXT_ENGINE_DOGFOOD_ORGANIZATION_ID" \ + --user-id "$CONTEXT_ENGINE_DOGFOOD_USER_ID" \ + --membership-id "$CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_ID" \ + --provision-release-operator-grant \ + --file-import-service-principal-id \ + "$CONTEXT_ENGINE_WORKER_SERVICE_PRINCIPAL_ID" +uv run context-engine-control register-file-source \ + --organization-id "$CONTEXT_ENGINE_OPERATOR_ORGANIZATION_ID" \ + --display-name '' \ + --root-ref '' \ + --idempotency-key '' +``` + +Put the returned `sourceRef` in `CONTEXT_ENGINE_OPERATOR_SOURCE_REF`, then run +the README's explicit change-feed/delete-observation activation, first scan, +queue drain, status, and reviewed release-promotion sequence. Promotion is +intentionally absent from every scheduled unit. + +## 4. Validate, install, and load the launchd units + +The renderer never writes `~/Library/LaunchAgents`. Review its ignored output, +prove every file parses, and search only for credential variable names (never +for live values): + +```bash +find "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/launchd" \ + -name '*.plist' -type f -exec plutil -lint {} + +! rg -n \ + 'POSTGRES_PASSWORD|CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET|CONTEXT_ENGINE_DOGFOOD_SECRET' \ + "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/launchd" +``` + +The tracked database agent opens Docker Desktop at login and retries the +existing idempotent `db-up` harness until Docker is available; it creates no +second database topology. Install idempotently as the logged-in user, then +bootstrap the six rendered agents. `install -m 600` overwrites only the exact +chosen labels. + +```bash +export CONTEXT_ENGINE_LAUNCH_AGENTS="$HOME/Library/LaunchAgents" +install -d -m 700 "$CONTEXT_ENGINE_LAUNCH_AGENTS" +for source in "$CONTEXT_ENGINE_DEPLOY_CHECKOUT"/.context-engine/launchd/*.plist; do + install -m 600 "$source" "$CONTEXT_ENGINE_LAUNCH_AGENTS/$(basename "$source")" +done +for service in database api worker backup scan health; do + plist="$CONTEXT_ENGINE_LAUNCH_AGENTS/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.$service.plist" + launchctl bootout "gui/$(id -u)" "$plist" 2>/dev/null || true + launchctl bootstrap "gui/$(id -u)" "$plist" +done +``` + +Verify both daemons and the health carrier: + +```bash +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.api" +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.worker" +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.database" +curl --fail --silent --show-error \ + "http://127.0.0.1:$CONTEXT_ENGINE_API_PORT/health" | \ + "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.venv/bin/python" -c \ + 'import json, sys; health = json.load(sys.stdin); assert health["status"] == "ready"; assert health["runtime_delivery"] == "ACTIVE"' +``` + +## 5. Prove failure restart and nightly backup visibility + +Force-kill each daemon by launchd label and confirm launchd assigns a new PID: + +```bash +launchctl kill SIGKILL \ + "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.api" +launchctl kill SIGKILL \ + "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.worker" +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.api" +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.worker" +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.database" +curl --fail --silent --show-error \ + "http://127.0.0.1:$CONTEXT_ENGINE_API_PORT/health" | \ + "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.venv/bin/python" -c \ + 'import json, sys; health = json.load(sys.stdin); assert health["status"] == "ready"; assert health["runtime_delivery"] == "ACTIVE"' +``` + +Kick the backup once instead of waiting for its calendar interval: + +```bash +launchctl kickstart -k \ + "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.backup" +launchctl print \ + "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.backup" +for dump in "$CONTEXT_ENGINE_DATABASE_BACKUP_ROOT"/*.dump; do + test -f "$dump" || continue + test "$(stat -f '%Lp' "$dump")" = 600 + printf '%s\n' "$dump" +done +``` + +A nonzero scheduled exit persists an owner-only marker at +`.context-engine/scheduled-failures//.json`, and launchd also +records the last exit status. Standard output/error logs live under +`.context-engine/logs/`. Inspect all three surfaces: + +```bash +find "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/scheduled-failures" \ + -name '*.json' -type f -print 2>/dev/null || true +tail -n 100 "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/logs/backup.error.log" +launchctl print \ + "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.backup" | \ + rg 'last exit code|state' +``` + +## 6. Prove volume persistence and rehearse a scratch restore + +First prove the live Organization rows survive the non-destructive harness +round trip. The durable-deployment marker makes `make db-reset` unavailable, +while `make db-down` and `make db-up` retain the exact compose volume: + +```bash +cd "$CONTEXT_ENGINE_DEPLOY_CHECKOUT" +set -a +source .context-engine/database.env +set +a +CONTEXT_ENGINE_ORGANIZATION_COUNT_BEFORE="$(docker compose \ + --env-file .context-engine/database.env \ + --project-name "$CONTEXT_ENGINE_COMPOSE_PROJECT" exec -T postgres \ + psql --tuples-only --no-align --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" --command 'SELECT count(*) FROM organization')" +make db-down +make db-up +CONTEXT_ENGINE_ORGANIZATION_COUNT_AFTER="$(docker compose \ + --env-file .context-engine/database.env \ + --project-name "$CONTEXT_ENGINE_COMPOSE_PROJECT" exec -T postgres \ + psql --tuples-only --no-align --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" --command 'SELECT count(*) FROM organization')" +test "$CONTEXT_ENGINE_ORGANIZATION_COUNT_BEFORE" = \ + "$CONTEXT_ENGINE_ORGANIZATION_COUNT_AFTER" +unset CONTEXT_ENGINE_ORGANIZATION_COUNT_BEFORE CONTEXT_ENGINE_ORGANIZATION_COUNT_AFTER +``` + +Do not use `make db-reset` and do not target the live database. Select one dump +privately, then run the tracked restore helper against the reserved scratch +namespace. The helper drops and recreates only the exact +`context_engine_restore_*` database, restores through the running pinned PG17 +container, and refuses success unless the restored Alembic schema is visible. + +```bash +export CONTEXT_ENGINE_RESTORE_DUMP='' +cd "$CONTEXT_ENGINE_DEPLOY_CHECKOUT" +uv run python -c \ + 'import os; from pathlib import Path; from scripts.daily_driver.backup import restore_database_backup; restore_database_backup(checkout=Path.cwd(), dump_path=Path(os.environ["CONTEXT_ENGINE_RESTORE_DUMP"]), scratch_database="context_engine_restore_drill")' +set -a +source .context-engine/database.env +set +a +docker compose --env-file .context-engine/database.env \ + --project-name "$CONTEXT_ENGINE_COMPOSE_PROJECT" exec -T postgres \ + psql --username "$POSTGRES_USER" --dbname context_engine_restore_drill \ + --command 'SELECT count(*) FROM organization' +docker compose --env-file .context-engine/database.env \ + --project-name "$CONTEXT_ENGINE_COMPOSE_PROJECT" exec -T postgres \ + dropdb --if-exists --force --username "$POSTGRES_USER" \ + context_engine_restore_drill +``` + +Record the date, selected dump's private inventory reference, row count, and +successful schema revision in the maintainer's private operational log. Never +put a dump path, credential, or tenant content in a pull request. + +## 7. Reboot-survival drill and deployed security veto + +Reboot only when the maintainer is ready, then run: + +```bash +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.api" +launchctl print "gui/$(id -u)/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.worker" +curl --fail --silent --show-error \ + "http://127.0.0.1:$CONTEXT_ENGINE_API_PORT/health" | \ + "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.venv/bin/python" -c \ + 'import json, sys; health = json.load(sys.stdin); assert health["status"] == "ready"; assert health["runtime_delivery"] == "ACTIVE"' +cd "$CONTEXT_ENGINE_DEPLOY_CHECKOUT" +make security-gate +``` + +The deployed-instance `make security-gate` and reboot confirmation are +maintainer-owned evidence. Repository verification cannot substitute for them. + +## Uninstall + +This stops and removes only the six exact launchd labels; it preserves the +checkout, database volume, backups, logs, and failure markers: + +```bash +for service in database api worker backup scan health; do + plist="$CONTEXT_ENGINE_LAUNCH_AGENTS/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.$service.plist" + launchctl bootout "gui/$(id -u)" "$plist" 2>/dev/null || true + rm -f "$plist" +done +``` + +Removing the durable checkout, compose volume, backup root, or password-manager +item is a separate destructive operator decision and is deliberately not part +of this uninstall procedure. + +To change the label prefix after uninstalling, keep the old prefix set while +removing the six exact ignored renders plus their renderer-owned manifest; then +set the new prefix and rerun setup: + +```bash +for service in database api worker backup scan health; do + rm -f "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/launchd/$CONTEXT_ENGINE_LAUNCHD_LABEL_PREFIX.$service.plist" +done +rm -f "$CONTEXT_ENGINE_DEPLOY_CHECKOUT/.context-engine/launchd/render-manifest.json" +``` diff --git a/engine/learning/golden_storage.py b/engine/learning/golden_storage.py index 3a456c57..538a4072 100644 --- a/engine/learning/golden_storage.py +++ b/engine/learning/golden_storage.py @@ -11,30 +11,40 @@ REPOSITORY_ROOT: Final = Path(__file__).resolve().parents[2] -def _configured_durable_root(variable: str) -> Path: - try: - configured = os.environ[variable] - except KeyError: - raise ValueError("durable golden root is unavailable") from None - if not configured or configured != configured.strip(): - raise ValueError("durable golden root is unavailable") - root = Path(configured) +def require_durable_storage_root(root: Path) -> Path: + """Apply the one durable, worktree-external root convention.""" + if ( - not root.is_absolute() + not isinstance(root, Path) + or not root.is_absolute() or not root.is_dir() or root.is_symlink() or ".context-engine" in root.parts ): - raise ValueError("durable golden root is unavailable") + raise ValueError("durable storage root is unavailable") resolved = root.resolve(strict=True) if any( (candidate / ".git").exists() for candidate in (resolved, *resolved.parents) ): - raise ValueError("durable golden root must be outside every git worktree") + raise ValueError("durable storage root must be outside every git worktree") return resolved +def _configured_durable_root(variable: str) -> Path: + try: + configured = os.environ[variable] + except KeyError: + raise ValueError("durable golden root is unavailable") from None + if not configured or configured != configured.strip(): + raise ValueError("durable golden root is unavailable") + try: + return require_durable_storage_root(Path(configured)) + except ValueError as error: + message = str(error).replace("storage", "golden") + raise ValueError(message) from None + + def durable_golden_root() -> Path: """Resolve the configured corpus root; no worktree-local default exists.""" diff --git a/scripts/daily_driver/__init__.py b/scripts/daily_driver/__init__.py new file mode 100644 index 00000000..6786f80d --- /dev/null +++ b/scripts/daily_driver/__init__.py @@ -0,0 +1 @@ +"""Tracked, operator-run artifacts for one durable daily-driver checkout.""" diff --git a/scripts/daily_driver/backup.py b/scripts/daily_driver/backup.py new file mode 100644 index 00000000..cd02f824 --- /dev/null +++ b/scripts/daily_driver/backup.py @@ -0,0 +1,329 @@ +"""Owner-only PostgreSQL logical backup and scratch-restore drill.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import BinaryIO, Final + +from engine.learning.golden_storage import ( + require_durable_golden_path, + require_durable_storage_root, +) +from scripts.daily_driver.environment import EnvironmentRefused, load_owner_environment + +BACKUP_DIRECTORY_MODE: Final = 0o700 +BACKUP_FILE_MODE: Final = 0o600 +_SCRATCH_DATABASE = re.compile(r"context_engine_restore_[a-z0-9_]+") +_CONTAINER_ID = re.compile(r"[a-f0-9]{12,64}") + + +class BackupRefused(ValueError): + """The backup could not be created or restored without weakening safety.""" + + +@dataclass(frozen=True) +class BackupOutcome: + """Path-free callers may inspect whether an exact backup was newly recorded.""" + + path: Path + created: bool + + +PgDump = Callable[[tuple[str, ...], BinaryIO], int] + + +def require_safe_backup_root(root: Path) -> Path: + """Reuse the golden-storage durable-root convention and seal permissions.""" + + try: + resolved = require_durable_storage_root(root) + except ValueError as error: + raise BackupRefused(str(error)) from None + if resolved.stat().st_uid != os.getuid(): + raise BackupRefused("backup root must be owned by the current user") + os.chmod(resolved, BACKUP_DIRECTORY_MODE) + return resolved + + +def _database_environment(checkout: Path) -> tuple[Path, dict[str, str]]: + environment_path = checkout / ".context-engine" / "database.env" + try: + loaded = load_owner_environment( + environment_path, + required=( + "CONTEXT_ENGINE_COMPOSE_PROJECT", + "POSTGRES_USER", + "POSTGRES_DB", + ), + ) + except EnvironmentRefused as error: + raise BackupRefused(str(error)) from None + return environment_path, dict(loaded) + + +def _compose_command( + *, + docker_executable: str, + checkout: Path, + environment_path: Path, + project: str, + postgres_arguments: tuple[str, ...], +) -> tuple[str, ...]: + return ( + docker_executable, + "compose", + "--project-directory", + str(checkout), + "--env-file", + str(environment_path), + "--project-name", + project, + "exec", + "-T", + "postgres", + *postgres_arguments, + ) + + +def _run_dump(command: tuple[str, ...], output: BinaryIO) -> int: + return subprocess.run(command, stdout=output, check=False).returncode + + +def create_database_backup( + *, + checkout: Path, + backup_root: Path, + recorded_at: datetime | None = None, + pg_dump: PgDump = _run_dump, + docker_executable: str = "docker", +) -> BackupOutcome: + """Stage one custom dump, fsync it, then atomically publish it.""" + + root = require_safe_backup_root(backup_root) + environment_path, environment = _database_environment(checkout.resolve()) + instant = datetime.now(UTC) if recorded_at is None else recorded_at + if instant.tzinfo is None or instant.utcoffset() is None: + raise BackupRefused("backup instant must be timezone-aware") + recorded_utc = instant.astimezone(UTC) + target = root / f"context-engine-{recorded_utc:%Y%m%dT%H%M%SZ}.dump" + try: + require_durable_golden_path(target, root=root) + except ValueError as error: + raise BackupRefused(str(error).replace("golden corpus", "backup")) from None + if target.exists(): + if target.is_symlink() or not target.is_file(): + raise BackupRefused("recorded backup target is unsafe") + if target.stat().st_mode & 0o777 != BACKUP_FILE_MODE: + raise BackupRefused("recorded backup must have mode 0600") + return BackupOutcome(path=target, created=False) + + descriptor, temporary_name = tempfile.mkstemp( + dir=root, + prefix=".context-engine-backup.partial-", + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, BACKUP_FILE_MODE) + with os.fdopen(descriptor, "wb") as output: + descriptor = -1 + command = _compose_command( + docker_executable=docker_executable, + checkout=checkout, + environment_path=environment_path, + project=environment["CONTEXT_ENGINE_COMPOSE_PROJECT"], + postgres_arguments=( + "pg_dump", + "--format=custom", + f"--username={environment['POSTGRES_USER']}", + f"--dbname={environment['POSTGRES_DB']}", + ), + ) + if pg_dump(command, output) != 0: + raise BackupRefused("pg_dump failed; no backup was published") + output.flush() + os.fsync(output.fileno()) + if temporary.stat().st_size == 0: + raise BackupRefused("pg_dump produced no backup") + os.replace(temporary, target) + os.chmod(target, BACKUP_FILE_MODE) + _fsync_directory(root) + return BackupOutcome(path=target, created=True) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + +def restore_database_backup( + *, + checkout: Path, + dump_path: Path, + scratch_database: str, +) -> None: + """Restore an owner-only dump into one exact, disposable scratch database.""" + + if _SCRATCH_DATABASE.fullmatch(scratch_database) is None: + raise BackupRefused("scratch database name is outside the restore namespace") + root = require_safe_backup_root(dump_path.parent) + try: + require_durable_golden_path(dump_path, root=root) + except ValueError as error: + raise BackupRefused(str(error).replace("golden corpus", "backup")) from None + if ( + dump_path.is_symlink() + or not dump_path.is_file() + or dump_path.stat().st_mode & 0o777 != BACKUP_FILE_MODE + ): + raise BackupRefused("restore input must be an owner-only regular backup") + environment_path, environment = _database_environment(checkout.resolve()) + if scratch_database == environment["POSTGRES_DB"]: + raise BackupRefused("restore drill may not target the live database") + + project = environment["CONTEXT_ENGINE_COMPOSE_PROJECT"] + user = environment["POSTGRES_USER"] + _checked_compose( + _compose_command( + docker_executable="docker", + checkout=checkout, + environment_path=environment_path, + project=project, + postgres_arguments=( + "dropdb", + "--if-exists", + "--force", + f"--username={user}", + scratch_database, + ), + ), + checkout=checkout, + ) + _checked_compose( + _compose_command( + docker_executable="docker", + checkout=checkout, + environment_path=environment_path, + project=project, + postgres_arguments=("createdb", f"--username={user}", scratch_database), + ), + checkout=checkout, + ) + container_id = _postgres_container_id( + checkout=checkout, + environment_path=environment_path, + project=project, + ) + restore_command = ( + "docker", + "exec", + "--interactive", + container_id, + "pg_restore", + "--exit-on-error", + "--username", + user, + "--dbname", + scratch_database, + ) + with dump_path.open("rb") as dump: + result = subprocess.run( + restore_command, + cwd=checkout, + stdin=dump, + check=False, + ) + if result.returncode != 0: + raise BackupRefused("pg_restore failed") + _verify_restored_schema( + checkout=checkout, + environment_path=environment_path, + project=project, + user=user, + scratch_database=scratch_database, + ) + + +def _checked_compose(command: tuple[str, ...], *, checkout: Path) -> None: + if subprocess.run(command, cwd=checkout, check=False).returncode != 0: + raise BackupRefused("scratch database preparation failed") + + +def _postgres_container_id( + *, + checkout: Path, + environment_path: Path, + project: str, +) -> str: + result = subprocess.run( + ( + "docker", + "compose", + "--project-directory", + str(checkout), + "--env-file", + str(environment_path), + "--project-name", + project, + "ps", + "--quiet", + "postgres", + ), + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + container_id = result.stdout.strip() + if result.returncode != 0 or _CONTAINER_ID.fullmatch(container_id) is None: + raise BackupRefused("PostgreSQL container is unavailable") + return container_id + + +def _verify_restored_schema( + *, + checkout: Path, + environment_path: Path, + project: str, + user: str, + scratch_database: str, +) -> None: + command = _compose_command( + docker_executable="docker", + checkout=checkout, + environment_path=environment_path, + project=project, + postgres_arguments=( + "psql", + "--tuples-only", + "--no-align", + "--username", + user, + "--dbname", + scratch_database, + "--command", + "SELECT to_regclass('public.alembic_version') IS NOT NULL", + ), + ) + result = subprocess.run( + command, + cwd=checkout, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0 or result.stdout.strip() != "t": + raise BackupRefused("restored database failed its schema check") + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/scripts/daily_driver/environment.py b/scripts/daily_driver/environment.py new file mode 100644 index 00000000..e0737b2b --- /dev/null +++ b/scripts/daily_driver/environment.py @@ -0,0 +1,129 @@ +"""Strict loading of the ignored single-source deployment environments.""" + +from __future__ import annotations + +import os +import re +import stat +from collections.abc import Iterable, Mapping +from pathlib import Path + +_NAME = re.compile(r"[A-Z][A-Z0-9_]*") +_SAFE_UNQUOTED_VALUE = re.compile(r"[A-Za-z0-9_./:@%+,=-]+") + + +class EnvironmentRefused(ValueError): + """The ignored environment source is absent, exposed, or malformed.""" + + +def load_owner_environment( + path: Path, + *, + required: Iterable[str] = (), +) -> Mapping[str, str]: + """Load plain KEY=VALUE records without evaluating shell syntax.""" + + if not path.is_absolute(): + raise EnvironmentRefused("environment path must be absolute") + try: + metadata = path.lstat() + except FileNotFoundError: + raise EnvironmentRefused("required environment source is unavailable") from None + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + ): + raise EnvironmentRefused( + "environment source must be a current-user-owned regular file" + ) + if stat.S_IMODE(metadata.st_mode) != 0o600: + raise EnvironmentRefused("environment source must have mode 0600") + + values: dict[str, str] = {} + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError): + raise EnvironmentRefused("environment source is unreadable") from None + for line in lines: + if not line or line.startswith("#"): + continue + name, separator, value = line.partition("=") + if ( + not separator + or _NAME.fullmatch(name) is None + or not value + or "\x00" in value + ): + raise EnvironmentRefused("environment source is malformed") + if "'" in value: + if ( + len(value) < 2 + or not value.startswith("'") + or not value.endswith("'") + or "'" in value[1:-1] + ): + raise EnvironmentRefused("environment source is malformed") + value = value[1:-1] + elif _SAFE_UNQUOTED_VALUE.fullmatch(value) is None: + raise EnvironmentRefused("environment source is malformed") + if not value: + raise EnvironmentRefused("environment source is malformed") + if name in values: + raise EnvironmentRefused("environment source contains a duplicate key") + values[name] = value + + missing = sorted(name for name in required if not values.get(name)) + if missing: + raise EnvironmentRefused("environment source lacks required values") + return values + + +def combined_environment(*sources: Mapping[str, str]) -> dict[str, str]: + """Return the process environment with explicit sources layered once.""" + + combined = dict(os.environ) + owned_names: set[str] = set() + for source in sources: + overlap = owned_names.intersection(source) + if overlap: + raise EnvironmentRefused( + "environment sources may not redefine the single live contract" + ) + combined.update(source) + owned_names.update(source) + return combined + + +def project_environment( + *sources: Mapping[str, str], + allowed: Iterable[str], + required: Iterable[str], +) -> dict[str, str]: + """Project the single live sources into one least-privilege child contract.""" + + allowed_names = frozenset(allowed) + required_names = frozenset(required) + if not required_names <= allowed_names: + raise EnvironmentRefused("process environment projection is invalid") + projected_source: dict[str, str] = {} + for source in sources: + overlap = projected_source.keys() & source.keys() + if overlap: + raise EnvironmentRefused( + "environment sources may not redefine the single live contract" + ) + projected_source.update(source) + missing = required_names - projected_source.keys() + if missing: + raise EnvironmentRefused("process environment lacks required values") + inherited = { + name: os.environ[name] + for name in ("HOME", "LANG", "LC_ALL", "PATH", "TMPDIR", "USER") + if name in os.environ + } + return inherited | { + name: projected_source[name] + for name in allowed_names + if name in projected_source + } diff --git a/scripts/daily_driver/jobs.py b/scripts/daily_driver/jobs.py new file mode 100644 index 00000000..b522db96 --- /dev/null +++ b/scripts/daily_driver/jobs.py @@ -0,0 +1,445 @@ +"""Closed scheduled-job runner with durable failure visibility.""" + +from __future__ import annotations + +import argparse +import hmac +import json +import os +import re +import subprocess +import tempfile +import urllib.request +from collections.abc import Callable, Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +from applications.operator_authentication import ( + CONTROL_OPERATOR_SECRET_ENV, + DOGFOOD_SECRET_ENV, + DOGFOOD_SECRET_FINGERPRINT_ENV, + RELEASE_OPERATOR_SECRET_ENV, + RELEASE_OPERATOR_SECRET_FINGERPRINT_ENV, + WORKER_SECRET_ENV, + LocalOperatorConfiguration, + local_secret_fingerprint, +) +from scripts.daily_driver.backup import create_database_backup +from scripts.daily_driver.environment import ( + EnvironmentRefused, + load_owner_environment, + project_environment, +) + +SCHEDULED_OPERATION_CATEGORIES: Final = frozenset( + {"scan", "refresh", "drain", "health", "backup"} +) +RUNNABLE_SCHEDULED_OPERATIONS: Final = ( + SCHEDULED_OPERATION_CATEGORIES - {"refresh"} +) +_JOB = re.compile(r"[a-z][a-z0-9-]*") +_PROVIDER_SIGNING_KEY_ENV = "CONTEXT_ENGINE_FILE_CHANGE_PROVIDER_SIGNING_KEY_HEX" +_CHECKPOINT_SIGNING_KEY_ENV = ( + "CONTEXT_ENGINE_FILE_CHANGE_CHECKPOINT_SIGNING_KEY_HEX" +) + +_API_DATABASE_ENVIRONMENT = frozenset( + {"CONTEXT_ENGINE_RUNTIME_DATABASE_URL", "CONTEXT_ENGINE_RUNTIME_ROLE"} +) +_API_OPERATOR_ENVIRONMENT = frozenset( + { + "CONTEXT_ENGINE_API_COMPOSITION", + "CONTEXT_ENGINE_DOGFOOD_AGENT_VERSION_REF", + "CONTEXT_ENGINE_DOGFOOD_APPLICATION_REF", + "CONTEXT_ENGINE_DOGFOOD_AUTHENTICATION_BINDING_REF", + "CONTEXT_ENGINE_DOGFOOD_EMBEDDING_PROVIDER", + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_ID", + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_VERSION", + "CONTEXT_ENGINE_DOGFOOD_ORGANIZATION_ID", + "CONTEXT_ENGINE_DOGFOOD_PRINCIPAL_REF", + "CONTEXT_ENGINE_DOGFOOD_SECRET", + "CONTEXT_ENGINE_DOGFOOD_USER_ID", + } +) +_WORKER_DATABASE_ENVIRONMENT = frozenset( + { + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL", + "CONTEXT_ENGINE_SCHEDULER_ROLE", + "CONTEXT_ENGINE_WORKER_DATABASE_URL", + "CONTEXT_ENGINE_WORKER_ROLE", + } +) +_WORKER_REQUIRED_ENVIRONMENT = frozenset( + { + "CONTEXT_ENGINE_WORKER_EMBEDDING_DIMENSION", + "CONTEXT_ENGINE_WORKER_EMBEDDING_PROVIDER", + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON", + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX", + } +) +_WORKER_OPTIONAL_ENVIRONMENT = frozenset( + { + "CONTEXT_ENGINE_WORKER_EMBEDDING_API_KEY", + "CONTEXT_ENGINE_WORKER_EMBEDDING_BATCH_SIZE", + "CONTEXT_ENGINE_WORKER_EMBEDDING_ENDPOINT", + "CONTEXT_ENGINE_WORKER_EMBEDDING_MODEL", + "CONTEXT_ENGINE_WORKER_EMBEDDING_TIMEOUT_SECONDS", + "CONTEXT_ENGINE_WORKER_MAX_FILE_BYTES", + } +) +_SCAN_DATABASE_ENVIRONMENT = frozenset( + {"CONTEXT_ENGINE_CONTROL_DATABASE_URL", "CONTEXT_ENGINE_CONTROL_ROLE"} +) +_SCAN_OPERATOR_ENVIRONMENT = frozenset( + { + "CONTEXT_ENGINE_CONTROL_OPERATOR_OPERATIONS", + "CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET", + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_ID", + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_VERSION", + "CONTEXT_ENGINE_DOGFOOD_PRINCIPAL_REF", + "CONTEXT_ENGINE_FILE_CHANGE_CHECKPOINT_SIGNING_KEY_HEX", + "CONTEXT_ENGINE_FILE_CHANGE_PROVIDER_SIGNING_KEY_HEX", + "CONTEXT_ENGINE_OPERATOR_ORGANIZATION_ID", + "CONTEXT_ENGINE_OPERATOR_SOURCE_REF", + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON", + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX", + "CONTEXT_ENGINE_WORKER_MAX_FILE_BYTES", + "CONTEXT_ENGINE_WORKER_SERVICE_PRINCIPAL_ID", + } +) +_PROCESS_ENVIRONMENT_CONTRACTS = { + "api": ( + _API_DATABASE_ENVIRONMENT | _API_OPERATOR_ENVIRONMENT, + _API_DATABASE_ENVIRONMENT | _API_OPERATOR_ENVIRONMENT, + ), + "worker": ( + _WORKER_DATABASE_ENVIRONMENT + | _WORKER_REQUIRED_ENVIRONMENT + | _WORKER_OPTIONAL_ENVIRONMENT, + _WORKER_DATABASE_ENVIRONMENT | _WORKER_REQUIRED_ENVIRONMENT, + ), + "scan": ( + _SCAN_DATABASE_ENVIRONMENT | _SCAN_OPERATOR_ENVIRONMENT, + _SCAN_DATABASE_ENVIRONMENT + | (_SCAN_OPERATOR_ENVIRONMENT - {"CONTEXT_ENGINE_WORKER_MAX_FILE_BYTES"}), + ), +} + + +def run_visible_job( + *, + job: str, + signal_root: Path, + action: Callable[[], int], + recorded_at: datetime | None = None, +) -> int: + """Run one allowlisted job and durably record every non-zero outcome.""" + + if job not in SCHEDULED_OPERATION_CATEGORIES or _JOB.fullmatch(job) is None: + raise ValueError("scheduled job is outside the closed allowlist") + try: + exit_code = action() + except Exception: + exit_code = 1 + if exit_code == 0: + return 0 + instant = datetime.now(UTC) if recorded_at is None else recorded_at.astimezone(UTC) + signal_root.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(signal_root, 0o700) + job_root = signal_root / job + job_root.mkdir(mode=0o700, exist_ok=True) + os.chmod(job_root, 0o700) + marker_stem = f"{instant:%Y%m%dT%H%M%S%fZ}" + marker = job_root / f"{marker_stem}.json" + collision = 0 + while marker.exists() or marker.is_symlink(): + collision += 1 + marker = job_root / f"{marker_stem}-{collision}.json" + payload = { + "attemptedAt": instant.isoformat().replace("+00:00", "Z"), + "exitCode": exit_code, + "job": job, + "status": "FAILED", + } + descriptor, temporary_name = tempfile.mkstemp(dir=job_root, prefix=".failure-") + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + descriptor = -1 + json.dump(payload, output, sort_keys=True, separators=(",", ":")) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, marker) + _fsync_directory(job_root) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + return exit_code + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="ContextEngine daily-driver runner") + subparsers = parser.add_subparsers(dest="mode", required=True) + run = subparsers.add_parser("run") + run.add_argument( + "--job", + choices=sorted(RUNNABLE_SCHEDULED_OPERATIONS), + required=True, + ) + run.add_argument("--checkout", type=Path, required=True) + run.add_argument("--database-environment", type=Path) + run.add_argument("--operator-environment", type=Path) + run.add_argument("--failure-root", type=Path, required=True) + run.add_argument("--backup-root", type=Path) + run.add_argument("--docker-executable", type=Path) + run.add_argument("--health-url") + + daemon = subparsers.add_parser("daemon") + daemon.add_argument("--service", choices=("api", "worker"), required=True) + daemon.add_argument("--checkout", type=Path, required=True) + daemon.add_argument("--database-environment", type=Path, required=True) + daemon.add_argument("--operator-environment", type=Path, required=True) + daemon.add_argument("--api-port", type=int) + + bootstrap = subparsers.add_parser("bootstrap") + bootstrap.add_argument("--service", choices=("database",), required=True) + bootstrap.add_argument("--checkout", type=Path, required=True) + bootstrap.add_argument("--docker-executable", type=Path, required=True) + bootstrap.add_argument("--uv-executable", type=Path, required=True) + return parser + + +def main(arguments: Sequence[str] | None = None) -> int: + parsed = _parser().parse_args(arguments) + if parsed.mode == "bootstrap": + return _run_database_bootstrap(parsed) + if parsed.mode == "daemon": + return _run_daemon(parsed) + return run_visible_job( + job=parsed.job, + signal_root=parsed.failure_root, + action=lambda: _run_scheduled(parsed), + ) + + +def _run_scheduled(arguments: argparse.Namespace) -> int: + python = arguments.checkout / ".venv" / "bin" / "python" + command: tuple[str, ...] + if arguments.job == "backup": + if arguments.backup_root is None or arguments.docker_executable is None: + return 2 + create_database_backup( + checkout=arguments.checkout, + backup_root=arguments.backup_root, + docker_executable=str(arguments.docker_executable), + ) + return 0 + if arguments.job == "health": + if arguments.health_url is None: + return 2 + with urllib.request.urlopen(arguments.health_url, timeout=10) as response: + return 0 if response.status == 200 else 1 + if arguments.job == "drain": + database, operator = _live_environments(arguments) + environment = process_environment("worker", database, operator) + command = ( + str(python), + "-m", + "applications.worker", + "--dispatch-file-once", + ) + elif arguments.job == "scan": + database, operator = _live_environments(arguments) + fingerprints = validate_scan_secret_separation(operator) + organization = operator.get("CONTEXT_ENGINE_OPERATOR_ORGANIZATION_ID") + source = operator.get("CONTEXT_ENGINE_OPERATOR_SOURCE_REF") + if not organization or not source: + return 2 + environment = process_environment("scan", database, operator) | fingerprints + command = ( + str(python), + "-m", + "applications.control", + "scan", + "--organization-id", + organization, + "--source-ref", + source, + ) + else: + return 2 + return subprocess.run( + command, + cwd=arguments.checkout, + env=environment, + check=False, + ).returncode + + +def _run_daemon(arguments: argparse.Namespace) -> int: + database = load_owner_environment(arguments.database_environment) + operator = load_owner_environment(arguments.operator_environment) + python = arguments.checkout / ".venv" / "bin" / "python" + command: tuple[str, ...] + if arguments.service == "api": + if arguments.api_port is None: + return 2 + environment = process_environment("api", database, operator) + command = ( + str(python), + "-m", + "applications.api", + "--host", + "127.0.0.1", + "--port", + str(arguments.api_port), + ) + else: + environment = process_environment("worker", database, operator) + command = (str(python), "-m", "applications.worker", "--dispatch-files") + os.execve(command[0], command, environment) + + +def _live_environments( + arguments: argparse.Namespace, +) -> tuple[dict[str, str], dict[str, str]]: + if arguments.database_environment is None or arguments.operator_environment is None: + raise ValueError("scheduled process environment is unavailable") + return ( + dict(load_owner_environment(arguments.database_environment)), + dict(load_owner_environment(arguments.operator_environment)), + ) + + +def process_environment( + process: str, + database: Mapping[str, str], + operator: Mapping[str, str], +) -> dict[str, str]: + """Project the closed API, worker, or scan child-process contract.""" + + try: + allowed, required = _PROCESS_ENVIRONMENT_CONTRACTS[process] + except KeyError: + raise ValueError("deployment process is outside the closed set") from None + return project_environment( + database, + operator, + allowed=allowed, + required=required, + ) + + +def validate_scan_secret_separation(operator: Mapping[str, str]) -> dict[str, str]: + """Validate ADR-0071 collisions before projecting away release credentials.""" + + try: + configuration = LocalOperatorConfiguration.load(operator) + if configuration is None: + raise ValueError + proof_values = tuple( + operator[name] + for name in (_PROVIDER_SIGNING_KEY_ENV, _CHECKPOINT_SIGNING_KEY_ENV) + ) + if any( + len(value) != 64 + or len(bytes.fromhex(value)) != 32 + for value in proof_values + ): + raise ValueError + operator_secret_values = ( + operator[CONTROL_OPERATOR_SECRET_ENV], + operator[RELEASE_OPERATOR_SECRET_ENV], + operator[DOGFOOD_SECRET_ENV], + ) + if any( + hmac.compare_digest(proof_value.lower(), operator_secret.lower()) + for proof_value in proof_values + for operator_secret in operator_secret_values + ): + raise ValueError + separated = ( + *(bytes.fromhex(value) for value in proof_values), + bytes.fromhex(operator[WORKER_SECRET_ENV]), + configuration.control_secret, + configuration.release_secret, + operator[DOGFOOD_SECRET_ENV].encode("utf-8"), + ) + for index, secret in enumerate(separated): + if any( + hmac.compare_digest(secret, other) + for other in separated[index + 1 :] + ): + raise ValueError + return { + RELEASE_OPERATOR_SECRET_FINGERPRINT_ENV: local_secret_fingerprint( + operator[RELEASE_OPERATOR_SECRET_ENV] + ), + DOGFOOD_SECRET_FINGERPRINT_ENV: local_secret_fingerprint( + operator[DOGFOOD_SECRET_ENV] + ), + } + except ( + KeyError, + TypeError, + ValueError, + UnicodeError, + ): + raise EnvironmentRefused("scan secret separation is invalid") from None + + +def _run_database_bootstrap(arguments: argparse.Namespace) -> int: + docker = _absolute_executable(arguments.docker_executable) + uv = _absolute_executable(arguments.uv_executable) + checkout = arguments.checkout.resolve(strict=True) + harness = checkout / "scripts" / "database_harness.sh" + if not harness.is_file(): + return 2 + environment = dict(os.environ) + executable_directories = (docker.parent, uv.parent) + environment["PATH"] = os.pathsep.join( + (*map(str, executable_directories), "/usr/bin", "/bin", "/usr/sbin", "/sbin") + ) + if subprocess.run( + ("/usr/bin/open", "-gja", "Docker"), + cwd=checkout, + env=environment, + check=False, + ).returncode != 0: + return 1 + return subprocess.run( + ("/bin/bash", str(harness), "up"), + cwd=checkout, + env=environment, + check=False, + ).returncode + + +def _absolute_executable(path: Path) -> Path: + try: + resolved = path.resolve(strict=True) + except (OSError, RuntimeError): + raise ValueError("bootstrap executable is unavailable") from None + if ( + not path.is_absolute() + or not resolved.is_file() + or not os.access(resolved, os.X_OK) + ): + raise ValueError("bootstrap executable is unavailable") + return resolved + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/daily_driver/launchd.py b/scripts/daily_driver/launchd.py new file mode 100644 index 00000000..63a32241 --- /dev/null +++ b/scripts/daily_driver/launchd.py @@ -0,0 +1,277 @@ +"""Deterministically render tracked launchd templates without installing them.""" + +from __future__ import annotations + +import json +import os +import plistlib +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path +from string import Template +from xml.sax.saxutils import escape + +from engine.learning.golden_storage import require_durable_storage_root +from scripts.daily_driver.backup import require_safe_backup_root +from scripts.daily_driver.environment import EnvironmentRefused, load_owner_environment + +_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9.-]+") +_RENDER_MANIFEST = "render-manifest.json" + + +class LaunchdRenderRefused(ValueError): + """A required render input or tracked template is invalid.""" + + +@dataclass(frozen=True) +class LaunchdRenderConfiguration: + checkout: Path + backup_root: Path + docker_executable: Path + uv_executable: Path + label_prefix: str + backup_hour: int + scan_hour: int + health_interval_seconds: int + api_port: int + + def __post_init__(self) -> None: + required = ( + self.checkout, + self.backup_root, + self.docker_executable, + self.uv_executable, + self.label_prefix, + self.backup_hour, + self.scan_hour, + self.health_interval_seconds, + self.api_port, + ) + if any(value is None or value == "" for value in required): + raise LaunchdRenderRefused("every launchd render input is required") + if _LABEL.fullmatch(self.label_prefix) is None: + raise LaunchdRenderRefused("launchd label prefix is invalid") + if not 0 <= self.backup_hour <= 23 or not 0 <= self.scan_hour <= 23: + raise LaunchdRenderRefused("launchd calendar hour is invalid") + if self.health_interval_seconds < 60: + raise LaunchdRenderRefused("health interval must be at least 60 seconds") + if not 1 <= self.api_port <= 65535: + raise LaunchdRenderRefused("API port is invalid") + _require_executable(self.docker_executable, name="Docker") + _require_executable(self.uv_executable, name="uv") + + +def render_launchd_templates( + configuration: LaunchdRenderConfiguration, +) -> dict[str, str]: + """Render each tracked template from explicit, non-secret inputs only.""" + + checkout = _require_plain_checkout(configuration.checkout) + backup_root = require_safe_backup_root(configuration.backup_root) + state = checkout / ".context-engine" + for environment_path in ( + state / "database.env", + state / "operators.env", + ): + try: + load_owner_environment(environment_path) + except EnvironmentRefused as error: + raise LaunchdRenderRefused(str(error)) from None + values = { + "checkout": escape(str(checkout)), + "python": escape(str(checkout / ".venv" / "bin" / "python")), + "backup_root": escape(str(backup_root)), + "docker_executable": escape( + str(configuration.docker_executable.resolve(strict=True)) + ), + "uv_executable": escape(str(configuration.uv_executable.resolve(strict=True))), + "database_environment": escape(str(state / "database.env")), + "operator_environment": escape(str(state / "operators.env")), + "log_root": escape(str(state / "logs")), + "failure_root": escape(str(state / "scheduled-failures")), + "label_prefix": escape(configuration.label_prefix), + "backup_hour": str(configuration.backup_hour), + "scan_hour": str(configuration.scan_hour), + "health_interval_seconds": str(configuration.health_interval_seconds), + "health_url": escape(f"http://127.0.0.1:{configuration.api_port}/health"), + "api_port": str(configuration.api_port), + } + rendered: dict[str, str] = {} + templates = sorted( + (checkout / "deploy" / "daily-driver").glob("*.plist.template") + ) + if not templates: + raise LaunchdRenderRefused("tracked launchd templates are unavailable") + for path in templates: + try: + content = Template(path.read_text(encoding="utf-8")).substitute(values) + parsed = plistlib.loads(content.encode("utf-8")) + except (KeyError, OSError, plistlib.InvalidFileException, UnicodeError): + raise LaunchdRenderRefused("tracked launchd template is invalid") from None + label = parsed.get("Label") + if not isinstance(label, str) or not label.startswith( + f"{configuration.label_prefix}." + ): + raise LaunchdRenderRefused("rendered launchd label is invalid") + rendered[f"{label}.plist"] = content + return rendered + + +def write_rendered_templates( + configuration: LaunchdRenderConfiguration, + destination: Path, +) -> tuple[Path, ...]: + """Idempotently publish owner-only rendered plists to ignored state.""" + + checkout = _require_plain_checkout(configuration.checkout) + expected_destination = checkout / ".context-engine" / "launchd" + if destination.resolve(strict=False) != expected_destination.resolve(strict=False): + raise LaunchdRenderRefused("rendered templates must stay in ignored state") + for path in (expected_destination.parent, expected_destination): + if path.is_symlink(): + raise LaunchdRenderRefused("rendered template state may not be a symlink") + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(destination, 0o700) + published: list[Path] = [] + rendered = render_launchd_templates(configuration) + manifest = destination / _RENDER_MANIFEST + previously_owned = _read_render_manifest( + manifest, + label_prefix=configuration.label_prefix, + ) + discovered = {path.name for path in destination.glob("*.plist")} + if previously_owned is None and discovered: + raise LaunchdRenderRefused("unowned rendered template state is present") + owned = frozenset() if previously_owned is None else previously_owned + if not discovered <= owned: + raise LaunchdRenderRefused("unowned rendered template state is present") + _write_render_manifest( + manifest, + label_prefix=configuration.label_prefix, + plists=owned | frozenset(rendered), + ) + for stale_name in sorted(owned - rendered.keys()): + stale = destination / stale_name + if stale.is_symlink() or not stale.is_file(): + raise LaunchdRenderRefused("stale rendered template is unsafe") + stale.unlink() + for name, content in rendered.items(): + target = destination / name + if target.is_symlink() or (target.exists() and not target.is_file()): + raise LaunchdRenderRefused("rendered template target is unsafe") + if target.exists() and target.read_text(encoding="utf-8") == content: + os.chmod(target, 0o600) + published.append(target) + continue + descriptor, temporary_name = tempfile.mkstemp(dir=destination, prefix=".plist-") + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + descriptor = -1 + output.write(content) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, target) + published.append(target) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + _write_render_manifest( + manifest, + label_prefix=configuration.label_prefix, + plists=frozenset(rendered), + ) + return tuple(sorted(published)) + + +def _read_render_manifest( + path: Path, + *, + label_prefix: str, +) -> frozenset[str] | None: + if not path.exists() and not path.is_symlink(): + return None + if path.is_symlink() or not path.is_file() or path.stat().st_mode & 0o777 != 0o600: + raise LaunchdRenderRefused("render manifest is unsafe") + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + raise LaunchdRenderRefused("render manifest is invalid") from None + if ( + type(document) is not dict + or set(document) != {"labelPrefix", "plists", "schemaVersion"} + or document["schemaVersion"] != 1 + or document["labelPrefix"] != label_prefix + or type(document["plists"]) is not list + or not document["plists"] + or any( + type(name) is not str + or Path(name).name != name + or not name.endswith(".plist") + for name in document["plists"] + ) + or len(set(document["plists"])) != len(document["plists"]) + ): + raise LaunchdRenderRefused( + "launchd label prefix is immutable; uninstall before replacing it" + ) + return frozenset(document["plists"]) + + +def _write_render_manifest( + path: Path, + *, + label_prefix: str, + plists: frozenset[str], +) -> None: + document = { + "labelPrefix": label_prefix, + "plists": sorted(plists), + "schemaVersion": 1, + } + descriptor, temporary_name = tempfile.mkstemp(dir=path.parent, prefix=".manifest-") + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + descriptor = -1 + json.dump(document, output, sort_keys=True, separators=(",", ":")) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + +def _require_plain_checkout(checkout: Path) -> Path: + if not checkout.is_absolute() or checkout.is_symlink(): + raise LaunchdRenderRefused("dedicated checkout is invalid") + resolved = checkout.resolve(strict=True) + if not (resolved / ".git").is_dir(): + raise LaunchdRenderRefused("dedicated checkout must be a plain checkout") + try: + require_durable_storage_root(resolved.parent) + except ValueError as error: + raise LaunchdRenderRefused(str(error)) from None + return resolved + + +def _require_executable(path: Path, *, name: str) -> Path: + try: + resolved = path.resolve(strict=True) + except (OSError, RuntimeError): + raise LaunchdRenderRefused(f"{name} executable is invalid") from None + if ( + not isinstance(path, Path) + or not path.is_absolute() + or not resolved.is_file() + or not os.access(resolved, os.X_OK) + ): + raise LaunchdRenderRefused(f"{name} executable is invalid") + return resolved diff --git a/scripts/daily_driver/setup.py b/scripts/daily_driver/setup.py new file mode 100644 index 00000000..d36f94e5 --- /dev/null +++ b/scripts/daily_driver/setup.py @@ -0,0 +1,187 @@ +"""Idempotently prepare, but never install, one daily-driver deployment.""" + +from __future__ import annotations + +import argparse +import os +import stat +import subprocess +from collections.abc import Sequence +from pathlib import Path + +from engine.learning.golden_storage import ( + require_durable_golden_path, + require_durable_storage_root, +) +from scripts.daily_driver.environment import EnvironmentRefused, load_owner_environment +from scripts.daily_driver.launchd import ( + LaunchdRenderConfiguration, + write_rendered_templates, +) + + +class SetupRefused(ValueError): + """The requested checkout could be disposable or overwrite operator work.""" + + +DURABLE_DEPLOYMENT_MARKER = "daily-driver-v1\n" + + +def require_setup_target(*, target: Path, current_directory: Path) -> Path: + """Refuse execution in worktrees and accept only a plain dedicated clone.""" + + current = current_directory.resolve(strict=True) + if _inside_git_worktree(current): + raise SetupRefused("run from outside every git worktree") + if not target.is_absolute() or target.is_symlink(): + raise SetupRefused("dedicated checkout path must be absolute and non-symlink") + if ".context-engine" in target.parts: + raise SetupRefused("dedicated checkout must follow the durable root contract") + try: + parent = require_durable_storage_root(target.parent) + require_durable_golden_path(target, root=parent) + except ValueError as error: + raise SetupRefused(str(error)) from None + if not target.exists(): + return target + resolved = target.resolve(strict=True) + git_entry = resolved / ".git" + if git_entry.is_file(): + raise SetupRefused("durable target must be a plain dedicated checkout") + if not git_entry.is_dir(): + if any(resolved.iterdir()): + raise SetupRefused("durable target must be absent or a dedicated checkout") + return resolved + return resolved + + +def _inside_git_worktree(path: Path) -> bool: + return any((candidate / ".git").exists() for candidate in (path, *path.parents)) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Prepare tracked ContextEngine daily-driver artifacts" + ) + parser.add_argument("--checkout", type=Path, required=True) + parser.add_argument("--origin", required=True) + parser.add_argument("--branch", required=True) + parser.add_argument("--backup-root", type=Path, required=True) + parser.add_argument("--docker-executable", type=Path, required=True) + parser.add_argument("--uv-executable", type=Path, required=True) + parser.add_argument("--label-prefix", required=True) + parser.add_argument("--api-port", type=int, required=True) + parser.add_argument("--backup-hour", type=int, required=True) + parser.add_argument("--scan-hour", type=int, required=True) + parser.add_argument("--health-interval-seconds", type=int, required=True) + return parser + + +def main(arguments: Sequence[str] | None = None) -> int: + parsed = _parser().parse_args(arguments) + checkout = require_setup_target( + target=parsed.checkout, + current_directory=Path.cwd(), + ) + if not checkout.exists() or not (checkout / ".git").is_dir(): + subprocess.run( + ( + "git", + "clone", + "--branch", + parsed.branch, + "--single-branch", + parsed.origin, + str(checkout), + ), + check=True, + ) + else: + _update_existing_checkout(checkout, parsed.origin, parsed.branch) + + state = checkout / ".context-engine" + _prepare_state_directory(state) + _write_durable_deployment_marker(state) + _ensure_operator_environment(state / "operators.env") + subprocess.run(("make", "install"), cwd=checkout, check=True) + subprocess.run(("make", "db-up"), cwd=checkout, check=True) + (state / "logs").mkdir(mode=0o700, exist_ok=True) + write_rendered_templates( + LaunchdRenderConfiguration( + checkout=checkout, + backup_root=parsed.backup_root, + docker_executable=parsed.docker_executable, + uv_executable=parsed.uv_executable, + label_prefix=parsed.label_prefix, + backup_hour=parsed.backup_hour, + scan_hour=parsed.scan_hour, + health_interval_seconds=parsed.health_interval_seconds, + api_port=parsed.api_port, + ), + state / "launchd", + ) + return 0 + + +def _prepare_state_directory(state: Path) -> None: + if state.is_symlink() or (state.exists() and not state.is_dir()): + raise SetupRefused("durable deployment state is unsafe") + state.mkdir(mode=0o700, exist_ok=True) + metadata = state.lstat() + if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.getuid(): + raise SetupRefused("durable deployment state is unsafe") + state.chmod(0o700) + + +def _ensure_operator_environment(path: Path) -> None: + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + try: + load_owner_environment(path) + except EnvironmentRefused as error: + raise SetupRefused(str(error)) from None + return + os.fchmod(descriptor, 0o600) + os.close(descriptor) + + +def _write_durable_deployment_marker(state: Path) -> None: + marker = state / "durable-deployment" + if marker.is_symlink() or (marker.exists() and not marker.is_file()): + raise SetupRefused("durable deployment marker is unsafe") + if marker.exists() and marker.read_text(encoding="utf-8") != ( + DURABLE_DEPLOYMENT_MARKER + ): + raise SetupRefused("durable deployment marker is invalid") + marker.write_text(DURABLE_DEPLOYMENT_MARKER, encoding="utf-8") + marker.chmod(0o600) + + +def _update_existing_checkout(checkout: Path, origin: str, branch: str) -> None: + remote = subprocess.run( + ("git", "-C", str(checkout), "remote", "get-url", "origin"), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if remote != origin: + raise SetupRefused("existing checkout origin does not match") + status = subprocess.run( + ("git", "-C", str(checkout), "status", "--porcelain"), + check=True, + capture_output=True, + text=True, + ).stdout + if status: + raise SetupRefused("existing checkout contains uncommitted changes") + subprocess.run(("git", "-C", str(checkout), "fetch", "origin", branch), check=True) + subprocess.run(("git", "-C", str(checkout), "checkout", branch), check=True) + subprocess.run( + ("git", "-C", str(checkout), "merge", "--ff-only", f"origin/{branch}"), + check=True, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/daily_driver_setup.py b/scripts/daily_driver_setup.py new file mode 100644 index 00000000..50dda78f --- /dev/null +++ b/scripts/daily_driver_setup.py @@ -0,0 +1,16 @@ +"""Run daily-driver setup by absolute path while outside every worktree.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from scripts.daily_driver.setup import main as _main # noqa: E402, I001 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/scripts/database_harness.sh b/scripts/database_harness.sh index c672d108..b75d2006 100755 --- a/scripts/database_harness.sh +++ b/scripts/database_harness.sh @@ -6,6 +6,7 @@ readonly ROOT_DIR readonly STATE_DIR="$ROOT_DIR/.context-engine" readonly ENV_FILE="$STATE_DIR/database.env" readonly ENV_MIGRATION_LOCK="$STATE_DIR/database.env.migration.lock" +readonly DURABLE_DEPLOYMENT_MARKER="$STATE_DIR/durable-deployment" readonly LEGACY_PROJECT_FILE="$STATE_DIR/compose-project" readonly COMPOSE_FILE="$ROOT_DIR/compose.yaml" COMPOSE_PROJECT='' @@ -671,6 +672,10 @@ database_down() { } database_reset() { + if [[ -e "$DURABLE_DEPLOYMENT_MARKER" || -L "$DURABLE_DEPLOYMENT_MARKER" ]]; then + printf 'refusing to reset a durable deployment database\n' >&2 + exit 1 + fi require_command docker require_command uv load_environment diff --git a/tests/integration/test_backup_restore_roundtrip.py b/tests/integration/test_backup_restore_roundtrip.py new file mode 100644 index 00000000..92699f7c --- /dev/null +++ b/tests/integration/test_backup_restore_roundtrip.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import hashlib +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from uuid import UUID + +import pytest +from sqlalchemy import Engine, text + +from engine.persistence import DatabaseConfiguration, create_database_engine +from scripts.daily_driver.backup import create_database_backup, restore_database_backup + +pytestmark = pytest.mark.integration + +ORGANIZATION_ID = UUID("14900000-0000-4000-8000-000000000001") +FIRST_RECORD_ID = UUID("14900000-0000-4000-8000-000000000002") +SECOND_RECORD_ID = UUID("14900000-0000-4000-8000-000000000003") +SCRATCH_DATABASE = "context_engine_restore_149" +ROOT = Path(__file__).resolve().parents[2] + + +def _digest(rows: list[tuple[str, str]]) -> str: + payload = "\n".join(f"{record_id}:{value}" for record_id, value in rows) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _representative_rows(engine: Engine) -> list[tuple[str, str]]: + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT record_id::text, payload " + "FROM organization_record " + "WHERE organization_id = :organization_id " + "ORDER BY record_id" + ), + {"organization_id": ORGANIZATION_ID}, + ).tuples() + return [(record_id, payload) for record_id, payload in rows] + + +def test_pg_dump_restores_representative_rows_into_a_fresh_database( + tmp_path: Path, + migration_configuration: DatabaseConfiguration, +) -> None: + source_engine = create_database_engine(migration_configuration) + with source_engine.begin() as connection: + connection.execute( + text( + "INSERT INTO organization (organization_id) VALUES (:organization_id) " + "ON CONFLICT DO NOTHING" + ), + {"organization_id": ORGANIZATION_ID}, + ) + connection.execute( + text( + "DELETE FROM organization_record " + "WHERE organization_id = :organization_id" + ), + {"organization_id": ORGANIZATION_ID}, + ) + connection.execute( + text( + "INSERT INTO organization_record " + "(organization_id, record_id, parent_record_id, payload) VALUES " + "(:organization_id, :first_record_id, NULL, :first_payload), " + "(:organization_id, :second_record_id, :first_record_id, " + ":second_payload)" + ), + { + "organization_id": ORGANIZATION_ID, + "first_record_id": FIRST_RECORD_ID, + "second_record_id": SECOND_RECORD_ID, + "first_payload": "daily-driver-backup-parent", + "second_payload": "daily-driver-backup-child", + }, + ) + expected = _representative_rows(source_engine) + + backup_root = tmp_path / "database-backups" + backup_root.mkdir() + outcome = create_database_backup( + checkout=ROOT, + backup_root=backup_root, + recorded_at=datetime(2026, 7, 30, 20, 0, tzinfo=UTC), + ) + restore_database_backup( + checkout=ROOT, + dump_path=outcome.path, + scratch_database=SCRATCH_DATABASE, + ) + + scratch_url = migration_configuration.url.set(database=SCRATCH_DATABASE) + scratch_engine = create_database_engine( + DatabaseConfiguration( + purpose=migration_configuration.purpose, + url=scratch_url, + expected_role=migration_configuration.expected_role, + ) + ) + try: + restored = _representative_rows(scratch_engine) + assert len(restored) == 2 + assert _digest(restored) == _digest(expected) + finally: + scratch_engine.dispose() + source_engine.dispose() + subprocess.run( + ( + "docker", + "compose", + "--env-file", + str(ROOT / ".context-engine" / "database.env"), + "--project-name", + _compose_project(), + "exec", + "-T", + "postgres", + "dropdb", + "--if-exists", + "--force", + "--username", + _database_value("POSTGRES_USER"), + SCRATCH_DATABASE, + ), + cwd=ROOT, + check=True, + ) + + +def _database_value(name: str) -> str: + for line in (ROOT / ".context-engine" / "database.env").read_text( + encoding="utf-8" + ).splitlines(): + key, separator, value = line.partition("=") + if key == name and separator: + return value + raise AssertionError(f"missing harness value: {name}") + + +def _compose_project() -> str: + return _database_value("CONTEXT_ENGINE_COMPOSE_PROJECT") diff --git a/tests/unit/test_backup_paths_are_safe.py b/tests/unit/test_backup_paths_are_safe.py new file mode 100644 index 00000000..1ce7053d --- /dev/null +++ b/tests/unit/test_backup_paths_are_safe.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import os +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from typing import BinaryIO + +import pytest + +from scripts.daily_driver.backup import ( + BackupRefused, + create_database_backup, + require_safe_backup_root, +) + + +def _checkout(tmp_path: Path) -> Path: + checkout = tmp_path / "checkout" + state = checkout / ".context-engine" + state.mkdir(parents=True) + (checkout / ".git").mkdir() + environment = state / "database.env" + environment.write_text( + "CONTEXT_ENGINE_COMPOSE_PROJECT=synthetic-project\n" + "POSTGRES_USER=synthetic-user\n" + "POSTGRES_DB=synthetic-database\n", + encoding="utf-8", + ) + environment.chmod(0o600) + return checkout + + +def test_backup_root_is_absolute_owner_only_and_outside_every_worktree( + tmp_path: Path, +) -> None: + root = tmp_path / "backups" + root.mkdir(mode=0o755) + + resolved = require_safe_backup_root(root) + + assert resolved == root.resolve() + assert resolved.stat().st_mode & 0o777 == 0o700 + + +@pytest.mark.parametrize("unsafe_part", (".context-engine", "nested/.context-engine")) +def test_backup_root_under_context_engine_state_is_refused( + tmp_path: Path, + unsafe_part: str, +) -> None: + root = tmp_path / unsafe_part / "backups" + root.mkdir(parents=True) + + with pytest.raises(BackupRefused, match=r"durable .*root"): + require_safe_backup_root(root) + + +def test_backup_root_inside_a_git_worktree_is_refused(tmp_path: Path) -> None: + worktree = tmp_path / "worktree" + worktree.mkdir() + subprocess.run(("git", "init", "--quiet", str(worktree)), check=True) + root = worktree / "backups" + root.mkdir() + + with pytest.raises(BackupRefused, match="outside every git worktree"): + require_safe_backup_root(root) + + +def test_backup_root_symlink_is_refused(tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + linked = tmp_path / "linked" + linked.symlink_to(real, target_is_directory=True) + + with pytest.raises(BackupRefused, match=r"durable .*root"): + require_safe_backup_root(linked) + + +def test_pg_dump_is_staged_owner_only_then_atomically_published( + tmp_path: Path, +) -> None: + checkout = _checkout(tmp_path) + backup_root = tmp_path / "backups" + backup_root.mkdir() + + def pg_dump(command: tuple[str, ...], output: BinaryIO) -> int: + assert command[-4:] == ( + "pg_dump", + "--format=custom", + "--username=synthetic-user", + "--dbname=synthetic-database", + ) + output.write(b"synthetic custom dump") + return 0 + + outcome = create_database_backup( + checkout=checkout, + backup_root=backup_root, + recorded_at=datetime(2026, 7, 30, 18, 0, tzinfo=UTC), + pg_dump=pg_dump, + ) + + assert outcome.path.parent == backup_root + assert outcome.path.name == "context-engine-20260730T180000Z.dump" + assert outcome.path.read_bytes() == b"synthetic custom dump" + assert outcome.path.stat().st_mode & 0o777 == 0o600 + assert not tuple(backup_root.glob("*.partial-*")) + + +def test_failed_pg_dump_leaves_no_partial_or_published_dump(tmp_path: Path) -> None: + checkout = _checkout(tmp_path) + backup_root = tmp_path / "backups" + backup_root.mkdir() + + def failing_dump(command: tuple[str, ...], output: BinaryIO) -> int: + del command + output.write(b"partial secret content") + return 9 + + with pytest.raises(BackupRefused, match="pg_dump failed"): + create_database_backup( + checkout=checkout, + backup_root=backup_root, + recorded_at=datetime(2026, 7, 30, 18, 0, tzinfo=UTC), + pg_dump=failing_dump, + ) + + assert list(backup_root.iterdir()) == [] + + +def test_backup_refuses_a_group_readable_database_environment( + tmp_path: Path, +) -> None: + checkout = _checkout(tmp_path) + (checkout / ".context-engine" / "database.env").chmod(0o640) + backup_root = tmp_path / "backups" + backup_root.mkdir() + + with pytest.raises(BackupRefused, match="mode 0600"): + create_database_backup( + checkout=checkout, + backup_root=backup_root, + recorded_at=datetime.now(UTC), + pg_dump=lambda _command, _output: os.EX_OK, + ) diff --git a/tests/unit/test_database_harness_behavior.py b/tests/unit/test_database_harness_behavior.py index cd18f8d5..c7814df7 100644 --- a/tests/unit/test_database_harness_behavior.py +++ b/tests/unit/test_database_harness_behavior.py @@ -53,6 +53,42 @@ def _stub_harness_dependencies(stub_directory: Path) -> None: _write_executable(stub_directory / "uv", "#!/usr/bin/env bash\nexit 0\n") +def test_reset_refuses_a_checkout_marked_as_a_durable_deployment( + tmp_path: Path, +) -> None: + stub_directory = tmp_path / "bin" + stub_directory.mkdir() + _stub_harness_dependencies(stub_directory) + checkout = tmp_path / "checkout" + scripts = checkout / "scripts" + scripts.mkdir(parents=True) + shutil.copy2(ROOT / "scripts/database_harness.sh", scripts) + (checkout / "compose.yaml").write_text("services: {}\n", encoding="utf-8") + state = checkout / ".context-engine" + state.mkdir() + (state / "durable-deployment").write_text( + "daily-driver-v1\n", + encoding="utf-8", + ) + command_log = checkout / "docker-command.log" + + completed = subprocess.run( + ["/bin/bash", str(scripts / "database_harness.sh"), "reset"], + check=False, + capture_output=True, + text=True, + env={ + **os.environ, + "HARNESS_COMMAND_LOG": str(command_log), + "PATH": f"{stub_directory}{os.pathsep}{os.environ['PATH']}", + }, + ) + + assert completed.returncode != 0 + assert "refusing to reset a durable deployment database" in completed.stderr + assert not command_log.exists() + + def test_two_checkouts_generate_distinct_persistent_compose_projects( tmp_path: Path, ) -> None: diff --git a/tests/unit/test_failed_job_is_visible.py b/tests/unit/test_failed_job_is_visible.py new file mode 100644 index 00000000..7890a65e --- /dev/null +++ b/tests/unit/test_failed_job_is_visible.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +from scripts.daily_driver.jobs import run_visible_job + + +def test_a_failed_scheduled_job_leaves_a_durable_non_silent_signal( + tmp_path: Path, +) -> None: + signal_root = tmp_path / "signals" + + exit_code = run_visible_job( + job="backup", + signal_root=signal_root, + action=lambda: 23, + recorded_at=datetime(2026, 7, 30, 19, 45, tzinfo=UTC), + ) + + assert exit_code == 23 + markers = list((signal_root / "backup").glob("*.json")) + assert len(markers) == 1 + marker = markers[0] + assert marker.stat().st_mode & 0o777 == 0o600 + assert signal_root.stat().st_mode & 0o777 == 0o700 + assert marker.parent.stat().st_mode & 0o777 == 0o700 + assert json.loads(marker.read_text(encoding="utf-8")) == { + "attemptedAt": "2026-07-30T19:45:00Z", + "exitCode": 23, + "job": "backup", + "status": "FAILED", + } + + +def test_a_successful_job_does_not_fabricate_a_failure_signal( + tmp_path: Path, +) -> None: + signal_root = tmp_path / "signals" + + assert ( + run_visible_job( + job="health", + signal_root=signal_root, + action=lambda: 0, + recorded_at=datetime(2026, 7, 30, 19, 45, tzinfo=UTC), + ) + == 0 + ) + assert not signal_root.exists() + + +def test_repeated_failures_each_leave_a_durable_signal(tmp_path: Path) -> None: + signal_root = tmp_path / "signals" + + first = run_visible_job( + job="backup", + signal_root=signal_root, + action=lambda: 7, + recorded_at=datetime(2026, 7, 30, 19, 45, tzinfo=UTC), + ) + second = run_visible_job( + job="backup", + signal_root=signal_root, + action=lambda: 9, + recorded_at=datetime(2026, 7, 30, 19, 45, tzinfo=UTC), + ) + + assert (first, second) == (7, 9) + markers = sorted((signal_root / "backup").glob("*.json")) + assert len(markers) == 2 + assert { + json.loads(path.read_text())["exitCode"] for path in markers + } == {7, 9} diff --git a/tests/unit/test_launchd_template_render.py b/tests/unit/test_launchd_template_render.py new file mode 100644 index 00000000..b5e7f433 --- /dev/null +++ b/tests/unit/test_launchd_template_render.py @@ -0,0 +1,455 @@ +from __future__ import annotations + +import plistlib +import shutil +import subprocess +from argparse import Namespace +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from applications.file_scan import _proof_keys +from applications.operator_authentication import local_secret_fingerprint +from engine.control import SourceNotAvailable +from scripts.daily_driver.environment import ( + EnvironmentRefused, + combined_environment, + load_owner_environment, +) +from scripts.daily_driver.jobs import ( + _run_database_bootstrap, + process_environment, + validate_scan_secret_separation, +) +from scripts.daily_driver.launchd import ( + LaunchdRenderConfiguration, + LaunchdRenderRefused, + render_launchd_templates, + write_rendered_templates, +) + +ROOT = Path(__file__).resolve().parents[2] + + +def _configuration(tmp_path: Path) -> LaunchdRenderConfiguration: + checkout = tmp_path / "checkout" + checkout.mkdir() + (checkout / ".git").mkdir() + (checkout / ".venv" / "bin").mkdir(parents=True) + (checkout / ".venv" / "bin" / "python").touch() + state = checkout / ".context-engine" + state.mkdir(mode=0o700) + for environment in (state / "database.env", state / "operators.env"): + environment.write_text("SYNTHETIC_VALUE=present\n", encoding="utf-8") + environment.chmod(0o600) + shutil.copytree( + ROOT / "deploy" / "daily-driver", + checkout / "deploy" / "daily-driver", + ) + backup_root = tmp_path / "database-backups" + backup_root.mkdir(mode=0o700) + docker_executable = tmp_path / "docker" + docker_executable.write_text("#!/bin/sh\n", encoding="utf-8") + docker_executable.chmod(0o700) + uv_executable = tmp_path / "uv" + uv_executable.write_text("#!/bin/sh\n", encoding="utf-8") + uv_executable.chmod(0o700) + return LaunchdRenderConfiguration( + checkout=checkout, + backup_root=backup_root, + docker_executable=docker_executable, + uv_executable=uv_executable, + label_prefix="org.example.context-engine", + backup_hour=2, + scan_hour=3, + health_interval_seconds=300, + api_port=8137, + ) + + +def test_render_is_deterministic_and_contains_no_credentials( + tmp_path: Path, +) -> None: + configuration = _configuration(tmp_path) + database_secret = "DATABASE_SECRET_MUST_NOT_RENDER" + operator_secret = "OPERATOR_SECRET_MUST_NOT_RENDER" + state = configuration.checkout / ".context-engine" + (state / "database.env").write_text( + f"POSTGRES_PASSWORD={database_secret}\n", encoding="utf-8" + ) + (state / "operators.env").write_text( + f"CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET={operator_secret}\n", + encoding="utf-8", + ) + + first = render_launchd_templates(configuration) + second = render_launchd_templates(configuration) + + assert first == second + assert set(first) == { + "org.example.context-engine.api.plist", + "org.example.context-engine.backup.plist", + "org.example.context-engine.database.plist", + "org.example.context-engine.health.plist", + "org.example.context-engine.scan.plist", + "org.example.context-engine.worker.plist", + } + for rendered in first.values(): + assert database_secret not in rendered + assert operator_secret not in rendered + parsed = plistlib.loads(rendered.encode("utf-8")) + assert parsed["Label"].startswith("org.example.context-engine.") + database = plistlib.loads( + first["org.example.context-engine.database.plist"].encode("utf-8") + ) + assert database["RunAtLoad"] is True + assert database["KeepAlive"] == {"SuccessfulExit": False} + assert "bootstrap" in database["ProgramArguments"] + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("label_prefix", ""), + ("docker_executable", None), + ("uv_executable", None), + ("backup_hour", None), + ("scan_hour", None), + ("health_interval_seconds", None), + ("api_port", None), + ), +) +def test_missing_required_render_input_refuses( + tmp_path: Path, + field: str, + value: str | int | None, +) -> None: + values = _configuration(tmp_path).__dict__ | {field: value} + + with pytest.raises(LaunchdRenderRefused, match="required"): + LaunchdRenderConfiguration(**values) + + +def test_rendered_units_reference_the_single_live_environment_sources( + tmp_path: Path, +) -> None: + configuration = _configuration(tmp_path) + + rendered = render_launchd_templates(configuration) + + combined = "\n".join(rendered.values()) + assert ".context-engine/database.env" in combined + assert ".context-engine/operators.env" in combined + assert "POSTGRES_PASSWORD" not in combined + assert "CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET" not in combined + + +def test_rendered_templates_pass_the_platform_plist_validator( + tmp_path: Path, +) -> None: + configuration = _configuration(tmp_path) + rendered = render_launchd_templates(configuration) + + for name, content in rendered.items(): + plist = tmp_path / name + plist.write_text(content, encoding="utf-8") + completed = subprocess.run( + ("plutil", "-lint", str(plist)), + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + + +def test_writing_the_same_render_twice_is_idempotent(tmp_path: Path) -> None: + configuration = _configuration(tmp_path) + destination = configuration.checkout / ".context-engine" / "launchd" + + first = write_rendered_templates(configuration, destination) + first_inodes = {path.name: path.stat().st_ino for path in first} + second = write_rendered_templates(configuration, destination) + + assert first == second + assert {path.name: path.stat().st_ino for path in second} == first_inodes + + +def test_render_refuses_a_label_change_until_the_old_services_are_uninstalled( + tmp_path: Path, +) -> None: + configuration = _configuration(tmp_path) + destination = configuration.checkout / ".context-engine" / "launchd" + first = write_rendered_templates(configuration, destination) + changed = LaunchdRenderConfiguration( + **( + configuration.__dict__ + | {"label_prefix": "org.example.context-engine-v2"} + ) + ) + + with pytest.raises(LaunchdRenderRefused, match="prefix is immutable"): + write_rendered_templates(changed, destination) + + assert set(destination.glob("*.plist")) == set(first) + + +def test_render_refuses_to_delete_an_unknown_plist(tmp_path: Path) -> None: + configuration = _configuration(tmp_path) + destination = configuration.checkout / ".context-engine" / "launchd" + first = write_rendered_templates(configuration, destination) + unknown = destination / "maintainer-owned.plist" + unknown.write_text("preserve me", encoding="utf-8") + + with pytest.raises(LaunchdRenderRefused, match="unowned"): + write_rendered_templates(configuration, destination) + + assert unknown.read_text(encoding="utf-8") == "preserve me" + assert set(first) <= set(destination.glob("*.plist")) + + +def test_render_refuses_a_symbolic_link_at_an_owned_target(tmp_path: Path) -> None: + configuration = _configuration(tmp_path) + destination = configuration.checkout / ".context-engine" / "launchd" + first = write_rendered_templates(configuration, destination) + external = tmp_path / "external" + external.write_text("must remain unchanged", encoding="utf-8") + target = destination / "org.example.context-engine.api.plist" + target.unlink() + target.symlink_to(external) + + with pytest.raises(LaunchdRenderRefused, match="target is unsafe"): + write_rendered_templates(configuration, destination) + + assert external.read_text(encoding="utf-8") == "must remain unchanged" + assert set(path.name for path in first) - {target.name} <= { + path.name for path in destination.glob("*.plist") + } + + +def test_shell_quoted_json_environment_remains_one_live_source( + tmp_path: Path, +) -> None: + configuration = _configuration(tmp_path) + operator_environment = ( + configuration.checkout / ".context-engine" / "operators.env" + ) + operator_environment.write_text( + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON=" + "'{\"maintainer-notes\":\"/private/notes\"}'\n", + encoding="utf-8", + ) + operator_environment.chmod(0o600) + + rendered = render_launchd_templates(configuration) + + assert rendered + assert "/private/notes" not in "\n".join(rendered.values()) + + +@pytest.mark.parametrize( + "malformed", + ( + "value'", + "'value", + "'value'continued", + '"value"', + "$(touch /tmp/not-allowed)", + "`touch /tmp/not-allowed`", + "value;false", + ), +) +def test_environment_parser_refuses_unmatched_or_interior_shell_quotes( + tmp_path: Path, + malformed: str, +) -> None: + environment = tmp_path / "operators.env" + environment.write_text(f"SYNTHETIC={malformed}\n", encoding="utf-8") + environment.chmod(0o600) + + with pytest.raises(EnvironmentRefused, match="malformed"): + load_owner_environment(environment) + + +def test_operator_environment_cannot_override_the_database_contract() -> None: + with pytest.raises(EnvironmentRefused, match="single live contract"): + combined_environment( + {"CONTEXT_ENGINE_RUNTIME_DATABASE_URL": "database-source"}, + {"CONTEXT_ENGINE_RUNTIME_DATABASE_URL": "operator-source"}, + ) + + +def test_child_processes_receive_only_their_closed_credential_projection() -> None: + database = { + "CONTEXT_ENGINE_RUNTIME_DATABASE_URL": "runtime-url", + "CONTEXT_ENGINE_RUNTIME_ROLE": "runtime-role", + "CONTEXT_ENGINE_WORKER_DATABASE_URL": "worker-url", + "CONTEXT_ENGINE_WORKER_ROLE": "worker-role", + "CONTEXT_ENGINE_SCHEDULER_DATABASE_URL": "scheduler-url", + "CONTEXT_ENGINE_SCHEDULER_ROLE": "scheduler-role", + "CONTEXT_ENGINE_CONTROL_DATABASE_URL": "control-url", + "CONTEXT_ENGINE_CONTROL_ROLE": "control-role", + "CONTEXT_ENGINE_MIGRATION_DATABASE_URL": "must-not-leak", + "POSTGRES_PASSWORD": "must-not-leak", + } + operator = { + name: "configured" + for name in { + "CONTEXT_ENGINE_API_COMPOSITION", + "CONTEXT_ENGINE_DOGFOOD_AGENT_VERSION_REF", + "CONTEXT_ENGINE_DOGFOOD_APPLICATION_REF", + "CONTEXT_ENGINE_DOGFOOD_AUTHENTICATION_BINDING_REF", + "CONTEXT_ENGINE_DOGFOOD_EMBEDDING_PROVIDER", + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_ID", + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_VERSION", + "CONTEXT_ENGINE_DOGFOOD_ORGANIZATION_ID", + "CONTEXT_ENGINE_DOGFOOD_PRINCIPAL_REF", + "CONTEXT_ENGINE_DOGFOOD_SECRET", + "CONTEXT_ENGINE_DOGFOOD_USER_ID", + "CONTEXT_ENGINE_CONTROL_OPERATOR_OPERATIONS", + "CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET", + "CONTEXT_ENGINE_FILE_CHANGE_CHECKPOINT_SIGNING_KEY_HEX", + "CONTEXT_ENGINE_FILE_CHANGE_PROVIDER_SIGNING_KEY_HEX", + "CONTEXT_ENGINE_OPERATOR_ORGANIZATION_ID", + "CONTEXT_ENGINE_OPERATOR_SOURCE_REF", + "CONTEXT_ENGINE_WORKER_EMBEDDING_DIMENSION", + "CONTEXT_ENGINE_WORKER_EMBEDDING_PROVIDER", + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON", + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX", + "CONTEXT_ENGINE_WORKER_SERVICE_PRINCIPAL_ID", + "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET", + } + } + + api = process_environment("api", database, operator) + worker = process_environment("worker", database, operator) + scan = process_environment("scan", database, operator) + + assert api["CONTEXT_ENGINE_RUNTIME_DATABASE_URL"] == "runtime-url" + assert worker["CONTEXT_ENGINE_WORKER_DATABASE_URL"] == "worker-url" + assert scan["CONTEXT_ENGINE_CONTROL_DATABASE_URL"] == "control-url" + assert "POSTGRES_PASSWORD" not in api | worker | scan + assert "CONTEXT_ENGINE_MIGRATION_DATABASE_URL" not in api | worker | scan + assert "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET" not in api | worker | scan + assert "CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET" not in api | worker + assert "CONTEXT_ENGINE_DOGFOOD_SECRET" not in worker | scan + + +def test_scan_validates_cross_plane_key_collisions_before_projection() -> None: + proof_key = "11" * 32 + operator = { + "CONTEXT_ENGINE_CONTROL_OPERATOR_OPERATIONS": "read_source", + "CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET": "control-" + "a" * 32, + "CONTEXT_ENGINE_DOGFOOD_SECRET": "dogfood-" + "b" * 32, + "CONTEXT_ENGINE_FILE_CHANGE_CHECKPOINT_SIGNING_KEY_HEX": "22" * 32, + "CONTEXT_ENGINE_FILE_CHANGE_PROVIDER_SIGNING_KEY_HEX": proof_key, + "CONTEXT_ENGINE_OPERATOR_ORGANIZATION_ID": ( + "14900000-0000-4000-8000-000000000001" + ), + "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET": proof_key.upper(), + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX": "33" * 32, + } + + with pytest.raises(EnvironmentRefused, match="separation"): + validate_scan_secret_separation(operator) + + separated_operator = operator | { + "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET": "release-" + "c" * 32, + } + fingerprints = validate_scan_secret_separation(separated_operator) + assert set(fingerprints) == { + "CONTEXT_ENGINE_DOGFOOD_SECRET_SHA256", + "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET_SHA256", + } + assert proof_key not in fingerprints.values() + assert "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET" not in process_environment( + "scan", + { + "CONTEXT_ENGINE_CONTROL_DATABASE_URL": "control-url", + "CONTEXT_ENGINE_CONTROL_ROLE": "control-role", + }, + separated_operator + | { + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_ID": "membership", + "CONTEXT_ENGINE_DOGFOOD_MEMBERSHIP_VERSION": "1", + "CONTEXT_ENGINE_DOGFOOD_PRINCIPAL_REF": "principal", + "CONTEXT_ENGINE_OPERATOR_SOURCE_REF": "source", + "CONTEXT_ENGINE_WORKER_FILE_ROOTS_JSON": "{}", + "CONTEXT_ENGINE_WORKER_SERVICE_PRINCIPAL_ID": "service", + }, + ) + + +def test_scan_child_uses_fingerprints_without_receiving_release_secrets( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider_key = "11" * 32 + checkpoint_key = "22" * 32 + release_secret = "release-" + "c" * 32 + dogfood_secret = "dogfood-" + "d" * 32 + environment = { + "CONTEXT_ENGINE_CONTROL_OPERATOR_SECRET": "control-" + "a" * 32, + "CONTEXT_ENGINE_DOGFOOD_SECRET_SHA256": local_secret_fingerprint( + dogfood_secret + ), + "CONTEXT_ENGINE_FILE_CHANGE_CHECKPOINT_SIGNING_KEY_HEX": checkpoint_key, + "CONTEXT_ENGINE_FILE_CHANGE_PROVIDER_SIGNING_KEY_HEX": provider_key, + "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET_SHA256": local_secret_fingerprint( + release_secret + ), + "CONTEXT_ENGINE_WORKER_LEASE_SIGNING_KEY_HEX": "33" * 32, + } + for name, value in environment.items(): + monkeypatch.setenv(name, value) + monkeypatch.delenv("CONTEXT_ENGINE_DOGFOOD_SECRET", raising=False) + monkeypatch.delenv("CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET", raising=False) + + assert len(_proof_keys()) == 2 + + monkeypatch.setenv( + "CONTEXT_ENGINE_RELEASE_OPERATOR_SECRET_SHA256", + local_secret_fingerprint(provider_key), + ) + with pytest.raises(SourceNotAvailable): + _proof_keys() + + +def test_database_bootstrap_opens_docker_then_runs_the_idempotent_harness( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + checkout = tmp_path / "checkout" + harness = checkout / "scripts" / "database_harness.sh" + harness.parent.mkdir(parents=True) + harness.write_text("#!/bin/bash\n", encoding="utf-8") + docker = tmp_path / "docker" + uv = tmp_path / "uv" + for executable in (docker, uv): + executable.write_text("#!/bin/sh\n", encoding="utf-8") + executable.chmod(0o700) + run = Mock( + side_effect=( + subprocess.CompletedProcess(("open",), 0), + subprocess.CompletedProcess(("database_harness.sh",), 0), + ) + ) + monkeypatch.setattr(subprocess, "run", run) + + result = _run_database_bootstrap( + Namespace( + checkout=checkout, + docker_executable=docker, + uv_executable=uv, + ) + ) + + assert result == 0 + assert run.call_args_list[0].args[0] == ("/usr/bin/open", "-gja", "Docker") + assert run.call_args_list[1].args[0] == ( + "/bin/bash", + str(harness), + "up", + ) + child_environment = run.call_args_list[1].kwargs["env"] + assert str(docker.parent) in child_environment["PATH"] + assert str(uv.parent) in child_environment["PATH"] diff --git a/tests/unit/test_local_operator_authentication.py b/tests/unit/test_local_operator_authentication.py index 5a3119b4..dab3ea6b 100644 --- a/tests/unit/test_local_operator_authentication.py +++ b/tests/unit/test_local_operator_authentication.py @@ -7,7 +7,7 @@ import pytest -from applications.control import local_operator_authorities +from applications.control import local_control_operator_authority from applications.operator_authentication import ( CONTROL_OPERATOR_OPERATIONS_ENV, CONTROL_OPERATOR_SECRET_ENV, @@ -16,6 +16,7 @@ RELEASE_OPERATOR_SECRET_ENV, WORKER_SECRET_ENV, LocalControlOperatorAuthenticator, + LocalControlOperatorConfiguration, LocalOperatorAuthorities, LocalOperatorConfiguration, LocalOperatorConfigurationUnavailable, @@ -71,7 +72,7 @@ def test_operator_configuration_is_absent_by_default_and_partial_values_fail_clo for name in environment(): monkeypatch.delenv(name, raising=False) assert LocalOperatorConfiguration.load({}) is None - assert local_operator_authorities() is None + assert local_control_operator_authority() is None for missing_name in environment(): partial = environment() @@ -92,6 +93,26 @@ def test_operator_configuration_is_absent_by_default_and_partial_values_fail_clo assert RELEASE_SECRET not in repr(_configuration()) +def test_routine_control_configuration_does_not_require_release_secrets() -> None: + projected = { + name: value + for name, value in environment().items() + if name + in { + OPERATOR_ORGANIZATION_ENV, + CONTROL_OPERATOR_SECRET_ENV, + CONTROL_OPERATOR_OPERATIONS_ENV, + } + } + + configuration = LocalControlOperatorConfiguration.load(projected) + + assert configuration is not None + assert configuration.organization_id == ORGANIZATION_ID + assert RELEASE_OPERATOR_SECRET_ENV not in projected + assert DOGFOOD_SECRET_ENV not in projected + assert WORKER_SECRET_ENV not in projected + def test_control_operations_are_an_exact_enumerated_set() -> None: configuration = _configuration() assert configuration.control_operations == frozenset( diff --git a/tests/unit/test_scheduled_jobs_exclude_promotion.py b/tests/unit/test_scheduled_jobs_exclude_promotion.py new file mode 100644 index 00000000..11243ae1 --- /dev/null +++ b/tests/unit/test_scheduled_jobs_exclude_promotion.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from scripts.daily_driver.jobs import SCHEDULED_OPERATION_CATEGORIES + +ROOT = Path(__file__).resolve().parents[2] +DEFINITION = ROOT / "deploy" / "daily-driver" / "scheduled-jobs.json" +TEMPLATES = ROOT / "deploy" / "daily-driver" +FORBIDDEN_TOKENS = {"promote", "promotion", "activate", "activation", "rollback"} + + +def test_scheduled_jobs_use_only_the_closed_non_publication_allowlist() -> None: + document = json.loads(DEFINITION.read_text(encoding="utf-8")) + + assert document["allowedOperations"] == sorted(SCHEDULED_OPERATION_CATEGORIES) + assert document["jobs"] + assert {job["operation"] for job in document["jobs"]} <= ( + SCHEDULED_OPERATION_CATEGORIES + ) + assert all(job["publicationAuthority"] == "NONE" for job in document["jobs"]) + + +def test_scheduled_job_definition_contains_no_publication_operation() -> None: + scheduled_artifacts = [ + DEFINITION, + ROOT / "scripts" / "daily_driver" / "jobs.py", + *sorted(TEMPLATES.glob("*.plist.template")), + ] + for artifact in scheduled_artifacts: + content = artifact.read_text(encoding="utf-8").lower() + assert all(token not in content for token in FORBIDDEN_TOKENS), artifact diff --git a/tests/unit/test_setup_refuses_worktree.py b/tests/unit/test_setup_refuses_worktree.py new file mode 100644 index 00000000..0d6dd1a3 --- /dev/null +++ b/tests/unit/test_setup_refuses_worktree.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from scripts.daily_driver.setup import ( + DURABLE_DEPLOYMENT_MARKER, + SetupRefused, + _ensure_operator_environment, + _prepare_state_directory, + _write_durable_deployment_marker, + require_setup_target, +) + + +def _git_repository(path: Path) -> None: + subprocess.run(("git", "init", "--quiet", str(path)), check=True) + + +def test_setup_refuses_to_run_from_inside_any_git_worktree( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + _git_repository(source) + target = tmp_path / "durable" / "context-engine" + target.parent.mkdir() + + with pytest.raises(SetupRefused, match="run from outside every git worktree"): + require_setup_target(target=target, current_directory=source) + + +def test_setup_refuses_a_target_inside_an_existing_git_worktree( + tmp_path: Path, +) -> None: + parent = tmp_path / "disposable" + parent.mkdir() + _git_repository(parent) + + with pytest.raises(SetupRefused, match="outside every git worktree"): + require_setup_target( + target=parent / "context-engine", + current_directory=tmp_path, + ) + + +def test_setup_refuses_a_linked_worktree_as_the_durable_checkout( + tmp_path: Path, +) -> None: + current_directory = tmp_path / "operator" + current_directory.mkdir() + target = tmp_path / "linked-worktree" + target.mkdir() + (target / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + + with pytest.raises(SetupRefused, match="plain dedicated checkout"): + require_setup_target(target=target, current_directory=current_directory) + + +def test_setup_accepts_an_existing_plain_dedicated_checkout( + tmp_path: Path, +) -> None: + current_directory = tmp_path / "operator" + current_directory.mkdir() + target = tmp_path / "context-engine" + target.mkdir() + (target / ".git").mkdir() + + assert ( + require_setup_target(target=target, current_directory=current_directory) + == target + ) + + +def test_setup_refuses_an_absent_checkout_under_context_engine_state( + tmp_path: Path, +) -> None: + current_directory = tmp_path / "operator" + current_directory.mkdir() + + with pytest.raises(SetupRefused, match="durable root contract"): + require_setup_target( + target=tmp_path / ".context-engine" / "daily-driver", + current_directory=current_directory, + ) + + +def test_setup_marks_the_checkout_so_database_reset_refuses(tmp_path: Path) -> None: + state = tmp_path / ".context-engine" + state.mkdir() + + _write_durable_deployment_marker(state) + _write_durable_deployment_marker(state) + + marker = state / "durable-deployment" + assert marker.read_text(encoding="utf-8") == DURABLE_DEPLOYMENT_MARKER + assert marker.stat().st_mode & 0o777 == 0o600 + + +def test_setup_prepares_owner_only_state_without_overwriting_operator_values( + tmp_path: Path, +) -> None: + state = tmp_path / ".context-engine" + + _prepare_state_directory(state) + operator_environment = state / "operators.env" + _ensure_operator_environment(operator_environment) + operator_environment.write_text("SYNTHETIC=value\n", encoding="utf-8") + _ensure_operator_environment(operator_environment) + + assert state.stat().st_mode & 0o777 == 0o700 + assert operator_environment.stat().st_mode & 0o777 == 0o600 + assert operator_environment.read_text(encoding="utf-8") == "SYNTHETIC=value\n" + + +def test_setup_refuses_a_symbolic_link_state_directory(tmp_path: Path) -> None: + actual = tmp_path / "actual" + actual.mkdir() + state = tmp_path / ".context-engine" + state.symlink_to(actual, target_is_directory=True) + + with pytest.raises(SetupRefused, match="state is unsafe"): + _prepare_state_directory(state)