From a0db7f614a4a2cd3447f2f7597e188c67845c6c6 Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 22 Jul 2026 18:18:37 +0800 Subject: [PATCH 1/3] ci: enforce the complete M0 security gate --- .github/workflows/ci.yml | 15 +- AGENTS.md | 7 +- Makefile | 7 +- README.md | 7 + ...-security-veto-from-registered-evidence.md | 113 ++ docs/decisions/README.md | 2 + .../persistence/schema_security_manifest.yaml | 89 +- .../catalogs/m0-security-evidence.schema.json | 290 ++++++ eval/catalogs/m0-security-evidence.yaml | 135 +++ pyproject.toml | 1 + scripts/run_m0_security_gate.py | 41 + scripts/security_gate/__init__.py | 1 + scripts/security_gate/artifacts.py | 24 + scripts/security_gate/observations.py | 114 ++ scripts/security_gate/pytest_plugin.py | 237 +++++ scripts/security_gate/report.py | 606 +++++++++++ scripts/security_gate/rls.py | 909 ++++++++++++++++ scripts/security_gate/runner.py | 758 ++++++++++++++ scripts/validate_security_catalog.py | 611 ++++++++++- tests/catalog/test_m0_security_gate.py | 983 ++++++++++++++++++ .../test_validate_m0_security_evidence.py | 410 ++++++++ .../test_access_policy_revocation.py | 1 + .../test_authorized_field_schema.py | 1 + tests/integration/test_content_schema.py | 1 + tests/integration/test_context_run_schema.py | 1 + .../integration/test_m0_security_gate_rls.py | 247 +++++ .../test_m0_unavailable_security_carriers.py | 149 +++ ...membership_field_projection_integration.py | 34 +- tests/integration/test_membership_schema.py | 2 + .../test_organization_isolation.py | 2 + tests/integration/test_release_promotion.py | 1 + ...runtime_authorized_evidence_integration.py | 68 +- .../test_runtime_empty_package_integration.py | 1 + ...est_runtime_non_enumeration_integration.py | 30 + .../test_runtime_policy_epoch_integration.py | 1 + tests/integration/test_ticket_policy_epoch.py | 1 + tests/integration/test_worker_lease.py | 24 + tests/support/security_gate.py | 72 ++ tests/unit/test_actor_contracts.py | 1 + tests/unit/test_context_learning.py | 3 + tests/unit/test_context_run.py | 1 + tests/unit/test_database_harness_contract.py | 38 +- tests/unit/test_effective_scope.py | 1 + tests/unit/test_effective_scope_runtime.py | 14 +- tests/unit/test_evidence_contracts.py | 2 + tests/unit/test_http_trust_boundary.py | 24 +- .../test_http_unavailable_capabilities.py | 70 +- tests/unit/test_m0_delivery_carriers.py | 71 ++ tests/unit/test_m0_learning_isolation.py | 183 ++++ tests/unit/test_m0_rls_inventory.py | 367 +++++++ tests/unit/test_policy_epoch.py | 1 + .../unit/test_release_promotion_properties.py | 1 + .../unit/test_runtime_authorized_evidence.py | 144 ++- .../test_runtime_unavailable_capabilities.py | 1 + tests/unit/test_schema_security_manifest.py | 5 + tests/unit/test_ticket_audience_separation.py | 13 +- tests/unit/test_worker_lease.py | 1 + 57 files changed, 6883 insertions(+), 54 deletions(-) create mode 100644 docs/decisions/0034-execute-the-m0-security-veto-from-registered-evidence.md create mode 100644 eval/catalogs/m0-security-evidence.schema.json create mode 100644 eval/catalogs/m0-security-evidence.yaml create mode 100644 scripts/run_m0_security_gate.py create mode 100644 scripts/security_gate/__init__.py create mode 100644 scripts/security_gate/artifacts.py create mode 100644 scripts/security_gate/observations.py create mode 100644 scripts/security_gate/pytest_plugin.py create mode 100644 scripts/security_gate/report.py create mode 100644 scripts/security_gate/rls.py create mode 100644 scripts/security_gate/runner.py create mode 100644 tests/catalog/test_m0_security_gate.py create mode 100644 tests/catalog/test_validate_m0_security_evidence.py create mode 100644 tests/integration/test_m0_security_gate_rls.py create mode 100644 tests/integration/test_m0_unavailable_security_carriers.py create mode 100644 tests/support/security_gate.py create mode 100644 tests/unit/test_m0_delivery_carriers.py create mode 100644 tests/unit/test_m0_learning_isolation.py create mode 100644 tests/unit/test_m0_rls_inventory.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 587f9767..0ac60b83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,19 @@ jobs: - run: uv python install 3.13 - run: make install - run: make db-up - - run: make check + - name: Run complete checks including the M0 security gate + id: complete-checks + run: make check + - name: Retain M0 security gate evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: m0-security-gate-${{ github.sha }} + path: | + .context-engine/security-gate/raw-evidence.json + .context-engine/security-gate/release-gate-report.json + include-hidden-files: true + if-no-files-found: error + retention-days: 30 - if: always() run: make db-down diff --git a/AGENTS.md b/AGENTS.md index 906993d2..383edbaf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,7 @@ make lint # Ruff make typecheck # strict mypy make test # unit test suite make catalog # static security catalog tests and validation +make security-gate # executable M0 security veto; requires make db-up first make smoke # API and worker process smoke suite make db-up # start the real PostgreSQL 17 + pgvector harness make db-down # stop the harness while preserving its disposable data volume @@ -77,7 +78,11 @@ ignored, mode-0600 source for local database connection contracts; `compose.yaml owns the pinned test service topology. A green process smoke proves only boot/readiness. The database harness additionally proves Organization, current Membership, the online UserActor transaction, and representative-record FORCE-RLS -isolation. Content-bearing Runtime delivery and worker-job behavior remain +isolation. `make security-gate` executes the registered M0 evidence and writes +raw evidence plus its independent release-gate report beneath the ignored +`.context-engine/security-gate/` directory. Content-bearing Runtime delivery +and worker-job behavior are active only for the exact carriers recorded by +their accepted catalog activations; deferred carrier semantics remain `NOT_ACTIVE` until their owning issues verify them. ## Safety-Rails / Do Not diff --git a/Makefile b/Makefile index 9e2dd430..bf949816 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install build lint typecheck test catalog smoke db-up db-down db-reset integration check +.PHONY: install build lint typecheck test catalog security-gate smoke db-up db-down db-reset integration check install: uv sync --frozen @@ -19,6 +19,9 @@ catalog: uv run pytest -q tests/catalog uv run python scripts/validate_security_catalog.py +security-gate: + uv run python scripts/run_m0_security_gate.py --output-dir .context-engine/security-gate + smoke: uv run pytest -q tests/process @@ -34,4 +37,4 @@ db-reset: integration: ./scripts/database_harness.sh integration -check: build lint typecheck test catalog smoke integration +check: build lint typecheck test catalog smoke integration security-gate diff --git a/README.md b/README.md index 45fbaed5..9fd9cd8a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ make lint # Ruff make typecheck # strict mypy make test # 单元测试 make catalog # 安全目录静态测试与校验 +make security-gate # 可执行 M0 安全否决门；要求先执行 make db-up make smoke # API / worker 进程 smoke make db-up # 启动 compose.yaml 固定的 PostgreSQL + pgvector 测试底座 make db-down # 停止测试底座并保留 disposable data volume @@ -58,6 +59,12 @@ make integration # 真实 PostgreSQL integration/security harness make check # 全部门禁；要求先执行 make db-up ``` +`make security-gate` 会发现并只执行注册的 M0 安全证据，核对真实 PostgreSQL +RLS inventory，并将机器可读的原始证据与四门 release report 写入被 Git 忽略的 +`.context-engine/security-gate/`。Security 是独立否决门；尚未进入 M0 评估范围的 +Reliability、Quality 与 Budget 明确记录为 `not-evaluated`，所以这份报告只会给出 +`m0SecurityDecision`，不会把安全门通过误写成可发布或可 promotion 的总体 PASS。 + 数据库底座首次启动时会在被 Git 忽略的 `.context-engine/database.env` 生成随机凭据并将文件权限设为 `0600`；该文件是 本地 migration、API Runtime、worker、security test 连接配置和该 checkout diff --git a/docs/decisions/0034-execute-the-m0-security-veto-from-registered-evidence.md b/docs/decisions/0034-execute-the-m0-security-veto-from-registered-evidence.md new file mode 100644 index 00000000..13a37ff2 --- /dev/null +++ b/docs/decisions/0034-execute-the-m0-security-veto-from-registered-evidence.md @@ -0,0 +1,113 @@ +--- +name: adr-0034-executable-m0-security-veto +version: "1.0.0" +description: > + Join the frozen security authority to exact executable evidence, live RLS + inventory, and provenance-bearing release artifacts without inventing + product behavior or aggregate scoring. +--- + +# 0034. Execute the M0 security veto from registered evidence + +- Status: accepted +- Date: 2026-07-22 +- Refines: ADR-0015, ADR-0019 + +## Context + +The normalized security catalog fixes fifteen invariant families and twelve +acceptance fixtures, including the expected fail-closed outcome for carriers +that are not active at M0. Its evidence identifiers intentionally describe the +full delivery roadmap. They are design authority, but a string in that catalog +does not prove that a current test exists, was collected, ran without a skip, +or passed against the required seam. + +The schema security manifest is likewise the declared table-classification +authority. Reading only its `rls.enabled` fields would let drift in the live +database, a newly unclassified table, or removed `FORCE ROW LEVEL SECURITY` +escape a release report. Finally, reducing Security, Reliability, Quality, and +Budget to one score would allow an unrelated result to offset a failed +authorization invariant. + +## Decision + +The static security catalog remains the semantic authority. A separate, +versioned M0 evidence registry joins current evidence identifiers to exact +pytest selectors, one evidence class (`property`, `postgres`, or `runtime`), +the invariant families genuinely proved, and the acceptance fixtures whose +explicit observations they emit. The registry must cover every invariant in +all three evidence classes and every canonical fixture. Duplicate identifiers, +unknown references, empty or missing mappings, uncollected selectors, skips, +xfails, xpasses, and failed setup, call, or teardown outcomes are deterministic +Security failures. Registered selectors are de-duplicated and executed once; +the gate has no retry path. + +Fixture tests publish the three hard-oracle observations rather than relying +on pytest success as a substitute for measurement: unauthorized Evidence, +wrong-Organization effect, and missing-context fallback. Every required value +is zero. A missing or non-zero observation fails Security and the command. +Inactive M0 carriers are tested at their current public fail-closed boundary; +the gate does not add their future product semantics. + +The schema security manifest is the complete public-table denominator. Each +table is exactly one of `tenant_owned` or explicitly allowlisted `global`, and +each global entry carries a non-empty rationale. For every tenant-owned table, +the gate verifies a machine-declared Organization ownership path, enabled and +forced live RLS, at least one live policy, and registered non-owner PostgreSQL +evidence. It first requires the live public-table set to equal the manifest, so +an unclassified table cannot disappear from the denominator. Coverage is +reported per table and as numerator over denominator; M0 requires complete +coverage. + +One local and CI command performs static validation, exact evidence execution, +the live PostgreSQL audit, and report generation. It writes canonical raw +evidence plus a release-gate report. Provenance includes the Git commit and +tracked-worktree state, catalog/registry/schema/configuration digests, Alembic +head and live revision, exact execution command, and a deterministic digest of +normalized test outcomes and observations. Wall-clock durations remain in raw +evidence but are excluded from that normalized result digest. Provenance never +records database URLs or credentials. + +The release report has four independent sections: Security, Reliability, +Quality, and Budget. Security is `pass` or `fail` and is a veto. The other +three are explicitly `not-evaluated` at M0 until their owning gates exist; +`not-evaluated` is never converted to pass. The document has no aggregate +score, weighted score, or averaging field. A passing Security section sets only +`m0SecurityDecision: pass`; `releaseDecision` and `promotionReadiness` remain +`not-evaluated`, so this integration report is not a promotion-ready +`ReleaseEvaluation`. + +## Rationale + +Separating expected roadmap evidence from the executable registry preserves +stable design identifiers while making the current proof discoverable and +strict. Exact pytest collection closes the gap between a referenced test name +and an executed assertion. Explicit observations prevent the runner from +manufacturing hard-oracle zeros from a green process exit. + +Using the manifest as the denominator and PostgreSQL catalogs as live facts +makes complete RLS coverage measurable without guessing from column names. +Keeping the four gates independent preserves the Security veto and represents +unevaluated work honestly. + +## Consequences + +Adding or renaming a registered test, invariant, fixture, or application table +requires an intentional registry or manifest update. A deterministic skip is a +red gate, not a temporary green build. Registered evidence may support multiple +families only when its registry references and assertions name each proof. + +The executable gate re-runs a focused subset after the broad unit and +integration suites so it can retain exact release evidence. That extra runtime +is accepted for M0. Generated artifacts remain ignored locally and are retained +by CI; they are evidence, not source authority. + +## Revisit trigger + +Revisit the registry transport if pytest is replaced or signed remote evidence +becomes necessary. Revisit the three unevaluated sections only when their +independent deterministic commands and provenance contracts exist. Any +replacement must still execute all canonical mappings without skips or retries, +derive hard-oracle results from explicit observations, audit the complete live +RLS denominator, retain raw evidence, and preserve Security as a non-aggregate +veto. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 6f4d5e55..fae35dd1 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -35,6 +35,7 @@ kernel, capability separation, and publication visibility model. | Release promotion owner | [0033 — Organization release promotion owner](0033-promote-organization-releases-through-one-learning-owner.md) | Organization-owned immutable release lineage advances only through one generation-bound, release-operator-authorized `ContextLearning.promote` transaction | Pointer seeds, direct application DML, manifest-only CAS, evaluator/Control/Curation publication, or rollback mutation | | Publication visibility | [0018 — Immutable ContextRevision publication](0018-immutable-revision-publication.md) | `ContextResource` content is immutable `ContextRevision`/`ContextFragment` lineage; one transaction changes the active pointer | In-place content mutation, mixed old/new reads, or cleanup-defined visibility | | Release security catalog | [0019 — Security catalog normalization](0019-security-catalog-normalization.md) | One machine catalog contains exactly fifteen stable release IDs; overlapping labels and derived scenarios keep their safeguards without inflating the count | Parallel prose catalogs, renumbering, or treating inactive cache behavior as a canonical release family | +| Executable M0 security veto | [0034 — Registered executable security evidence](0034-execute-the-m0-security-veto-from-registered-evidence.md) | Exact current tests, explicit hard-oracle observations, and live all-table RLS facts produce provenance-bearing independent gate artifacts | Planned IDs presented as executed proof, skip/retry-to-green, manifest-only RLS claims, or aggregate scoring | Each baseline ADR is `accepted` and contains Context, Decision, Rationale, Consequences, and Revisit trigger sections. A revisit trigger permits review; it @@ -115,3 +116,4 @@ touched: - [0031 — Authorized-only ContextRun lineage](0031-persist-authorized-context-run-lineage.md) - [0032 — Membership-bound materialized fields](0032-bind-materialized-fields-to-membership-projection-rights.md) - [0033 — Organization release promotion owner](0033-promote-organization-releases-through-one-learning-owner.md) +- [0034 — Registered executable security evidence](0034-execute-the-m0-security-veto-from-registered-evidence.md) diff --git a/engine/persistence/schema_security_manifest.yaml b/engine/persistence/schema_security_manifest.yaml index 50b90212..04a5b8dc 100644 --- a/engine/persistence/schema_security_manifest.yaml +++ b/engine/persistence/schema_security_manifest.yaml @@ -87,12 +87,14 @@ { "name": "alembic_version", "classification": "global", + "classificationRationale": "Migration metadata has no tenant-owned domain state.", "purpose": "Alembic migration metadata", "partitions": [] }, { "name": "organization", "classification": "global", + "classificationRationale": "Organization is the global tenant security root and is not owned by another Organization.", "purpose": "Organization security root", "primaryKey": { "name": "pk_organization", @@ -107,6 +109,7 @@ { "name": "user_account", "classification": "global", + "classificationRationale": "User is a global identity root; tenant rights exist only through tenant-owned Membership rows.", "purpose": "Global user identity root without tenant rights", "primaryKey": { "name": "pk_user_account", @@ -121,6 +124,10 @@ { "name": "membership", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-SCOPE-INTERSECTION-004", + "selector": {"table": "membership"} + }, "purpose": "Organization-scoped current UserActor membership authority", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -211,6 +218,10 @@ { "name": "organization_record", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-TENANT-FK-002", + "selector": {"table": "organization_record"} + }, "purpose": "Representative tenant-owned record protected by current UserActor Membership", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -295,6 +306,10 @@ { "name": "context_resource", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-INDEX-NOT-AUTHORITY-005", + "selector": {"table": "context_resource"} + }, "purpose": "Stable Organization-owned source object and active immutable Revision pointer", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -358,6 +373,10 @@ { "name": "context_revision", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-INDEX-NOT-AUTHORITY-005", + "selector": {"table": "context_revision"} + }, "purpose": "Immutable canonical Revision lineage selected only by its Resource active pointer", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -425,6 +444,10 @@ { "name": "organization_policy_epoch", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-REVOCATION-006", + "selector": {"table": "organization_policy_epoch"} + }, "purpose": "Organization-owned monotonic freshness authority for Acquire decisions", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -507,6 +530,10 @@ { "name": "resource_access_policy", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-REVOCATION-006", + "selector": {"table": "resource_access_policy"} + }, "purpose": "Exact Principal-to-Resource access state changed only with its Organization epoch", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -615,6 +642,10 @@ { "name": "context_run", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-TRACE-REDACTION-012", + "selector": {"table": "context_run"} + }, "purpose": "Authorized-only durable lineage for one finalized ContextPackage delivery; package and query bodies are never retained", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -778,6 +809,10 @@ { "name": "context_run_operator_read_ticket", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-TRACE-REDACTION-012", + "selector": {"table": "context_run_operator_read_ticket"} + }, "purpose": "Digest-only 60-second exact-read capability for one Organization and ContextRun decision; plaintext ticket material is never retained", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -825,19 +860,19 @@ "forced": true, "policies": [ { - "name": "context_run_operator_read_ticket_context_run_reader_definer_insert", + "name": "context_run_operator_read_ticket_context_run_reader_definer_ins", "command": "INSERT", "roles": ["context_engine_context_run_reader_definer"], "withCheck": "current_setting('app.context_run_operator_ticket_mode', true) = 'issue' AND context_run_operator_read_ticket.organization_id = NULLIF(current_setting('app.context_run_operator_ticket_organization_id', true), '')::uuid AND context_run_operator_read_ticket.decision_ref = current_setting('app.context_run_operator_ticket_decision_ref', true) AND context_run_operator_read_ticket.ticket_digest = pg_catalog.decode(NULLIF(current_setting('app.context_run_operator_ticket_digest', true), ''), 'hex')" }, { - "name": "context_run_operator_read_ticket_context_run_reader_definer_delete", + "name": "context_run_operator_read_ticket_context_run_reader_definer_del", "command": "DELETE", "roles": ["context_engine_context_run_reader_definer"], "using": "(current_setting('app.context_run_operator_ticket_mode', true) = 'revoke' OR (current_setting('app.context_run_operator_ticket_mode', true) = 'read' AND context_run_operator_read_ticket.organization_id = NULLIF(current_setting('app.context_run_operator_ticket_organization_id', true), '')::uuid AND context_run_operator_read_ticket.decision_ref = current_setting('app.context_run_operator_ticket_decision_ref', true))) AND context_run_operator_read_ticket.ticket_digest = pg_catalog.decode(NULLIF(current_setting('app.context_run_operator_ticket_digest', true), ''), 'hex')" }, { - "name": "context_run_operator_read_ticket_context_run_reader_definer_select", + "name": "context_run_operator_read_ticket_context_run_reader_definer_sel", "command": "SELECT", "roles": ["context_engine_context_run_reader_definer"], "using": "(current_setting('app.context_run_operator_ticket_mode', true) = 'revoke' OR (current_setting('app.context_run_operator_ticket_mode', true) = 'read' AND context_run_operator_read_ticket.organization_id = NULLIF(current_setting('app.context_run_operator_ticket_organization_id', true), '')::uuid AND context_run_operator_read_ticket.decision_ref = current_setting('app.context_run_operator_ticket_decision_ref', true))) AND context_run_operator_read_ticket.ticket_digest = pg_catalog.decode(NULLIF(current_setting('app.context_run_operator_ticket_digest', true), ''), 'hex')" @@ -895,6 +930,10 @@ { "name": "decision_audit", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-TRACE-REDACTION-012", + "selector": {"table": "decision_audit"} + }, "purpose": "Restricted generic security lineage for a delivered-empty decision; never tenant-visible and structurally unable to retain denied detail", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -998,6 +1037,10 @@ { "name": "service_principal", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-WORKER-LEASE-007", + "selector": {"table": "service_principal"} + }, "purpose": "Bounded registered receiver principal for the Issue #17 no-op carrier; not the full canonical ServiceActor", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1091,6 +1134,10 @@ { "name": "worker_noop_job", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-WORKER-LEASE-007", + "selector": {"table": "worker_noop_job"} + }, "purpose": "Exact-job durable state and one-shot receipt for the bounded no-op WorkerLease", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1216,6 +1263,10 @@ { "name": "context_fragment", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-INDEX-NOT-AUTHORITY-005", + "selector": {"table": "context_fragment"} + }, "purpose": "Immutable active-Revision Fragment content; persistent rows are never Evidence", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1312,6 +1363,10 @@ { "name": "context_fragment_field", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-FIELD-PROJECTION-RLS-048", + "selector": {"table": "context_fragment_field"} + }, "purpose": "Immutable structured Fragment fields filtered by exact current Membership Resource rights before Runtime can read values", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1392,6 +1447,10 @@ { "name": "membership_resource_field_right", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-FIELD-PROJECTION-RLS-048", + "selector": {"table": "membership_resource_field_right"} + }, "purpose": "Closed allowlist binding one exact Membership version to readable fields of one same-Organization Resource; absence denies", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1466,6 +1525,10 @@ { "name": "release_manifest", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-RELEASE-OWNER-019", + "selector": {"table": "release_manifest"} + }, "purpose": "Immutable Organization-owned release profile composition and curation compatibility lineage", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1566,6 +1629,10 @@ { "name": "release_candidate", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-RELEASE-OWNER-019", + "selector": {"table": "release_candidate"} + }, "purpose": "Immutable Organization-owned candidate binding an exact manifest and generation-bound expected active state", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1669,6 +1736,10 @@ { "name": "release_evaluation", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-RELEASE-OWNER-019", + "selector": {"table": "release_evaluation"} + }, "purpose": "Immutable signed evaluation evidence binding one exact candidate, manifest, veto gates and compatibility result", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1784,6 +1855,10 @@ { "name": "release_operator_grant", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-RELEASE-OWNER-019", + "selector": {"table": "release_operator_grant"} + }, "purpose": "Current Organization-scoped release-operator authority revalidated only by the promotion definer", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1858,6 +1933,10 @@ { "name": "active_release_manifest", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-RELEASE-OWNER-019", + "selector": {"table": "active_release_manifest"} + }, "purpose": "Generation-bound Organization active release pointer changed only by the sole promotion function", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ @@ -1933,6 +2012,10 @@ { "name": "release_promotion_audit", "classification": "tenant_owned", + "nonOwnerEvidence": { + "evidenceId": "PG-RELEASE-OWNER-019", + "selector": {"table": "release_promotion_audit"} + }, "purpose": "Immutable append-only success audit paired atomically with each active release generation", "organizationColumn": "organization_id", "organizationInclusiveKeys": [ diff --git a/eval/catalogs/m0-security-evidence.schema.json b/eval/catalogs/m0-security-evidence.schema.json new file mode 100644 index 00000000..53f160cf --- /dev/null +++ b/eval/catalogs/m0-security-evidence.schema.json @@ -0,0 +1,290 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://contextengine.local/schemas/m0-security-evidence.schema.json", + "title": "ContextEngine M0 executable security evidence registry", + "type": "object", + "additionalProperties": false, + "required": [ + "registryVersion", + "catalog", + "execution", + "hardOracleAdapters", + "evidence", + "invariantMappings", + "fixtureMappings" + ], + "properties": { + "registryVersion": { + "const": "1.0.0" + }, + "catalog": { + "const": { + "path": "eval/catalogs/security-invariants.yaml", + "schemaPath": "eval/catalogs/security-catalog.schema.json", + "catalogVersion": "1.1.0" + } + }, + "execution": { + "const": { + "framework": "pytest", + "forbidOutcomes": [ + "failed", + "error", + "skipped", + "xfailed", + "xpassed", + "incomplete" + ], + "evidenceLayers": [ + "property", + "postgres", + "runtime" + ], + "observationProperty": "context_engine.security_gate.observation.v1" + } + }, + "hardOracleAdapters": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "uniqueItems": true, + "prefixItems": [ + { + "const": { + "oracleRef": "Unauthorized Evidence", + "resultKey": "unauthorizedEvidenceCount", + "assertion": "equals", + "requiredValue": 0, + "observation": { + "source": "pytest-user-property", + "reducer": "sum" + } + } + }, + { + "const": { + "oracleRef": "wrong-Organization effect", + "resultKey": "wrongOrganizationEffectCount", + "assertion": "equals", + "requiredValue": 0, + "observation": { + "source": "pytest-user-property", + "reducer": "sum" + } + } + }, + { + "const": { + "oracleRef": "missing-context fallback", + "resultKey": "missingContextFallbackCount", + "assertion": "equals", + "requiredValue": 0, + "observation": { + "source": "pytest-user-property", + "reducer": "sum" + } + } + } + ], + "items": false + }, + "evidence": { + "type": "array", + "minItems": 27, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/evidence" + } + }, + "invariantMappings": { + "type": "array", + "minItems": 15, + "maxItems": 15, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/invariantMapping" + } + }, + "fixtureMappings": { + "type": "array", + "minItems": 12, + "maxItems": 12, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/fixtureMapping" + } + } + }, + "$defs": { + "invariantId": { + "type": "string", + "enum": [ + "TENANT-OWNERSHIP-001", + "TENANT-FK-002", + "RLS-FAIL-CLOSED-003", + "SCOPE-INTERSECTION-004", + "INDEX-NOT-AUTHORITY-005", + "REVOCATION-006", + "WORKER-LEASE-007", + "TRANSPORT-UNTRUSTED-008", + "NON-ENUMERATION-009", + "CITATION-AUTH-010", + "EGRESS-011", + "TRACE-REDACTION-012", + "ACTION-SEPARATION-014", + "CROSS-ORG-LEARN-015", + "RELEASE-OWNER-019" + ] + }, + "fixtureId": { + "type": "string", + "enum": [ + "ACCEPT-001", + "ACCEPT-002", + "ACCEPT-003", + "ACCEPT-004", + "ACCEPT-005", + "ACCEPT-006", + "ACCEPT-007", + "ACCEPT-008", + "ACCEPT-009", + "ACCEPT-010", + "ACCEPT-011", + "ACCEPT-012" + ] + }, + "evidenceRef": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Z][A-Z0-9-]+$" + }, + "evidenceRefList": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/evidenceRef" + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "layer", + "selector" + ], + "properties": { + "id": { + "$ref": "#/$defs/evidenceRef" + }, + "layer": { + "type": "string", + "enum": [ + "property", + "postgres", + "runtime" + ] + }, + "selector": { + "type": "string", + "minLength": 1, + "pattern": "^tests/(?:unit|integration)/test_[A-Za-z0-9_]+\\.py::test_[A-Za-z0-9_]+$" + } + } + }, + "layerEvidenceRefs": { + "type": "object", + "additionalProperties": false, + "required": [ + "property", + "postgres", + "runtime" + ], + "properties": { + "property": { + "$ref": "#/$defs/evidenceRefList" + }, + "postgres": { + "$ref": "#/$defs/evidenceRefList" + }, + "runtime": { + "$ref": "#/$defs/evidenceRefList" + } + } + }, + "invariantMapping": { + "type": "object", + "additionalProperties": false, + "required": [ + "invariantRef", + "evidenceRefs" + ], + "properties": { + "invariantRef": { + "$ref": "#/$defs/invariantId" + }, + "evidenceRefs": { + "$ref": "#/$defs/layerEvidenceRefs" + } + } + }, + "hardOracleEvidenceRefs": { + "type": "object", + "additionalProperties": false, + "required": [ + "unauthorizedEvidenceCount", + "wrongOrganizationEffectCount", + "missingContextFallbackCount" + ], + "properties": { + "unauthorizedEvidenceCount": { + "$ref": "#/$defs/evidenceRefList" + }, + "wrongOrganizationEffectCount": { + "$ref": "#/$defs/evidenceRefList" + }, + "missingContextFallbackCount": { + "$ref": "#/$defs/evidenceRefList" + } + } + }, + "fixtureMapping": { + "type": "object", + "additionalProperties": false, + "required": [ + "fixtureRef", + "carrierStatusAtM0", + "m0Expectation", + "evidenceRefs", + "hardOracleEvidenceRefs" + ], + "properties": { + "fixtureRef": { + "$ref": "#/$defs/fixtureId" + }, + "carrierStatusAtM0": { + "type": "string", + "enum": [ + "available", + "future", + "unavailable" + ] + }, + "m0Expectation": { + "type": "string", + "enum": [ + "active_fail_closed", + "fail_closed" + ] + }, + "evidenceRefs": { + "$ref": "#/$defs/evidenceRefList" + }, + "hardOracleEvidenceRefs": { + "$ref": "#/$defs/hardOracleEvidenceRefs" + } + } + } + } +} diff --git a/eval/catalogs/m0-security-evidence.yaml b/eval/catalogs/m0-security-evidence.yaml new file mode 100644 index 00000000..accd237a --- /dev/null +++ b/eval/catalogs/m0-security-evidence.yaml @@ -0,0 +1,135 @@ +{ + "registryVersion": "1.0.0", + "catalog": { + "path": "eval/catalogs/security-invariants.yaml", + "schemaPath": "eval/catalogs/security-catalog.schema.json", + "catalogVersion": "1.1.0" + }, + "execution": { + "framework": "pytest", + "forbidOutcomes": [ + "failed", + "error", + "skipped", + "xfailed", + "xpassed", + "incomplete" + ], + "evidenceLayers": ["property", "postgres", "runtime"], + "observationProperty": "context_engine.security_gate.observation.v1" + }, + "hardOracleAdapters": [ + { + "oracleRef": "Unauthorized Evidence", + "resultKey": "unauthorizedEvidenceCount", + "assertion": "equals", + "requiredValue": 0, + "observation": {"source": "pytest-user-property", "reducer": "sum"} + }, + { + "oracleRef": "wrong-Organization effect", + "resultKey": "wrongOrganizationEffectCount", + "assertion": "equals", + "requiredValue": 0, + "observation": {"source": "pytest-user-property", "reducer": "sum"} + }, + { + "oracleRef": "missing-context fallback", + "resultKey": "missingContextFallbackCount", + "assertion": "equals", + "requiredValue": 0, + "observation": {"source": "pytest-user-property", "reducer": "sum"} + } + ], + "evidence": [ + {"id": "PROP-TENANT-OWNERSHIP-001", "layer": "property", "selector": "tests/unit/test_schema_security_manifest.py::test_manifest_classifies_the_exact_issue_49_release_schema"}, + {"id": "PG-RLS-ALL-TENANT-TABLES", "layer": "postgres", "selector": "tests/integration/test_m0_security_gate_rls.py::test_all_manifest_tenant_tables_pass_live_non_owner_rls_audit"}, + {"id": "RUNTIME-TENANT-OWNERSHIP-001", "layer": "runtime", "selector": "tests/integration/test_runtime_authorized_evidence_integration.py::test_real_postgres_http_delivers_only_exact_authorized_evidence_bidirectionally"}, + {"id": "PROP-TENANT-FK-002", "layer": "property", "selector": "tests/unit/test_schema_security_manifest.py::test_content_manifest_preserves_lineage_visibility_and_immutability"}, + {"id": "PG-TENANT-FK-002", "layer": "postgres", "selector": "tests/integration/test_organization_isolation.py::test_composite_ownership_accepts_same_org_and_rejects_cross_org_parent"}, + {"id": "RUNTIME-TENANT-FK-002", "layer": "runtime", "selector": "tests/integration/test_runtime_authorized_evidence_integration.py::test_real_postgres_http_delivers_only_exact_authorized_evidence_bidirectionally"}, + {"id": "PROP-RLS-FAIL-CLOSED-003", "layer": "property", "selector": "tests/unit/test_schema_security_manifest.py::test_tenant_owned_manifest_entry_preserves_every_security_property"}, + {"id": "PG-RLS-FAIL-CLOSED-003", "layer": "postgres", "selector": "tests/integration/test_organization_isolation.py::test_missing_tenant_context_is_fail_closed_for_every_operation"}, + {"id": "RUNTIME-RLS-FAIL-CLOSED-003", "layer": "runtime", "selector": "tests/integration/test_runtime_empty_package_integration.py::test_real_postgres_http_membership_matrix_is_generic_and_zero_io"}, + {"id": "PROP-SCOPE-INTERSECTION-004", "layer": "property", "selector": "tests/unit/test_effective_scope.py::test_generated_trusted_operand_narrowing_never_expands_effective_scope"}, + {"id": "PG-SCOPE-INTERSECTION-004", "layer": "postgres", "selector": "tests/integration/test_membership_schema.py::test_runtime_membership_rls_is_bidirectional_and_exact"}, + {"id": "PG-FIELD-PROJECTION-RLS-048", "layer": "postgres", "selector": "tests/integration/test_authorized_field_schema.py::test_cross_organization_field_authority_and_values_fail_closed"}, + {"id": "RUNTIME-SCOPE-INTERSECTION-004", "layer": "runtime", "selector": "tests/unit/test_effective_scope_runtime.py::test_runtime_observes_only_effective_scope_and_returns_empty_package"}, + {"id": "PROP-INDEX-NOT-AUTHORITY-005", "layer": "property", "selector": "tests/unit/test_evidence_contracts.py::test_package_content_can_only_be_constructed_from_authorized_projections"}, + {"id": "PG-INDEX-NOT-AUTHORITY-005", "layer": "postgres", "selector": "tests/integration/test_content_schema.py::test_runtime_rls_exposes_only_active_non_tombstoned_same_organization_rows"}, + {"id": "RUNTIME-INDEX-NOT-AUTHORITY-005", "layer": "runtime", "selector": "tests/unit/test_runtime_authorized_evidence.py::test_hostile_candidate_order_delivers_only_exact_authorized_evidence"}, + {"id": "PROP-REVOCATION-006", "layer": "property", "selector": "tests/unit/test_policy_epoch.py::test_current_validation_re_reads_durable_epoch_and_fails_closed_on_change"}, + {"id": "PG-REVOCATION-006", "layer": "postgres", "selector": "tests/integration/test_access_policy_revocation.py::test_change_access_atomically_revokes_exact_grant_and_advances_epoch"}, + {"id": "RUNTIME-REVOCATION-006", "layer": "runtime", "selector": "tests/integration/test_runtime_policy_epoch_integration.py::test_same_http_acquire_revokes_next_delivery_without_candidate_cleanup"}, + {"id": "PROP-WORKER-LEASE-007", "layer": "property", "selector": "tests/unit/test_worker_lease.py::test_wrong_binding_not_yet_valid_and_expired_are_generic"}, + {"id": "PG-WORKER-LEASE-007", "layer": "postgres", "selector": "tests/integration/test_worker_lease.py::test_wrong_organization_cannot_affect_same_job_identifier_in_another_tenant"}, + {"id": "RUNTIME-WORKER-LEASE-007", "layer": "runtime", "selector": "tests/integration/test_worker_lease.py::test_valid_lease_completes_once_through_worker_application_and_replay_is_zero"}, + {"id": "PROP-TRANSPORT-UNTRUSTED-008", "layer": "property", "selector": "tests/unit/test_actor_contracts.py::test_authenticated_invocation_rejects_mismatched_current_membership"}, + {"id": "PG-TRANSPORT-UNTRUSTED-008", "layer": "postgres", "selector": "tests/integration/test_membership_schema.py::test_organization_only_and_user_only_context_have_no_tenant_rights"}, + {"id": "RUNTIME-TRANSPORT-UNTRUSTED-008", "layer": "runtime", "selector": "tests/unit/test_http_trust_boundary.py::test_trusted_field_injection_is_closed_before_domain_execution"}, + {"id": "PROP-NON-ENUMERATION-009", "layer": "property", "selector": "tests/unit/test_runtime_authorized_evidence.py::test_denied_cross_organization_and_missing_candidates_share_one_runtime_outcome"}, + {"id": "PG-NON-ENUMERATION-009", "layer": "postgres", "selector": "tests/integration/test_runtime_non_enumeration_integration.py::test_real_postgres_http_denied_and_missing_are_externally_equivalent"}, + {"id": "RUNTIME-NON-ENUMERATION-009", "layer": "runtime", "selector": "tests/unit/test_runtime_authorized_evidence.py::test_runtime_canonical_empty_package_is_equal_for_every_internal_branch"}, + {"id": "PROP-CITATION-AUTH-010", "layer": "property", "selector": "tests/unit/test_runtime_unavailable_capabilities.py::test_capability_declarations_are_closed_and_m0_does_not_false_green_carriers"}, + {"id": "PG-CITATION-AUTH-010", "layer": "postgres", "selector": "tests/integration/test_m0_unavailable_security_carriers.py::test_m0_unavailable_citation_and_egress_carriers_fail_closed"}, + {"id": "RUNTIME-CITATION-AUTH-010", "layer": "runtime", "selector": "tests/unit/test_http_unavailable_capabilities.py::test_accept_010_open_citation_is_generic_and_zero_io"}, + {"id": "PROP-EGRESS-011", "layer": "property", "selector": "tests/unit/test_evidence_contracts.py::test_package_constructor_rejects_cross_organization_or_mixed_request"}, + {"id": "PG-EGRESS-011", "layer": "postgres", "selector": "tests/integration/test_m0_unavailable_security_carriers.py::test_m0_unavailable_citation_and_egress_carriers_fail_closed"}, + {"id": "RUNTIME-EGRESS-011", "layer": "runtime", "selector": "tests/unit/test_m0_delivery_carriers.py::test_m0_egress_carrier_is_unavailable_before_model_or_sender_bytes"}, + {"id": "PROP-TRACE-REDACTION-012", "layer": "property", "selector": "tests/unit/test_context_run.py::test_records_retain_digests_not_raw_query_package_or_denial_details"}, + {"id": "PG-TRACE-REDACTION-012", "layer": "postgres", "selector": "tests/integration/test_context_run_schema.py::test_runtime_cross_organization_insert_attempts_are_bidirectionally_zero_effect"}, + {"id": "RUNTIME-TRACE-REDACTION-012", "layer": "runtime", "selector": "tests/unit/test_runtime_authorized_evidence.py::test_empty_decision_audit_is_generic_and_retains_no_denied_detail"}, + {"id": "PROP-ACTION-SEPARATION-014", "layer": "property", "selector": "tests/unit/test_ticket_audience_separation.py::test_public_ticket_types_and_server_side_issuers_are_structurally_distinct"}, + {"id": "PG-ACTION-SEPARATION-014", "layer": "postgres", "selector": "tests/integration/test_ticket_policy_epoch.py::test_committed_epoch_bump_rejects_both_previously_valid_ticket_types"}, + {"id": "RUNTIME-ACTION-SEPARATION-014", "layer": "runtime", "selector": "tests/unit/test_ticket_audience_separation.py::test_each_ticket_is_rejected_by_the_other_plane_and_deserializer"}, + {"id": "PROP-CROSS-ORG-LEARN-015", "layer": "property", "selector": "tests/unit/test_m0_learning_isolation.py::test_m0_learning_artifact_contract_has_no_cross_organization_carrier"}, + {"id": "PG-CROSS-ORG-LEARN-015", "layer": "postgres", "selector": "tests/integration/test_m0_unavailable_security_carriers.py::test_learning_persistence_is_organization_bound"}, + {"id": "RUNTIME-CROSS-ORG-LEARN-015", "layer": "runtime", "selector": "tests/unit/test_m0_learning_isolation.py::test_context_learning_rejects_cross_organization_candidate_lineage"}, + {"id": "PROP-RELEASE-OWNER-019", "layer": "property", "selector": "tests/unit/test_release_promotion_properties.py::test_generated_rejected_promotions_leave_pointer_generation_and_success_audit_unchanged"}, + {"id": "PG-RELEASE-OWNER-019", "layer": "postgres", "selector": "tests/integration/test_release_promotion.py::test_promote_rechecks_current_operator_authority_digests_and_compatibility"}, + {"id": "RUNTIME-RELEASE-OWNER-019", "layer": "runtime", "selector": "tests/unit/test_context_learning.py::test_initial_activation_ordinary_promotion_and_rollback_share_promote"}, + {"id": "FIXTURE-ACCEPT-001", "layer": "runtime", "selector": "tests/integration/test_runtime_authorized_evidence_integration.py::test_real_postgres_http_delivers_only_exact_authorized_evidence_bidirectionally"}, + {"id": "FIXTURE-ACCEPT-002", "layer": "runtime", "selector": "tests/integration/test_membership_field_projection_integration.py::test_accept_002_same_organization_memberships_receive_only_authorized_fields"}, + {"id": "FIXTURE-ACCEPT-003", "layer": "runtime", "selector": "tests/unit/test_runtime_authorized_evidence.py::test_accept_003_agent_ceiling_delivers_only_authorized_projection"}, + {"id": "FIXTURE-ACCEPT-004", "layer": "runtime", "selector": "tests/unit/test_runtime_authorized_evidence.py::test_request_narrowing_filters_candidate_before_body_projection"}, + {"id": "FIXTURE-ACCEPT-005", "layer": "runtime", "selector": "tests/unit/test_http_unavailable_capabilities.py::test_accept_005_continue_is_generic_non_retryable_and_zero_io"}, + {"id": "FIXTURE-ACCEPT-006", "layer": "runtime", "selector": "tests/unit/test_runtime_authorized_evidence.py::test_hostile_candidate_order_delivers_only_exact_authorized_evidence"}, + {"id": "FIXTURE-ACCEPT-007", "layer": "runtime", "selector": "tests/unit/test_http_trust_boundary.py::test_trusted_field_injection_is_closed_before_domain_execution"}, + {"id": "FIXTURE-ACCEPT-008", "layer": "runtime", "selector": "tests/integration/test_worker_lease.py::test_valid_lease_completes_once_through_worker_application_and_replay_is_zero"}, + {"id": "FIXTURE-ACCEPT-009", "layer": "runtime", "selector": "tests/unit/test_http_unavailable_capabilities.py::test_accept_009_server_owned_unavailable_source_paths_are_generic_and_zero_io"}, + {"id": "FIXTURE-ACCEPT-010", "layer": "runtime", "selector": "tests/unit/test_http_unavailable_capabilities.py::test_accept_010_open_citation_is_generic_and_zero_io"}, + {"id": "FIXTURE-ACCEPT-011", "layer": "runtime", "selector": "tests/integration/test_runtime_non_enumeration_integration.py::test_real_postgres_http_denied_and_missing_are_externally_equivalent"}, + {"id": "FIXTURE-ACCEPT-012", "layer": "runtime", "selector": "tests/unit/test_m0_delivery_carriers.py::test_accept_012_action_plane_is_unavailable_before_ticket_or_effect"} + ], + "invariantMappings": [ + {"invariantRef": "TENANT-OWNERSHIP-001", "evidenceRefs": {"property": ["PROP-TENANT-OWNERSHIP-001"], "postgres": ["PG-RLS-ALL-TENANT-TABLES"], "runtime": ["RUNTIME-TENANT-OWNERSHIP-001"]}}, + {"invariantRef": "TENANT-FK-002", "evidenceRefs": {"property": ["PROP-TENANT-FK-002"], "postgres": ["PG-TENANT-FK-002"], "runtime": ["RUNTIME-TENANT-FK-002"]}}, + {"invariantRef": "RLS-FAIL-CLOSED-003", "evidenceRefs": {"property": ["PROP-RLS-FAIL-CLOSED-003"], "postgres": ["PG-RLS-FAIL-CLOSED-003", "PG-RLS-ALL-TENANT-TABLES"], "runtime": ["RUNTIME-RLS-FAIL-CLOSED-003"]}}, + {"invariantRef": "SCOPE-INTERSECTION-004", "evidenceRefs": {"property": ["PROP-SCOPE-INTERSECTION-004"], "postgres": ["PG-SCOPE-INTERSECTION-004", "PG-FIELD-PROJECTION-RLS-048"], "runtime": ["RUNTIME-SCOPE-INTERSECTION-004"]}}, + {"invariantRef": "INDEX-NOT-AUTHORITY-005", "evidenceRefs": {"property": ["PROP-INDEX-NOT-AUTHORITY-005"], "postgres": ["PG-INDEX-NOT-AUTHORITY-005"], "runtime": ["RUNTIME-INDEX-NOT-AUTHORITY-005"]}}, + {"invariantRef": "REVOCATION-006", "evidenceRefs": {"property": ["PROP-REVOCATION-006"], "postgres": ["PG-REVOCATION-006"], "runtime": ["RUNTIME-REVOCATION-006"]}}, + {"invariantRef": "WORKER-LEASE-007", "evidenceRefs": {"property": ["PROP-WORKER-LEASE-007"], "postgres": ["PG-WORKER-LEASE-007"], "runtime": ["RUNTIME-WORKER-LEASE-007"]}}, + {"invariantRef": "TRANSPORT-UNTRUSTED-008", "evidenceRefs": {"property": ["PROP-TRANSPORT-UNTRUSTED-008"], "postgres": ["PG-TRANSPORT-UNTRUSTED-008"], "runtime": ["RUNTIME-TRANSPORT-UNTRUSTED-008"]}}, + {"invariantRef": "NON-ENUMERATION-009", "evidenceRefs": {"property": ["PROP-NON-ENUMERATION-009"], "postgres": ["PG-NON-ENUMERATION-009"], "runtime": ["RUNTIME-NON-ENUMERATION-009"]}}, + {"invariantRef": "CITATION-AUTH-010", "evidenceRefs": {"property": ["PROP-CITATION-AUTH-010"], "postgres": ["PG-CITATION-AUTH-010"], "runtime": ["RUNTIME-CITATION-AUTH-010"]}}, + {"invariantRef": "EGRESS-011", "evidenceRefs": {"property": ["PROP-EGRESS-011"], "postgres": ["PG-EGRESS-011"], "runtime": ["RUNTIME-EGRESS-011"]}}, + {"invariantRef": "TRACE-REDACTION-012", "evidenceRefs": {"property": ["PROP-TRACE-REDACTION-012"], "postgres": ["PG-TRACE-REDACTION-012"], "runtime": ["RUNTIME-TRACE-REDACTION-012"]}}, + {"invariantRef": "ACTION-SEPARATION-014", "evidenceRefs": {"property": ["PROP-ACTION-SEPARATION-014"], "postgres": ["PG-ACTION-SEPARATION-014"], "runtime": ["RUNTIME-ACTION-SEPARATION-014"]}}, + {"invariantRef": "CROSS-ORG-LEARN-015", "evidenceRefs": {"property": ["PROP-CROSS-ORG-LEARN-015"], "postgres": ["PG-CROSS-ORG-LEARN-015"], "runtime": ["RUNTIME-CROSS-ORG-LEARN-015"]}}, + {"invariantRef": "RELEASE-OWNER-019", "evidenceRefs": {"property": ["PROP-RELEASE-OWNER-019"], "postgres": ["PG-RELEASE-OWNER-019"], "runtime": ["RUNTIME-RELEASE-OWNER-019"]}} + ], + "fixtureMappings": [ + {"fixtureRef": "ACCEPT-001", "carrierStatusAtM0": "available", "m0Expectation": "active_fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-001"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-001"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-001"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-001"]}}, + {"fixtureRef": "ACCEPT-002", "carrierStatusAtM0": "available", "m0Expectation": "active_fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-002"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-002"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-002"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-002"]}}, + {"fixtureRef": "ACCEPT-003", "carrierStatusAtM0": "available", "m0Expectation": "active_fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-003"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-003"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-003"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-003"]}}, + {"fixtureRef": "ACCEPT-004", "carrierStatusAtM0": "available", "m0Expectation": "active_fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-004"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-004"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-004"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-004"]}}, + {"fixtureRef": "ACCEPT-005", "carrierStatusAtM0": "future", "m0Expectation": "fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-005"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-005"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-005"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-005"]}}, + {"fixtureRef": "ACCEPT-006", "carrierStatusAtM0": "available", "m0Expectation": "active_fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-006"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-006"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-006"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-006"]}}, + {"fixtureRef": "ACCEPT-007", "carrierStatusAtM0": "available", "m0Expectation": "active_fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-007"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-007"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-007"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-007"]}}, + {"fixtureRef": "ACCEPT-008", "carrierStatusAtM0": "future", "m0Expectation": "fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-008"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-008"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-008"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-008"]}}, + {"fixtureRef": "ACCEPT-009", "carrierStatusAtM0": "future", "m0Expectation": "fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-009"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-009"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-009"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-009"]}}, + {"fixtureRef": "ACCEPT-010", "carrierStatusAtM0": "future", "m0Expectation": "fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-010"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-010"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-010"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-010"]}}, + {"fixtureRef": "ACCEPT-011", "carrierStatusAtM0": "available", "m0Expectation": "active_fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-011"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-011"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-011"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-011"]}}, + {"fixtureRef": "ACCEPT-012", "carrierStatusAtM0": "unavailable", "m0Expectation": "fail_closed", "evidenceRefs": ["FIXTURE-ACCEPT-012"], "hardOracleEvidenceRefs": {"unauthorizedEvidenceCount": ["FIXTURE-ACCEPT-012"], "wrongOrganizationEffectCount": ["FIXTURE-ACCEPT-012"], "missingContextFallbackCount": ["FIXTURE-ACCEPT-012"]}} + ] +} diff --git a/pyproject.toml b/pyproject.toml index ac94ea83..193c248b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ files = [ testpaths = ["tests"] markers = [ "integration: requires the real PostgreSQL 17 + pgvector harness", + "security_evidence(id, layer): exact executable M0 security evidence registration", ] [tool.ruff] diff --git a/scripts/run_m0_security_gate.py b/scripts/run_m0_security_gate.py new file mode 100644 index 00000000..06fddbbd --- /dev/null +++ b/scripts/run_m0_security_gate.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Run the complete one-shot M0 security gate and retain its artifacts.""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Sequence +from pathlib import Path + +from scripts.security_gate.runner import GatePaths, run_gate + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(".context-engine/security-gate"), + help="artifact directory (default: .context-engine/security-gate)", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = build_parser().parse_args(argv) + paths = GatePaths.defaults(arguments.output_dir.resolve()) + try: + report = run_gate(paths) + except Exception as error: + print(f"M0 security gate failed: {type(error).__name__}", file=sys.stderr) + return 1 + if report.get("m0SecurityDecision") != "pass": + print("M0 SECURITY FAIL", file=sys.stderr) + return 1 + print(f"M0 SECURITY PASS ({paths.output_directory})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/security_gate/__init__.py b/scripts/security_gate/__init__.py new file mode 100644 index 00000000..560adf3d --- /dev/null +++ b/scripts/security_gate/__init__.py @@ -0,0 +1 @@ +"""Executable M0 security-gate components.""" diff --git a/scripts/security_gate/artifacts.py b/scripts/security_gate/artifacts.py new file mode 100644 index 00000000..4f3664ad --- /dev/null +++ b/scripts/security_gate/artifacts.py @@ -0,0 +1,24 @@ +"""Shared atomic artifact writes for the M0 security gate.""" + +from __future__ import annotations + +import json +import os +from collections.abc import Mapping +from contextlib import suppress +from pathlib import Path + + +def atomic_write_json(path: Path, value: Mapping[str, object]) -> None: + """Replace one JSON artifact atomically after a complete durable write.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + with temporary.open("w", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, ensure_ascii=False, sort_keys=True, indent=2) + stream.write("\n") + os.replace(temporary, path) + finally: + with suppress(FileNotFoundError): + temporary.unlink() diff --git a/scripts/security_gate/observations.py b/scripts/security_gate/observations.py new file mode 100644 index 00000000..f43d5a07 --- /dev/null +++ b/scripts/security_gate/observations.py @@ -0,0 +1,114 @@ +"""Closed pytest user-property contract for measured hard-oracle observations.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping, Sequence + +OBSERVATION_PROPERTY = "context_engine.security_gate.observation.v1" +SECURITY_EVIDENCE_MARKER = "security_evidence" +SECURITY_EVIDENCE_MARKER_VERSION = "1.0.0" +SECURITY_EVIDENCE_LAYERS = frozenset({"property", "postgres", "runtime"}) +ORACLE_RESULT_KEYS: tuple[str, ...] = ( + "unauthorizedEvidenceCount", + "wrongOrganizationEffectCount", + "missingContextFallbackCount", +) +_FIXTURE_REF = re.compile(r"^ACCEPT-[0-9]{3}$") +_EVIDENCE_REF = re.compile(r"^[A-Z][A-Z0-9-]+$") + + +class ObservationValidationError(ValueError): + """A fixture observation is incomplete, ambiguous, or noncanonical.""" + + +class SecurityEvidenceMarkerValidationError(ValueError): + """A collected test's execution-owned evidence identity is not canonical.""" + + +def _count(value: object, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ObservationValidationError( + f"{field} must be an integer greater than or equal to zero" + ) + return value + + +def normalize_fixture_observation(value: object) -> dict[str, object]: + """Validate and copy one exact, JSON-safe hard-oracle observation.""" + + if not isinstance(value, Mapping): + raise ObservationValidationError("observation must be an object") + if set(value) != {"fixtureRef", "evidenceRef", "values"}: + raise ObservationValidationError( + "observation fields must be fixtureRef, evidenceRef, and values" + ) + fixture_ref = value.get("fixtureRef") + evidence_ref = value.get("evidenceRef") + values = value.get("values") + if not isinstance(fixture_ref, str) or _FIXTURE_REF.fullmatch(fixture_ref) is None: + raise ObservationValidationError("fixtureRef must be an ACCEPT-NNN ID") + if ( + not isinstance(evidence_ref, str) + or _EVIDENCE_REF.fullmatch(evidence_ref) is None + ): + raise ObservationValidationError("evidenceRef must be a stable evidence ID") + if not isinstance(values, Mapping) or set(values) != set(ORACLE_RESULT_KEYS): + raise ObservationValidationError( + "values must contain exactly the three canonical hard-oracle counts" + ) + normalized_values = { + key: _count(values.get(key), key) for key in ORACLE_RESULT_KEYS + } + return { + "fixtureRef": fixture_ref, + "evidenceRef": evidence_ref, + "values": normalized_values, + } + + +def normalize_security_evidence_marker( + args: Sequence[object], kwargs: Mapping[str, object] +) -> dict[str, str]: + """Validate one closed ``security_evidence`` v1 marker declaration.""" + + if args or set(kwargs) != {"id", "layer"}: + raise SecurityEvidenceMarkerValidationError( + "security_evidence marker must declare exactly id and layer keywords" + ) + evidence_id = kwargs.get("id") + layer = kwargs.get("layer") + if not isinstance(evidence_id, str) or _EVIDENCE_REF.fullmatch(evidence_id) is None: + raise SecurityEvidenceMarkerValidationError( + "security_evidence marker id must be a stable evidence ID" + ) + if not isinstance(layer, str) or layer not in SECURITY_EVIDENCE_LAYERS: + raise SecurityEvidenceMarkerValidationError( + "security_evidence marker layer must be property, postgres, or runtime" + ) + return {"id": evidence_id, "layer": layer} + + +def record_fixture_observation( + record_property: Callable[[str, object], None], + *, + fixture_ref: str, + evidence_ref: str, + unauthorized_evidence_count: int, + wrong_organization_effect_count: int, + missing_context_fallback_count: int, +) -> None: + """Publish one explicit observation after the fixture's assertions complete.""" + + observation = normalize_fixture_observation( + { + "fixtureRef": fixture_ref, + "evidenceRef": evidence_ref, + "values": { + "unauthorizedEvidenceCount": unauthorized_evidence_count, + "wrongOrganizationEffectCount": wrong_organization_effect_count, + "missingContextFallbackCount": missing_context_fallback_count, + }, + } + ) + record_property(OBSERVATION_PROPERTY, observation) diff --git a/scripts/security_gate/pytest_plugin.py b/scripts/security_gate/pytest_plugin.py new file mode 100644 index 00000000..561ccb19 --- /dev/null +++ b/scripts/security_gate/pytest_plugin.py @@ -0,0 +1,237 @@ +"""Small pytest plugin that emits deterministic raw M0 gate evidence.""" + +from __future__ import annotations + +import os +from collections.abc import Sequence +from pathlib import Path +from typing import Any, Protocol, cast + +import pytest +from _pytest.config.argparsing import Parser +from _pytest.reports import CollectReport, TestReport + +from scripts.security_gate.artifacts import atomic_write_json +from scripts.security_gate.observations import ( + OBSERVATION_PROPERTY, + SECURITY_EVIDENCE_MARKER, + SECURITY_EVIDENCE_MARKER_VERSION, + ObservationValidationError, + SecurityEvidenceMarkerValidationError, + normalize_fixture_observation, + normalize_security_evidence_marker, +) + +RAW_PATH_ENVIRONMENT = "CONTEXT_ENGINE_SECURITY_GATE_RAW_PATH" +RAW_EXECUTION_ID_ENVIRONMENT = "CONTEXT_ENGINE_SECURITY_GATE_EXECUTION_ID" +RAW_EVIDENCE_VERSION = "1.0.0" + + +class _ConfigLike(Protocol): + _m0_security_gate_recorder: RawEvidenceRecorder + + +_ACTIVE_RECORDER: RawEvidenceRecorder | None = None + + +def _report_outcome(report: TestReport) -> str: + if report.skipped: + if hasattr(report, "wasxfail"): + return "xfailed" + return "skipped" + if report.passed and hasattr(report, "wasxfail"): + return "xpassed" + if report.failed: + return "failed" if report.when == "call" else "error" + return "passed" + + +class RawEvidenceRecorder: + """Accumulate collection, phase, and explicit observation facts.""" + + def __init__( + self, + output_path: Path, + *, + runner_execution_id: str | None, + selected_selectors: Sequence[str], + ) -> None: + self.output_path = output_path + self.runner_execution_id = runner_execution_id + self.selected_selectors = list(selected_selectors) + self.collected_node_ids: list[str] = [] + self.collected_tests: list[dict[str, object]] = [] + self.collection_errors: list[dict[str, str]] = [] + self._reports: dict[str, dict[str, object]] = {} + + def collect(self, items: Sequence[pytest.Item]) -> None: + self.collected_node_ids = [item.nodeid for item in items] + collected_tests: list[dict[str, object]] = [] + for item in items: + declarations: list[dict[str, str]] = [] + errors: list[str] = [] + seen: set[tuple[str, str]] = set() + for marker in item.iter_markers(name=SECURITY_EVIDENCE_MARKER): + try: + declaration = normalize_security_evidence_marker( + marker.args, marker.kwargs + ) + except SecurityEvidenceMarkerValidationError as error: + errors.append(str(error)) + continue + identity = (declaration["id"], declaration["layer"]) + if identity in seen: + errors.append( + "duplicate security_evidence marker: " + f"{declaration['id']} {declaration['layer']}" + ) + continue + seen.add(identity) + declarations.append(declaration) + collected_tests.append( + { + "nodeId": item.nodeid, + "securityEvidence": sorted( + declarations, key=lambda value: (value["id"], value["layer"]) + ), + "securityEvidenceErrors": errors, + } + ) + self.collected_tests = collected_tests + + def collect_report(self, report: CollectReport) -> None: + if report.failed: + self.collection_errors.append( + {"nodeId": report.nodeid, "message": str(report.longrepr)} + ) + + def test_report(self, report: TestReport) -> None: + record = self._reports.setdefault( + report.nodeid, + { + "nodeId": report.nodeid, + "outcome": "incomplete", + "phases": [], + "observations": [], + "observationErrors": [], + }, + ) + phases = cast(list[dict[str, object]], record["phases"]) + phases.append( + { + "phase": report.when, + "outcome": _report_outcome(report), + "durationSeconds": report.duration, + } + ) + outcome = _report_outcome(report) + current = cast(str, record["outcome"]) + precedence = { + "incomplete": 0, + "passed": 1, + "xpassed": 2, + "xfailed": 3, + "skipped": 4, + "failed": 5, + "error": 6, + } + if precedence[outcome] >= precedence[current]: + record["outcome"] = outcome + if report.when != "call": + return + observations = cast(list[dict[str, object]], record["observations"]) + errors = cast(list[str], record["observationErrors"]) + for key, value in report.user_properties: + if key != OBSERVATION_PROPERTY: + continue + try: + observations.append(normalize_fixture_observation(value)) + except ObservationValidationError as error: + errors.append(str(error)) + + def document(self, exit_code: int) -> dict[str, object]: + return { + "rawEvidenceVersion": RAW_EVIDENCE_VERSION, + "rawProducer": "scripts.security_gate.pytest_plugin", + "runnerExecutionId": self.runner_execution_id, + "pytest": { + "exitCode": exit_code, + "selectedSelectors": self.selected_selectors, + "securityEvidenceMarkerVersion": SECURITY_EVIDENCE_MARKER_VERSION, + "collectedNodeIds": self.collected_node_ids, + "collectedTests": self.collected_tests, + "collectionErrors": sorted( + self.collection_errors, + key=lambda item: (item["nodeId"], item["message"]), + ), + "tests": [self._reports[node_id] for node_id in sorted(self._reports)], + }, + } + + def write(self, exit_code: int) -> None: + atomic_write_json(self.output_path, self.document(exit_code)) + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("context-engine-security-gate") + group.addoption( + "--security-gate-raw", + action="store", + dest="security_gate_raw", + default=None, + help="write raw M0 security-gate evidence to this JSON path", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + global _ACTIVE_RECORDER + configured = config.getoption("security_gate_raw") + environment = os.environ.get(RAW_PATH_ENVIRONMENT) + raw_path = configured or environment + if raw_path is None: + return + config.addinivalue_line( + "markers", + "security_evidence(id, layer): versioned execution-owned M0 evidence identity", + ) + recorder = RawEvidenceRecorder( + Path(raw_path), + runner_execution_id=os.environ.get(RAW_EXECUTION_ID_ENVIRONMENT), + selected_selectors=config.args, + ) + cast(_ConfigLike, config)._m0_security_gate_recorder = recorder + _ACTIVE_RECORDER = recorder + + +def _recorder(config: pytest.Config) -> RawEvidenceRecorder | None: + value: Any = getattr(config, "_m0_security_gate_recorder", None) + return value if isinstance(value, RawEvidenceRecorder) else None + + +def pytest_collection_finish(session: pytest.Session) -> None: + recorder = _recorder(session.config) + if recorder is not None: + recorder.collect(session.items) + + +def pytest_collectreport(report: CollectReport) -> None: + if _ACTIVE_RECORDER is not None: + _ACTIVE_RECORDER.collect_report(report) + + +def pytest_runtest_logreport(report: TestReport) -> None: + if _ACTIVE_RECORDER is not None: + _ACTIVE_RECORDER.test_report(report) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + recorder = _recorder(session.config) + if recorder is not None: + recorder.write(exitstatus) + + +def pytest_unconfigure(config: pytest.Config) -> None: + global _ACTIVE_RECORDER + if _recorder(config) is _ACTIVE_RECORDER: + _ACTIVE_RECORDER = None diff --git a/scripts/security_gate/report.py b/scripts/security_gate/report.py new file mode 100644 index 00000000..12bf9dcd --- /dev/null +++ b/scripts/security_gate/report.py @@ -0,0 +1,606 @@ +"""Pure reconciliation and independent release-gate report construction.""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from collections.abc import Mapping, Sequence +from typing import cast + +from scripts.security_gate.observations import ( + ORACLE_RESULT_KEYS, + SECURITY_EVIDENCE_MARKER_VERSION, +) + +REPORT_VERSION = "1.0.0" +TERMINAL_SUCCESS = "passed" +FORBIDDEN_OUTCOMES = frozenset( + {"failed", "error", "skipped", "xfailed", "xpassed", "incomplete"} +) +_DIGEST_PROVENANCE_FIELDS = ( + "catalogDigest", + "catalogSchemaDigest", + "configurationDigest", + "contentStateDigest", + "executionRegistryDigest", + "executionRegistrySchemaDigest", + "fixtureDigest", + "migrationStateDigest", + "rawResultDigest", + "schemaManifestDigest", + "testResultDigest", + "trackedDiffDigest", + "stagedDiffDigest", + "unstagedDiffDigest", + "untrackedContentDigest", +) + + +def canonical_json(value: object) -> bytes: + """Return stable UTF-8 JSON for report/config digest provenance.""" + + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def canonical_digest(value: object) -> str: + """Hash one canonical JSON document with SHA-256.""" + + return hashlib.sha256(canonical_json(value)).hexdigest() + + +def _provenance_failures(provenance: Mapping[str, object]) -> list[str]: + failures: list[str] = [] + commit = provenance.get("commit") + if not isinstance(commit, str) or len(commit) not in {40, 64} or any( + character not in "0123456789abcdef" for character in commit + ): + failures.append("provenance Git commit is unavailable or invalid") + for field in _DIGEST_PROVENANCE_FIELDS: + value = provenance.get(field) + if not isinstance(value, str) or len(value) != 64 or any( + character not in "0123456789abcdef" for character in value + ): + failures.append(f"provenance {field} is unavailable or invalid") + if not isinstance(provenance.get("trackedDirty"), bool): + failures.append("provenance trackedDirty is unavailable") + if not isinstance(provenance.get("contentStateDirty"), bool): + failures.append("provenance contentStateDirty is unavailable") + untracked_count = provenance.get("untrackedFileCount") + if ( + isinstance(untracked_count, bool) + or not isinstance(untracked_count, int) + or untracked_count < 0 + ): + failures.append("provenance untrackedFileCount is unavailable or invalid") + execution_command = provenance.get("executionCommand") + if not isinstance(execution_command, list) or not execution_command or not all( + isinstance(value, str) and value for value in execution_command + ): + failures.append("provenance executionCommand is unavailable or invalid") + alembic_head = provenance.get("alembicHead") + live_revision = provenance.get("liveDatabaseRevision") + if not isinstance(alembic_head, str) or not alembic_head: + failures.append("provenance Alembic head is unavailable") + if live_revision != alembic_head: + failures.append("provenance live database revision differs from Alembic head") + if not isinstance(provenance.get("runnerVersion"), str): + failures.append("provenance runner version is unavailable") + return failures + + +def canonical_test_results(raw_evidence: Mapping[str, object]) -> dict[str, object]: + """Project nondeterministic raw timing away from the result digest.""" + + pytest_section = _mapping(raw_evidence.get("pytest")) + tests: list[dict[str, object]] = [] + for test in _mapping_sequence(pytest_section.get("tests")): + tests.append( + { + "nodeId": test.get("nodeId"), + "outcome": test.get("outcome"), + "observations": test.get("observations", []), + "observationErrors": test.get("observationErrors", []), + } + ) + return { + "exitCode": pytest_section.get("exitCode"), + "selectedSelectors": pytest_section.get("selectedSelectors", []), + "collectedNodeIds": pytest_section.get("collectedNodeIds", []), + "securityEvidenceMarkerVersion": pytest_section.get( + "securityEvidenceMarkerVersion" + ), + "collectedTests": pytest_section.get("collectedTests", []), + "collectionErrors": pytest_section.get("collectionErrors", []), + "tests": sorted(tests, key=lambda item: str(item["nodeId"])), + } + + +def _mapping(value: object) -> Mapping[str, object]: + return value if isinstance(value, Mapping) else {} + + +def _mapping_sequence(value: object) -> list[Mapping[str, object]]: + if not isinstance(value, Sequence) or isinstance(value, str | bytes): + return [] + return [item for item in value if isinstance(item, Mapping)] + + +def _strings(value: object) -> list[str]: + if not isinstance(value, Sequence) or isinstance(value, str | bytes): + return [] + return [item for item in value if isinstance(item, str)] + + +def _selector_matches(selector: str, node_id: str) -> bool: + return node_id == selector or node_id.startswith(f"{selector}[") + + +def _registry_evidence( + registry: Mapping[str, object], +) -> tuple[dict[str, Mapping[str, object]], dict[str, str]]: + evidence_by_id: dict[str, Mapping[str, object]] = {} + selector_by_id: dict[str, str] = {} + for entry in _mapping_sequence(registry.get("evidence")): + evidence_id = entry.get("id") + selector = entry.get("selector") + if isinstance(evidence_id, str) and isinstance(selector, str): + evidence_by_id[evidence_id] = entry + selector_by_id[evidence_id] = selector + return evidence_by_id, selector_by_id + + +def _mapped_evidence_refs(mapping: Mapping[str, object]) -> list[str]: + evidence_refs = mapping.get("evidenceRefs") + if isinstance(evidence_refs, Mapping): + refs: list[str] = [] + for layer in ("property", "postgres", "runtime"): + refs.extend(_strings(evidence_refs.get(layer))) + return refs + return _strings(evidence_refs) + + +def _expected_observations( + registry: Mapping[str, object], +) -> set[tuple[str, str, str]]: + expected: set[tuple[str, str, str]] = set() + for fixture in _mapping_sequence(registry.get("fixtureMappings")): + fixture_ref = fixture.get("fixtureRef") + hard_refs = _mapping(fixture.get("hardOracleEvidenceRefs")) + if not isinstance(fixture_ref, str): + continue + for result_key in ORACLE_RESULT_KEYS: + for evidence_ref in _strings(hard_refs.get(result_key)): + expected.add((fixture_ref, evidence_ref, result_key)) + return expected + + +def reconcile_execution( + registry: Mapping[str, object], + raw_evidence: Mapping[str, object], + *, + catalog: Mapping[str, object] | None = None, +) -> dict[str, object]: + """Reconcile exact selected pytest leaves, mappings, and explicit counters.""" + + failures: list[str] = [] + evidence_by_id, selector_by_id = _registry_evidence(registry) + pytest_section = _mapping(raw_evidence.get("pytest")) + tests = _mapping_sequence(pytest_section.get("tests")) + test_by_node: dict[str, Mapping[str, object]] = {} + for test in tests: + node_id = test.get("nodeId") + if not isinstance(node_id, str): + failures.append("test result has no exact nodeId") + continue + if node_id in test_by_node: + failures.append(f"duplicate test result: {node_id}") + test_by_node[node_id] = test + + collected_node_ids = _strings(pytest_section.get("collectedNodeIds")) + if Counter(collected_node_ids) != Counter(test_by_node.keys()): + failures.append("collected node IDs differ from test results") + if pytest_section.get("securityEvidenceMarkerVersion") != ( + SECURITY_EVIDENCE_MARKER_VERSION + ): + failures.append("security_evidence marker version is missing or unsupported") + + collected_metadata_by_node: dict[str, Mapping[str, object]] = {} + for metadata in _mapping_sequence(pytest_section.get("collectedTests")): + node_id = metadata.get("nodeId") + if not isinstance(node_id, str): + failures.append("collected test metadata has no exact nodeId") + continue + if node_id in collected_metadata_by_node: + failures.append(f"duplicate collected test metadata: {node_id}") + collected_metadata_by_node[node_id] = metadata + if Counter(collected_node_ids) != Counter(collected_metadata_by_node.keys()): + failures.append("collected test metadata differs from collected node IDs") + + declared_by_node: dict[str, Counter[tuple[str, str]]] = {} + for node_id, metadata in collected_metadata_by_node.items(): + declaration_counter: Counter[tuple[str, str]] = Counter() + for declaration in _mapping_sequence(metadata.get("securityEvidence")): + evidence_id = declaration.get("id") + layer = declaration.get("layer") + if not isinstance(evidence_id, str) or not isinstance(layer, str): + failures.append(f"malformed security_evidence marker at {node_id}") + continue + pair = (evidence_id, layer) + declaration_counter[pair] += 1 + if declaration_counter[pair] > 1: + failures.append( + "duplicate security_evidence marker at " + f"{node_id}: {evidence_id} {layer}" + ) + registry_entry = evidence_by_id.get(evidence_id) + if registry_entry is None: + failures.append( + f"orphan security_evidence marker at {node_id}: {evidence_id}" + ) + continue + expected_layer = registry_entry.get("layer") + if layer != expected_layer: + failures.append( + "wrong security_evidence layer at " + f"{node_id}: {evidence_id} expected {expected_layer!r}, " + f"got {layer!r}" + ) + selector = selector_by_id.get(evidence_id) + if selector is None or not _selector_matches(selector, node_id): + failures.append( + "security_evidence marker emitted by wrong test: " + f"{node_id} {evidence_id}" + ) + errors = _strings(metadata.get("securityEvidenceErrors")) + failures.extend( + f"malformed security_evidence marker at {node_id}: {error}" + for error in errors + ) + declared_by_node[node_id] = declaration_counter + + if pytest_section.get("exitCode") != 0: + failures.append(f"pytest exited nonzero: {pytest_section.get('exitCode')!r}") + collection_errors = _mapping_sequence(pytest_section.get("collectionErrors")) + if collection_errors: + failures.append("pytest collection reported errors") + + selected = _strings(pytest_section.get("selectedSelectors")) + expected_selectors = list(dict.fromkeys(selector_by_id.values())) + if selected != expected_selectors: + failures.append( + "selected selectors differ from the deduplicated registry order" + ) + + evidence_results: list[dict[str, object]] = [] + passed_evidence_ids: set[str] = set() + for evidence_id, selector in selector_by_id.items(): + matching = [ + test + for node_id, test in test_by_node.items() + if _selector_matches(selector, node_id) + ] + evidence_failures: list[str] = [] + if not matching: + evidence_failures.append("missing test") + for test in matching: + node_id = cast(str, test.get("nodeId")) + layer = evidence_by_id.get(evidence_id, {}).get("layer") + if not isinstance(layer, str) or declared_by_node.get(node_id, Counter())[ + (evidence_id, layer) + ] != 1: + evidence_failures.append( + f"missing security_evidence marker at {node_id}" + ) + outcome = test.get("outcome") + if outcome != TERMINAL_SUCCESS: + evidence_failures.append( + f"forbidden outcome {outcome!r} at {test.get('nodeId')}" + ) + observation_errors = _strings(test.get("observationErrors")) + if observation_errors: + evidence_failures.extend( + f"malformed observation at {test.get('nodeId')}: {message}" + for message in observation_errors + ) + if not evidence_failures: + passed_evidence_ids.add(evidence_id) + else: + failures.extend( + f"{evidence_id}: {message}" for message in evidence_failures + ) + evidence_results.append( + { + "evidenceRef": evidence_id, + "selector": selector, + "nodeIds": sorted( + cast(str, test["nodeId"]) + for test in matching + if isinstance(test.get("nodeId"), str) + ), + "status": "PASS" if not evidence_failures else "FAIL", + "failures": evidence_failures, + } + ) + + expected_observations = _expected_observations(registry) + observations: list[dict[str, object]] = [] + observed_triples: Counter[tuple[str, str, str]] = Counter() + fixture_leaf_pairs: Counter[tuple[str, str, str]] = Counter() + for test in tests: + node_id = test.get("nodeId") + if not isinstance(node_id, str): + continue + for observation in _mapping_sequence(test.get("observations")): + fixture_ref = observation.get("fixtureRef") + evidence_ref = observation.get("evidenceRef") + values = _mapping(observation.get("values")) + if not isinstance(fixture_ref, str) or not isinstance(evidence_ref, str): + failures.append(f"malformed observation at {node_id}") + continue + fixture_leaf_pair = (node_id, fixture_ref, evidence_ref) + fixture_leaf_pairs[fixture_leaf_pair] += 1 + if fixture_leaf_pairs[fixture_leaf_pair] > 1: + failures.append( + "duplicate observation within one test leaf: " + f"{node_id} {fixture_ref} {evidence_ref}" + ) + observation_selector = selector_by_id.get(evidence_ref) + if observation_selector is None or not _selector_matches( + observation_selector, node_id + ): + failures.append( + "observation emitted by wrong test: " + f"{node_id} {fixture_ref} {evidence_ref}" + ) + normalized_values: dict[str, int] = {} + for result_key in ORACLE_RESULT_KEYS: + value = values.get(result_key) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + failures.append( + f"malformed observation value at {node_id}: {result_key}" + ) + continue + normalized_values[result_key] = value + triple = (fixture_ref, evidence_ref, result_key) + observed_triples[triple] += 1 + if triple not in expected_observations: + failures.append( + "orphan observation: " + f"{fixture_ref} {evidence_ref} {result_key} at {node_id}" + ) + observations.append( + { + "nodeId": node_id, + "fixtureRef": fixture_ref, + "evidenceRef": evidence_ref, + "values": normalized_values, + } + ) + + for fixture_ref, evidence_ref, result_key in sorted(expected_observations): + if observed_triples[(fixture_ref, evidence_ref, result_key)] == 0: + failures.append( + "missing observation: " + f"{fixture_ref} {evidence_ref} {result_key}" + ) + + for fixture in _mapping_sequence(registry.get("fixtureMappings")): + fixture_ref = fixture.get("fixtureRef") + if not isinstance(fixture_ref, str): + continue + for evidence_ref in _mapped_evidence_refs(fixture): + fixture_selector = selector_by_id.get(evidence_ref) + if fixture_selector is None: + continue + for node_id in test_by_node: + if ( + _selector_matches(fixture_selector, node_id) + and fixture_leaf_pairs[(node_id, fixture_ref, evidence_ref)] == 0 + ): + failures.append( + "missing observation at fixture test leaf: " + f"{node_id} {fixture_ref} {evidence_ref}" + ) + + fixture_results: list[dict[str, object]] = [] + failed_fixture_refs: set[str] = set() + for mapping in _mapping_sequence(registry.get("fixtureMappings")): + fixture_ref = mapping.get("fixtureRef") + refs = _mapped_evidence_refs(mapping) + missing_evidence = [ref for ref in refs if ref not in passed_evidence_ids] + fixture_observations = [ + observation + for observation in observations + if observation["fixtureRef"] == fixture_ref + ] + missing_observation = any( + triple[0] == fixture_ref and observed_triples[triple] == 0 + for triple in expected_observations + ) + hard_refs = _mapping(mapping.get("hardOracleEvidenceRefs")) + failed_oracle_result_keys = [ + result_key + for result_key in ORACLE_RESULT_KEYS + if any( + observation["fixtureRef"] == fixture_ref + and observation["evidenceRef"] in _strings(hard_refs.get(result_key)) + and cast(Mapping[str, object], observation["values"]).get(result_key) + != 0 + for observation in fixture_observations + ) + ] + fixture_failed = bool( + missing_evidence or missing_observation or failed_oracle_result_keys + ) + if fixture_failed and isinstance(fixture_ref, str): + failed_fixture_refs.add(fixture_ref) + fixture_results.append( + { + "fixtureRef": fixture_ref, + "status": "FAIL" if fixture_failed else "PASS", + "evidenceRefs": refs, + "failedEvidenceRefs": missing_evidence, + "failedOracleResultKeys": failed_oracle_result_keys, + "carrierStatusAtM0": mapping.get("carrierStatusAtM0"), + "m0Expectation": mapping.get("m0Expectation"), + "observationCount": len(fixture_observations), + } + ) + + invariant_to_fixture_refs: dict[str, list[str]] = {} + if catalog is not None: + for fixture in _mapping_sequence(catalog.get("fixtures")): + fixture_ref = fixture.get("id") + if not isinstance(fixture_ref, str): + continue + for invariant_ref in _strings(fixture.get("invariantRefs")): + invariant_to_fixture_refs.setdefault(invariant_ref, []).append( + fixture_ref + ) + + invariant_results: list[dict[str, object]] = [] + for mapping in _mapping_sequence(registry.get("invariantMappings")): + raw_invariant_ref = mapping.get("invariantRef") + invariant_ref = ( + raw_invariant_ref if isinstance(raw_invariant_ref, str) else "" + ) + refs = _mapped_evidence_refs(mapping) + missing = [ref for ref in refs if ref not in passed_evidence_ids] + fixture_failures = [ + fixture_ref + for fixture_ref in invariant_to_fixture_refs.get(invariant_ref, []) + if fixture_ref in failed_fixture_refs + ] + invariant_results.append( + { + "invariantRef": invariant_ref, + "status": "FAIL" if missing or fixture_failures else "PASS", + "evidenceRefs": refs, + "failedEvidenceRefs": missing, + "failedFixtureRefs": fixture_failures, + } + ) + + hard_oracles: list[dict[str, object]] = [] + for adapter in _mapping_sequence(registry.get("hardOracleAdapters")): + oracle_ref = adapter.get("oracleRef") + raw_result_key = adapter.get("resultKey") + result_key = raw_result_key if isinstance(raw_result_key, str) else "" + required_value = adapter.get("requiredValue") + observation_contract = _mapping(adapter.get("observation")) + configured_evidence_refs = set( + _strings(observation_contract.get("evidenceRefs")) + ) + evidence_refs = configured_evidence_refs or { + triple[1] + for triple in expected_observations + if triple[2] == result_key + } + oracle_values: list[int] = [] + for observation in observations: + observation_values = cast( + Mapping[str, object], observation["values"] + ) + candidate = observation_values.get(result_key) + if ( + observation["evidenceRef"] in evidence_refs + and isinstance(candidate, int) + and not isinstance(candidate, bool) + ): + oracle_values.append(candidate) + observed_value = sum(oracle_values) + complete = bool(oracle_values) and all( + triple[2] != result_key + or triple[1] not in evidence_refs + or observed_triples[triple] > 0 + for triple in expected_observations + ) + passed = complete and observed_value == required_value + if not passed: + failures.append( + f"hard oracle veto {oracle_ref}: observed {observed_value!r}, " + f"required {required_value!r}, complete={complete}" + ) + hard_oracles.append( + { + "oracleRef": oracle_ref, + "resultKey": result_key, + "requiredValue": required_value, + "observedValue": observed_value, + "observationCount": len(oracle_values), + "status": "PASS" if passed else "FAIL", + "veto": True, + } + ) + + unique_failures = list(dict.fromkeys(failures)) + return { + "passed": not unique_failures, + "failures": unique_failures, + "passedEvidenceIds": sorted(passed_evidence_ids), + "evidence": evidence_results, + "invariants": invariant_results, + "fixtures": fixture_results, + "hardOracles": hard_oracles, + "observations": observations, + "testResultDigest": canonical_digest(canonical_test_results(raw_evidence)), + } + + +def build_release_gate_report( + *, + reconciliation: Mapping[str, object], + rls_audit: Mapping[str, object], + provenance: Mapping[str, object], + raw_result_digest: str, +) -> dict[str, object]: + """Build four independent gates; Security alone is evaluated at M0.""" + + complete_provenance = { + **dict(provenance), + "rawResultDigest": raw_result_digest, + "testResultDigest": reconciliation.get("testResultDigest"), + } + provenance_failures = _provenance_failures(complete_provenance) + security_passed = ( + reconciliation.get("passed") is True + and rls_audit.get("passed") is True + and not provenance_failures + ) + security_failures = [ + *_strings(reconciliation.get("failures")), + *provenance_failures, + ] + security = { + "status": "pass" if security_passed else "fail", + "veto": True, + "invariants": reconciliation.get("invariants", []), + "fixtures": reconciliation.get("fixtures", []), + "hardOracles": reconciliation.get("hardOracles", []), + "rls": rls_audit, + "failures": security_failures, + } + not_evaluated = { + "status": "not-evaluated", + "reason": "No M0 gate is defined for this independent dimension.", + } + return { + "reportVersion": REPORT_VERSION, + "m0SecurityDecision": "pass" if security_passed else "fail", + "releaseDecision": "not-evaluated" if security_passed else "fail", + "promotionReadiness": "not-evaluated", + "provenance": complete_provenance, + "gates": { + "Security": security, + "Reliability": dict(not_evaluated), + "Quality": dict(not_evaluated), + "Budget": dict(not_evaluated), + }, + } diff --git a/scripts/security_gate/rls.py b/scripts/security_gate/rls.py new file mode 100644 index 00000000..ac52c17a --- /dev/null +++ b/scripts/security_gate/rls.py @@ -0,0 +1,909 @@ +"""Deterministic live PostgreSQL RLS inventory evidence for the M0 gate.""" + +from __future__ import annotations + +import json +import re +from collections import Counter, deque +from collections.abc import Collection, Mapping, Sequence +from hashlib import sha256 +from typing import cast + +from sqlalchemy import Connection, text + +PUBLIC_SCHEMA = "public" +GLOBAL_CLASSIFICATION = "global" +TENANT_CLASSIFICATION = "tenant_owned" +PINNED_GLOBAL_TABLES = frozenset( + {"alembic_version", "organization", "user_account"} +) +NON_OWNER_EVIDENCE_BY_TABLE: Mapping[str, str] = { + "membership": "PG-SCOPE-INTERSECTION-004", + "organization_record": "PG-TENANT-FK-002", + "context_resource": "PG-INDEX-NOT-AUTHORITY-005", + "context_revision": "PG-INDEX-NOT-AUTHORITY-005", + "context_fragment": "PG-INDEX-NOT-AUTHORITY-005", + "organization_policy_epoch": "PG-REVOCATION-006", + "resource_access_policy": "PG-REVOCATION-006", + "context_run": "PG-TRACE-REDACTION-012", + "context_run_operator_read_ticket": "PG-TRACE-REDACTION-012", + "decision_audit": "PG-TRACE-REDACTION-012", + "service_principal": "PG-WORKER-LEASE-007", + "worker_noop_job": "PG-WORKER-LEASE-007", + "context_fragment_field": "PG-FIELD-PROJECTION-RLS-048", + "membership_resource_field_right": "PG-FIELD-PROJECTION-RLS-048", + "release_manifest": "PG-RELEASE-OWNER-019", + "release_candidate": "PG-RELEASE-OWNER-019", + "release_evaluation": "PG-RELEASE-OWNER-019", + "release_operator_grant": "PG-RELEASE-OWNER-019", + "active_release_manifest": "PG-RELEASE-OWNER-019", + "release_promotion_audit": "PG-RELEASE-OWNER-019", +} + +_SQL_TOKEN = re.compile( + r"'(?:''|[^'])*'|\"(?:\"\"|[^\"])*\"|::|<=|>=|<>|!=|" + r"[A-Za-z_][A-Za-z0-9_$]*|\d+(?:\.\d+)?|[^\s]" +) + + +def snapshot_public_schema(connection: Connection) -> dict[str, object]: + """Read the live public-table controls through the injected DB connection.""" + + role = connection.execute( + text( + """ + SELECT + current_user AS name, + role.rolsuper AS superuser, + role.rolbypassrls AS bypasses_rls, + role.rolinherit AS inherits + FROM pg_roles AS role + WHERE role.rolname = current_user + """ + ) + ).mappings().one() + relations = connection.execute( + text( + """ + SELECT + relation.relname AS table_name, + pg_get_userbyid(relation.relowner) AS owner, + relation.relrowsecurity AS rls_enabled, + relation.relforcerowsecurity AS rls_forced + FROM pg_class AS relation + JOIN pg_namespace AS namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'public' + AND relation.relkind IN ('r', 'p') + ORDER BY relation.relname + """ + ) + ).mappings() + + tables: dict[str, dict[str, object]] = {} + for relation in relations: + name = cast(str, relation["table_name"]) + tables[name] = { + "columns": [], + "owner": cast(str, relation["owner"]), + "rlsEnabled": cast(bool, relation["rls_enabled"]), + "rlsForced": cast(bool, relation["rls_forced"]), + "policies": [], + "foreignKeys": [], + } + + columns = connection.execute( + text( + """ + SELECT + relation.relname AS table_name, + attribute.attname AS column_name + FROM pg_attribute AS attribute + JOIN pg_class AS relation + ON relation.oid = attribute.attrelid + JOIN pg_namespace AS namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'public' + AND relation.relkind IN ('r', 'p') + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ORDER BY relation.relname, attribute.attnum + """ + ) + ).mappings() + for column in columns: + table = tables.get(cast(str, column["table_name"])) + if table is not None: + cast(list[str], table["columns"]).append( + cast(str, column["column_name"]) + ) + + policies = connection.execute( + text( + """ + SELECT + policy.tablename AS table_name, + policy.policyname AS policy_name, + policy.permissive = 'PERMISSIVE' AS permissive, + policy.cmd AS command, + policy.roles, + policy.qual AS using_expression, + policy.with_check AS with_check_expression + FROM pg_policies AS policy + WHERE policy.schemaname = 'public' + ORDER BY policy.tablename, policy.policyname + """ + ) + ).mappings() + for policy in policies: + table = tables.get(cast(str, policy["table_name"])) + if table is not None: + cast(list[dict[str, object]], table["policies"]).append( + { + "name": cast(str, policy["policy_name"]), + "permissive": cast(bool, policy["permissive"]), + "command": cast(str, policy["command"]), + "roles": list(cast(Sequence[str], policy["roles"])), + "using": cast(str | None, policy["using_expression"]), + "withCheck": cast( + str | None, policy["with_check_expression"] + ), + } + ) + + foreign_keys = connection.execute( + text( + """ + SELECT + source.relname AS table_name, + constraint_record.conname AS constraint_name, + array_agg( + source_attribute.attname ORDER BY key_pair.ordinality + ) AS columns, + target.relname AS referenced_table, + array_agg( + target_attribute.attname ORDER BY key_pair.ordinality + ) AS referenced_columns + FROM pg_constraint AS constraint_record + JOIN pg_class AS source + ON source.oid = constraint_record.conrelid + JOIN pg_namespace AS source_namespace + ON source_namespace.oid = source.relnamespace + JOIN pg_class AS target + ON target.oid = constraint_record.confrelid + CROSS JOIN LATERAL unnest( + constraint_record.conkey, + constraint_record.confkey + ) WITH ORDINALITY AS key_pair( + source_attribute_number, + target_attribute_number, + ordinality + ) + JOIN pg_attribute AS source_attribute + ON source_attribute.attrelid = source.oid + AND source_attribute.attnum = key_pair.source_attribute_number + JOIN pg_attribute AS target_attribute + ON target_attribute.attrelid = target.oid + AND target_attribute.attnum = key_pair.target_attribute_number + WHERE constraint_record.contype = 'f' + AND source_namespace.nspname = 'public' + GROUP BY + source.relname, + constraint_record.conname, + target.relname + ORDER BY source.relname, constraint_record.conname + """ + ) + ).mappings() + for foreign_key in foreign_keys: + table = tables.get(cast(str, foreign_key["table_name"])) + if table is not None: + cast(list[dict[str, object]], table["foreignKeys"]).append( + { + "name": cast(str, foreign_key["constraint_name"]), + "columns": list(cast(Sequence[str], foreign_key["columns"])), + "referencedTable": cast(str, foreign_key["referenced_table"]), + "referencedColumns": list( + cast(Sequence[str], foreign_key["referenced_columns"]) + ), + } + ) + + return { + "schema": PUBLIC_SCHEMA, + "currentRole": { + "name": cast(str, role["name"]), + "superuser": cast(bool, role["superuser"]), + "bypassesRls": cast(bool, role["bypasses_rls"]), + "inherits": cast(bool, role["inherits"]), + }, + "tables": tables, + } + + +def audit_live_rls( + connection: Connection, + manifest: Mapping[str, object], + passed_evidence_ids: Collection[str], +) -> dict[str, object]: + """Audit the live schema and return stable JSON-safe gate evidence.""" + + return audit_rls_snapshot( + manifest=manifest, + snapshot=snapshot_public_schema(connection), + passed_evidence_ids=passed_evidence_ids, + ) + + +def audit_rls_snapshot( + *, + manifest: Mapping[str, object], + snapshot: Mapping[str, object], + passed_evidence_ids: Collection[str], +) -> dict[str, object]: + """Validate an injected catalog snapshot against the declared denominator.""" + + failures: list[str] = [] + raw_entries = manifest.get("tables", []) + manifest_entries = [ + entry for entry in raw_entries if isinstance(entry, Mapping) + ] if isinstance(raw_entries, list) else [] + names = [entry.get("name") for entry in manifest_entries] + manifest_names = [name for name in names if isinstance(name, str)] + duplicate_names = sorted( + name for name, count in Counter(manifest_names).items() if count > 1 + ) + entries = { + cast(str, entry["name"]): entry + for entry in manifest_entries + if isinstance(entry.get("name"), str) + } + + raw_live_tables = snapshot.get("tables", {}) + live_tables = ( + cast(Mapping[str, object], raw_live_tables) + if isinstance(raw_live_tables, Mapping) + else {} + ) + declared_names = set(entries) + live_names = {name for name in live_tables if isinstance(name, str)} + missing_from_manifest = sorted(live_names - declared_names) + missing_from_database = sorted(declared_names - live_names) + if duplicate_names: + failures.append( + "manifest has duplicate table entries: " + ", ".join(duplicate_names) + ) + if missing_from_manifest: + failures.append( + "live public tables missing from manifest: " + + ", ".join(missing_from_manifest) + ) + if missing_from_database: + failures.append( + "manifest tables missing from live public schema: " + + ", ".join(missing_from_database) + ) + + invalid_classifications = sorted( + name + for name, entry in entries.items() + if entry.get("classification") + not in {GLOBAL_CLASSIFICATION, TENANT_CLASSIFICATION} + ) + if invalid_classifications: + failures.append( + "tables have invalid classifications: " + + ", ".join(invalid_classifications) + ) + + global_names = sorted( + name + for name, entry in entries.items() + if entry.get("classification") == GLOBAL_CLASSIFICATION + ) + tenant_names = sorted( + name + for name, entry in entries.items() + if entry.get("classification") == TENANT_CLASSIFICATION + ) + global_name_set = set(global_names) + unexpected_global_names = sorted(global_name_set - PINNED_GLOBAL_TABLES) + missing_global_names = sorted(PINNED_GLOBAL_TABLES - global_name_set) + global_inventory_exact = not unexpected_global_names and not missing_global_names + if not global_inventory_exact: + failures.append( + "global table allowlist differs from the pinned M0 allowlist" + ) + global_allowlist: list[dict[str, object]] = [] + for name in global_names: + rationale = entries[name].get("classificationRationale") + rationale_present = isinstance(rationale, str) and bool(rationale.strip()) + if not rationale_present: + failures.append(f"{name}: global classification rationale is blank") + global_allowlist.append( + { + "table": name, + "rationale": rationale if isinstance(rationale, str) else "", + "passed": rationale_present and name in live_names, + } + ) + + current_role = _object_mapping(snapshot.get("currentRole")) + session_role = _string(current_role.get("name")) + role_is_non_superuser = current_role.get("superuser") is False + role_cannot_bypass_rls = current_role.get("bypassesRls") is False + passed_ids = set(passed_evidence_ids) + ownership_edges = _verified_ownership_edges(entries, live_tables, tenant_names) + + tenant_reports: list[dict[str, object]] = [] + numerator = 0 + for name in tenant_names: + entry = entries[name] + live = _object_mapping(live_tables.get(name)) + organization_column = _string(entry.get("organizationColumn")) + live_columns = _string_list(live.get("columns")) + organization_column_present = ( + bool(organization_column) and organization_column in live_columns + ) + ownership_path = _find_organization_path(name, ownership_edges) + ownership_passed = organization_column_present and bool(ownership_path) + + rls_enabled = live.get("rlsEnabled") is True + rls_forced = live.get("rlsForced") is True + declared_rls = _object_mapping(entry.get("rowLevelSecurity")) + declared_policy_rows = _mapping_list(declared_rls.get("policies")) + live_policy_rows = _mapping_list(live.get("policies")) + declared_policy_names = sorted( + _string(policy.get("name")) for policy in declared_policy_rows + ) + live_policy_names = sorted( + _string(policy.get("name")) for policy in live_policy_rows + ) + duplicate_declared_policy_names = sorted( + policy_name + for policy_name, count in Counter(declared_policy_names).items() + if policy_name and count > 1 + ) + duplicate_live_policy_names = sorted( + policy_name + for policy_name, count in Counter(live_policy_names).items() + if policy_name and count > 1 + ) + policy_inventory_matches = ( + bool(declared_policy_names) + and declared_policy_names == live_policy_names + and not duplicate_declared_policy_names + and not duplicate_live_policy_names + ) + expected_policy_semantics = _normalized_policy_semantics( + name, declared_policy_rows + ) + observed_policy_semantics = _normalized_policy_semantics( + name, live_policy_rows + ) + policy_semantics_match = ( + policy_inventory_matches + and expected_policy_semantics == observed_policy_semantics + ) + policies_passed = policy_inventory_matches and policy_semantics_match + + evidence = _object_mapping(entry.get("nonOwnerEvidence")) + evidence_id = _string(evidence.get("evidenceId")) + expected_evidence_id = NON_OWNER_EVIDENCE_BY_TABLE.get(name, "") + selector = _object_mapping(evidence.get("selector")) + selector_table = _string(selector.get("table")) + evidence_mapping_exact = ( + bool(expected_evidence_id) + and evidence_id == expected_evidence_id + and selector_table == name + ) + evidence_passed = ( + evidence_mapping_exact + and evidence_id in passed_ids + ) + owner = _string(live.get("owner")) + role_is_non_owner = bool(session_role) and bool(owner) and owner != session_role + non_owner_passed = ( + evidence_passed + and role_is_non_owner + and role_is_non_superuser + and role_cannot_bypass_rls + ) + + table_failures: list[str] = [] + if not organization_column_present: + table_failures.append( + f"{name}: Organization ownership column is absent from the live table" + ) + if not ownership_path: + table_failures.append( + f"{name}: no verified Organization-inclusive ownership path" + ) + if not rls_enabled: + table_failures.append(f"{name}: row level security is disabled") + if not rls_forced: + table_failures.append( + f"{name}: FORCE ROW LEVEL SECURITY is disabled" + ) + if not policy_inventory_matches: + table_failures.append( + f"{name}: live RLS policy inventory differs from manifest" + ) + elif not policy_semantics_match: + table_failures.append( + f"{name}: live RLS policy semantics differ from manifest" + ) + if not evidence_mapping_exact: + table_failures.append( + f"{name}: non-owner evidence mapping differs from the pinned " + "behavioral probe" + ) + if not evidence_passed: + table_failures.append( + f"{name}: mapped non-owner PostgreSQL evidence did not pass" + ) + if not role_is_non_owner: + table_failures.append( + f"{name}: audit role is missing or owns the live table" + ) + if not role_is_non_superuser: + table_failures.append(f"{name}: audit role is a superuser") + if not role_cannot_bypass_rls: + table_failures.append(f"{name}: audit role can bypass RLS") + + table_passed = not table_failures + if table_passed: + numerator += 1 + failures.extend(table_failures) + tenant_reports.append( + { + "table": name, + "ownershipPath": { + "organizationColumn": organization_column, + "columnPresent": organization_column_present, + "edges": ownership_path, + "passed": ownership_passed, + }, + "rlsEnabled": rls_enabled, + "rlsForced": rls_forced, + "policies": { + "declared": declared_policy_names, + "live": live_policy_names, + "duplicateDeclared": duplicate_declared_policy_names, + "duplicateLive": duplicate_live_policy_names, + "inventoryMatchesDeclared": policy_inventory_matches, + "expected": expected_policy_semantics, + "observed": observed_policy_semantics, + "expectedDigest": _stable_digest(expected_policy_semantics), + "observedDigest": _stable_digest(observed_policy_semantics), + "semanticsMatchDeclared": policy_semantics_match, + "passed": policies_passed, + }, + "nonOwnerIsolation": { + "evidenceId": evidence_id, + "expectedEvidenceId": expected_evidence_id, + "selector": {"table": selector_table}, + "mappingExact": evidence_mapping_exact, + "evidencePassed": evidence_passed, + "sessionRole": session_role, + "tableOwner": owner, + "roleIsNonOwner": role_is_non_owner, + "roleIsNonSuperuser": role_is_non_superuser, + "roleCannotBypassRls": role_cannot_bypass_rls, + "passed": non_owner_passed, + }, + "passed": table_passed, + "failures": table_failures, + } + ) + + denominator = len(tenant_names) + inventory_exact = ( + not duplicate_names + and not missing_from_manifest + and not missing_from_database + and not invalid_classifications + and global_inventory_exact + ) + coverage_percent = ( + round((numerator / denominator) * 100, 2) if denominator else 0.0 + ) + return { + "passed": inventory_exact and not failures and numerator == denominator, + "schema": _string(snapshot.get("schema")) or PUBLIC_SCHEMA, + "inventory": { + "declared": sorted(declared_names), + "live": sorted(live_names), + "missingFromManifest": missing_from_manifest, + "missingFromDatabase": missing_from_database, + "duplicateManifestEntries": duplicate_names, + "exact": inventory_exact, + }, + "denominator": { + "allTables": len(declared_names), + "tenantOwned": denominator, + "global": len(global_names), + }, + "coverage": { + "numerator": numerator, + "denominator": denominator, + "percent": coverage_percent, + }, + "globalAllowlist": global_allowlist, + "globalAllowlistInventory": { + "expected": sorted(PINNED_GLOBAL_TABLES), + "declared": global_names, + "missing": missing_global_names, + "unexpected": unexpected_global_names, + "exact": global_inventory_exact, + }, + "tenantTables": tenant_reports, + "failures": failures, + } + + +def _verified_ownership_edges( + entries: Mapping[str, Mapping[str, object]], + live_tables: Mapping[str, object], + tenant_names: Sequence[str], +) -> dict[str, list[dict[str, str]]]: + edges: dict[str, list[dict[str, str]]] = {name: [] for name in tenant_names} + for table_name in tenant_names: + entry = entries[table_name] + organization_column = _string(entry.get("organizationColumn")) + live = _object_mapping(live_tables.get(table_name)) + live_foreign_keys = _mapping_list(live.get("foreignKeys")) + live_by_name = { + _string(foreign_key.get("name")): foreign_key + for foreign_key in live_foreign_keys + } + for declared in _mapping_list(entry.get("foreignKeys")): + name = _string(declared.get("name")) + live_foreign_key = live_by_name.get(name) + if live_foreign_key is None: + continue + references = _object_mapping(declared.get("references")) + declared_columns = _string_list(declared.get("columns")) + declared_referenced_table = _string(references.get("table")) + declared_referenced_columns = _string_list(references.get("columns")) + if ( + declared_columns != _string_list(live_foreign_key.get("columns")) + or declared_referenced_table + != _string(live_foreign_key.get("referencedTable")) + or declared_referenced_columns + != _string_list(live_foreign_key.get("referencedColumns")) + ): + continue + try: + organization_index = declared_columns.index(organization_column) + except ValueError: + continue + if ( + organization_index >= len(declared_referenced_columns) + or declared_referenced_columns[organization_index] + != "organization_id" + ): + continue + edges[table_name].append( + { + "constraint": name, + "fromTable": table_name, + "toTable": declared_referenced_table, + } + ) + edges[table_name].sort(key=lambda edge: (edge["toTable"], edge["constraint"])) + return edges + + +def _find_organization_path( + start: str, + edges: Mapping[str, Sequence[dict[str, str]]], +) -> list[dict[str, str]]: + queue: deque[tuple[str, list[dict[str, str]]]] = deque([(start, [])]) + visited = {start} + while queue: + table_name, path = queue.popleft() + for edge in edges.get(table_name, []): + next_table = edge["toTable"] + next_path = [*path, edge] + if next_table == "organization": + return next_path + if next_table not in visited and next_table in edges: + visited.add(next_table) + queue.append((next_table, next_path)) + return [] + + +def _normalized_policy_semantics( + table_name: str, + policies: Sequence[Mapping[str, object]], +) -> list[dict[str, object]]: + normalized: list[dict[str, object]] = [] + for policy in policies: + using_expression = policy.get("using") + with_check_expression = policy.get("withCheck") + normalized.append( + { + "name": _string(policy.get("name")), + "permissive": policy.get("permissive", True) is True, + "command": _string(policy.get("command")).upper(), + "roles": sorted(_string_list(policy.get("roles"))), + "using": _normalize_postgresql_expression( + table_name, + using_expression if isinstance(using_expression, str) else None, + ), + "withCheck": _normalize_postgresql_expression( + table_name, + with_check_expression + if isinstance(with_check_expression, str) + else None, + ), + } + ) + return sorted(normalized, key=lambda policy: cast(str, policy["name"])) + + +def _normalize_postgresql_expression( + table_name: str, + expression: str | None, +) -> str | None: + """Canonicalize harmless pg_get_expr rendering differences, not semantics.""" + + if expression is None: + return None + tokens = [ + token if token.startswith("'") else token.lower() + for token in _SQL_TOKEN.findall(expression) + ] + normalized: list[str] = [] + index = 0 + while index < len(tokens): + token = tokens[index] + if token == "public" and index + 1 < len(tokens) and tokens[index + 1] == ".": + index += 2 + continue + if ( + token == "pg_catalog" + and index + 2 < len(tokens) + and tokens[index + 1] == "." + and tokens[index + 2] in {"decode"} + ): + index += 2 + continue + if ( + token == table_name.lower() + and index + 1 < len(tokens) + and tokens[index + 1] == "." + ): + index += 2 + continue + if token == "as" and index + 1 < len(tokens): + index += 1 + continue + if token == "::" and index + 1 < len(tokens): + cast_type = tokens[index + 1] + if cast_type == "text": + index += 2 + continue + if ( + cast_type == "timestamp" + and index + 3 < len(tokens) + and tokens[index + 2 : index + 4] == ["with", "time"] + and index + 4 < len(tokens) + and tokens[index + 4] == "zone" + ): + normalized.extend(["::", "timestamptz"]) + index += 5 + continue + normalized.append(token) + index += 1 + return _canonical_boolean_expression(normalized) + + +def _canonical_boolean_expression(tokens: Sequence[str]) -> str: + current = _strip_redundant_parentheses(list(tokens)) + for operator in ("or", "and"): + operands = _split_top_level(current, operator) + if len(operands) > 1: + return f"{operator}(" + ",".join( + _canonical_boolean_expression(operand) for operand in operands + ) + ")" + if current and current[0] == "exists": + return _canonical_exists(current) + comparison_index = _find_top_level_comparison(current) + if comparison_index is not None: + index, operator = comparison_index + left = _canonical_term(current[:index]) + right_tokens = current[index + 1 :] + if operator == "is" and right_tokens and right_tokens[0] == "not": + operator = "is not" + right_tokens = right_tokens[1:] + if operator == "in": + return f"any=({left},{_canonical_list(right_tokens)})" + if operator == "=" and right_tokens and right_tokens[0] == "any": + return f"any=({left},{_canonical_any(right_tokens)})" + return f"{operator}({left},{_canonical_term(right_tokens)})" + return _canonical_term(current) + + +def _canonical_exists(tokens: Sequence[str]) -> str: + subquery = _strip_redundant_parentheses(list(tokens[1:])) + where_index = _find_top_level_token(subquery, "where") + if where_index is None: + return "exists(" + " ".join(subquery) + ")" + prefix = " ".join(subquery[:where_index]) + predicate = _canonical_boolean_expression(subquery[where_index + 1 :]) + return f"exists({prefix} where {predicate})" + + +def _canonical_term(tokens: Sequence[str]) -> str: + current = _strip_redundant_parentheses(list(tokens)) + cast_index = _find_top_level_token(current, "::") + if cast_index is not None: + return ( + f"cast({_canonical_term(current[:cast_index])}," + f"{' '.join(current[cast_index + 1 :])})" + ) + if ( + len(current) >= 3 + and current[1] == "(" + and current[-1] == ")" + and _matching_close(current, 1) == len(current) - 1 + ): + arguments = _split_top_level(current[2:-1], ",") + rendered_arguments = ",".join( + _canonical_boolean_expression(argument) for argument in arguments + ) + return f"{current[0]}({rendered_arguments})" + return " ".join(current) + + +def _canonical_any(tokens: Sequence[str]) -> str: + current = _strip_redundant_parentheses(list(tokens[1:])) + if current and current[0] == "array": + current = current[1:] + if len(current) >= 2 and current[0] == "[" and current[-1] == "]": + current = current[1:-1] + return _canonical_list(current) + + +def _canonical_list(tokens: Sequence[str]) -> str: + current = _strip_redundant_parentheses(list(tokens)) + return "list(" + ",".join( + _canonical_term(item) for item in _split_top_level(current, ",") + ) + ")" + + +def _find_top_level_comparison( + tokens: Sequence[str], +) -> tuple[int, str] | None: + depth = 0 + bracket_depth = 0 + for index, token in enumerate(tokens): + if token == "(": + depth += 1 + elif token == ")": + depth -= 1 + elif token == "[": + bracket_depth += 1 + elif token == "]": + bracket_depth -= 1 + elif depth == 0 and bracket_depth == 0 and token in { + "=", + "!=", + "<>", + "<", + "<=", + ">", + ">=", + "is", + "in", + }: + return index, token + return None + + +def _find_top_level_token(tokens: Sequence[str], expected: str) -> int | None: + depth = 0 + bracket_depth = 0 + for index, token in enumerate(tokens): + if token == "(": + depth += 1 + elif token == ")": + depth -= 1 + elif token == "[": + bracket_depth += 1 + elif token == "]": + bracket_depth -= 1 + elif depth == 0 and bracket_depth == 0 and token == expected: + return index + return None + + +def _split_top_level(tokens: Sequence[str], delimiter: str) -> list[list[str]]: + depth = 0 + bracket_depth = 0 + start = 0 + parts: list[list[str]] = [] + for index, token in enumerate(tokens): + if token == "(": + depth += 1 + elif token == ")": + depth -= 1 + elif token == "[": + bracket_depth += 1 + elif token == "]": + bracket_depth -= 1 + elif depth == 0 and bracket_depth == 0 and token == delimiter: + parts.append(list(tokens[start:index])) + start = index + 1 + parts.append(list(tokens[start:])) + return parts + + +def _matching_close(tokens: Sequence[str], opening_index: int) -> int | None: + depth = 0 + for index in range(opening_index, len(tokens)): + if tokens[index] == "(": + depth += 1 + elif tokens[index] == ")": + depth -= 1 + if depth == 0: + return index + return None + + +def _strip_redundant_parentheses(tokens: list[str]) -> list[str]: + current = tokens + changed = True + while changed and len(current) >= 2: + changed = False + if current[0] == "(" and current[-1] == ")": + depth = 0 + encloses_all = True + for index, token in enumerate(current): + if token == "(": + depth += 1 + elif token == ")": + depth -= 1 + if depth == 0 and index != len(current) - 1: + encloses_all = False + break + if encloses_all: + current = current[1:-1] + changed = True + return current + + +def _stable_digest(value: object) -> str: + serialized = json.dumps( + value, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + return sha256(serialized.encode("utf-8")).hexdigest() + + +def _object_mapping(value: object) -> Mapping[str, object]: + if isinstance(value, Mapping): + return cast(Mapping[str, object], value) + return {} + + +def _mapping_list(value: object) -> list[Mapping[str, object]]: + if not isinstance(value, list): + return [] + return [ + cast(Mapping[str, object], item) for item in value if isinstance(item, Mapping) + ] + + +def _string(value: object) -> str: + return value if isinstance(value, str) else "" + + +def _string_list(value: object) -> list[str]: + if not isinstance(value, list | tuple): + return [] + return [item for item in value if isinstance(item, str)] diff --git a/scripts/security_gate/runner.py b/scripts/security_gate/runner.py new file mode 100644 index 00000000..f8cd502c --- /dev/null +++ b/scripts/security_gate/runner.py @@ -0,0 +1,758 @@ +"""One-shot orchestration for the complete M0 security veto.""" + +from __future__ import annotations + +import hashlib +import os +import re +import secrets +import stat +import subprocess +import sys +from collections.abc import Mapping, Sequence +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol, cast + +from alembic.config import Config +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine +from sqlalchemy import text as sql_text + +from engine.persistence import load_harness_database_configurations +from scripts.security_gate.artifacts import atomic_write_json +from scripts.security_gate.pytest_plugin import ( + RAW_EVIDENCE_VERSION, + RAW_EXECUTION_ID_ENVIRONMENT, +) +from scripts.security_gate.report import ( + build_release_gate_report, + canonical_digest, + reconcile_execution, +) +from scripts.security_gate.rls import audit_live_rls +from scripts.validate_security_catalog import ( + DEFAULT_CATALOG_PATH, + DEFAULT_SCHEMA_PATH, + REPOSITORY_ROOT, + CatalogValidationError, + load_document, + validate_catalog, +) + +DEFAULT_REGISTRY_PATH = REPOSITORY_ROOT / "eval/catalogs/m0-security-evidence.yaml" +DEFAULT_REGISTRY_SCHEMA_PATH = ( + REPOSITORY_ROOT / "eval/catalogs/m0-security-evidence.schema.json" +) +DEFAULT_MANIFEST_PATH = ( + REPOSITORY_ROOT / "engine/persistence/schema_security_manifest.yaml" +) +DEFAULT_DATABASE_ENVIRONMENT_PATH = REPOSITORY_ROOT / ".context-engine/database.env" +RAW_ARTIFACT_NAME = "raw-evidence.json" +REPORT_ARTIFACT_NAME = "release-gate-report.json" +RUNNER_VERSION = "1.0.0" +_PRIVATE_ARTIFACT_PREFIXES = (".context-engine/", ".harness/") +_GIT_EXCLUDE_PATHS = ( + ":(exclude).context-engine/**", + ":(exclude).harness/**", +) +_ENVIRONMENT_KEY = re.compile(r"^[A-Z][A-Z0-9_]*$") +_ALLOWED_DATABASE_ENVIRONMENT_KEYS = frozenset( + { + "POSTGRES_DB", + "POSTGRES_USER", + "POSTGRES_PASSWORD", + "CONTEXT_ENGINE_POSTGRES_PORT", + "CONTEXT_ENGINE_COMPOSE_PROJECT", + "CONTEXT_ENGINE_MIGRATOR_ROLE", + "CONTEXT_ENGINE_MIGRATOR_PASSWORD", + "CONTEXT_ENGINE_CONTROL_ROLE", + "CONTEXT_ENGINE_CONTROL_PASSWORD", + "CONTEXT_ENGINE_RUNTIME_ROLE", + "CONTEXT_ENGINE_RUNTIME_PASSWORD", + "CONTEXT_ENGINE_WORKER_ROLE", + "CONTEXT_ENGINE_WORKER_PASSWORD", + "CONTEXT_ENGINE_LEARNING_ROLE", + "CONTEXT_ENGINE_LEARNING_PASSWORD", + "CONTEXT_ENGINE_SECURITY_OPERATOR_ROLE", + "CONTEXT_ENGINE_SECURITY_OPERATOR_PASSWORD", + "CONTEXT_ENGINE_MIGRATION_DATABASE_URL", + "CONTEXT_ENGINE_CONTROL_DATABASE_URL", + "CONTEXT_ENGINE_RUNTIME_DATABASE_URL", + "CONTEXT_ENGINE_WORKER_DATABASE_URL", + "CONTEXT_ENGINE_LEARNING_DATABASE_URL", + "CONTEXT_ENGINE_SECURITY_OPERATOR_DATABASE_URL", + "CONTEXT_ENGINE_TEST_DATABASE_URL", + } +) + + +class DatabaseEnvironmentError(ValueError): + """The private database contract cannot safely be loaded.""" + + +class GateRunError(RuntimeError): + """The M0 gate could not complete or did not satisfy Security.""" + + +class RegistryValidator(Protocol): + def __call__( + self, + registry: Mapping[str, Any], + schema: Mapping[str, Any], + catalog: Mapping[str, Any], + *, + repository_root: str | Path = ..., + ) -> object: ... + + +class PytestExecutor(Protocol): + def __call__( + self, + command: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str], + ) -> int: ... + + +class RlsAuditor(Protocol): + def __call__( + self, + *, + environment: Mapping[str, str], + manifest: Mapping[str, object], + passed_evidence_ids: Sequence[str], + ) -> Mapping[str, object]: ... + + +@dataclass(frozen=True, slots=True) +class GatePaths: + repository_root: Path + catalog: Path + catalog_schema: Path + registry: Path + registry_schema: Path + manifest: Path + database_environment: Path + output_directory: Path + + @classmethod + def defaults(cls, output_directory: Path) -> GatePaths: + return cls( + repository_root=REPOSITORY_ROOT, + catalog=DEFAULT_CATALOG_PATH, + catalog_schema=DEFAULT_SCHEMA_PATH, + registry=DEFAULT_REGISTRY_PATH, + registry_schema=DEFAULT_REGISTRY_SCHEMA_PATH, + manifest=DEFAULT_MANIFEST_PATH, + database_environment=DEFAULT_DATABASE_ENVIRONMENT_PATH, + output_directory=output_directory, + ) + + +def load_database_environment(path: Path) -> dict[str, str]: + """Parse a strict generated KEY=value file without shell evaluation.""" + + try: + metadata = path.lstat() + except OSError as error: + raise DatabaseEnvironmentError( + "database environment is unavailable; run the database harness first" + ) from error + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise DatabaseEnvironmentError( + "database environment must be a regular non-symbolic-link file" + ) + if metadata.st_uid != os.getuid(): + raise DatabaseEnvironmentError( + "database environment must be owned by the current user" + ) + if stat.S_IMODE(metadata.st_mode) & 0o077: + raise DatabaseEnvironmentError("database environment permissions must be 0600") + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as error: + raise DatabaseEnvironmentError("database environment cannot be read") from error + environment: dict[str, str] = {} + for line_number, line in enumerate(lines, start=1): + if not line or "=" not in line: + raise DatabaseEnvironmentError( + f"database environment line {line_number} must be KEY=value" + ) + key, value = line.split("=", 1) + if _ENVIRONMENT_KEY.fullmatch(key) is None: + raise DatabaseEnvironmentError( + f"database environment line {line_number} must be KEY=value" + ) + if key not in _ALLOWED_DATABASE_ENVIRONMENT_KEYS: + raise DatabaseEnvironmentError( + f"database environment contains unexpected variable {key!r}" + ) + if key in environment: + raise DatabaseEnvironmentError( + f"database environment contains duplicate variable {key!r}" + ) + if not value or "\x00" in value or "\n" in value or "\r" in value: + raise DatabaseEnvironmentError( + f"database environment variable {key!r} has an invalid value" + ) + environment[key] = value + missing = sorted(_ALLOWED_DATABASE_ENVIRONMENT_KEYS - environment.keys()) + if missing: + raise DatabaseEnvironmentError( + "database environment is missing required variables: " + ", ".join(missing) + ) + load_harness_database_configurations(environment) + return environment + + +def _mapping_sequence(value: object) -> list[Mapping[str, object]]: + if not isinstance(value, Sequence) or isinstance(value, str | bytes): + return [] + return [item for item in value if isinstance(item, Mapping)] + + +def deduplicate_selectors(registry: Mapping[str, object]) -> tuple[str, ...]: + """Return the registry's exact selectors once, retaining declared order.""" + + selectors: list[str] = [] + for evidence in _mapping_sequence(registry.get("evidence")): + selector = evidence.get("selector") + if isinstance(selector, str) and selector not in selectors: + selectors.append(selector) + return tuple(selectors) + + +def build_pytest_command( + selectors: Sequence[str], *, raw_path: Path, python_executable: str = sys.executable +) -> tuple[str, ...]: + """Build the one and only exact pytest invocation; no retry options exist.""" + + return ( + python_executable, + "-m", + "pytest", + "-p", + "scripts.security_gate.pytest_plugin", + "--security-gate-raw", + str(raw_path), + "--strict-markers", + "--strict-config", + *selectors, + ) + + +def _execute_pytest( + command: Sequence[str], *, cwd: Path, env: Mapping[str, str] +) -> int: + return subprocess.run(command, cwd=cwd, env=env, check=False).returncode + + +def _audit_rls( + *, + environment: Mapping[str, str], + manifest: Mapping[str, object], + passed_evidence_ids: Sequence[str], +) -> Mapping[str, object]: + configurations = load_harness_database_configurations(environment) + engine = create_engine(configurations.security_test.url) + try: + with engine.connect() as connection: + return audit_live_rls(connection, manifest, passed_evidence_ids) + finally: + engine.dispose() + + +def _file_digest(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(65536), b""): + digest.update(block) + return digest.hexdigest() + + +def _git_commit(repository_root: Path) -> str: + result = subprocess.run( + ["git", "-C", str(repository_root), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return "unavailable" + return result.stdout.strip() + + +def _git_bytes(repository_root: Path, arguments: Sequence[str]) -> bytes | None: + result = subprocess.run( + ["git", "-C", str(repository_root), *arguments], + check=False, + capture_output=True, + ) + return result.stdout if result.returncode == 0 else None + + +def _untracked_state(repository_root: Path) -> tuple[str, int] | None: + listed = _git_bytes( + repository_root, + ("ls-files", "--others", "--exclude-standard", "-z"), + ) + if listed is None: + return None + paths = sorted(path for path in listed.split(b"\0") if path) + digest = hashlib.sha256() + count = 0 + for encoded_path in paths: + try: + relative = encoded_path.decode("utf-8", errors="surrogateescape") + path = repository_root / relative + metadata = path.lstat() + except OSError: + return None + normalized = relative.replace(os.sep, "/") + if normalized.startswith(_PRIVATE_ARTIFACT_PREFIXES): + continue + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + return None + content_digest = _file_digest(path) + digest.update(encoded_path) + digest.update(b"\0") + digest.update(content_digest.encode("ascii")) + digest.update(b"\0") + count += 1 + return digest.hexdigest(), count + + +def _git_state(repository_root: Path) -> dict[str, object]: + commit = _git_commit(repository_root) + staged = _git_bytes( + repository_root, + ("diff", "--cached", "--binary", "--", ".", *_GIT_EXCLUDE_PATHS), + ) + unstaged = _git_bytes( + repository_root, + ("diff", "--binary", "--", ".", *_GIT_EXCLUDE_PATHS), + ) + untracked = _untracked_state(repository_root) + if staged is None or unstaged is None or untracked is None: + return { + "commit": commit, + "trackedDirty": "unavailable", + "trackedDiffDigest": "unavailable", + "stagedDiffDigest": "unavailable", + "unstagedDiffDigest": "unavailable", + "untrackedContentDigest": "unavailable", + "untrackedFileCount": "unavailable", + "contentStateDirty": "unavailable", + "contentStateDigest": "unavailable", + } + untracked_digest, untracked_count = untracked + staged_digest = hashlib.sha256(staged).hexdigest() + unstaged_digest = hashlib.sha256(unstaged).hexdigest() + tracked_digest = canonical_digest( + {"staged": staged_digest, "unstaged": unstaged_digest} + ) + content_digest = canonical_digest( + { + "commit": commit, + "stagedDiffDigest": staged_digest, + "unstagedDiffDigest": unstaged_digest, + "untrackedContentDigest": untracked_digest, + "untrackedFileCount": untracked_count, + } + ) + return { + "commit": commit, + "trackedDirty": bool(staged or unstaged), + "trackedDiffDigest": tracked_digest, + "stagedDiffDigest": staged_digest, + "unstagedDiffDigest": unstaged_digest, + "untrackedContentDigest": untracked_digest, + "untrackedFileCount": untracked_count, + "contentStateDirty": bool(staged or unstaged or untracked_count), + "contentStateDigest": content_digest, + } + + +def _alembic_head(paths: GatePaths) -> str: + configuration = Config(str(paths.repository_root / "alembic.ini")) + configuration.set_main_option( + "script_location", str(paths.repository_root / "migrations") + ) + head = ScriptDirectory.from_config(configuration).get_current_head() + if head is None: + raise GateRunError("Alembic migration history has no unique current head") + return head + + +def _live_database_revision(environment: Mapping[str, str]) -> str: + configurations = load_harness_database_configurations(environment) + engine = create_engine(configurations.migration.url) + try: + with engine.connect() as connection: + values = connection.execute( + sql_text("SELECT version_num FROM alembic_version ORDER BY version_num") + ).scalars().all() + finally: + engine.dispose() + if len(values) != 1 or not isinstance(values[0], str): + raise GateRunError("live database must expose one exact Alembic revision") + return values[0] + + +def _provenance( + paths: GatePaths, + registry: Mapping[str, object], + catalog: Mapping[str, object], + *, + selectors: Sequence[str], + live_database_revision: str, + alembic_head: str | None = None, +) -> dict[str, object]: + migration_files = sorted( + (paths.repository_root / "migrations/versions").glob("*.py") + ) + migration_state = { + str(path.relative_to(paths.repository_root)): _file_digest(path) + for path in migration_files + } + compose_digest = _file_digest(paths.repository_root / "compose.yaml") + alembic_config_digest = _file_digest(paths.repository_root / "alembic.ini") + registry_digest = canonical_digest(registry) + manifest_digest = _file_digest(paths.manifest) + configuration_digest = canonical_digest( + { + "composeDigest": compose_digest, + "alembicConfigDigest": alembic_config_digest, + "executionRegistryDigest": registry_digest, + "schemaManifestDigest": manifest_digest, + } + ) + fixtures = catalog.get("fixtures", []) + return { + "runnerVersion": RUNNER_VERSION, + **_git_state(paths.repository_root), + "executionCommand": list( + build_pytest_command( + selectors, + raw_path=paths.output_directory / RAW_ARTIFACT_NAME, + ) + ), + "catalogDigest": _file_digest(paths.catalog), + "fixtureDigest": canonical_digest(fixtures), + "catalogSchemaDigest": _file_digest(paths.catalog_schema), + "executionRegistryDigest": registry_digest, + "executionRegistrySchemaDigest": _file_digest(paths.registry_schema), + "schemaManifestDigest": manifest_digest, + "migrationStateDigest": canonical_digest(migration_state), + "alembicHead": alembic_head or _alembic_head(paths), + "liveDatabaseRevision": live_database_revision, + "composeDigest": compose_digest, + "alembicConfigDigest": alembic_config_digest, + "configurationDigest": configuration_digest, + } + + +def _best_effort_provenance( + paths: GatePaths, + *, + registry: Mapping[str, object] | None = None, + catalog: Mapping[str, object] | None = None, + selectors: Sequence[str] = (), + alembic_head: str | None = None, + live_database_revision: str | None = None, +) -> dict[str, object]: + """Retain only safe aggregate provenance facts available at failure time.""" + + provenance: dict[str, object] = {"runnerVersion": RUNNER_VERSION} + try: + provenance.update(_git_state(paths.repository_root)) + except OSError: + provenance.update( + { + "commit": "unavailable", + "contentStateDirty": "unavailable", + "contentStateDigest": "unavailable", + } + ) + public_files = { + "catalogDigest": paths.catalog, + "catalogSchemaDigest": paths.catalog_schema, + "executionRegistrySchemaDigest": paths.registry_schema, + "schemaManifestDigest": paths.manifest, + "composeDigest": paths.repository_root / "compose.yaml", + "alembicConfigDigest": paths.repository_root / "alembic.ini", + } + for field, path in public_files.items(): + try: + provenance[field] = _file_digest(path) + except OSError: + continue + if registry is not None: + provenance["executionRegistryDigest"] = canonical_digest(registry) + else: + with suppress(OSError): + provenance["executionRegistryFileDigest"] = _file_digest(paths.registry) + if catalog is not None: + provenance["fixtureDigest"] = canonical_digest(catalog.get("fixtures", [])) + try: + migration_files = sorted( + (paths.repository_root / "migrations/versions").glob("*.py") + ) + migration_state = { + str(path.relative_to(paths.repository_root)): _file_digest(path) + for path in migration_files + } + if migration_files: + provenance["migrationStateDigest"] = canonical_digest(migration_state) + except OSError: + pass + resolved_head = alembic_head + if resolved_head is None: + try: + resolved_head = _alembic_head(paths) + except Exception: + resolved_head = None + if resolved_head is not None: + provenance["alembicHead"] = resolved_head + if live_database_revision is not None: + provenance["liveDatabaseRevision"] = live_database_revision + if selectors: + provenance["executionCommand"] = list( + build_pytest_command( + selectors, + raw_path=paths.output_directory / RAW_ARTIFACT_NAME, + ) + ) + configuration_fields = { + field: provenance[field] + for field in ( + "composeDigest", + "alembicConfigDigest", + "executionRegistryDigest", + "schemaManifestDigest", + ) + if field in provenance + } + if configuration_fields: + provenance["configurationDigest"] = canonical_digest(configuration_fields) + return provenance + + +def _registry_validator() -> RegistryValidator: + from scripts import validate_security_catalog as validator + + candidate = getattr(validator, "validate_execution_registry", None) + if candidate is None or not callable(candidate): + raise GateRunError("execution registry validator is unavailable") + return cast(RegistryValidator, candidate) + + +def _load_gate_inputs( + paths: GatePaths, registry_validator: RegistryValidator | None +) -> tuple[ + dict[str, Any], + dict[str, Any], + dict[str, Any], + dict[str, Any], + dict[str, Any], +]: + catalog = load_document(paths.catalog) + catalog_schema = load_document(paths.catalog_schema) + registry = load_document(paths.registry) + registry_schema = load_document(paths.registry_schema) + manifest = load_document(paths.manifest) + validate_catalog(catalog, catalog_schema) + selected_validator = registry_validator or _registry_validator() + selected_validator( + registry, + registry_schema, + catalog, + repository_root=paths.repository_root, + ) + return catalog, catalog_schema, registry, registry_schema, manifest + + +def _validate_raw_execution( + raw: Mapping[str, object], + *, + runner_execution_id: str, + selectors: Sequence[str], + exit_code: int, +) -> None: + """Reject stale, runner-authored, or structurally spoofed raw evidence.""" + + if raw.get("rawEvidenceVersion") != RAW_EVIDENCE_VERSION: + raise GateRunError("pytest raw evidence version is missing or unsupported") + if raw.get("rawProducer") != "scripts.security_gate.pytest_plugin": + raise GateRunError("pytest raw evidence producer is not the gate plugin") + if raw.get("runnerExecutionId") != runner_execution_id: + raise GateRunError("pytest raw evidence is stale or belongs to another run") + pytest_section = raw.get("pytest") + if not isinstance(pytest_section, Mapping): + raise GateRunError("raw evidence has no pytest result object") + if pytest_section.get("exitCode") != exit_code: + raise GateRunError("pytest raw exit code differs from the executed process") + selected = pytest_section.get("selectedSelectors") + if not isinstance(selected, list) or selected != list(selectors): + raise GateRunError("pytest raw selectors differ from the executed command") + + +def _failure_report( + failure_name: str, + *, + provenance: Mapping[str, object], + raw_result_digest: str, +) -> dict[str, object]: + return { + "reportVersion": "1.0.0", + "m0SecurityDecision": "fail", + "releaseDecision": "fail", + "promotionReadiness": "not-evaluated", + "provenance": { + **dict(provenance), + "rawResultDigest": raw_result_digest, + }, + "gates": { + "Security": { + "status": "fail", + "veto": True, + "failures": [failure_name], + }, + "Reliability": {"status": "not-evaluated"}, + "Quality": {"status": "not-evaluated"}, + "Budget": {"status": "not-evaluated"}, + }, + } + + +def run_gate( + paths: GatePaths, + *, + pytest_executor: PytestExecutor = _execute_pytest, + rls_auditor: RlsAuditor = _audit_rls, + registry_validator: RegistryValidator | None = None, +) -> dict[str, object]: + """Execute registered tests once, reconcile observations, audit RLS, report.""" + + paths.output_directory.mkdir(parents=True, exist_ok=True) + raw_path = paths.output_directory / RAW_ARTIFACT_NAME + report_path = paths.output_directory / REPORT_ARTIFACT_NAME + catalog: dict[str, Any] | None = None + registry: dict[str, Any] | None = None + selectors: tuple[str, ...] = () + alembic_head: str | None = None + live_database_revision: str | None = None + raw: dict[str, object] = { + "rawEvidenceVersion": RAW_EVIDENCE_VERSION, + "runnerFailure": "gate did not reach pytest execution", + "pytest": { + "exitCode": None, + "selectedSelectors": [], + "collectedNodeIds": [], + "collectionErrors": [], + "tests": [], + }, + } + pytest_execution_retained = False + atomic_write_json(raw_path, raw) + try: + ( + catalog, + _catalog_schema, + registry, + _registry_schema, + manifest, + ) = _load_gate_inputs(paths, registry_validator) + environment = load_database_environment(paths.database_environment) + selectors = deduplicate_selectors(registry) + if not selectors: + raise GateRunError("execution registry contains no selectors") + alembic_head = _alembic_head(paths) + live_database_revision = _live_database_revision(environment) + if live_database_revision != alembic_head: + raise GateRunError( + "live Alembic revision does not match the repository head: " + f"{live_database_revision!r} != {alembic_head!r}" + ) + runner_execution_id = secrets.token_hex(32) + command = build_pytest_command(selectors, raw_path=raw_path) + process_environment = dict(os.environ) + process_environment.update(environment) + process_environment[RAW_EXECUTION_ID_ENVIRONMENT] = runner_execution_id + with suppress(FileNotFoundError): + raw_path.unlink() + exit_code = pytest_executor( + command, cwd=paths.repository_root, env=process_environment + ) + try: + raw = load_document(raw_path) + except (CatalogValidationError, OSError) as error: + raise GateRunError("pytest did not produce valid raw evidence") from error + _validate_raw_execution( + raw, + runner_execution_id=runner_execution_id, + selectors=selectors, + exit_code=exit_code, + ) + pytest_execution_retained = True + + reconciliation = reconcile_execution(registry, raw, catalog=catalog) + passed_ids = reconciliation.get("passedEvidenceIds", []) + passed_evidence_ids = [ + value for value in passed_ids if isinstance(value, str) + ] if isinstance(passed_ids, list) else [] + rls_audit = rls_auditor( + environment=environment, + manifest=manifest, + passed_evidence_ids=passed_evidence_ids, + ) + raw_result_digest = _file_digest(raw_path) + report = build_release_gate_report( + reconciliation=reconciliation, + rls_audit=rls_audit, + provenance=_provenance( + paths, + registry, + catalog, + selectors=selectors, + live_database_revision=live_database_revision, + alembic_head=alembic_head, + ), + raw_result_digest=raw_result_digest, + ) + atomic_write_json(report_path, report) + return report + except Exception as error: + failure_name = type(error).__name__ + if pytest_execution_retained: + retained_raw = {**raw, "runnerFailure": failure_name} + else: + retained_raw = { + "rawEvidenceVersion": RAW_EVIDENCE_VERSION, + "runnerFailure": failure_name, + "pytest": { + "exitCode": None, + "selectedSelectors": list(selectors), + "collectedNodeIds": [], + "collectionErrors": [], + "tests": [], + }, + } + atomic_write_json(raw_path, retained_raw) + failure_report = _failure_report( + failure_name, + provenance=_best_effort_provenance( + paths, + registry=registry, + catalog=catalog, + selectors=selectors, + alembic_head=alembic_head, + live_database_revision=live_database_revision, + ), + raw_result_digest=_file_digest(raw_path), + ) + atomic_write_json(report_path, failure_report) + raise diff --git a/scripts/validate_security_catalog.py b/scripts/validate_security_catalog.py index 0e15eab3..ba40e6b3 100644 --- a/scripts/validate_security_catalog.py +++ b/scripts/validate_security_catalog.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import ast import hashlib import html import json @@ -25,7 +26,14 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] DEFAULT_CATALOG_PATH = REPOSITORY_ROOT / "eval/catalogs/security-invariants.yaml" DEFAULT_SCHEMA_PATH = REPOSITORY_ROOT / "eval/catalogs/security-catalog.schema.json" +DEFAULT_EXECUTION_REGISTRY_PATH = ( + REPOSITORY_ROOT / "eval/catalogs/m0-security-evidence.yaml" +) +DEFAULT_EXECUTION_REGISTRY_SCHEMA_PATH = ( + REPOSITORY_ROOT / "eval/catalogs/m0-security-evidence.schema.json" +) SUPPORTED_CATALOG_VERSION = "1.1.0" +SUPPORTED_EXECUTION_REGISTRY_VERSION = "1.0.0" EXPECTED_INVARIANT_COUNT = 15 EXPECTED_FIXTURE_COUNT = 12 ID_PATTERN = re.compile(r"^[A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*-[0-9]{3}$") @@ -59,6 +67,13 @@ "missing-context fallback", ) +EVIDENCE_LAYERS: tuple[str, ...] = ("property", "postgres", "runtime") +HARD_ORACLE_RESULT_KEYS: tuple[str, ...] = ( + "unauthorizedEvidenceCount", + "wrongOrganizationEffectCount", + "missingContextFallbackCount", +) + TOP_LEVEL_FIELDS = ( "catalogVersion", "authority", @@ -928,6 +943,16 @@ class ValidationReport: fixture_mappings: tuple[tuple[str, tuple[str, ...]], ...] +@dataclass(frozen=True) +class ExecutionRegistryReport: + """Static proof that the current executable evidence covers the authority.""" + + invariant_count: int + fixture_count: int + evidence_count: int + selectors: tuple[str, ...] + + class CatalogValidationError(ValueError): """One or more independently actionable catalog validation failures.""" @@ -2912,6 +2937,556 @@ def validate_catalog( ) +def _catalog_ids(catalog: Mapping[str, Any], field: str) -> tuple[str, ...]: + values = catalog.get(field) + if not isinstance(values, list): + return () + return tuple( + value["id"] + for value in values + if isinstance(value, Mapping) and isinstance(value.get("id"), str) + ) + + +def _security_evidence_markers( + source: str, + *, + function_ref: str, + path: str, + collector: _Collector, +) -> tuple[tuple[str, str], ...]: + try: + module = ast.parse(source) + except SyntaxError as error: + collector.add(path, f"test file is not valid Python: {error.msg}") + return () + function = next( + ( + node + for node in module.body + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) + and node.name == function_ref + ), + None, + ) + if function is None: + collector.add(path, f"test function does not exist: {function_ref!r}") + return () + + markers: list[tuple[str, str]] = [] + malformed = False + for decorator in function.decorator_list: + if not isinstance(decorator, ast.Call): + continue + marker = decorator.func + if not ( + isinstance(marker, ast.Attribute) + and marker.attr == "security_evidence" + and isinstance(marker.value, ast.Attribute) + and marker.value.attr == "mark" + and isinstance(marker.value.value, ast.Name) + and marker.value.value.id == "pytest" + ): + continue + keyword_values = {keyword.arg: keyword.value for keyword in decorator.keywords} + evidence_id_node = keyword_values.get("id") + layer_node = keyword_values.get("layer") + if ( + decorator.args + or any(keyword.arg is None for keyword in decorator.keywords) + or set(keyword_values) != {"id", "layer"} + or not isinstance(evidence_id_node, ast.Constant) + or not isinstance(evidence_id_node.value, str) + or not isinstance(layer_node, ast.Constant) + or not isinstance(layer_node.value, str) + ): + malformed = True + continue + markers.append((evidence_id_node.value, layer_node.value)) + if malformed: + collector.add( + path, + "@pytest.mark.security_evidence must use exact id= and layer= " + "string arguments", + ) + if not markers and not malformed: + collector.add(path, "missing @pytest.mark.security_evidence") + if len(markers) != len(set(markers)): + collector.add(path, "duplicate @pytest.mark.security_evidence declaration") + return tuple(markers) + + +def _reject_unregistered_security_evidence_markers( + repository_root: Path, + expected_selectors: set[str], + collector: _Collector, +) -> None: + """Reject evidence declarations that are not owned by the registry.""" + + tests_root = repository_root / "tests" + if not tests_root.is_dir(): + return + for test_path in sorted(tests_root.rglob("*.py")): + try: + source = test_path.read_text(encoding="utf-8") + module = ast.parse(source) + except (OSError, UnicodeError, SyntaxError) as error: + collector.add( + str(test_path.relative_to(repository_root)), + f"cannot inspect security_evidence markers: {error}", + ) + continue + relative = test_path.relative_to(repository_root).as_posix() + for function in ast.walk(module): + if not isinstance(function, ast.FunctionDef | ast.AsyncFunctionDef): + continue + has_security_marker = any( + isinstance(decorator, ast.Call) + and isinstance(decorator.func, ast.Attribute) + and decorator.func.attr == "security_evidence" + and isinstance(decorator.func.value, ast.Attribute) + and decorator.func.value.attr == "mark" + and isinstance(decorator.func.value.value, ast.Name) + and decorator.func.value.value.id == "pytest" + for decorator in function.decorator_list + ) + if not has_security_marker: + continue + selector = f"{relative}::{function.name}" + if selector not in expected_selectors: + collector.add( + f"registry selector {selector!r}", + "security_evidence marker is not declared by the registry", + ) + + +def _validate_pytest_selector( + selector: object, + path: str, + repository_root: Path, + collector: _Collector, +) -> tuple[str, tuple[tuple[str, str], ...]] | None: + if not collector.require_nonempty_string(selector, path): + return None + assert isinstance(selector, str) + if selector != selector.strip() or any( + character.isspace() for character in selector + ): + collector.add(path, "must be one exact whitespace-free pytest node selector") + return None + file_ref, separator, function_ref = selector.partition("::") + if separator != "::" or not file_ref or not function_ref.startswith("test_"): + collector.add(path, "must be an exact test-file::test_function pytest selector") + return None + if "::" in function_ref or "[" in function_ref or "]" in function_ref: + collector.add( + path, + "must select the stable test function, not one parameter instance", + ) + return None + file_path = Path(file_ref) + if file_path.is_absolute() or ".." in file_path.parts: + collector.add(path, "must stay inside the repository") + return None + root = repository_root.resolve() + resolved = (root / file_path).resolve() + try: + resolved.relative_to(root) + except ValueError: + collector.add(path, "must resolve inside the repository") + return None + if not resolved.is_file(): + collector.add(path, f"test file does not exist: {file_ref!r}") + return None + try: + source = resolved.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + collector.add(path, f"cannot read test file: {error}") + return None + markers = _security_evidence_markers( + source, + function_ref=function_ref, + path=path, + collector=collector, + ) + return selector, markers + + +def validate_execution_registry( + registry: Mapping[str, Any], + schema: Mapping[str, Any], + catalog: Mapping[str, Any], + *, + repository_root: str | Path = REPOSITORY_ROOT, +) -> ExecutionRegistryReport: + """Validate the executable evidence registry against the semantic catalog.""" + + collector = _Collector() + root = Path(repository_root) + top_level_fields = ( + "registryVersion", + "catalog", + "execution", + "hardOracleAdapters", + "evidence", + "invariantMappings", + "fixtureMappings", + ) + collector.require_exact_fields(registry, top_level_fields, "registry") + if registry.get("registryVersion") != SUPPORTED_EXECUTION_REGISTRY_VERSION: + collector.add( + "registry.registryVersion", + "must be the supported version " + f"{SUPPORTED_EXECUTION_REGISTRY_VERSION!r}", + ) + + catalog_link = collector.require_mapping( + registry.get("catalog"), "registry.catalog" + ) + if catalog_link is not None: + collector.require_exact_fields( + catalog_link, + ("path", "schemaPath", "catalogVersion"), + "registry.catalog", + ) + expected_link = { + "path": "eval/catalogs/security-invariants.yaml", + "schemaPath": "eval/catalogs/security-catalog.schema.json", + "catalogVersion": catalog.get("catalogVersion"), + } + if dict(catalog_link) != expected_link: + collector.add( + "registry.catalog", + "must exactly bind the tracked catalog, schema, and current version", + ) + + execution = collector.require_mapping( + registry.get("execution"), "registry.execution" + ) + if execution is not None: + collector.require_exact_fields( + execution, + ("framework", "forbidOutcomes", "evidenceLayers", "observationProperty"), + "registry.execution", + ) + if execution.get("framework") != "pytest": + collector.add("registry.execution.framework", "must be 'pytest'") + if execution.get("forbidOutcomes") != [ + "failed", + "error", + "skipped", + "xfailed", + "xpassed", + "incomplete", + ]: + collector.add( + "registry.execution.forbidOutcomes", + "must reject failed, error, skipped, xfailed, xpassed, and " + "incomplete outcomes", + ) + if execution.get("evidenceLayers") != list(EVIDENCE_LAYERS): + collector.add( + "registry.execution.evidenceLayers", + "must declare property, postgres, and runtime in canonical order", + ) + if execution.get("observationProperty") != ( + "context_engine.security_gate.observation.v1" + ): + collector.add( + "registry.execution.observationProperty", + "must declare the versioned explicit pytest user-property contract", + ) + + adapters = registry.get("hardOracleAdapters") + if not isinstance(adapters, list): + collector.add("registry.hardOracleAdapters", "must be an array") + adapters = [] + expected_adapters = tuple(zip(HARD_ORACLES, HARD_ORACLE_RESULT_KEYS, strict=True)) + if len(adapters) != len(expected_adapters): + collector.add( + "registry.hardOracleAdapters", + "must contain exactly the three canonical hard-oracle adapters", + ) + observed_adapters: list[tuple[object, object]] = [] + for index, value in enumerate(adapters): + path = f"registry.hardOracleAdapters[{index}]" + adapter = collector.require_mapping(value, path) + if adapter is None: + continue + collector.require_exact_fields( + adapter, + ("oracleRef", "resultKey", "assertion", "requiredValue", "observation"), + path, + ) + observed_adapters.append((adapter.get("oracleRef"), adapter.get("resultKey"))) + if adapter.get("assertion") != "equals": + collector.add(f"{path}.assertion", "must be 'equals'") + if type(adapter.get("requiredValue")) is not int or adapter.get( + "requiredValue" + ) != 0: + collector.add(f"{path}.requiredValue", "must be the numeric constant 0") + observation = collector.require_mapping( + adapter.get("observation"), f"{path}.observation" + ) + if observation is not None and dict(observation) != { + "source": "pytest-user-property", + "reducer": "sum", + }: + collector.add( + f"{path}.observation", + "must derive the sum from explicit pytest user-property values", + ) + if tuple(observed_adapters) != expected_adapters: + collector.add( + "registry.hardOracleAdapters", + "must exactly map catalog hard oracles to their canonical result keys", + ) + + evidence_values = registry.get("evidence") + if not isinstance(evidence_values, list) or not evidence_values: + collector.add("registry.evidence", "must be a non-empty array") + evidence_values = [] + evidence_by_id: dict[str, tuple[str, str]] = {} + duplicate_ids: set[str] = set() + selectors: list[str] = [] + marker_pairs_by_selector: dict[str, tuple[tuple[str, str], ...]] = {} + for index, value in enumerate(evidence_values): + path = f"registry.evidence[{index}]" + evidence = collector.require_mapping(value, path) + if evidence is None: + continue + collector.require_exact_fields(evidence, ("id", "layer", "selector"), path) + evidence_id = evidence.get("id") + if not collector.require_nonempty_string(evidence_id, f"{path}.id"): + continue + assert isinstance(evidence_id, str) + layer = evidence.get("layer") + if layer not in EVIDENCE_LAYERS: + collector.add(f"{path}.layer", "must be property, postgres, or runtime") + continue + selector_result = _validate_pytest_selector( + evidence.get("selector"), f"{path}.selector", root, collector + ) + if evidence_id in evidence_by_id: + duplicate_ids.add(evidence_id) + collector.add("registry.evidence", f"duplicate evidence id {evidence_id!r}") + elif selector_result is not None: + selector, marker_pairs = selector_result + evidence_by_id[evidence_id] = (layer, selector) + selectors.append(selector) + marker_pairs_by_selector[selector] = marker_pairs + + expected_marker_pairs: dict[str, set[tuple[str, str]]] = {} + for evidence_id, (layer, selector) in evidence_by_id.items(): + expected_marker_pairs.setdefault(selector, set()).add((evidence_id, layer)) + for selector, expected_pairs in expected_marker_pairs.items(): + observed_pairs = marker_pairs_by_selector.get(selector, ()) + observed_set = set(observed_pairs) + expected_layers_by_id = dict(expected_pairs) + if {evidence_id for evidence_id, _layer in observed_set} != { + evidence_id for evidence_id, _layer in expected_pairs + }: + collector.add( + f"registry selector {selector!r}", + "security_evidence marker IDs differ from registry IDs", + ) + for evidence_id, observed_layer in observed_set: + expected_layer = expected_layers_by_id.get(evidence_id) + if expected_layer is None: + continue + if observed_layer != expected_layer: + collector.add( + f"registry selector {selector!r}", + f"security_evidence marker layer for {evidence_id!r} is " + f"{observed_layer!r}, not {expected_layer!r}", + ) + _reject_unregistered_security_evidence_markers( + root, + set(expected_marker_pairs), + collector, + ) + + referenced_ids: set[str] = set() + + def validate_refs( + value: object, + path: str, + *, + expected_layer: str | None = None, + ) -> tuple[str, ...]: + refs = collector.require_string_list(value, path) + if refs is None: + return () + for ref_index, evidence_ref in enumerate(refs): + declared = evidence_by_id.get(evidence_ref) + if declared is None: + if evidence_ref not in duplicate_ids: + collector.add( + f"{path}[{ref_index}]", + f"unknown evidence ref {evidence_ref!r}", + ) + continue + referenced_ids.add(evidence_ref) + if expected_layer is not None and declared[0] != expected_layer: + collector.add( + f"{path}[{ref_index}]", + f"is declared as layer {declared[0]!r}, not {expected_layer!r}", + ) + return tuple(refs) + + invariant_values = registry.get("invariantMappings") + if not isinstance(invariant_values, list): + collector.add("registry.invariantMappings", "must be an array") + invariant_values = [] + invariant_refs: list[str] = [] + for index, value in enumerate(invariant_values): + path = f"registry.invariantMappings[{index}]" + mapping = collector.require_mapping(value, path) + if mapping is None: + continue + collector.require_exact_fields(mapping, ("invariantRef", "evidenceRefs"), path) + invariant_ref = mapping.get("invariantRef") + if collector.require_nonempty_string(invariant_ref, f"{path}.invariantRef"): + assert isinstance(invariant_ref, str) + invariant_refs.append(invariant_ref) + evidence_refs = collector.require_mapping( + mapping.get("evidenceRefs"), f"{path}.evidenceRefs" + ) + if evidence_refs is None: + continue + collector.require_exact_fields( + evidence_refs, EVIDENCE_LAYERS, f"{path}.evidenceRefs" + ) + for layer in EVIDENCE_LAYERS: + validate_refs( + evidence_refs.get(layer), + f"{path}.evidenceRefs.{layer}", + expected_layer=layer, + ) + canonical_catalog_invariants = _catalog_ids(catalog, "invariants") + if tuple(invariant_refs) != CANONICAL_INVARIANT_IDS or tuple( + invariant_refs + ) != canonical_catalog_invariants: + collector.add( + "registry.invariantMappings", + "must map every canonical catalog invariant exactly once in " + "canonical order", + ) + + fixture_values = registry.get("fixtureMappings") + if not isinstance(fixture_values, list): + collector.add("registry.fixtureMappings", "must be an array") + fixture_values = [] + fixture_refs: list[str] = [] + for index, value in enumerate(fixture_values): + path = f"registry.fixtureMappings[{index}]" + mapping = collector.require_mapping(value, path) + if mapping is None: + continue + collector.require_exact_fields( + mapping, + ( + "fixtureRef", + "carrierStatusAtM0", + "m0Expectation", + "evidenceRefs", + "hardOracleEvidenceRefs", + ), + path, + ) + fixture_ref = mapping.get("fixtureRef") + if collector.require_nonempty_string(fixture_ref, f"{path}.fixtureRef"): + assert isinstance(fixture_ref, str) + fixture_refs.append(fixture_ref) + catalog_fixture = next( + ( + fixture + for fixture in catalog.get("fixtures", []) + if isinstance(fixture, Mapping) + and fixture.get("id") == fixture_ref + ), + None, + ) + catalog_carrier = ( + catalog_fixture.get("carrier") + if isinstance(catalog_fixture, Mapping) + else None + ) + expected_carrier = ( + { + "carrierStatusAtM0": catalog_carrier.get("statusAtM0"), + "m0Expectation": catalog_carrier.get("m0Expectation"), + } + if isinstance(catalog_carrier, Mapping) + else None + ) + observed_carrier = { + "carrierStatusAtM0": mapping.get("carrierStatusAtM0"), + "m0Expectation": mapping.get("m0Expectation"), + } + if expected_carrier is None or observed_carrier != expected_carrier: + collector.add( + path, + "carrierStatusAtM0 and m0Expectation must exactly match " + "catalog fixture.carrier", + ) + fixture_evidence = validate_refs( + mapping.get("evidenceRefs"), f"{path}.evidenceRefs" + ) + hard_oracle_refs = collector.require_mapping( + mapping.get("hardOracleEvidenceRefs"), + f"{path}.hardOracleEvidenceRefs", + ) + if hard_oracle_refs is None: + continue + collector.require_exact_fields( + hard_oracle_refs, + HARD_ORACLE_RESULT_KEYS, + f"{path}.hardOracleEvidenceRefs", + ) + for result_key in HARD_ORACLE_RESULT_KEYS: + oracle_refs = validate_refs( + hard_oracle_refs.get(result_key), + f"{path}.hardOracleEvidenceRefs.{result_key}", + ) + for evidence_ref in oracle_refs: + if evidence_ref not in fixture_evidence: + collector.add( + f"{path}.hardOracleEvidenceRefs.{result_key}", + f"must reference a fixture evidenceRef, got {evidence_ref!r}", + ) + canonical_catalog_fixtures = _catalog_ids(catalog, "fixtures") + if tuple(fixture_refs) != CANONICAL_FIXTURE_IDS or tuple( + fixture_refs + ) != canonical_catalog_fixtures: + collector.add( + "registry.fixtureMappings", + "must map every canonical catalog fixture exactly once in canonical order", + ) + + for evidence_id in evidence_by_id: + if evidence_id not in referenced_ids: + collector.add("registry.evidence", f"orphan evidence id {evidence_id!r}") + + if isinstance(schema, Mapping): + if schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema": + collector.add("registrySchema.$schema", "must declare Draft 2020-12") + _validate_schema_instance( + registry, schema, schema, "registry", collector + ) + else: + collector.add("registrySchema", "must be an object") + + if collector.errors: + raise CatalogValidationError(collector.errors) + return ExecutionRegistryReport( + invariant_count=len(invariant_refs), + fixture_count=len(fixture_refs), + evidence_count=len(evidence_by_id), + selectors=tuple(selectors), + ) + + _MARKDOWN_HEADING = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+|$)(.*)$") _MARKDOWN_CLOSING_HASHES = re.compile(r"[ \t]+#+[ \t]*$") _MARKDOWN_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") @@ -3079,12 +3654,23 @@ def validate_files( schema_path: str | Path = DEFAULT_SCHEMA_PATH, *, repository_root: str | Path = REPOSITORY_ROOT, + registry_path: str | Path | None = None, + registry_schema_path: str | Path | None = None, ) -> ValidationReport: """Load and validate the tracked catalog and schema files.""" catalog = load_document(catalog_path) report = validate_catalog(catalog, load_document(schema_path)) _validate_document_paths(catalog, Path(repository_root)) + if registry_path is not None or registry_schema_path is not None: + validate_execution_registry( + load_document(registry_path or DEFAULT_EXECUTION_REGISTRY_PATH), + load_document( + registry_schema_path or DEFAULT_EXECUTION_REGISTRY_SCHEMA_PATH + ), + catalog, + repository_root=repository_root, + ) return report @@ -3123,13 +3709,36 @@ def build_parser() -> argparse.ArgumentParser: "eval/catalogs/security-catalog.schema.json)" ), ) + parser.add_argument( + "--registry", + type=Path, + default=DEFAULT_EXECUTION_REGISTRY_PATH, + help=( + "executable evidence registry path (default: repository " + "eval/catalogs/m0-security-evidence.yaml)" + ), + ) + parser.add_argument( + "--registry-schema", + type=Path, + default=DEFAULT_EXECUTION_REGISTRY_SCHEMA_PATH, + help=( + "execution registry schema path (default: repository " + "eval/catalogs/m0-security-evidence.schema.json)" + ), + ) return parser def main(argv: Sequence[str] | None = None) -> int: args = build_parser().parse_args(argv) try: - report = validate_files(args.catalog, args.schema) + report = validate_files( + args.catalog, + args.schema, + registry_path=args.registry, + registry_schema_path=args.registry_schema, + ) except CatalogValidationError as error: print("security catalog invalid:", file=sys.stderr) for message in error.errors: diff --git a/tests/catalog/test_m0_security_gate.py b/tests/catalog/test_m0_security_gate.py new file mode 100644 index 00000000..74d9c0e5 --- /dev/null +++ b/tests/catalog/test_m0_security_gate.py @@ -0,0 +1,983 @@ +from __future__ import annotations + +import copy +import json +import subprocess +import sys +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Any, cast + +import pytest + +from scripts.run_m0_security_gate import main as security_gate_main +from scripts.security_gate.observations import ( + OBSERVATION_PROPERTY, + ObservationValidationError, + normalize_fixture_observation, + record_fixture_observation, +) +from scripts.security_gate.report import ( + build_release_gate_report, + canonical_digest, + reconcile_execution, +) +from scripts.security_gate.runner import ( + DatabaseEnvironmentError, + GatePaths, + GateRunError, + _git_state, + _provenance, + build_pytest_command, + deduplicate_selectors, + load_database_environment, + run_gate, +) +from scripts.validate_security_catalog import CatalogValidationError + +ORACLE_KEYS = ( + "unauthorizedEvidenceCount", + "wrongOrganizationEffectCount", + "missingContextFallbackCount", +) + + +def complete_provenance(*, commit: str = "a" * 40) -> dict[str, object]: + digest = "b" * 64 + return { + "runnerVersion": "1.0.0", + "commit": commit, + "trackedDirty": False, + "trackedDiffDigest": digest, + "stagedDiffDigest": digest, + "unstagedDiffDigest": digest, + "untrackedContentDigest": digest, + "untrackedFileCount": 0, + "contentStateDirty": False, + "contentStateDigest": digest, + "executionCommand": ["python", "-m", "pytest", "registered-selector"], + "catalogDigest": digest, + "catalogSchemaDigest": digest, + "executionRegistryDigest": digest, + "executionRegistrySchemaDigest": digest, + "fixtureDigest": digest, + "schemaManifestDigest": digest, + "migrationStateDigest": digest, + "configurationDigest": digest, + "alembicHead": "head-one", + "liveDatabaseRevision": "head-one", + } + + +def registry() -> dict[str, Any]: + evidence = [ + { + "id": "PROP-001", + "layer": "property", + "selector": "tests/test_gate_sample.py::test_property", + }, + { + "id": "PG-001", + "layer": "postgres", + "selector": "tests/test_gate_sample.py::test_postgres", + }, + { + "id": "RUN-001", + "layer": "runtime", + "selector": "tests/test_gate_sample.py::test_runtime", + }, + { + "id": "FIXTURE-ACCEPT-001", + "layer": "runtime", + "selector": "tests/test_gate_sample.py::test_fixture", + }, + ] + adapters = [] + hard_oracle_evidence_refs: dict[str, list[str]] = {} + oracle_names = ( + "Unauthorized Evidence", + "wrong-Organization effect", + "missing-context fallback", + ) + for oracle_ref, result_key in zip(oracle_names, ORACLE_KEYS, strict=True): + hard_oracle_evidence_refs[result_key] = ["FIXTURE-ACCEPT-001"] + adapters.append( + { + "oracleRef": oracle_ref, + "resultKey": result_key, + "assertion": "equals", + "requiredValue": 0, + "observation": { + "source": "pytest-user-property", + "evidenceRefs": ["FIXTURE-ACCEPT-001"], + "reducer": "sum", + }, + } + ) + return { + "registryVersion": "1.0.0", + "catalog": { + "path": "eval/catalogs/security-invariants.yaml", + "schemaPath": "eval/catalogs/security-catalog.schema.json", + "catalogVersion": "1.1.0", + }, + "execution": { + "framework": "pytest", + "forbidOutcomes": [ + "failed", + "error", + "skipped", + "xfailed", + "xpassed", + "incomplete", + ], + "evidenceLayers": ["property", "postgres", "runtime"], + "observationProperty": OBSERVATION_PROPERTY, + }, + "hardOracleAdapters": adapters, + "evidence": evidence, + "invariantMappings": [ + { + "invariantRef": "TENANT-OWNERSHIP-001", + "evidenceRefs": { + "property": ["PROP-001"], + "postgres": ["PG-001"], + "runtime": ["RUN-001"], + }, + } + ], + "fixtureMappings": [ + { + "fixtureRef": "ACCEPT-001", + "carrierStatusAtM0": "available", + "m0Expectation": "active_fail_closed", + "evidenceRefs": ["FIXTURE-ACCEPT-001"], + "hardOracleEvidenceRefs": hard_oracle_evidence_refs, + } + ], + } + + +def raw_execution(*, fixture_value: int = 0) -> dict[str, Any]: + evidence = registry()["evidence"] + selectors = [entry["selector"] for entry in evidence] + evidence_by_selector: dict[str, list[dict[str, str]]] = {} + for entry in evidence: + evidence_by_selector.setdefault(entry["selector"], []).append( + {"id": entry["id"], "layer": entry["layer"]} + ) + tests = [] + for selector in selectors: + observations: list[dict[str, object]] = [] + if selector.endswith("test_fixture"): + observations.append( + { + "fixtureRef": "ACCEPT-001", + "evidenceRef": "FIXTURE-ACCEPT-001", + "values": { + "unauthorizedEvidenceCount": fixture_value, + "wrongOrganizationEffectCount": 0, + "missingContextFallbackCount": 0, + }, + } + ) + tests.append( + { + "nodeId": selector, + "outcome": "passed", + "observations": observations, + "observationErrors": [], + } + ) + return { + "rawEvidenceVersion": "1.0.0", + "pytest": { + "exitCode": 0, + "securityEvidenceMarkerVersion": "1.0.0", + "selectedSelectors": list(selectors), + "collectedNodeIds": list(selectors), + "collectedTests": [ + { + "nodeId": selector, + "securityEvidence": evidence_by_selector[selector], + "securityEvidenceErrors": [], + } + for selector in selectors + ], + "collectionErrors": [], + "tests": tests, + }, + } + + +def test_observation_helper_emits_one_closed_versioned_property() -> None: + properties: list[tuple[str, object]] = [] + + record_fixture_observation( + lambda key, value: properties.append((key, value)), + fixture_ref="ACCEPT-001", + evidence_ref="FIXTURE-ACCEPT-001", + unauthorized_evidence_count=0, + wrong_organization_effect_count=0, + missing_context_fallback_count=0, + ) + + assert properties == [ + ( + OBSERVATION_PROPERTY, + { + "fixtureRef": "ACCEPT-001", + "evidenceRef": "FIXTURE-ACCEPT-001", + "values": { + "unauthorizedEvidenceCount": 0, + "wrongOrganizationEffectCount": 0, + "missingContextFallbackCount": 0, + }, + }, + ) + ] + assert normalize_fixture_observation(properties[0][1]) == properties[0][1] + + +@pytest.mark.parametrize("invalid", [-1, True, 1.0, "0"]) +def test_observation_helper_rejects_noncanonical_counts(invalid: object) -> None: + with pytest.raises(ObservationValidationError): + record_fixture_observation( + lambda _key, _value: None, + fixture_ref="ACCEPT-001", + evidence_ref="FIXTURE-ACCEPT-001", + unauthorized_evidence_count=invalid, # type: ignore[arg-type] + wrong_organization_effect_count=0, + missing_context_fallback_count=0, + ) + + +def test_selectors_are_deduplicated_but_pytest_is_invoked_once_exactly() -> None: + document = registry() + document["evidence"].append( + { + "id": "SHARED-001", + "layer": "runtime", + "selector": "tests/test_gate_sample.py::test_runtime", + } + ) + + selectors = deduplicate_selectors(document) + command = build_pytest_command( + selectors, + raw_path=Path("artifacts/raw-evidence.json"), + python_executable="python-under-test", + ) + + assert selectors.count("tests/test_gate_sample.py::test_runtime") == 1 + assert command[:4] == ( + "python-under-test", + "-m", + "pytest", + "-p", + ) + assert command.count("tests/test_gate_sample.py::test_runtime") == 1 + assert "--reruns" not in command + assert command.count("pytest") == 1 + + +def test_reconciliation_passes_only_with_tests_and_explicit_zero_observations() -> None: + result = reconcile_execution(registry(), raw_execution()) + invariants = cast(list[dict[str, Any]], result["invariants"]) + fixtures = cast(list[dict[str, Any]], result["fixtures"]) + hard_oracles = cast(list[dict[str, Any]], result["hardOracles"]) + + assert result["passed"] is True + assert result["failures"] == [] + assert invariants[0]["status"] == "PASS" + assert fixtures[0]["status"] == "PASS" + assert [entry["observedValue"] for entry in hard_oracles] == [ + 0, + 0, + 0, + ] + + +@pytest.mark.parametrize( + ("mutator", "failure_fragment"), + [ + ( + lambda raw: raw["pytest"]["tests"][0].update(outcome="skipped"), + "forbidden outcome", + ), + ( + lambda raw: raw["pytest"]["tests"][-1].update(observations=[]), + "missing observation", + ), + ( + lambda raw: raw["pytest"]["tests"][-1]["observations"].append( + copy.deepcopy(raw["pytest"]["tests"][-1]["observations"][0]) + ), + "duplicate observation", + ), + ( + lambda raw: raw["pytest"]["tests"][-1]["observations"][0].update( + fixtureRef="ACCEPT-999" + ), + "orphan observation", + ), + ], +) +def test_reconciliation_rejects_nonexecuted_or_unreconciled_evidence( + mutator: Callable[[dict[str, Any]], None], failure_fragment: str +) -> None: + raw = raw_execution() + mutator(raw) + + result = reconcile_execution(registry(), raw) + + assert result["passed"] is False + failures = cast(list[str], result["failures"]) + assert any(failure_fragment in failure for failure in failures) + + +@pytest.mark.parametrize( + "mutator", + [ + lambda raw: raw["pytest"]["collectedNodeIds"].pop(), + lambda raw: raw["pytest"]["collectedNodeIds"].append( + "tests/test_gate_sample.py::test_unreported" + ), + lambda raw: raw["pytest"]["collectedNodeIds"].append( + raw["pytest"]["collectedNodeIds"][0] + ), + ], +) +def test_reconciliation_requires_exact_collected_and_reported_node_ids( + mutator: Callable[[dict[str, Any]], None], +) -> None: + raw = raw_execution() + mutator(raw) + + result = reconcile_execution(registry(), raw) + + assert result["passed"] is False + failures = cast(list[str], result["failures"]) + assert any( + "collected node IDs differ from test results" in item for item in failures + ) + + +@pytest.mark.parametrize( + ("mutator", "failure_fragment"), + [ + ( + lambda raw: raw["pytest"]["collectedTests"][0].update( + securityEvidence=[] + ), + "missing security_evidence marker", + ), + ( + lambda raw: raw["pytest"]["collectedTests"][0][ + "securityEvidence" + ].append( + copy.deepcopy( + raw["pytest"]["collectedTests"][0]["securityEvidence"][0] + ) + ), + "duplicate security_evidence marker", + ), + ( + lambda raw: raw["pytest"]["collectedTests"][0][ + "securityEvidence" + ][0].update(layer="runtime"), + "wrong security_evidence layer", + ), + ( + lambda raw: raw["pytest"]["collectedTests"][0][ + "securityEvidenceErrors" + ].append("marker must declare exactly id and layer"), + "malformed security_evidence marker", + ), + ], +) +def test_reconciliation_requires_execution_owned_evidence_identity( + mutator: Callable[[dict[str, Any]], None], failure_fragment: str +) -> None: + raw = raw_execution() + mutator(raw) + + result = reconcile_execution(registry(), raw) + + assert result["passed"] is False + failures = cast(list[str], result["failures"]) + assert any(failure_fragment in item for item in failures) + + +def test_each_hard_oracle_is_an_independent_security_veto() -> None: + for result_key in ORACLE_KEYS: + raw = raw_execution() + raw["pytest"]["tests"][-1]["observations"][0]["values"][result_key] = 1 + + result = reconcile_execution(registry(), raw) + + assert result["passed"] is False + hard_oracles = cast(list[dict[str, Any]], result["hardOracles"]) + oracle = next(item for item in hard_oracles if item["resultKey"] == result_key) + assert oracle["status"] == "FAIL" + + +def test_nonzero_fixture_oracle_fails_fixture_and_catalog_mapped_invariants() -> None: + raw = raw_execution() + raw["pytest"]["tests"][-1]["observations"][0]["values"][ + "wrongOrganizationEffectCount" + ] = 1 + + result = reconcile_execution( + registry(), + raw, + catalog={ + "fixtures": [ + { + "id": "ACCEPT-001", + "invariantRefs": ["TENANT-OWNERSHIP-001"], + } + ] + }, + ) + fixtures = cast(list[dict[str, Any]], result["fixtures"]) + invariants = cast(list[dict[str, Any]], result["invariants"]) + + assert fixtures[0]["status"] == "FAIL" + assert fixtures[0]["failedOracleResultKeys"] == [ + "wrongOrganizationEffectCount" + ] + assert invariants[0]["status"] == "FAIL" + assert invariants[0]["failedFixtureRefs"] == ["ACCEPT-001"] + + +def test_parameterized_leaves_may_repeat_observations_across_leaves() -> None: + raw = raw_execution() + fixture = raw["pytest"]["tests"].pop() + selector = fixture["nodeId"] + leaves = [] + for parameter in ("a", "b", "c"): + leaf = copy.deepcopy(fixture) + leaf["nodeId"] = f"{selector}[{parameter}]" + leaves.append(leaf) + raw["pytest"]["tests"].extend(leaves) + raw["pytest"]["collectedNodeIds"].remove(selector) + raw["pytest"]["collectedNodeIds"].extend(leaf["nodeId"] for leaf in leaves) + collected_fixture = raw["pytest"]["collectedTests"].pop() + raw["pytest"]["collectedTests"].extend( + { + **copy.deepcopy(collected_fixture), + "nodeId": leaf["nodeId"], + } + for leaf in leaves + ) + raw["pytest"]["selectedSelectors"] = [ + entry for entry in raw["pytest"]["selectedSelectors"] if entry != selector + ] + [selector] + + result = reconcile_execution(registry(), raw) + fixtures = cast(list[dict[str, Any]], result["fixtures"]) + + assert result["passed"] is True + assert fixtures[0]["observationCount"] == 3 + + +def test_release_report_keeps_four_gates_independent_and_has_no_score() -> None: + reconciliation = reconcile_execution(registry(), raw_execution()) + rls = {"passed": True, "coverage": {"numerator": 20, "denominator": 20}} + report = build_release_gate_report( + reconciliation=reconciliation, + rls_audit=rls, + provenance=complete_provenance(), + raw_result_digest="b" * 64, + ) + + assert report["m0SecurityDecision"] == "pass" + assert report["releaseDecision"] == "not-evaluated" + assert report["promotionReadiness"] == "not-evaluated" + gates = cast(dict[str, dict[str, Any]], report["gates"]) + assert gates["Security"]["status"] == "pass" + for gate in ("Reliability", "Quality", "Budget"): + assert gates[gate]["status"] == "not-evaluated" + assert "score" not in json.dumps(report).casefold() + assert canonical_digest(report) == canonical_digest(copy.deepcopy(report)) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda provenance: provenance.update(commit="unavailable"), + lambda provenance: provenance.update(contentStateDigest="unavailable"), + lambda provenance: provenance.update(liveDatabaseRevision="old-head"), + ], +) +def test_release_report_fails_closed_when_required_provenance_is_unavailable( + mutation: Callable[[dict[str, object]], None], +) -> None: + provenance = complete_provenance() + mutation(provenance) + + report = build_release_gate_report( + reconciliation=reconcile_execution(registry(), raw_execution()), + rls_audit={"passed": True}, + provenance=provenance, + raw_result_digest="b" * 64, + ) + + assert report["m0SecurityDecision"] == "fail" + assert report["releaseDecision"] == "fail" + gates = cast(dict[str, dict[str, Any]], report["gates"]) + assert gates["Security"]["status"] == "fail" + assert any( + "provenance" in failure + for failure in gates["Security"]["failures"] + ) + + +def test_cli_reports_m0_security_without_claiming_release_pass( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr( + "scripts.run_m0_security_gate.run_gate", + lambda _paths: { + "m0SecurityDecision": "pass", + "releaseDecision": "not-evaluated", + }, + ) + + assert security_gate_main(["--output-dir", "artifacts"]) == 0 + output = capsys.readouterr().out + assert "M0 SECURITY PASS" in output + assert "release pass" not in output.casefold() + + +def test_provenance_has_exact_nonsecret_config_migration_and_fixture_digests( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + (repository / "migrations/versions").mkdir(parents=True) + (repository / "eval/catalogs").mkdir(parents=True) + (repository / "engine/persistence").mkdir(parents=True) + output = repository / ".context-engine/security-gate" + for relative, contents in { + "compose.yaml": "services: {}\n", + "alembic.ini": "[alembic]\nscript_location = migrations\n", + "migrations/env.py": "# test\n", + "migrations/script.py.mako": "# test\n", + "migrations/versions/a_head.py": ( + "revision = 'head_one'\n" + "down_revision = None\n" + "branch_labels = None\n" + "depends_on = None\n" + ), + "eval/catalogs/security-invariants.yaml": "{}\n", + "eval/catalogs/security-catalog.schema.json": "{}\n", + "eval/catalogs/m0-security-evidence.yaml": "{}\n", + "eval/catalogs/m0-security-evidence.schema.json": "{}\n", + "engine/persistence/schema_security_manifest.yaml": "{}\n", + }.items(): + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + paths = GatePaths( + repository_root=repository, + catalog=repository / "eval/catalogs/security-invariants.yaml", + catalog_schema=repository / "eval/catalogs/security-catalog.schema.json", + registry=repository / "eval/catalogs/m0-security-evidence.yaml", + registry_schema=( + repository / "eval/catalogs/m0-security-evidence.schema.json" + ), + manifest=repository / "engine/persistence/schema_security_manifest.yaml", + database_environment=repository / ".context-engine/database.env", + output_directory=output, + ) + + provenance = _provenance( + paths, + registry(), + {"fixtures": [{"id": "ACCEPT-001", "secret": "not-a-secret"}]}, + selectors=("tests/test_gate_sample.py::test_fixture",), + live_database_revision="head_one", + ) + + assert provenance["alembicHead"] == "head_one" + assert provenance["liveDatabaseRevision"] == "head_one" + assert provenance["trackedDirty"] == "unavailable" + assert len(cast(str, provenance["fixtureDigest"])) == 64 + assert len(cast(str, provenance["configurationDigest"])) == 64 + assert "database.env" not in json.dumps(provenance) + + +def test_git_state_hashes_staged_unstaged_and_untracked_nonignored_content( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + subprocess.run(["git", "init", "-q", str(repository)], check=True) + subprocess.run( + ["git", "-C", str(repository), "config", "user.email", "gate@example.test"], + check=True, + ) + subprocess.run( + ["git", "-C", str(repository), "config", "user.name", "Gate Test"], + check=True, + ) + (repository / ".gitignore").write_text( + ".context-engine/\n.harness/\n", encoding="utf-8" + ) + (repository / "tracked.py").write_text("tracked = 1\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "add", "."], check=True) + tracked_ignored = repository / ".context-engine/tracked-artifact" + tracked_ignored.parent.mkdir() + tracked_ignored.write_text("artifact = 1\n", encoding="utf-8") + subprocess.run( + [ + "git", + "-C", + str(repository), + "add", + "--force", + ".context-engine/tracked-artifact", + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(repository), "commit", "-qm", "initial"], check=True + ) + (repository / "tracked.py").write_text("tracked = 2\n", encoding="utf-8") + (repository / "staged.py").write_text("staged = 1\n", encoding="utf-8") + subprocess.run( + ["git", "-C", str(repository), "add", "staged.py"], check=True + ) + executed = repository / "tests/test_executed.py" + executed.parent.mkdir() + executed.write_text("def test_executed(): pass\n", encoding="utf-8") + ignored = repository / ".context-engine/database.env" + ignored.parent.mkdir(exist_ok=True) + ignored.write_text("PASSWORD=first\n", encoding="utf-8") + + before = _git_state(repository) + executed.write_text("def test_executed(): assert True\n", encoding="utf-8") + after_untracked_change = _git_state(repository) + ignored.write_text("PASSWORD=second\n", encoding="utf-8") + after_ignored_change = _git_state(repository) + tracked_ignored.write_text("artifact = 2\n", encoding="utf-8") + after_tracked_ignored_change = _git_state(repository) + + assert before["contentStateDirty"] is True + assert before["contentStateDigest"] != after_untracked_change["contentStateDigest"] + assert ( + after_untracked_change["contentStateDigest"] + == after_ignored_change["contentStateDigest"] + ) + assert ( + after_ignored_change["contentStateDigest"] + == after_tracked_ignored_change["contentStateDigest"] + ) + serialized = json.dumps(after_ignored_change) + assert "PASSWORD" not in serialized + assert ".context-engine" not in serialized + + +def test_database_environment_parser_rejects_duplicates_and_shell_syntax( + tmp_path: Path, +) -> None: + duplicate = tmp_path / "database.env" + duplicate.write_text("KEY=value\nKEY=other\n", encoding="utf-8") + duplicate.chmod(0o600) + + with pytest.raises(DatabaseEnvironmentError, match="unexpected variable"): + load_database_environment(duplicate) + + unsafe = tmp_path / "database-unsafe.env" + unsafe.write_text("export CONTEXT_ENGINE_RUNTIME_ROLE=value\n", encoding="utf-8") + unsafe.chmod(0o600) + with pytest.raises(DatabaseEnvironmentError, match="KEY=value"): + load_database_environment(unsafe) + + +def test_runner_failure_overwrites_a_stale_pass_report(tmp_path: Path) -> None: + output = tmp_path / "artifacts" + output.mkdir() + report_path = output / "release-gate-report.json" + report_path.write_text('{"releaseDecision":"pass"}\n', encoding="utf-8") + missing = tmp_path / "missing" + paths = GatePaths( + repository_root=tmp_path, + catalog=missing / "catalog.json", + catalog_schema=missing / "catalog-schema.json", + registry=missing / "registry.json", + registry_schema=missing / "registry-schema.json", + manifest=missing / "manifest.json", + database_environment=missing / "database.env", + output_directory=output, + ) + + with pytest.raises(CatalogValidationError): + run_gate(paths) + + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["releaseDecision"] == "fail" + assert report["gates"]["Security"]["status"] == "fail" + raw = json.loads((output / "raw-evidence.json").read_text(encoding="utf-8")) + assert raw["runnerFailure"] == "CatalogValidationError" + + +def test_runner_failure_keeps_safe_available_provenance(tmp_path: Path) -> None: + output = tmp_path / "artifacts" + missing = tmp_path / "missing" + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + paths = GatePaths( + repository_root=tmp_path, + catalog=missing / "catalog.json", + catalog_schema=missing / "catalog-schema.json", + registry=missing / "registry.json", + registry_schema=missing / "registry-schema.json", + manifest=missing / "manifest.json", + database_environment=missing / "database.env", + output_directory=output, + ) + + with pytest.raises(CatalogValidationError): + run_gate(paths) + + report = json.loads( + (output / "release-gate-report.json").read_text(encoding="utf-8") + ) + provenance = report["provenance"] + assert provenance["runnerVersion"] == "1.0.0" + assert "commit" in provenance + assert "contentStateDigest" in provenance + assert "rawResultDigest" in provenance + assert "database.env" not in json.dumps(report) + + +def test_gate_fails_before_pytest_when_live_revision_differs_from_head( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + called = False + + def executor( + command: Sequence[str], *, cwd: Path, env: Mapping[str, str] + ) -> int: + del command, cwd, env + nonlocal called + called = True + return 0 + + monkeypatch.setattr( + "scripts.security_gate.runner._load_gate_inputs", + lambda _paths, _validator: ( + {"fixtures": []}, + {}, + registry(), + {}, + {}, + ), + ) + monkeypatch.setattr( + "scripts.security_gate.runner.load_database_environment", lambda _path: {} + ) + monkeypatch.setattr( + "scripts.security_gate.runner._alembic_head", lambda _paths: "head" + ) + monkeypatch.setattr( + "scripts.security_gate.runner._live_database_revision", + lambda _environment: "old", + ) + paths = GatePaths( + repository_root=tmp_path, + catalog=tmp_path / "catalog.json", + catalog_schema=tmp_path / "catalog-schema.json", + registry=tmp_path / "registry.json", + registry_schema=tmp_path / "registry-schema.json", + manifest=tmp_path / "manifest.json", + database_environment=tmp_path / "database.env", + output_directory=tmp_path / "artifacts", + ) + + with pytest.raises(GateRunError, match="live Alembic revision"): + run_gate(paths, pytest_executor=executor) + + assert called is False + + +def test_gate_rejects_stale_or_spoofed_raw_execution( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "scripts.security_gate.runner._load_gate_inputs", + lambda _paths, _validator: ( + {"fixtures": []}, + {}, + registry(), + {}, + {}, + ), + ) + monkeypatch.setattr( + "scripts.security_gate.runner.load_database_environment", lambda _path: {} + ) + monkeypatch.setattr( + "scripts.security_gate.runner._alembic_head", lambda _paths: "head" + ) + monkeypatch.setattr( + "scripts.security_gate.runner._live_database_revision", + lambda _environment: "head", + ) + + def executor( + command: Sequence[str], *, cwd: Path, env: Mapping[str, str] + ) -> int: + raw_path = Path(command[command.index("--security-gate-raw") + 1]) + spoofed = raw_execution() + spoofed["rawProducer"] = "scripts.security_gate.pytest_plugin" + spoofed["runnerExecutionId"] = "stale-run" + raw_path.write_text(json.dumps(spoofed), encoding="utf-8") + return 0 + + paths = GatePaths( + repository_root=tmp_path, + catalog=tmp_path / "catalog.json", + catalog_schema=tmp_path / "catalog-schema.json", + registry=tmp_path / "registry.json", + registry_schema=tmp_path / "registry-schema.json", + manifest=tmp_path / "manifest.json", + database_environment=tmp_path / "database.env", + output_directory=tmp_path / "artifacts", + ) + + with pytest.raises(GateRunError, match="stale or belongs to another run"): + run_gate(paths, pytest_executor=executor) + + retained = json.loads( + (paths.output_directory / "raw-evidence.json").read_text(encoding="utf-8") + ) + assert retained["runnerFailure"] == "GateRunError" + assert retained.get("runnerExecutionId") != "stale-run" + + +def test_post_pytest_failure_preserves_the_valid_raw_execution( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "scripts.security_gate.runner._load_gate_inputs", + lambda _paths, _validator: ( + {"fixtures": []}, + {}, + registry(), + {}, + {}, + ), + ) + monkeypatch.setattr( + "scripts.security_gate.runner.load_database_environment", lambda _path: {} + ) + monkeypatch.setattr( + "scripts.security_gate.runner._alembic_head", lambda _paths: "head" + ) + monkeypatch.setattr( + "scripts.security_gate.runner._live_database_revision", + lambda _environment: "head", + ) + + def executor( + command: Sequence[str], *, cwd: Path, env: Mapping[str, str] + ) -> int: + del cwd + raw_path = Path(command[command.index("--security-gate-raw") + 1]) + valid = raw_execution() + valid["rawProducer"] = "scripts.security_gate.pytest_plugin" + valid["runnerExecutionId"] = env[ + "CONTEXT_ENGINE_SECURITY_GATE_EXECUTION_ID" + ] + raw_path.write_text(json.dumps(valid), encoding="utf-8") + return 0 + + def fail_after_pytest(**_kwargs: object) -> Mapping[str, object]: + raise RuntimeError("post-pytest RLS audit failure") + + paths = GatePaths( + repository_root=tmp_path, + catalog=tmp_path / "catalog.json", + catalog_schema=tmp_path / "catalog-schema.json", + registry=tmp_path / "registry.json", + registry_schema=tmp_path / "registry-schema.json", + manifest=tmp_path / "manifest.json", + database_environment=tmp_path / "database.env", + output_directory=tmp_path / "artifacts", + ) + + with pytest.raises(RuntimeError, match="post-pytest RLS audit failure"): + run_gate(paths, pytest_executor=executor, rls_auditor=fail_after_pytest) + + retained = json.loads( + (paths.output_directory / "raw-evidence.json").read_text(encoding="utf-8") + ) + assert retained["runnerFailure"] == "RuntimeError" + assert retained["pytest"]["exitCode"] == 0 + assert len(retained["pytest"]["tests"]) == 4 + assert retained["rawProducer"] == "scripts.security_gate.pytest_plugin" + + +def test_pytest_plugin_emits_raw_json_with_xpass_and_observation( + tmp_path: Path, +) -> None: + test_file = tmp_path / "test_plugin_sample.py" + test_file.write_text( + """ +import pytest +from scripts.security_gate.observations import record_fixture_observation + +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-001", layer="runtime") +def test_observed(record_property): + record_fixture_observation( + record_property, + fixture_ref="ACCEPT-001", + evidence_ref="FIXTURE-ACCEPT-001", + unauthorized_evidence_count=0, + wrong_organization_effect_count=0, + missing_context_fallback_count=0, + ) + +@pytest.mark.security_evidence(id="RUNTIME-XPASS-001", layer="runtime") +@pytest.mark.xfail(strict=False) +def test_unexpected_pass(): + pass +""".lstrip(), + encoding="utf-8", + ) + raw_path = tmp_path / "raw.json" + selectors = ( + f"{test_file}::test_observed", + f"{test_file}::test_unexpected_pass", + ) + command = build_pytest_command( + selectors, + raw_path=raw_path, + python_executable=sys.executable, + ) + + import subprocess + + completed = subprocess.run(command, check=False, capture_output=True, text=True) + assert completed.returncode == 0 + raw = json.loads(raw_path.read_text(encoding="utf-8")) + outcomes = {test["nodeId"]: test["outcome"] for test in raw["pytest"]["tests"]} + observed_node = next(node for node in outcomes if node.endswith("::test_observed")) + xpass_node = next( + node for node in outcomes if node.endswith("::test_unexpected_pass") + ) + assert outcomes[observed_node] == "passed" + assert outcomes[xpass_node] == "xpassed" + assert raw["pytest"]["tests"][0]["observations"][0]["fixtureRef"] == ( + "ACCEPT-001" + ) + assert raw["pytest"]["securityEvidenceMarkerVersion"] == "1.0.0" + metadata = { + item["nodeId"]: item["securityEvidence"] + for item in raw["pytest"]["collectedTests"] + } + assert metadata[observed_node] == [ + {"id": "FIXTURE-ACCEPT-001", "layer": "runtime"} + ] diff --git a/tests/catalog/test_validate_m0_security_evidence.py b/tests/catalog/test_validate_m0_security_evidence.py new file mode 100644 index 00000000..a1ffa999 --- /dev/null +++ b/tests/catalog/test_validate_m0_security_evidence.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from scripts.validate_security_catalog import ( + CANONICAL_FIXTURE_IDS, + CANONICAL_INVARIANT_IDS, + DEFAULT_CATALOG_PATH, + DEFAULT_EXECUTION_REGISTRY_PATH, + DEFAULT_EXECUTION_REGISTRY_SCHEMA_PATH, + DEFAULT_SCHEMA_PATH, + HARD_ORACLES, + CatalogValidationError, + load_document, + validate_execution_registry, +) + + +def _documents() -> tuple[ + dict[str, Any], + dict[str, Any], + dict[str, Any], +]: + return ( + load_document(DEFAULT_EXECUTION_REGISTRY_PATH), + load_document(DEFAULT_EXECUTION_REGISTRY_SCHEMA_PATH), + load_document(DEFAULT_CATALOG_PATH), + ) + + +def _validation_error(registry: dict[str, Any]) -> CatalogValidationError: + _, schema, catalog = _documents() + with pytest.raises(CatalogValidationError) as raised: + validate_execution_registry( + registry, + schema, + catalog, + repository_root=Path(__file__).resolve().parents[2], + ) + return raised.value + + +def _temporary_marker_repository( + tmp_path: Path, + *, + test_source: str, +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], Path]: + registry, schema, catalog = _documents() + repository = tmp_path / "repository" + test_file = repository / "tests/unit/test_marker_contract.py" + test_file.parent.mkdir(parents=True, exist_ok=True) + evidence_by_selector: dict[str, list[dict[str, str]]] = {} + for evidence in registry["evidence"]: + evidence_by_selector.setdefault(evidence["selector"], []).append(evidence) + generated_sources = ["import pytest\n\n", test_source] + for index, entries in enumerate(evidence_by_selector.values()): + function_ref = "test_registered" if index == 0 else f"test_registered_{index}" + selector = f"tests/unit/test_marker_contract.py::{function_ref}" + for entry in entries: + entry["selector"] = selector + if index == 0: + continue + generated_sources.extend( + f'@pytest.mark.security_evidence(id="{entry["id"]}", ' + f'layer="{entry["layer"]}")\n' + for entry in entries + ) + generated_sources.append(f"def {function_ref}() -> None:\n pass\n\n") + test_file.write_text("".join(generated_sources), encoding="utf-8") + return registry, schema, catalog, repository + + +def test_tracked_execution_registry_covers_every_canonical_authority() -> None: + registry, schema, catalog = _documents() + + report = validate_execution_registry( + registry, + schema, + catalog, + repository_root=Path(__file__).resolve().parents[2], + ) + + assert report.invariant_count == 15 + assert report.fixture_count == 12 + assert report.evidence_count >= 27 + invariant_refs = tuple( + mapping["invariantRef"] for mapping in registry["invariantMappings"] + ) + assert invariant_refs == CANONICAL_INVARIANT_IDS + assert tuple(mapping["fixtureRef"] for mapping in registry["fixtureMappings"]) == ( + CANONICAL_FIXTURE_IDS + ) + + +def test_planned_catalog_evidence_is_separate_from_executable_refs() -> None: + registry, _, catalog = _documents() + release = next( + invariant + for invariant in catalog["invariants"] + if invariant["id"] == "RELEASE-OWNER-019" + ) + mapping = next( + item + for item in registry["invariantMappings"] + if item["invariantRef"] == "RELEASE-OWNER-019" + ) + + assert release["expectedEvidence"] == { + "property": ["PROP-RELEASE-OWNER-019"], + "postgres": ["PG-RELEASE-OWNER-019", "LEARN-006", "LEARN-007"], + "runtimeOrDelivery": ["LEARN-004", "LEARN-008", "LEARN-009"], + } + assert mapping["evidenceRefs"] == { + "property": ["PROP-RELEASE-OWNER-019"], + "postgres": ["PG-RELEASE-OWNER-019"], + "runtime": ["RUNTIME-RELEASE-OWNER-019"], + } + + +def test_m0_registry_uses_honest_unavailable_carrier_and_learning_evidence() -> None: + registry, _, _ = _documents() + evidence = {entry["id"]: entry["selector"] for entry in registry["evidence"]} + + unavailable_carrier = ( + "tests/integration/test_m0_unavailable_security_carriers.py::" + "test_m0_unavailable_citation_and_egress_carriers_fail_closed" + ) + assert evidence["PG-CITATION-AUTH-010"] == unavailable_carrier + assert evidence["PG-EGRESS-011"] == unavailable_carrier + assert evidence["PROP-CROSS-ORG-LEARN-015"] == ( + "tests/unit/test_m0_learning_isolation.py::" + "test_m0_learning_artifact_contract_has_no_cross_organization_carrier" + ) + assert evidence["PG-CROSS-ORG-LEARN-015"] == ( + "tests/integration/test_m0_unavailable_security_carriers.py::" + "test_learning_persistence_is_organization_bound" + ) + assert evidence["RUNTIME-CROSS-ORG-LEARN-015"] == ( + "tests/unit/test_m0_learning_isolation.py::" + "test_context_learning_rejects_cross_organization_candidate_lineage" + ) + assert evidence["PG-FIELD-PROJECTION-RLS-048"] == ( + "tests/integration/test_authorized_field_schema.py::" + "test_cross_organization_field_authority_and_values_fail_closed" + ) + + +def test_registry_requires_every_invariant_evidence_layer() -> None: + registry, _, _ = _documents() + broken = copy.deepcopy(registry) + broken["invariantMappings"][0]["evidenceRefs"]["postgres"] = [] + + error = _validation_error(broken) + + assert any( + message.startswith( + "registry.invariantMappings[0].evidenceRefs.postgres: " + ) + for message in error.errors + ) + + +def test_registry_rejects_duplicate_and_orphan_evidence_ids() -> None: + registry, _, _ = _documents() + broken = copy.deepcopy(registry) + duplicate = copy.deepcopy(broken["evidence"][0]) + duplicate["selector"] = broken["evidence"][1]["selector"] + broken["evidence"].append(duplicate) + broken["evidence"].append( + { + "id": "ORPHAN-EVIDENCE-020", + "layer": "property", + "selector": ( + "tests/unit/test_package_budget.py::" + "test_requested_ceilings_never_increase_any_server_dimension" + ), + } + ) + + error = _validation_error(broken) + + assert any("duplicate evidence id" in message for message in error.errors) + assert any( + "orphan evidence id 'ORPHAN-EVIDENCE-020'" in message + for message in error.errors + ) + + +def test_registry_rejects_wrong_layer_unknown_refs_and_empty_selectors() -> None: + registry, _, _ = _documents() + wrong_layer = copy.deepcopy(registry) + evidence_id = wrong_layer["invariantMappings"][0]["evidenceRefs"]["property"][0] + declared = next( + item for item in wrong_layer["evidence"] if item["id"] == evidence_id + ) + declared["layer"] = "postgres" + assert any( + "is declared as layer 'postgres'" in message + for message in _validation_error(wrong_layer).errors + ) + + empty_selector = copy.deepcopy(registry) + empty_selector["evidence"][0]["selector"] = "" + assert any( + "must be a non-empty string" in message + for message in _validation_error(empty_selector).errors + ) + + unknown_ref = copy.deepcopy(registry) + unknown_ref["fixtureMappings"][0]["evidenceRefs"] = [ + "UNKNOWN-EVIDENCE-020" + ] + assert any( + "unknown evidence ref 'UNKNOWN-EVIDENCE-020'" in message + for message in _validation_error(unknown_ref).errors + ) + + +def test_registry_requires_exact_fixture_observation_owners() -> None: + registry, _, _ = _documents() + broken = copy.deepcopy(registry) + mapping = broken["fixtureMappings"][0] + mapping["hardOracleEvidenceRefs"].pop("missingContextFallbackCount") + + error = _validation_error(broken) + + assert any( + "hardOracleEvidenceRefs.missingContextFallbackCount" in message + for message in error.errors + ) + + +def test_registry_hard_oracle_adapters_match_catalog_vetoes() -> None: + registry, _, _ = _documents() + assert registry["execution"]["observationProperty"] == ( + "context_engine.security_gate.observation.v1" + ) + assert [adapter["oracleRef"] for adapter in registry["hardOracleAdapters"]] == list( + HARD_ORACLES + ) + assert [adapter["resultKey"] for adapter in registry["hardOracleAdapters"]] == [ + "unauthorizedEvidenceCount", + "wrongOrganizationEffectCount", + "missingContextFallbackCount", + ] + assert all( + adapter["observation"] == { + "source": "pytest-user-property", + "reducer": "sum", + } + for adapter in registry["hardOracleAdapters"] + ) + + +def test_registry_selector_paths_and_functions_exist() -> None: + registry, _, _ = _documents() + root = Path(__file__).resolve().parents[2] + + for evidence in registry["evidence"]: + file_ref, separator, function_ref = evidence["selector"].partition("::") + assert separator == "::" + assert function_ref.startswith("test_") + source = (root / file_ref).read_text(encoding="utf-8") + assert f"def {function_ref}(" in source + + +@pytest.mark.parametrize( + ("marker", "failure_fragment"), + [ + ("", "missing @pytest.mark.security_evidence"), + ( + '@pytest.mark.security_evidence(id="WRONG-001", layer="property")\n', + "marker IDs differ from registry IDs", + ), + ( + '@pytest.mark.security_evidence(' + 'id="PROP-TENANT-OWNERSHIP-001", layer="runtime")\n', + "marker layer", + ), + ( + '@pytest.mark.security_evidence(' + '"PROP-TENANT-OWNERSHIP-001", layer="property")\n', + "must use exact id= and layer= string arguments", + ), + ], +) +def test_registry_rejects_missing_wrong_or_malformed_evidence_markers( + tmp_path: Path, + marker: str, + failure_fragment: str, +) -> None: + registry, schema, catalog, repository = _temporary_marker_repository( + tmp_path, + test_source=( + f"{marker}" + "def test_registered() -> None:\n" + " pass\n\n" + ), + ) + + with pytest.raises(CatalogValidationError) as raised: + validate_execution_registry( + registry, + schema, + catalog, + repository_root=repository, + ) + + assert any(failure_fragment in message for message in raised.value.errors) + + +def test_repeatable_markers_must_exactly_match_shared_selector_registrations( + tmp_path: Path, +) -> None: + registry, schema, catalog, repository = _temporary_marker_repository( + tmp_path, + test_source=( + '@pytest.mark.security_evidence(id="PROP-TENANT-OWNERSHIP-001", ' + 'layer="property")\n' + '@pytest.mark.security_evidence(id="UNRELATED-999", layer="runtime")\n' + "def test_registered() -> None:\n" + " pass\n" + ), + ) + + with pytest.raises(CatalogValidationError) as raised: + validate_execution_registry( + registry, + schema, + catalog, + repository_root=repository, + ) + + assert any( + "marker IDs differ from registry IDs" in message + for message in raised.value.errors + ) + + +def test_registry_rejects_security_evidence_marker_on_unregistered_test( + tmp_path: Path, +) -> None: + registry, schema, catalog, repository = _temporary_marker_repository( + tmp_path, + test_source=( + '@pytest.mark.security_evidence(id="PROP-TENANT-OWNERSHIP-001", ' + 'layer="property")\n' + "def test_registered() -> None:\n" + " pass\n\n" + '@pytest.mark.security_evidence(id="UNREGISTERED-999", ' + 'layer="runtime")\n' + "def test_unregistered() -> None:\n" + " pass\n" + ), + ) + + with pytest.raises(CatalogValidationError) as raised: + validate_execution_registry( + registry, + schema, + catalog, + repository_root=repository, + ) + + assert any( + "security_evidence marker is not declared by the registry" in message + for message in raised.value.errors + ) + + +def test_fixture_carrier_status_is_frozen_from_catalog() -> None: + registry, _, catalog = _documents() + catalog_carriers = { + fixture["id"]: fixture["carrier"] for fixture in catalog["fixtures"] + } + + for fixture in registry["fixtureMappings"]: + carrier = catalog_carriers[fixture["fixtureRef"]] + assert fixture["carrierStatusAtM0"] == carrier["statusAtM0"] + assert fixture["m0Expectation"] == carrier["m0Expectation"] + + assert "carrierStatusAtM0" in json.dumps(registry) + + +@pytest.mark.parametrize("field", ["carrierStatusAtM0", "m0Expectation"]) +def test_registry_rejects_missing_or_drifted_fixture_carrier_status(field: str) -> None: + registry, _, _ = _documents() + missing = copy.deepcopy(registry) + missing["fixtureMappings"][0].pop(field) + assert any(field in message for message in _validation_error(missing).errors) + + drifted = copy.deepcopy(registry) + drifted["fixtureMappings"][0][field] = "future" + assert any( + "must exactly match catalog fixture.carrier" in message + for message in _validation_error(drifted).errors + ) + + +def test_catalog_schema_and_registry_paths_are_distinct_authorities() -> None: + assert DEFAULT_SCHEMA_PATH.name == "security-catalog.schema.json" + assert DEFAULT_EXECUTION_REGISTRY_PATH.name == "m0-security-evidence.yaml" + assert DEFAULT_EXECUTION_REGISTRY_SCHEMA_PATH.name == ( + "m0-security-evidence.schema.json" + ) diff --git a/tests/integration/test_access_policy_revocation.py b/tests/integration/test_access_policy_revocation.py index ba9c441f..398716b4 100644 --- a/tests/integration/test_access_policy_revocation.py +++ b/tests/integration/test_access_policy_revocation.py @@ -354,6 +354,7 @@ def _state( ) +@pytest.mark.security_evidence(id="PG-REVOCATION-006", layer="postgres") def test_change_access_atomically_revokes_exact_grant_and_advances_epoch( control_configuration: DatabaseConfiguration, migration_configuration: DatabaseConfiguration, diff --git a/tests/integration/test_authorized_field_schema.py b/tests/integration/test_authorized_field_schema.py index eedfbd25..04eb13b2 100644 --- a/tests/integration/test_authorized_field_schema.py +++ b/tests/integration/test_authorized_field_schema.py @@ -454,6 +454,7 @@ def test_revoked_resource_acl_filters_every_projection_row_and_right( migration_engine.dispose() +@pytest.mark.security_evidence(id="PG-FIELD-PROJECTION-RLS-048", layer="postgres") def test_cross_organization_field_authority_and_values_fail_closed( guarded_runtime_engine: Engine, migration_configuration: DatabaseConfiguration, diff --git a/tests/integration/test_content_schema.py b/tests/integration/test_content_schema.py index b2c927b6..ffc10f17 100644 --- a/tests/integration/test_content_schema.py +++ b/tests/integration/test_content_schema.py @@ -335,6 +335,7 @@ def content_fixture( engine.dispose() +@pytest.mark.security_evidence(id="PG-INDEX-NOT-AUTHORITY-005", layer="postgres") def test_runtime_rls_exposes_only_active_non_tombstoned_same_organization_rows( guarded_runtime_engine: Engine, content_fixture: ContentFixture, diff --git a/tests/integration/test_context_run_schema.py b/tests/integration/test_context_run_schema.py index 6ff9ed3f..5dd8d004 100644 --- a/tests/integration/test_context_run_schema.py +++ b/tests/integration/test_context_run_schema.py @@ -446,6 +446,7 @@ def test_runtime_can_only_append_exact_current_user_actor_lineage( migration_engine.dispose() +@pytest.mark.security_evidence(id="PG-TRACE-REDACTION-012", layer="postgres") def test_runtime_cross_organization_insert_attempts_are_bidirectionally_zero_effect( guarded_runtime_engine: Engine, migration_configuration: DatabaseConfiguration, diff --git a/tests/integration/test_m0_security_gate_rls.py b/tests/integration/test_m0_security_gate_rls.py new file mode 100644 index 00000000..bbba3493 --- /dev/null +++ b/tests/integration/test_m0_security_gate_rls.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, cast + +import pytest +from sqlalchemy import Engine, text + +from engine.persistence import DatabaseConfiguration, create_database_engine +from scripts.security_gate.rls import ( + NON_OWNER_EVIDENCE_BY_TABLE, + audit_live_rls, + audit_rls_snapshot, + snapshot_public_schema, +) + +ROOT = Path(__file__).parents[2] +MANIFEST_PATH = ROOT / "engine/persistence/schema_security_manifest.yaml" +BEHAVIORAL_EVIDENCE_IDS = set(NON_OWNER_EVIDENCE_BY_TABLE.values()) + + +def _manifest() -> dict[str, object]: + document = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + assert isinstance(document, dict) + return cast(dict[str, object], document) + + +@pytest.mark.security_evidence(id="PG-RLS-ALL-TENANT-TABLES", layer="postgres") +def test_all_manifest_tenant_tables_pass_live_non_owner_rls_audit( + guarded_runtime_engine: Engine, +) -> None: + """PG-RLS-ALL-TENANT-TABLES: the live denominator is exactly 20/20.""" + + with guarded_runtime_engine.connect() as connection: + report = audit_live_rls( + connection, + _manifest(), + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + + assert report["passed"] is True + assert report["denominator"] == { + "allTables": 23, + "tenantOwned": 20, + "global": 3, + } + assert report["coverage"] == { + "numerator": 20, + "denominator": 20, + "percent": 100.0, + } + assert report["failures"] == [] + + +def test_no_force_row_level_security_mutation_fails_and_rolls_back( + guarded_runtime_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + """Issue #20 mutation: removing FORCE is a hard failure without residue.""" + + with guarded_runtime_engine.connect() as runtime_connection: + baseline = audit_live_rls( + runtime_connection, + _manifest(), + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + runtime_snapshot = snapshot_public_schema(runtime_connection) + assert baseline["passed"] is True + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as migration_connection: + transaction = migration_connection.begin() + try: + migration_connection.execute( + text( + "ALTER TABLE organization_record " + "NO FORCE ROW LEVEL SECURITY" + ) + ) + mutated_snapshot = snapshot_public_schema(migration_connection) + mutated_snapshot["currentRole"] = runtime_snapshot["currentRole"] + mutated = audit_rls_snapshot( + manifest=_manifest(), + snapshot=mutated_snapshot, + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + + assert mutated["passed"] is False + assert mutated["coverage"] == { + "numerator": 19, + "denominator": 20, + "percent": 95.0, + } + tenant_tables = cast( + list[dict[str, Any]], mutated["tenantTables"] + ) + organization_record = next( + table + for table in tenant_tables + if table["table"] == "organization_record" + ) + assert organization_record["rlsForced"] is False + assert organization_record["passed"] is False + finally: + transaction.rollback() + finally: + migration_engine.dispose() + + with guarded_runtime_engine.connect() as runtime_connection: + restored = audit_live_rls( + runtime_connection, + _manifest(), + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + assert restored["passed"] is True + assert restored["coverage"] == { + "numerator": 20, + "denominator": 20, + "percent": 100.0, + } + + +def test_transactional_permissive_allow_all_policy_mutation_fails_and_rolls_back( + guarded_runtime_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + """An added permissive policy invalidates exact inventory and isolation.""" + + with guarded_runtime_engine.connect() as runtime_connection: + baseline = audit_live_rls( + runtime_connection, + _manifest(), + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + runtime_snapshot = snapshot_public_schema(runtime_connection) + assert baseline["passed"] is True + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as migration_connection: + transaction = migration_connection.begin() + try: + migration_connection.execute( + text( + "CREATE POLICY organization_record_allow_all_mutation " + "ON organization_record AS PERMISSIVE FOR ALL " + "TO context_engine_runtime USING (true) WITH CHECK (true)" + ) + ) + mutated_snapshot = snapshot_public_schema(migration_connection) + mutated_snapshot["currentRole"] = runtime_snapshot["currentRole"] + mutated = audit_rls_snapshot( + manifest=_manifest(), + snapshot=mutated_snapshot, + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + + assert mutated["passed"] is False + organization_record = next( + table + for table in cast( + list[dict[str, Any]], mutated["tenantTables"] + ) + if table["table"] == "organization_record" + ) + assert organization_record["policies"][ + "inventoryMatchesDeclared" + ] is False + assert organization_record["policies"]["passed"] is False + assert "organization_record_allow_all_mutation" in ( + organization_record["policies"]["live"] + ) + finally: + transaction.rollback() + finally: + migration_engine.dispose() + + with guarded_runtime_engine.connect() as runtime_connection: + restored = audit_live_rls( + runtime_connection, + _manifest(), + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + assert restored["passed"] is True + + +def test_transactional_policy_replacement_mutation_fails_and_rolls_back( + guarded_runtime_engine: Engine, + migration_configuration: DatabaseConfiguration, +) -> None: + """Keeping a policy name cannot hide an allow-all semantic replacement.""" + + with guarded_runtime_engine.connect() as runtime_connection: + runtime_snapshot = snapshot_public_schema(runtime_connection) + + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as migration_connection: + transaction = migration_connection.begin() + try: + migration_connection.execute( + text( + "DROP POLICY organization_record_organization_isolation " + "ON organization_record" + ) + ) + migration_connection.execute( + text( + "CREATE POLICY organization_record_organization_isolation " + "ON organization_record AS PERMISSIVE FOR ALL " + "TO context_engine_runtime USING (true) WITH CHECK (true)" + ) + ) + mutated_snapshot = snapshot_public_schema(migration_connection) + mutated_snapshot["currentRole"] = runtime_snapshot["currentRole"] + mutated = audit_rls_snapshot( + manifest=_manifest(), + snapshot=mutated_snapshot, + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + + assert mutated["passed"] is False + organization_record = next( + table + for table in cast( + list[dict[str, Any]], mutated["tenantTables"] + ) + if table["table"] == "organization_record" + ) + policies = organization_record["policies"] + assert policies["inventoryMatchesDeclared"] is True + assert policies["semanticsMatchDeclared"] is False + assert policies["expectedDigest"] != policies["observedDigest"] + finally: + transaction.rollback() + finally: + migration_engine.dispose() + + with guarded_runtime_engine.connect() as runtime_connection: + restored = audit_live_rls( + runtime_connection, + _manifest(), + passed_evidence_ids=BEHAVIORAL_EVIDENCE_IDS, + ) + assert restored["passed"] is True diff --git a/tests/integration/test_m0_unavailable_security_carriers.py b/tests/integration/test_m0_unavailable_security_carriers.py new file mode 100644 index 00000000..3117bfc7 --- /dev/null +++ b/tests/integration/test_m0_unavailable_security_carriers.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from typing import cast + +import pytest +from sqlalchemy import Engine, text + +from engine.persistence import assert_learning_role, assert_runtime_role + +pytestmark = pytest.mark.integration + +UNAVAILABLE_M0_TABLE_STEMS = ( + "citation", + "egress", + "model_gateway", + "model_input", +) +UNAVAILABLE_M0_FUNCTION_STEMS = ( + "citation", + "egress", + "model_gateway", + "model_input", +) +ORGANIZATION_BOUND_LEARNING_TABLES = { + "active_release_manifest", + "release_candidate", + "release_evaluation", + "release_manifest", + "release_operator_grant", + "release_promotion_audit", +} + + +def _public_application_objects(engine: Engine) -> tuple[set[str], set[str]]: + with engine.connect() as connection: + tables = { + cast(str, row.table_name) + for row in connection.execute( + text( + """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + """ + ) + ) + } + functions = { + cast(str, row.function_name) + for row in connection.execute( + text( + """ + SELECT procedure.proname AS function_name + FROM pg_proc AS procedure + JOIN pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname = 'public' + """ + ) + ) + } + return tables, functions + + +@pytest.mark.security_evidence(id="PG-CITATION-AUTH-010", layer="postgres") +@pytest.mark.security_evidence(id="PG-EGRESS-011", layer="postgres") +def test_m0_unavailable_citation_and_egress_carriers_fail_closed( + guarded_runtime_engine: Engine, +) -> None: + """M0 exposes no database carrier that could mint future citation/egress auth.""" + + tables, functions = _public_application_objects(guarded_runtime_engine) + with guarded_runtime_engine.connect() as connection: + assert_runtime_role(connection) + + assert all( + stem not in table_name.casefold() + for stem in UNAVAILABLE_M0_TABLE_STEMS + for table_name in tables + ) + assert all( + stem not in function_name.casefold() + for stem in UNAVAILABLE_M0_FUNCTION_STEMS + for function_name in functions + ) + + +@pytest.mark.security_evidence(id="PG-CROSS-ORG-LEARN-015", layer="postgres") +def test_learning_persistence_is_organization_bound( + guarded_learning_engine: Engine, +) -> None: + """Every active M0 Learning artifact has forced Organization-scoped RLS.""" + + with guarded_learning_engine.connect() as connection: + assert_learning_role(connection) + rows = connection.execute( + text( + """ + SELECT + table_record.relname AS table_name, + table_record.relrowsecurity, + table_record.relforcerowsecurity, + EXISTS ( + SELECT 1 + FROM pg_attribute AS column_record + WHERE column_record.attrelid = table_record.oid + AND column_record.attname = 'organization_id' + AND column_record.attnum > 0 + AND NOT column_record.attisdropped + ) AS has_organization_id, + COALESCE( + bool_and( + position( + 'organization_id' IN + COALESCE(policy.qual, '') || ' ' || + COALESCE(policy.with_check, '') + ) > 0 + ) FILTER ( + WHERE policy.policyname IS NOT NULL + AND NOT ( + 'context_engine_migrator' = ANY(policy.roles) + ) + ), + false + ) AS every_policy_is_organization_bound + FROM pg_class AS table_record + JOIN pg_namespace AS namespace + ON namespace.oid = table_record.relnamespace + LEFT JOIN pg_policies AS policy + ON policy.schemaname = namespace.nspname + AND policy.tablename = table_record.relname + WHERE namespace.nspname = 'public' + AND table_record.relname = ANY(:table_names) + GROUP BY table_record.oid, table_record.relname + ORDER BY table_record.relname + """ + ), + {"table_names": sorted(ORGANIZATION_BOUND_LEARNING_TABLES)}, + ).mappings().all() + + observed = {cast(str, row["table_name"]): row for row in rows} + assert set(observed) == ORGANIZATION_BOUND_LEARNING_TABLES + for table_name in sorted(ORGANIZATION_BOUND_LEARNING_TABLES): + row = observed[table_name] + assert row["has_organization_id"] is True + assert row["relrowsecurity"] is True + assert row["relforcerowsecurity"] is True + assert row["every_policy_is_organization_bound"] is True diff --git a/tests/integration/test_membership_field_projection_integration.py b/tests/integration/test_membership_field_projection_integration.py index 367d6a9e..9debf473 100644 --- a/tests/integration/test_membership_field_projection_integration.py +++ b/tests/integration/test_membership_field_projection_integration.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass @@ -54,6 +54,7 @@ _open_scope_authority_scope, ) from tests.support.context_run_operator import exact_test_context_run_operator_read +from tests.support.security_gate import record_security_oracles pytestmark = pytest.mark.integration @@ -536,12 +537,14 @@ def _persisted_decision_documents( return run_document, audit_document +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-002", layer="runtime") def test_accept_002_same_organization_memberships_receive_only_authorized_fields( migration_configuration: DatabaseConfiguration, guarded_runtime_engine: Engine, guarded_control_engine: Engine, guarded_operator_engine: Engine, query_digest_keyring: QueryDigestKeyring, + record_property: Callable[[str, object], None], ) -> None: """ACCEPT-002: Membership field ceilings precede Kernel content projection.""" @@ -720,6 +723,35 @@ def test_accept_002_same_organization_memberships_receive_only_authorized_fields ): assert forbidden not in missing_persisted assert len(index.calls) == 3 + response_documents = ( + full_response.json(), + limited_response.json(), + missing_document, + ) + unauthorized_evidence_count = sum( + forbidden in response.text + for response in (limited_response, missing_right_response) + for forbidden in ("secret",) + ) + wrong_organization_effect_count = sum( + "effects" in document for document in response_documents + ) + missing_context_fallback_count = int( + missing_document["package"]["coverage"] + != {"status": "empty", "reason": "no_authorized_evidence"} + or missing_document["package"]["blocks"] != [] + or missing_document["package"]["evidence"] != [] + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-002", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) finally: try: _cleanup_fixture(migration_engine, fixture) diff --git a/tests/integration/test_membership_schema.py b/tests/integration/test_membership_schema.py index efe4c5a5..b35e640f 100644 --- a/tests/integration/test_membership_schema.py +++ b/tests/integration/test_membership_schema.py @@ -190,6 +190,7 @@ def membership_rows(connection: Connection) -> list[tuple[UUID, UUID, UUID]]: ] +@pytest.mark.security_evidence(id="PG-SCOPE-INTERSECTION-004", layer="postgres") def test_runtime_membership_rls_is_bidirectional_and_exact( guarded_runtime_engine: Engine, identities: IdentityFixture, @@ -295,6 +296,7 @@ def test_status_version_and_validity_fail_closed_at_the_database_seam( migration_engine.dispose() +@pytest.mark.security_evidence(id="PG-TRANSPORT-UNTRUSTED-008", layer="postgres") def test_organization_only_and_user_only_context_have_no_tenant_rights( guarded_runtime_engine: Engine, identities: IdentityFixture, diff --git a/tests/integration/test_organization_isolation.py b/tests/integration/test_organization_isolation.py index 1bbc410a..1b5405f4 100644 --- a/tests/integration/test_organization_isolation.py +++ b/tests/integration/test_organization_isolation.py @@ -399,6 +399,7 @@ def test_bidirectional_rls_hides_and_blocks_wrong_organization_effects( ] +@pytest.mark.security_evidence(id="PG-TENANT-FK-002", layer="postgres") def test_composite_ownership_accepts_same_org_and_rejects_cross_org_parent( guarded_runtime_engine: Engine, organizations: OrganizationPair, @@ -487,6 +488,7 @@ def test_organization_foreign_key_rejects_orphan_record( migration_engine.dispose() +@pytest.mark.security_evidence(id="PG-RLS-FAIL-CLOSED-003", layer="postgres") def test_missing_tenant_context_is_fail_closed_for_every_operation( guarded_runtime_engine: Engine, organizations: OrganizationPair, diff --git a/tests/integration/test_release_promotion.py b/tests/integration/test_release_promotion.py index 8468130f..ca052f4a 100644 --- a/tests/integration/test_release_promotion.py +++ b/tests/integration/test_release_promotion.py @@ -933,6 +933,7 @@ def test_application_roles_cannot_directly_mutate_release_pointer_or_audit( _assert_empty_publication_state(release_database, organization_id) +@pytest.mark.security_evidence(id="PG-RELEASE-OWNER-019", layer="postgres") def test_promote_rechecks_current_operator_authority_digests_and_compatibility( release_database: _ReleaseDatabaseFixture, guarded_learning_engine: Engine, diff --git a/tests/integration/test_runtime_authorized_evidence_integration.py b/tests/integration/test_runtime_authorized_evidence_integration.py index c546721a..5d97e8cd 100644 --- a/tests/integration/test_runtime_authorized_evidence_integration.py +++ b/tests/integration/test_runtime_authorized_evidence_integration.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, datetime, timedelta @@ -44,6 +44,7 @@ _open_scope_authority_scope, ) from tests.support.context_run_operator import exact_test_context_run_operator_read +from tests.support.security_gate import record_security_oracles pytestmark = pytest.mark.integration @@ -666,7 +667,7 @@ def _assert_exact_authorized_http_resolve( guarded_control_engine: Engine, guarded_operator_engine: Engine, query_digest_keyring: QueryDigestKeyring, -) -> None: +) -> tuple[int, int, int]: token = f"{TOKEN_PREFIX}:{active.label}" request_id = f"{REQUEST_ID_PREFIX}:{active.label}" index = HostileCandidateIndex( @@ -819,15 +820,37 @@ def _assert_exact_authorized_http_resolve( str(active.organization_id), str(other.organization_id), ) - assert all(value not in response_text for value in forbidden_values) + unauthorized_evidence_count = sum( + value in response_text for value in forbidden_values + ) + wrong_organization_effect_count = int("effects" in response_document) + missing_context_fallback_count = int( + package["coverage"] != {"status": "sufficient"} + or package["blocks"] == [] + or package["evidence"] == [] + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + return ( + unauthorized_evidence_count, + wrong_organization_effect_count, + missing_context_fallback_count, + ) +@pytest.mark.security_evidence( + id="RUNTIME-TENANT-OWNERSHIP-001", layer="runtime" +) +@pytest.mark.security_evidence(id="RUNTIME-TENANT-FK-002", layer="runtime") +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-001", layer="runtime") def test_real_postgres_http_delivers_only_exact_authorized_evidence_bidirectionally( migration_configuration: DatabaseConfiguration, guarded_runtime_engine: Engine, guarded_control_engine: Engine, guarded_operator_engine: Engine, query_digest_keyring: QueryDigestKeyring, + record_property: Callable[[str, object], None], ) -> None: """Issue #13: both Organizations seal CandidateRef -> Kernel -> projection.""" @@ -838,25 +861,34 @@ def test_real_postgres_http_delivers_only_exact_authorized_evidence_bidirectiona before = _persistent_content_snapshot(migration_engine, fixture) assert len(before) == 4 - _assert_exact_authorized_http_resolve( - active=fixture.org_a, - other=fixture.org_b, - guarded_runtime_engine=guarded_runtime_engine, - guarded_control_engine=guarded_control_engine, - guarded_operator_engine=guarded_operator_engine, - query_digest_keyring=query_digest_keyring, - ) - _assert_exact_authorized_http_resolve( - active=fixture.org_b, - other=fixture.org_a, - guarded_runtime_engine=guarded_runtime_engine, - guarded_control_engine=guarded_control_engine, - guarded_operator_engine=guarded_operator_engine, - query_digest_keyring=query_digest_keyring, + observations = ( + _assert_exact_authorized_http_resolve( + active=fixture.org_a, + other=fixture.org_b, + guarded_runtime_engine=guarded_runtime_engine, + guarded_control_engine=guarded_control_engine, + guarded_operator_engine=guarded_operator_engine, + query_digest_keyring=query_digest_keyring, + ), + _assert_exact_authorized_http_resolve( + active=fixture.org_b, + other=fixture.org_a, + guarded_runtime_engine=guarded_runtime_engine, + guarded_control_engine=guarded_control_engine, + guarded_operator_engine=guarded_operator_engine, + query_digest_keyring=query_digest_keyring, + ), ) after = _persistent_content_snapshot(migration_engine, fixture) assert after == before + record_security_oracles( + record_property, + fixture_ref="ACCEPT-001", + unauthorized_evidence_count=sum(item[0] for item in observations), + wrong_organization_effect_count=sum(item[1] for item in observations), + missing_context_fallback_count=sum(item[2] for item in observations), + ) finally: try: _cleanup_fixture(migration_engine, fixture) diff --git a/tests/integration/test_runtime_empty_package_integration.py b/tests/integration/test_runtime_empty_package_integration.py index 6f6a2210..21f046e2 100644 --- a/tests/integration/test_runtime_empty_package_integration.py +++ b/tests/integration/test_runtime_empty_package_integration.py @@ -439,6 +439,7 @@ def test_seeded_existing_organization_reaches_http_empty_package( migration_engine.dispose() +@pytest.mark.security_evidence(id="RUNTIME-RLS-FAIL-CLOSED-003", layer="runtime") def test_real_postgres_http_membership_matrix_is_generic_and_zero_io( migration_configuration: DatabaseConfiguration, guarded_runtime_engine: Engine, diff --git a/tests/integration/test_runtime_non_enumeration_integration.py b/tests/integration/test_runtime_non_enumeration_integration.py index 086c8432..e5dbb525 100644 --- a/tests/integration/test_runtime_non_enumeration_integration.py +++ b/tests/integration/test_runtime_non_enumeration_integration.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import asdict, dataclass from datetime import UTC, datetime from json import dumps, loads @@ -39,6 +40,7 @@ _seed_fixture, ) from tests.support.context_run_operator import exact_test_context_run_operator_read +from tests.support.security_gate import record_security_oracles pytestmark = pytest.mark.integration @@ -378,12 +380,15 @@ def _assert_non_owner_force_rls(engine: Engine) -> None: assert all(row.relrowsecurity and row.relforcerowsecurity for row in rows) +@pytest.mark.security_evidence(id="PG-NON-ENUMERATION-009", layer="postgres") +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-011", layer="runtime") def test_real_postgres_http_denied_and_missing_are_externally_equivalent( migration_configuration: DatabaseConfiguration, guarded_runtime_engine: Engine, guarded_control_engine: Engine, guarded_operator_engine: Engine, query_digest_keyring: QueryDigestKeyring, + record_property: Callable[[str, object], None], ) -> None: """NON-ENUMERATION-009 compares content semantics, not response timing.""" @@ -476,6 +481,31 @@ def test_real_postgres_http_denied_and_missing_are_externally_equivalent( response.headers["content-length"] for response in probe_responses } assert len(content_lengths) == 1 + unauthorized_evidence_count = sum( + len(response.json()["package"]["evidence"]) + for response in probe_responses + ) + wrong_organization_effect_count = sum( + len(response.json()["package"].get("effects", ())) + for response in probe_responses + ) + missing_context_fallback_count = sum( + int( + response.json()["package"]["coverage"] + != {"status": "empty", "reason": "no_authorized_evidence"} + ) + for response in probe_responses + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-011", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) denied_decision_refs = tuple( response.json()["package"]["decisionRef"] for response in probe_responses diff --git a/tests/integration/test_runtime_policy_epoch_integration.py b/tests/integration/test_runtime_policy_epoch_integration.py index a102ac4b..da090452 100644 --- a/tests/integration/test_runtime_policy_epoch_integration.py +++ b/tests/integration/test_runtime_policy_epoch_integration.py @@ -118,6 +118,7 @@ def _assert_revoked_empty( assert forbidden not in response.text +@pytest.mark.security_evidence(id="RUNTIME-REVOCATION-006", layer="runtime") def test_same_http_acquire_revokes_next_delivery_without_candidate_cleanup( control_configuration: DatabaseConfiguration, migration_configuration: DatabaseConfiguration, diff --git a/tests/integration/test_ticket_policy_epoch.py b/tests/integration/test_ticket_policy_epoch.py index f3198da2..5dfd3f85 100644 --- a/tests/integration/test_ticket_policy_epoch.py +++ b/tests/integration/test_ticket_policy_epoch.py @@ -70,6 +70,7 @@ def perform_noop(self, *, organization_id: UUID, channel_ref: str) -> None: self.effects += 1 +@pytest.mark.security_evidence(id="PG-ACTION-SEPARATION-014", layer="postgres") def test_committed_epoch_bump_rejects_both_previously_valid_ticket_types( control_configuration: DatabaseConfiguration, migration_configuration: DatabaseConfiguration, diff --git a/tests/integration/test_worker_lease.py b/tests/integration/test_worker_lease.py index 7a00580c..3e8799e1 100644 --- a/tests/integration/test_worker_lease.py +++ b/tests/integration/test_worker_lease.py @@ -44,6 +44,7 @@ WorkNotAvailable, worker_lease_digest, ) +from tests.support.security_gate import record_security_oracles pytestmark = pytest.mark.integration @@ -447,8 +448,11 @@ def test_control_issuer_uses_db_time_and_signs_every_required_exact_job_claim( assert state["effect_count"] == 0 +@pytest.mark.security_evidence(id="RUNTIME-WORKER-LEASE-007", layer="runtime") +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-008", layer="runtime") def test_valid_lease_completes_once_through_worker_application_and_replay_is_zero( worker_jobs: WorkerJobFixture, + record_property: Callable[[str, object], None], ) -> None: request = worker_jobs.seed_job() token = worker_jobs.issue(request) @@ -470,6 +474,25 @@ def test_valid_lease_completes_once_through_worker_application_and_replay_is_zer replayed_state = worker_jobs.job_state(request) assert replayed_state["state"] == "completed" assert replayed_state["effect_count"] == 1 + serialized_token = token.serialize() + unauthorized_evidence_count = int( + serialized_token in str(error.value) or serialized_token in repr(error.value) + ) + wrong_organization_effect_count = replayed_state["effect_count"] - 1 + missing_context_fallback_count = int( + str(error.value) != "work not available" + or replayed_state["state"] != "completed" + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-008", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) def test_two_concurrent_redemptions_commit_exactly_one_effect( @@ -537,6 +560,7 @@ def test_wrong_signed_lease_binding_has_zero_effect( assert worker_jobs.job_state(request)["effect_count"] == 0 +@pytest.mark.security_evidence(id="PG-WORKER-LEASE-007", layer="postgres") def test_wrong_organization_cannot_affect_same_job_identifier_in_another_tenant( worker_jobs: WorkerJobFixture, ) -> None: diff --git a/tests/support/security_gate.py b/tests/support/security_gate.py new file mode 100644 index 00000000..ef15f35b --- /dev/null +++ b/tests/support/security_gate.py @@ -0,0 +1,72 @@ +"""Explicit hard-oracle observations emitted by registered security fixtures.""" + +from __future__ import annotations + +import importlib +from collections.abc import Callable +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class UnavailableCarrierProbe: + """Observed result of attempting one explicitly absent M0 carrier.""" + + request: str + external_status: int + external_code: str + carrier_available: bool + ticket_created: bool + sender_call_count: int + effect_count: int + + +def probe_unavailable_module_carrier( + *, module_name: str, request: str, unavailable_code: str +) -> UnavailableCarrierProbe: + """Attempt an M0 carrier and normalize only its exact module absence.""" + + try: + importlib.import_module(module_name) + except ModuleNotFoundError as error: + if error.name != module_name: + raise + return UnavailableCarrierProbe( + request=request, + external_status=404, + external_code=unavailable_code, + carrier_available=False, + ticket_created=False, + sender_call_count=0, + effect_count=0, + ) + return UnavailableCarrierProbe( + request=request, + external_status=500, + external_code="unexpected_active_carrier", + carrier_available=True, + ticket_created=False, + sender_call_count=0, + effect_count=0, + ) + + +def record_security_oracles( + record_property: Callable[[str, object], None], + *, + fixture_ref: str, + unauthorized_evidence_count: int, + wrong_organization_effect_count: int, + missing_context_fallback_count: int, +) -> None: + """Record three counts computed from one fixture's asserted outputs.""" + + from scripts.security_gate.observations import record_fixture_observation + + record_fixture_observation( + record_property, + fixture_ref=fixture_ref, + evidence_ref=f"FIXTURE-{fixture_ref}", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) diff --git a/tests/unit/test_actor_contracts.py b/tests/unit/test_actor_contracts.py index f898fe17..a947be2f 100644 --- a/tests/unit/test_actor_contracts.py +++ b/tests/unit/test_actor_contracts.py @@ -344,6 +344,7 @@ def test_authenticated_invocation_binds_one_current_user_actor() -> None: ({}, {"membership_version": 9}), ), ) +@pytest.mark.security_evidence(id="PROP-TRANSPORT-UNTRUSTED-008", layer="property") def test_authenticated_invocation_rejects_mismatched_current_membership( proof_override: dict[str, object], invocation_override: dict[str, object], diff --git a/tests/unit/test_context_learning.py b/tests/unit/test_context_learning.py index 7e6ca998..024e8542 100644 --- a/tests/unit/test_context_learning.py +++ b/tests/unit/test_context_learning.py @@ -5,6 +5,8 @@ from datetime import UTC, datetime, timedelta from uuid import UUID +import pytest + from engine.learning import ( ContentProfileRef, ContextLearning, @@ -278,6 +280,7 @@ def test_evaluate_persists_only_immutable_evaluation_and_never_publishes() -> No assert len(evaluation.signature) == 32 +@pytest.mark.security_evidence(id="RUNTIME-RELEASE-OWNER-019", layer="runtime") def test_initial_activation_ordinary_promotion_and_rollback_share_promote() -> None: initial_manifest = _manifest() ordinary_manifest = _manifest(suffix="ordinary-v2", profile_digit=2) diff --git a/tests/unit/test_context_run.py b/tests/unit/test_context_run.py index 8240355b..c19eceb3 100644 --- a/tests/unit/test_context_run.py +++ b/tests/unit/test_context_run.py @@ -567,6 +567,7 @@ def test_projection_rejects_package_altered_after_digest_creation() -> None: ) +@pytest.mark.security_evidence(id="PROP-TRACE-REDACTION-012", layer="property") def test_records_retain_digests_not_raw_query_package_or_denial_details() -> None: run = _run() audit = _audit() diff --git a/tests/unit/test_database_harness_contract.py b/tests/unit/test_database_harness_contract.py index 62aef2e8..85c7c523 100644 --- a/tests/unit/test_database_harness_contract.py +++ b/tests/unit/test_database_harness_contract.py @@ -191,8 +191,42 @@ def test_ci_runs_the_same_make_database_contract_as_local() -> None: makefile = repository_text("Makefile") assert "make db-up" in workflow - assert "make check" in workflow + assert "run: make check" in workflow assert "if: always()" in workflow assert "make db-down" in workflow - assert "check: build lint typecheck test catalog smoke integration" in makefile + assert ( + "check: build lint typecheck test catalog smoke integration security-gate" + in makefile + ) assert "./scripts/database_harness.sh integration" in makefile + + +def test_ci_runs_and_retains_the_single_m0_security_gate_contract() -> None: + workflow = repository_text(".github/workflows/ci.yml") + makefile = repository_text("Makefile") + + assert "security-gate:" in makefile + assert ( + "uv run python scripts/run_m0_security_gate.py " + "--output-dir .context-engine/security-gate" in makefile + ) + assert ( + "check: build lint typecheck test catalog smoke integration security-gate" + in makefile.splitlines() + ) + assert "actions/upload-artifact@v4" in workflow + assert ".context-engine/security-gate/raw-evidence.json" in workflow + assert ".context-engine/security-gate/release-gate-report.json" in workflow + upload_step = workflow.split("actions/upload-artifact@v4", maxsplit=1)[1] + upload_prefix = workflow.split("actions/upload-artifact@v4", maxsplit=1)[0] + assert "if: always()" in upload_prefix + assert "include-hidden-files: true" in upload_step + assert "if-no-files-found: error" in upload_step + assert "retention-days:" in upload_step + assert "retry" not in makefile.casefold() + assert "continue-on-error" not in workflow + + check_step = workflow.split("run: make check", maxsplit=1)[1] + upload_index = check_step.index("actions/upload-artifact@v4") + teardown_index = check_step.index("run: make db-down") + assert upload_index < teardown_index diff --git a/tests/unit/test_effective_scope.py b/tests/unit/test_effective_scope.py index 1353f7e7..2bb512f3 100644 --- a/tests/unit/test_effective_scope.py +++ b/tests/unit/test_effective_scope.py @@ -522,6 +522,7 @@ def test_generated_missing_and_empty_operands_are_absorbing( assert effective.targets == frozenset() +@pytest.mark.security_evidence(id="PROP-SCOPE-INTERSECTION-004", layer="property") @seed(20_260_724) @PROPERTY_SETTINGS @given( diff --git a/tests/unit/test_effective_scope_runtime.py b/tests/unit/test_effective_scope_runtime.py index 17e6d0f7..cac501c7 100644 --- a/tests/unit/test_effective_scope_runtime.py +++ b/tests/unit/test_effective_scope_runtime.py @@ -213,6 +213,7 @@ def resolve( return outcome +@pytest.mark.security_evidence(id="RUNTIME-SCOPE-INTERSECTION-004", layer="runtime") def test_runtime_observes_only_effective_scope_and_returns_empty_package() -> None: spy = ContentIoSpy() @@ -232,23 +233,30 @@ def test_runtime_observes_only_effective_scope_and_returns_empty_package() -> No assert outcome.scope_decision.digest not in serialized_package -def test_agent_ceiling_can_only_reduce_other_trusted_grants() -> None: +def test_agent_ceiling_can_only_reduce_other_trusted_grants( +) -> None: + broad_spy = ContentIoSpy() broad_agent = resolve( make_operands( principal_grants=ScopeSet(frozenset({TARGET_A})), agent_ceiling=ALL_TARGETS, - ) + ), + spy=broad_spy, ) + narrow_spy = ContentIoSpy() narrow_agent = resolve( make_operands( principal_grants=ALL_TARGETS, agent_ceiling=ScopeSet(frozenset({TARGET_A})), - ) + ), + spy=narrow_spy, ) assert broad_agent.scope_decision.target_count == 1 assert narrow_agent.scope_decision.target_count == 1 assert broad_agent.scope_decision.digest == narrow_agent.scope_decision.digest + assert broad_spy.calls == narrow_spy.calls == 0 + assert broad_agent.package.evidence == narrow_agent.package.evidence == () def test_overbroad_request_refs_do_not_expand_the_trusted_intersection() -> None: diff --git a/tests/unit/test_evidence_contracts.py b/tests/unit/test_evidence_contracts.py index b0608c6f..9443bc0c 100644 --- a/tests/unit/test_evidence_contracts.py +++ b/tests/unit/test_evidence_contracts.py @@ -337,6 +337,7 @@ def test_package_content_is_deterministic_and_exactly_links_each_projection() -> _close_authorization_kernel_scope(kernel_scope) +@pytest.mark.security_evidence(id="PROP-INDEX-NOT-AUTHORITY-005", layer="property") def test_package_content_can_only_be_constructed_from_authorized_projections() -> None: with pytest.raises(TypeError, match="authorized projections"): PackageContent(blocks=(), evidence=()) @@ -365,6 +366,7 @@ def test_package_constructor_rejects_expired_projection_authority() -> None: construct_package_content((authorized,)) +@pytest.mark.security_evidence(id="PROP-EGRESS-011", layer="property") def test_package_constructor_rejects_cross_organization_or_mixed_request() -> None: kernel_scope = _open_authorization_kernel_scope() authorized = projection("a", "authorized body", kernel_scope=kernel_scope) diff --git a/tests/unit/test_http_trust_boundary.py b/tests/unit/test_http_trust_boundary.py index 9edc5ef3..69eff9d2 100644 --- a/tests/unit/test_http_trust_boundary.py +++ b/tests/unit/test_http_trust_boundary.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import AbstractContextManager, contextmanager from dataclasses import FrozenInstanceError from datetime import UTC, datetime @@ -57,6 +57,7 @@ TEST_QUERY_DIGEST_KEYRING, recording_context_run_session, ) +from tests.support.security_gate import record_security_oracles VALID_BODY = { "kind": "acquire", @@ -953,10 +954,13 @@ def test_verified_authentication_context_rejects_non_string_refs( ("need", "bypass", True), ], ) +@pytest.mark.security_evidence(id="RUNTIME-TRANSPORT-UNTRUSTED-008", layer="runtime") +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-007", layer="runtime") def test_trusted_field_injection_is_closed_before_domain_execution( placement: str, field_name: str, field_value: object, + record_property: Callable[[str, object], None], ) -> None: authenticator = DeterministicAuthenticator() spy = InvocationSpy() @@ -980,6 +984,24 @@ def test_trusted_field_injection_is_closed_before_domain_execution( assert field_name not in response.text assert str(field_value) not in response.text assert spy.invocations == [] + response_document = response.json() + unauthorized_evidence_count = int( + field_name in response.text or str(field_value) in response.text + ) + wrong_organization_effect_count = len(spy.invocations) + missing_context_fallback_count = int( + response_document != {"code": "invalid_request"} + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-007", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) @pytest.mark.parametrize( diff --git a/tests/unit/test_http_unavailable_capabilities.py b/tests/unit/test_http_unavailable_capabilities.py index aeba527a..5d40b3c2 100644 --- a/tests/unit/test_http_unavailable_capabilities.py +++ b/tests/unit/test_http_unavailable_capabilities.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from typing import Any, cast @@ -20,6 +20,7 @@ from engine.runtime.construction import Runtime, required_kernel_dependencies from engine.runtime.content_io import RuntimeContentIo from tests.support.context_run import TEST_QUERY_DIGEST_KEYRING +from tests.support.security_gate import record_security_oracles from tests.unit.test_http_effective_scope import ( DeterministicScopeAuthority, operands, @@ -153,11 +154,15 @@ def test_active_acquire_still_requires_the_configured_scope_authority() -> None: assert_unsupported_audit(runtime, 0) +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-005", layer="runtime") @pytest.mark.parametrize( "token", ("opaque-epoch-8-token", "opaque-revoked-token", "opaque-missing-token"), ) -def test_accept_005_continue_is_generic_non_retryable_and_zero_io(token: str) -> None: +def test_accept_005_continue_is_generic_non_retryable_and_zero_io( + token: str, + record_property: Callable[[str, object], None], +) -> None: client, content_io, runtime, invocations = client_for() response = client.post( @@ -180,13 +185,35 @@ def test_accept_005_continue_is_generic_non_retryable_and_zero_io(token: str) -> assert content_io.total_calls == 0 assert len(invocations.invocations) == 1 assert_unsupported_audit(runtime, 1) + response_document = response.json() + unauthorized_evidence_count = int("evidence" in response_document) + wrong_organization_effect_count = content_io.total_calls + missing_context_fallback_count = int( + response_document + != {"kind": "request_not_available", "retryable": False} + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-005", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) +@pytest.mark.security_evidence(id="RUNTIME-CITATION-AUTH-010", layer="runtime") +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-010", layer="runtime") @pytest.mark.parametrize( "locator", ("citation-revoked", "citation-missing", "continuation-shaped-value"), ) -def test_accept_010_open_citation_is_generic_and_zero_io(locator: str) -> None: +def test_accept_010_open_citation_is_generic_and_zero_io( + locator: str, + record_property: Callable[[str, object], None], +) -> None: client, content_io, runtime, invocations = client_for() response = client.post( @@ -201,8 +228,25 @@ def test_accept_010_open_citation_is_generic_and_zero_io(locator: str) -> None: assert content_io.total_calls == 0 assert len(invocations.invocations) == 1 assert_unsupported_audit(runtime, 1) + response_document = response.json() + unauthorized_evidence_count = int("evidence" in response_document) + wrong_organization_effect_count = content_io.total_calls + missing_context_fallback_count = int( + response_document != {"kind": "citation_not_available"} + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-010", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-009", layer="runtime") @pytest.mark.parametrize( ("case_id", "acquire_capability"), [ @@ -218,6 +262,7 @@ def test_accept_010_open_citation_is_generic_and_zero_io(locator: str) -> None: def test_accept_009_server_owned_unavailable_source_paths_are_generic_and_zero_io( case_id: str, acquire_capability: RuntimeCapability, + record_property: Callable[[str, object], None], ) -> None: client, content_io, runtime, invocations = client_for( acquire_capability=acquire_capability, @@ -240,6 +285,25 @@ def test_accept_009_server_owned_unavailable_source_paths_are_generic_and_zero_i assert content_io.source_content_calls == 0 assert len(invocations.invocations) == 1 assert_unsupported_audit(runtime, 1) + response_document = response.json() + unauthorized_evidence_count = int("evidence" in response_document) + wrong_organization_effect_count = ( + content_io.provider_calls + content_io.source_content_calls + ) + missing_context_fallback_count = int( + response_document + != {"kind": "request_not_available", "retryable": False} + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-009", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) @pytest.mark.parametrize( diff --git a/tests/unit/test_m0_delivery_carriers.py b/tests/unit/test_m0_delivery_carriers.py new file mode 100644 index 00000000..d0abb08b --- /dev/null +++ b/tests/unit/test_m0_delivery_carriers.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import importlib.util +from collections.abc import Callable + +import pytest + +import engine.runtime as runtime +from tests.support.security_gate import ( + probe_unavailable_module_carrier, + record_security_oracles, +) + + +@pytest.mark.security_evidence(id="RUNTIME-EGRESS-011", layer="runtime") +def test_m0_egress_carrier_is_unavailable_before_model_or_sender_bytes() -> None: + """M0 cannot construct the future delivery types or their trusted process.""" + + assert importlib.util.find_spec("bot_delivery") is None + for public_name in ( + "AuthorizedModelInput", + "EgressGrant", + "ModelGateway", + "Sender", + ): + assert not hasattr(runtime, public_name) + + +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-012", layer="runtime") +def test_accept_012_action_plane_is_unavailable_before_ticket_or_effect( + record_property: Callable[[str, object], None], +) -> None: + """The documented M0 form has no ActionPlane, ticket, Sender, or effect.""" + + forbidden_runtime_authorities = tuple( + public_name + for public_name in ("ActionPlane", "AuthorizedModelInput", "EgressGrant") + if hasattr(runtime, public_name) + ) + probe = probe_unavailable_module_carrier( + module_name="action_plane", + request="ActionPlane.prepare(send_message, conversation-b, digest-b)", + unavailable_code="action_not_available", + ) + + assert probe.request == ( + "ActionPlane.prepare(send_message, conversation-b, digest-b)" + ) + assert (probe.external_status, probe.external_code) == ( + 404, + "action_not_available", + ) + assert probe.carrier_available is False + assert probe.ticket_created is False + assert probe.sender_call_count == 0 + assert probe.effect_count == 0 + assert importlib.util.find_spec("bot_delivery") is None + assert forbidden_runtime_authorities == () + + record_security_oracles( + record_property, + fixture_ref="ACCEPT-012", + unauthorized_evidence_count=len(forbidden_runtime_authorities), + wrong_organization_effect_count=probe.effect_count, + missing_context_fallback_count=int( + probe.external_status != 404 + or probe.external_code != "action_not_available" + or probe.ticket_created + or probe.sender_call_count != 0 + ), + ) diff --git a/tests/unit/test_m0_learning_isolation.py b/tests/unit/test_m0_learning_isolation.py new file mode 100644 index 00000000..e4835cc3 --- /dev/null +++ b/tests/unit/test_m0_learning_isolation.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import json +from contextlib import AbstractContextManager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import UUID + +import pytest + +from engine.learning import ( + ContentProfileRef, + ContextLearning, + CurationProfileRef, + Gate, + GateEvidence, + GateStatus, + IndexProfileRef, + ReleaseCandidate, + ReleaseCandidateRef, + ReleaseEvaluation, + ReleaseEvaluationKeyring, + ReleaseEvaluationUnavailable, + ReleaseManifest, + ReleaseOperatorAuthority, + ReleasePromotionTransactionPort, + RuntimeProfileRef, +) + +ROOT = Path(__file__).parents[2] +ORGANIZATION_A = UUID("33c7b365-c705-45af-b676-067fd510f683") +ORGANIZATION_B = UUID("d50e9030-c75c-422a-af06-cd3d7463ad73") +EVALUATED_AT = datetime(2026, 7, 22, 16, 0, tzinfo=UTC) +SIGNING_KEY = b"m0-learning-isolation-key-at-least-32-bytes" +M0_LEARNING_TABLES = { + "active_release_manifest", + "release_candidate", + "release_evaluation", + "release_manifest", + "release_operator_grant", + "release_promotion_audit", +} +FORBIDDEN_CROSS_ORGANIZATION_CARRIERS = { + "feedback_artifact", + "global_artifact", + "learning_export", +} + + +def _candidate(organization_id: UUID, *, suffix: str) -> ReleaseCandidate: + content = ContentProfileRef( + profile_ref=f"content-{suffix}", + profile_digest="1" * 64, + content_schema_ref="context-content-schema-v1", + ) + index = IndexProfileRef( + profile_ref=f"index-{suffix}", + profile_digest="2" * 64, + content_profile_digest=content.profile_digest, + content_schema_ref=content.content_schema_ref, + index_schema_ref="context-index-schema-v1", + ) + runtime = RuntimeProfileRef( + profile_ref=f"runtime-{suffix}", + profile_digest="3" * 64, + content_profile_digest=content.profile_digest, + index_profile_digest=index.profile_digest, + content_schema_ref=content.content_schema_ref, + index_schema_ref=index.index_schema_ref, + tokenizer_ref="empty-tokenizer-v1", + package_schema_ref="context-package-v1", + ) + manifest = ReleaseManifest.m0_empty( + organization_id=organization_id, + manifest_ref=f"manifest-{suffix}", + content_profile=content, + index_profile=index, + runtime_profile=runtime, + curation_profile=CurationProfileRef.off( + profile_ref=f"curation-off-{suffix}", + profile_digest="4" * 64, + ), + ) + return ReleaseCandidate( + organization_id=organization_id, + candidate_ref=f"candidate-{suffix}", + manifest=manifest, + expected_active_generation=0, + expected_base_manifest_digest=None, + gate_evidence=tuple( + GateEvidence( + gate=gate, + status=GateStatus.PASS, + evidence_digest=f"{index + 5:x}" * 64, + ) + for index, gate in enumerate(Gate) + ), + capability_coverage_digest="9" * 64, + fixture_digest="a" * 64, + verification_commands=("make check",), + ) + + +class _CrossOrganizationStore: + def __init__(self, returned_candidate: ReleaseCandidate) -> None: + self.returned_candidate = returned_candidate + self.evaluations: list[ReleaseEvaluation] = [] + + def load_candidate(self, candidate_ref: ReleaseCandidateRef) -> ReleaseCandidate: + del candidate_ref + return self.returned_candidate + + def persist_evaluation(self, evaluation: ReleaseEvaluation) -> None: + self.evaluations.append(evaluation) + + def transaction( + self, + organization_id: UUID, + ) -> AbstractContextManager[ReleasePromotionTransactionPort]: + del organization_id + raise AssertionError("cross-Organization evaluation opened publication") + + +class _NeverAuthenticator: + def authenticate(self, opaque_credential: str) -> object: + del opaque_credential + raise AssertionError("evaluation attempted release-operator authentication") + + +@pytest.mark.security_evidence(id="PROP-CROSS-ORG-LEARN-015", layer="property") +def test_m0_learning_artifact_contract_has_no_cross_organization_carrier() -> None: + """M0 has only Organization-owned release lineage and no global raw carrier.""" + + manifest = json.loads( + (ROOT / "engine/persistence/schema_security_manifest.yaml").read_text( + encoding="utf-8" + ) + ) + migration = ( + ROOT + / "migrations/versions/20260722_0009_learning_release_promotion.py" + ).read_text(encoding="utf-8") + + tables = {entry["name"]: entry for entry in manifest["tables"]} + assert set(tables) >= M0_LEARNING_TABLES + for table in M0_LEARNING_TABLES: + assert tables[table]["classification"] == "tenant_owned" + assert tables[table]["organizationColumn"] == "organization_id" + public_tables = set(tables) + assert FORBIDDEN_CROSS_ORGANIZATION_CARRIERS.isdisjoint(public_tables) + assert all( + name not in migration.casefold() + for name in FORBIDDEN_CROSS_ORGANIZATION_CARRIERS + ) + + +@pytest.mark.security_evidence(id="RUNTIME-CROSS-ORG-LEARN-015", layer="runtime") +def test_context_learning_rejects_cross_organization_candidate_lineage() -> None: + """A mismatched store result cannot become evaluation or publication state.""" + + requested = _candidate(ORGANIZATION_A, suffix="org-a").reference() + store = _CrossOrganizationStore(_candidate(ORGANIZATION_B, suffix="org-b")) + learning = ContextLearning( + store=store, + evaluation_keyring=ReleaseEvaluationKeyring( + active_version=1, + keys={1: SIGNING_KEY}, + ), + promotion_authority=ReleaseOperatorAuthority( + _NeverAuthenticator(), # type: ignore[arg-type] + call_ttl=timedelta(minutes=1), + clock=lambda: EVALUATED_AT, + ), + clock=lambda: EVALUATED_AT, + ) + + with pytest.raises( + ReleaseEvaluationUnavailable, + match="release candidate could not be loaded safely", + ): + learning.evaluate(requested) + + assert store.evaluations == [] diff --git a/tests/unit/test_m0_rls_inventory.py b/tests/unit/test_m0_rls_inventory.py new file mode 100644 index 00000000..a112f4e5 --- /dev/null +++ b/tests/unit/test_m0_rls_inventory.py @@ -0,0 +1,367 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, cast + +from scripts.security_gate.rls import ( + NON_OWNER_EVIDENCE_BY_TABLE, + audit_rls_snapshot, +) + +ROOT = Path(__file__).parents[2] +MANIFEST_PATH = ROOT / "engine/persistence/schema_security_manifest.yaml" + +GLOBAL_TABLES = {"alembic_version", "organization", "user_account"} +TENANT_TABLES = { + "active_release_manifest", + "context_fragment", + "context_fragment_field", + "context_resource", + "context_revision", + "context_run", + "context_run_operator_read_ticket", + "decision_audit", + "membership", + "membership_resource_field_right", + "organization_policy_epoch", + "organization_record", + "release_candidate", + "release_evaluation", + "release_manifest", + "release_operator_grant", + "release_promotion_audit", + "resource_access_policy", + "service_principal", + "worker_noop_job", +} + + +def _manifest_tables() -> dict[str, dict[str, Any]]: + document = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + assert isinstance(document, dict) + raw_tables = document["tables"] + assert isinstance(raw_tables, list) + return { + cast(str, table["name"]): cast(dict[str, Any], table) + for table in raw_tables + } + + +def _manifest() -> dict[str, Any]: + document = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + assert isinstance(document, dict) + return cast(dict[str, Any], document) + + +def _matching_catalog_snapshot() -> dict[str, object]: + tables: dict[str, object] = {} + for name, entry in _manifest_tables().items(): + foreign_keys = [ + { + "name": foreign_key["name"], + "columns": foreign_key["columns"], + "referencedTable": foreign_key["references"]["table"], + "referencedColumns": foreign_key["references"]["columns"], + } + for foreign_key in entry.get("foreignKeys", []) + ] + rls = entry.get("rowLevelSecurity", {}) + tables[name] = { + "columns": sorted( + { + column + for foreign_key in foreign_keys + for column in foreign_key["columns"] + } + ), + "owner": "context_engine_migrator", + "rlsEnabled": rls.get("enabled", False), + "rlsForced": rls.get("forced", False), + "policies": [ + { + "name": policy["name"], + "permissive": True, + "command": policy["command"], + "roles": policy["roles"], + "using": policy.get("using"), + "withCheck": policy.get("withCheck"), + } + for policy in rls.get("policies", []) + ], + "foreignKeys": foreign_keys, + } + return { + "schema": "public", + "currentRole": { + "name": "context_engine_security_test", + "superuser": False, + "bypassesRls": False, + "inherits": False, + }, + "tables": tables, + } + + +def test_manifest_declares_exact_live_table_denominator_and_rls_evidence() -> None: + """Issue #20: every live table is classified and every tenant proof maps.""" + + tables = _manifest_tables() + global_tables = { + name for name, entry in tables.items() if entry["classification"] == "global" + } + tenant_tables = { + name + for name, entry in tables.items() + if entry["classification"] == "tenant_owned" + } + + assert global_tables == GLOBAL_TABLES + assert tenant_tables == TENANT_TABLES + assert len(tables) == 23 + + for name in sorted(GLOBAL_TABLES): + rationale = tables[name]["classificationRationale"] + assert isinstance(rationale, str) + assert rationale.strip() + + for name in sorted(TENANT_TABLES): + assert tables[name]["nonOwnerEvidence"] == { + "evidenceId": NON_OWNER_EVIDENCE_BY_TABLE[name], + "selector": {"table": name}, + } + + +def test_rls_auditor_requires_every_live_control_and_non_owner_evidence() -> None: + report = audit_rls_snapshot( + manifest=_manifest(), + snapshot=_matching_catalog_snapshot(), + passed_evidence_ids=set(NON_OWNER_EVIDENCE_BY_TABLE.values()), + ) + + assert report["passed"] is True + assert report["coverage"] == { + "numerator": 20, + "denominator": 20, + "percent": 100.0, + } + inventory = cast(dict[str, object], report["inventory"]) + global_allowlist = cast(list[dict[str, object]], report["globalAllowlist"]) + tenant_reports = cast(list[dict[str, object]], report["tenantTables"]) + assert inventory["exact"] is True + assert len(global_allowlist) == 3 + assert all(table["passed"] for table in tenant_reports) + + +def test_rls_auditor_does_not_count_force_rls_or_evidence_gaps() -> None: + snapshot = _matching_catalog_snapshot() + tables = cast(dict[str, dict[str, Any]], snapshot["tables"]) + tables["organization_record"]["rlsForced"] = False + + report = audit_rls_snapshot( + manifest=_manifest(), + snapshot=snapshot, + passed_evidence_ids=set(), + ) + + assert report["passed"] is False + assert report["coverage"] == { + "numerator": 0, + "denominator": 20, + "percent": 0.0, + } + tenant_reports = cast(list[dict[str, Any]], report["tenantTables"]) + organization_record = next( + table + for table in tenant_reports + if table["table"] == "organization_record" + ) + assert organization_record["rlsForced"] is False + assert organization_record["nonOwnerIsolation"]["passed"] is False + failures = cast(list[str], report["failures"]) + assert "organization_record: FORCE ROW LEVEL SECURITY is disabled" in failures + + +def test_rls_auditor_rejects_inventory_ownership_and_policy_mutations() -> None: + mutations: list[tuple[str, str]] = [ + ("missing_inventory", "live public tables missing from manifest"), + ("missing_ownership", "no verified Organization-inclusive ownership path"), + ("disabled_rls", "row level security is disabled"), + ("empty_policy", "live RLS policy inventory differs from manifest"), + ] + + for mutation, expected_failure in mutations: + manifest = _manifest() + snapshot = _matching_catalog_snapshot() + if mutation == "missing_inventory": + manifest_tables = cast(list[dict[str, Any]], manifest["tables"]) + manifest["tables"] = [ + table + for table in manifest_tables + if table["name"] != "organization_record" + ] + else: + live_tables = cast(dict[str, dict[str, Any]], snapshot["tables"]) + organization_record = live_tables["organization_record"] + if mutation == "missing_ownership": + organization_record["foreignKeys"] = [] + elif mutation == "disabled_rls": + organization_record["rlsEnabled"] = False + elif mutation == "empty_policy": + organization_record["policies"] = [] + + report = audit_rls_snapshot( + manifest=manifest, + snapshot=snapshot, + passed_evidence_ids=set(NON_OWNER_EVIDENCE_BY_TABLE.values()), + ) + + failures = cast(list[str], report["failures"]) + assert report["passed"] is False + assert any(expected_failure in failure for failure in failures) + + +def test_rls_auditor_pins_global_allowlist_independently_of_manifest() -> None: + manifest = _manifest() + manifest_tables = cast(list[dict[str, Any]], manifest["tables"]) + organization_record = next( + table for table in manifest_tables if table["name"] == "organization_record" + ) + organization_record["classification"] = "global" + organization_record["classificationRationale"] = "mutated to global" + + report = audit_rls_snapshot( + manifest=manifest, + snapshot=_matching_catalog_snapshot(), + passed_evidence_ids=set(NON_OWNER_EVIDENCE_BY_TABLE.values()), + ) + + assert report["passed"] is False + assert "organization_record" in cast( + dict[str, list[str]], report["globalAllowlistInventory"] + )["unexpected"] + assert any( + "global table allowlist differs from the pinned M0 allowlist" in failure + for failure in cast(list[str], report["failures"]) + ) + + +def test_rls_auditor_requires_exact_policy_inventory_and_semantics() -> None: + mutations: list[tuple[str, str]] = [ + ("extra", "live RLS policy inventory differs from manifest"), + ("permissive", "live RLS policy semantics differ from manifest"), + ("command", "live RLS policy semantics differ from manifest"), + ("roles", "live RLS policy semantics differ from manifest"), + ("using", "live RLS policy semantics differ from manifest"), + ("with_check", "live RLS policy semantics differ from manifest"), + ] + for mutation, expected_failure in mutations: + snapshot = _matching_catalog_snapshot() + live_tables = cast(dict[str, dict[str, Any]], snapshot["tables"]) + policies = cast( + list[dict[str, Any]], live_tables["organization_record"]["policies"] + ) + runtime_policy = next( + policy + for policy in policies + if policy["name"] == "organization_record_organization_isolation" + ) + if mutation == "extra": + policies.append( + { + "name": "organization_record_allow_all", + "permissive": True, + "command": "ALL", + "roles": ["context_engine_runtime"], + "using": "true", + "withCheck": "true", + } + ) + elif mutation == "permissive": + runtime_policy["permissive"] = False + elif mutation == "command": + runtime_policy["command"] = "SELECT" + elif mutation == "roles": + runtime_policy["roles"] = ["public"] + elif mutation == "using": + runtime_policy["using"] = "true" + elif mutation == "with_check": + runtime_policy["withCheck"] = "true" + + report = audit_rls_snapshot( + manifest=_manifest(), + snapshot=snapshot, + passed_evidence_ids=set(NON_OWNER_EVIDENCE_BY_TABLE.values()), + ) + + assert report["passed"] is False + assert any( + expected_failure in failure + for failure in cast(list[str], report["failures"]) + ) + + +def test_rls_auditor_accepts_postgresql_expression_rendering_only() -> None: + snapshot = _matching_catalog_snapshot() + live_tables = cast(dict[str, dict[str, Any]], snapshot["tables"]) + policies = cast( + list[dict[str, Any]], live_tables["context_run"]["policies"] + ) + reader_policy = next( + policy + for policy in policies + if policy["name"] == "context_run_context_run_reader_definer_read" + ) + using_expression = cast(str, reader_policy["using"]) + reader_policy["using"] = ( + "((" + + using_expression.replace( + " IN ('issue', 'read')", + " = ANY (ARRAY['issue'::text, 'read'::text])", + ) + .replace( + "'app.context_run_operator_ticket_mode'", + "'app.context_run_operator_ticket_mode'::text", + ) + .replace( + "'app.context_run_operator_ticket_decision_ref'", + "'app.context_run_operator_ticket_decision_ref'::text", + ) + .replace( + "'app.context_run_operator_ticket_organization_id'", + "'app.context_run_operator_ticket_organization_id'::text", + ) + .replace("''", "''::text") + + "))" + ) + + report = audit_rls_snapshot( + manifest=_manifest(), + snapshot=snapshot, + passed_evidence_ids=set(NON_OWNER_EVIDENCE_BY_TABLE.values()), + ) + + assert report["passed"] is True + + +def test_rls_auditor_requires_each_independent_behavioral_evidence_id() -> None: + passed_ids = set(NON_OWNER_EVIDENCE_BY_TABLE.values()) + passed_ids.remove("PG-WORKER-LEASE-007") + + report = audit_rls_snapshot( + manifest=_manifest(), + snapshot=_matching_catalog_snapshot(), + passed_evidence_ids=passed_ids, + ) + + assert report["passed"] is False + failed_tables = { + cast(str, table["table"]) + for table in cast(list[dict[str, Any]], report["tenantTables"]) + if table["passed"] is False + } + assert failed_tables == {"service_principal", "worker_noop_job"} + assert all( + table["nonOwnerIsolation"]["evidenceId"] != "PG-RLS-ALL-TENANT-TABLES" + for table in cast(list[dict[str, Any]], report["tenantTables"]) + ) diff --git a/tests/unit/test_policy_epoch.py b/tests/unit/test_policy_epoch.py index 94afd4d4..575b4cfd 100644 --- a/tests/unit/test_policy_epoch.py +++ b/tests/unit/test_policy_epoch.py @@ -74,6 +74,7 @@ def test_missing_or_malformed_policy_epoch_is_authority_unavailable( _close_policy_epoch_authority_scope(authority_scope) +@pytest.mark.security_evidence(id="PROP-REVOCATION-006", layer="property") def test_current_validation_re_reads_durable_epoch_and_fails_closed_on_change( ) -> None: authority_scope = _open_policy_epoch_authority_scope() diff --git a/tests/unit/test_release_promotion_properties.py b/tests/unit/test_release_promotion_properties.py index 1f3e8281..6aa65c6b 100644 --- a/tests/unit/test_release_promotion_properties.py +++ b/tests/unit/test_release_promotion_properties.py @@ -340,6 +340,7 @@ def _request( ) +@pytest.mark.security_evidence(id="PROP-RELEASE-OWNER-019", layer="property") @pytest.mark.parametrize("rejection_kind", REJECTION_KINDS) @seed(49) @settings(max_examples=3, deadline=None) diff --git a/tests/unit/test_runtime_authorized_evidence.py b/tests/unit/test_runtime_authorized_evidence.py index 4ccac5f0..4f32e423 100644 --- a/tests/unit/test_runtime_authorized_evidence.py +++ b/tests/unit/test_runtime_authorized_evidence.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from dataclasses import fields, replace from datetime import UTC, datetime @@ -78,6 +78,7 @@ RecordingContextRunPort, recording_context_run_session, ) +from tests.support.security_gate import record_security_oracles AS_OF = datetime(2026, 7, 21, 10, 0, tzinfo=UTC) ORGANIZATION_ID = UUID("81e18bca-86a1-478a-937d-7675c6fe69b0") @@ -247,6 +248,19 @@ def exact_operands() -> TrustedScopeOperands: ) +def scope_for(*candidates: CandidateRef) -> ScopeSet: + return ScopeSet( + frozenset( + ScopeTarget( + candidate.organization_id, + candidate.source_ref, + candidate.resource_ref, + ) + for candidate in candidates + ) + ) + + @contextmanager def trusted_operands( port: RecordingMaterializedPort, @@ -352,12 +366,15 @@ def trusted_operands( _close_membership_authority_scope(membership_scope) +@pytest.mark.security_evidence(id="RUNTIME-INDEX-NOT-AUTHORITY-005", layer="runtime") +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-006", layer="runtime") @pytest.mark.parametrize( "ranked", tuple(permutations((AUTHORIZED, DENIED, CROSS_ORGANIZATION))), ) def test_hostile_candidate_order_delivers_only_exact_authorized_evidence( ranked: tuple[CandidateRef, ...], + record_property: Callable[[str, object], None], ) -> None: index = HostileCandidateIndex(ranked) port = RecordingMaterializedPort() @@ -408,6 +425,29 @@ def test_hostile_candidate_order_delivers_only_exact_authorized_evidence( str(OTHER_ORGANIZATION_ID), ): assert forbidden not in rendered + unauthorized_evidence_count = sum( + item.fragment_ref + in {DENIED.fragment_ref, CROSS_ORGANIZATION.fragment_ref} + for item in package.evidence + ) + wrong_organization_effect_count = sum( + selected == locator(CROSS_ORGANIZATION) for selected in port.body_calls + ) + missing_context_fallback_count = int( + package.coverage.status != "sufficient" + or len(package.blocks) == 0 + or len(package.evidence) == 0 + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-006", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) def test_stale_scope_epoch_stops_before_candidate_or_body_io() -> None: @@ -741,6 +781,7 @@ def _canonical_empty_package(package: ContextPackage) -> tuple[object, ...]: ) +@pytest.mark.security_evidence(id="RUNTIME-NON-ENUMERATION-009", layer="runtime") def test_runtime_canonical_empty_package_is_equal_for_every_internal_branch() -> None: scenarios: tuple[ tuple[tuple[CandidateRef, ...], RecordingMaterializedPort], ... @@ -771,6 +812,7 @@ def test_runtime_canonical_empty_package_is_equal_for_every_internal_branch() -> assert packages == [packages[0]] * len(packages) +@pytest.mark.security_evidence(id="PROP-NON-ENUMERATION-009", layer="property") @pytest.mark.parametrize( "ranked", ( @@ -841,6 +883,7 @@ def test_denied_cross_organization_and_missing_candidates_share_one_runtime_outc assert forbidden_field not in rendered.casefold() +@pytest.mark.security_evidence(id="RUNTIME-TRACE-REDACTION-012", layer="runtime") def test_empty_decision_audit_is_generic_and_retains_no_denied_detail() -> None: receipt = DecisionAuditGate().record( DecisionProvenanceReceipt( @@ -1033,8 +1076,11 @@ def test_hostile_index_duplicate_candidates_are_deduplicated_before_projection() assert len(outcome.package.evidence) == 1 -def test_agent_ceiling_denial_keeps_candidate_content_out_of_package() -> None: - index = HostileCandidateIndex((AUTHORIZED,)) +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-003", layer="runtime") +def test_accept_003_agent_ceiling_delivers_only_authorized_projection( + record_property: Callable[[str, object], None], +) -> None: + index = HostileCandidateIndex((AUTHORIZED, AUTHORIZED_SECOND)) port = RecordingMaterializedPort() runtime = Runtime( required_kernel_dependencies(), @@ -1044,25 +1090,61 @@ def test_agent_ceiling_denial_keeps_candidate_content_out_of_package() -> None: ) with trusted_operands(port) as (invocation, delivery): + broad = scope_for(AUTHORIZED, AUTHORIZED_SECOND) + for operand_name in ( + "organization_boundary", + "membership_rights", + "principal_grants", + "source_native_acl", + "resource_acl", + "purpose_policy", + ): + object.__setattr__(invocation.trusted_scope_snapshot, operand_name, broad) object.__setattr__( invocation.trusted_scope_snapshot, "agent_ceiling", - ScopeSet(frozenset()), + scope_for(AUTHORIZED), ) outcome = runtime.resolve( invocation, delivery, - Acquire(need=ContextNeed(query="agent ceiling")), + Acquire(need=ContextNeed(query="agent ceiling cannot expand")), ) assert type(outcome) is Resolved - assert outcome.package.evidence == () - assert index.calls == 0 - assert port.body_calls == [] + assert index.calls == 1 + assert port.body_calls == [locator(AUTHORIZED)] + assert tuple(block.body for block in outcome.package.blocks) == ("A-safe",) + assert tuple(item.resource_ref for item in outcome.package.evidence) == ( + AUTHORIZED.resource_ref, + ) + unauthorized_evidence_count = sum( + item.resource_ref == AUTHORIZED_SECOND.resource_ref + for item in outcome.package.evidence + ) + wrong_organization_effect_count = 0 + missing_context_fallback_count = int( + outcome.package.coverage.status != "sufficient" + or not outcome.package.blocks + or not outcome.package.evidence + ) + assert unauthorized_evidence_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-003", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) -def test_request_narrowing_filters_candidate_before_body_projection() -> None: - index = HostileCandidateIndex((AUTHORIZED,)) +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-004", layer="runtime") +def test_request_narrowing_filters_candidate_before_body_projection( + record_property: Callable[[str, object], None], +) -> None: + excluded = replace(AUTHORIZED_SECOND, source_ref="source:other") + index = HostileCandidateIndex((AUTHORIZED, excluded)) port = RecordingMaterializedPort() runtime = Runtime( required_kernel_dependencies(), @@ -1072,16 +1154,50 @@ def test_request_narrowing_filters_candidate_before_body_projection() -> None: ) with trusted_operands(port) as (invocation, delivery): + broad = scope_for(AUTHORIZED, excluded) + for operand_name in ( + "organization_boundary", + "membership_rights", + "principal_grants", + "agent_ceiling", + "source_native_acl", + "resource_acl", + "purpose_policy", + ): + object.__setattr__(invocation.trusted_scope_snapshot, operand_name, broad) outcome = runtime.resolve( invocation, delivery, Acquire( need=ContextNeed(query="monotonic narrowing"), - narrowing=RequestNarrowing(source_refs=("source:other",)), + narrowing=RequestNarrowing(source_refs=(AUTHORIZED.source_ref,)), ), ) assert type(outcome) is Resolved - assert outcome.package.evidence == () - assert index.calls == 0 - assert port.body_calls == [] + assert index.calls == 1 + assert port.body_calls == [locator(AUTHORIZED)] + assert tuple(block.body for block in outcome.package.blocks) == ("A-safe",) + assert tuple(item.source_ref for item in outcome.package.evidence) == ( + AUTHORIZED.source_ref, + ) + unauthorized_evidence_count = sum( + item.source_ref == excluded.source_ref + for item in outcome.package.evidence + ) + wrong_organization_effect_count = 0 + missing_context_fallback_count = int( + outcome.package.coverage.status != "sufficient" + or not outcome.package.blocks + or not outcome.package.evidence + ) + assert unauthorized_evidence_count == 0 + assert wrong_organization_effect_count == 0 + assert missing_context_fallback_count == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-004", + unauthorized_evidence_count=unauthorized_evidence_count, + wrong_organization_effect_count=wrong_organization_effect_count, + missing_context_fallback_count=missing_context_fallback_count, + ) diff --git a/tests/unit/test_runtime_unavailable_capabilities.py b/tests/unit/test_runtime_unavailable_capabilities.py index 042b1d1c..063e2542 100644 --- a/tests/unit/test_runtime_unavailable_capabilities.py +++ b/tests/unit/test_runtime_unavailable_capabilities.py @@ -461,6 +461,7 @@ def test_replacing_the_mandatory_capability_gate_is_rejected_before_io() -> None assert twin.calls == (0, 0, 0) +@pytest.mark.security_evidence(id="PROP-CITATION-AUTH-010", layer="property") def test_capability_declarations_are_closed_and_m0_does_not_false_green_carriers( ) -> None: assert capability_module.__all__ == ["RuntimeCapability"] diff --git a/tests/unit/test_schema_security_manifest.py b/tests/unit/test_schema_security_manifest.py index 5c850029..7086e430 100644 --- a/tests/unit/test_schema_security_manifest.py +++ b/tests/unit/test_schema_security_manifest.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import Any, cast +import pytest + ROOT = Path(__file__).parents[2] MANIFEST_PATH = ROOT / "engine/persistence/schema_security_manifest.yaml" @@ -23,6 +25,7 @@ def table_entries(document: dict[str, Any]) -> dict[str, dict[str, Any]]: } +@pytest.mark.security_evidence(id="PROP-TENANT-OWNERSHIP-001", layer="property") def test_manifest_classifies_the_exact_issue_49_release_schema() -> None: """PROP-TENANT-OWNERSHIP-001: no current table is left unclassified.""" @@ -608,6 +611,7 @@ def test_membership_manifest_requires_exact_user_actor_and_read_only_runtime() - assert membership_property in expression +@pytest.mark.security_evidence(id="PROP-RLS-FAIL-CLOSED-003", layer="property") def test_tenant_owned_manifest_entry_preserves_every_security_property() -> None: """PROP-TENANT-FK-002/PROP-RLS-FAIL-CLOSED-003 structural properties.""" @@ -682,6 +686,7 @@ def test_tenant_owned_manifest_entry_preserves_every_security_property() -> None } +@pytest.mark.security_evidence(id="PROP-TENANT-FK-002", layer="property") def test_content_manifest_preserves_lineage_visibility_and_immutability() -> None: entries = table_entries(manifest()) resource = entries["context_resource"] diff --git a/tests/unit/test_ticket_audience_separation.py b/tests/unit/test_ticket_audience_separation.py index 0549e125..022e4d0e 100644 --- a/tests/unit/test_ticket_audience_separation.py +++ b/tests/unit/test_ticket_audience_separation.py @@ -339,6 +339,7 @@ def test_valid_tickets_invoke_only_their_bound_synthetic_effects() -> None: assert channel.effects == 1 +@pytest.mark.security_evidence(id="PROP-ACTION-SEPARATION-014", layer="property") def test_public_ticket_types_and_server_side_issuers_are_structurally_distinct() -> ( None ): @@ -540,7 +541,9 @@ def test_forged_exact_identity_is_normalized_before_any_effect() -> None: assert provider.effects == 0 -def test_each_ticket_is_rejected_by_the_other_plane_and_deserializer() -> None: +@pytest.mark.security_evidence(id="RUNTIME-ACTION-SEPARATION-014", layer="runtime") +def test_each_ticket_is_rejected_by_the_other_plane_and_deserializer( +) -> None: provider = _Provider() channel = _Channel() @@ -588,15 +591,17 @@ def test_each_ticket_is_rejected_by_the_other_plane_and_deserializer() -> None: read_ticket.serialize(), keyring=KEYRING ), ) + rejections: list[TicketNotAvailable] = [] for rejected_use in rejected_uses: with pytest.raises( TicketNotAvailable, match="^capability not available$", - ): + ) as error: rejected_use() + rejections.append(error.value) - assert provider.effects == 0 - assert channel.effects == 0 + assert len(rejections) == len(rejected_uses) + assert provider.effects == channel.effects == 0 @pytest.mark.parametrize("segment_index", [0, 1, 2]) diff --git a/tests/unit/test_worker_lease.py b/tests/unit/test_worker_lease.py index 530161df..6cc7ba37 100644 --- a/tests/unit/test_worker_lease.py +++ b/tests/unit/test_worker_lease.py @@ -137,6 +137,7 @@ def test_valid_lease_verifies_for_its_exact_job_and_worker_audience() -> None: assert claims.expires_at == NOW + timedelta(minutes=5) +@pytest.mark.security_evidence(id="PROP-WORKER-LEASE-007", layer="property") @pytest.mark.parametrize( "override", [ From 17796a6a232a39ba0800abd46e68b8b404943cd0 Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 22 Jul 2026 18:35:08 +0800 Subject: [PATCH 2/3] fix: isolate security gate pytest environment --- scripts/security_gate/runner.py | 3 ++ tests/catalog/test_m0_security_gate.py | 49 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/scripts/security_gate/runner.py b/scripts/security_gate/runner.py index f8cd502c..9a89fe1b 100644 --- a/scripts/security_gate/runner.py +++ b/scripts/security_gate/runner.py @@ -57,6 +57,7 @@ ":(exclude).context-engine/**", ":(exclude).harness/**", ) +_AMBIENT_PYTEST_CONTROL_KEYS = ("PYTEST_ADDOPTS", "PYTEST_PLUGINS") _ENVIRONMENT_KEY = re.compile(r"^[A-Z][A-Z0-9_]*$") _ALLOWED_DATABASE_ENVIRONMENT_KEYS = frozenset( { @@ -680,6 +681,8 @@ def run_gate( runner_execution_id = secrets.token_hex(32) command = build_pytest_command(selectors, raw_path=raw_path) process_environment = dict(os.environ) + for key in _AMBIENT_PYTEST_CONTROL_KEYS: + process_environment.pop(key, None) process_environment.update(environment) process_environment[RAW_EXECUTION_ID_ENVIRONMENT] = runner_execution_id with suppress(FileNotFoundError): diff --git a/tests/catalog/test_m0_security_gate.py b/tests/catalog/test_m0_security_gate.py index 74d9c0e5..1d525551 100644 --- a/tests/catalog/test_m0_security_gate.py +++ b/tests/catalog/test_m0_security_gate.py @@ -802,6 +802,55 @@ def executor( assert called is False +def test_gate_scrubs_ambient_pytest_control_variables( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("PYTEST_ADDOPTS", "-k injected-selector") + monkeypatch.setenv("PYTEST_PLUGINS", "hostile.pytest_plugin") + monkeypatch.setattr( + "scripts.security_gate.runner._load_gate_inputs", + lambda _paths, _validator: ( + {"fixtures": []}, + {}, + registry(), + {}, + {}, + ), + ) + monkeypatch.setattr( + "scripts.security_gate.runner.load_database_environment", lambda _path: {} + ) + monkeypatch.setattr( + "scripts.security_gate.runner._alembic_head", lambda _paths: "head" + ) + monkeypatch.setattr( + "scripts.security_gate.runner._live_database_revision", + lambda _environment: "head", + ) + + def executor( + command: Sequence[str], *, cwd: Path, env: Mapping[str, str] + ) -> int: + del command, cwd + assert "PYTEST_ADDOPTS" not in env + assert "PYTEST_PLUGINS" not in env + raise RuntimeError("environment inspected") + + paths = GatePaths( + repository_root=tmp_path, + catalog=tmp_path / "catalog.json", + catalog_schema=tmp_path / "catalog-schema.json", + registry=tmp_path / "registry.json", + registry_schema=tmp_path / "registry-schema.json", + manifest=tmp_path / "manifest.json", + database_environment=tmp_path / "database.env", + output_directory=tmp_path / "artifacts", + ) + + with pytest.raises(RuntimeError, match="environment inspected"): + run_gate(paths, pytest_executor=executor) + + def test_gate_rejects_stale_or_spoofed_raw_execution( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 06c8b9ab93a8131dd3e41d7815fd915b3961df6d Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 22 Jul 2026 18:42:37 +0800 Subject: [PATCH 3/3] fix: ground security gate failure evidence --- eval/catalogs/m0-security-evidence.yaml | 2 +- scripts/security_gate/report.py | 4 ++ tests/catalog/test_m0_security_gate.py | 25 +++++++++ tests/support/security_gate.py | 45 ---------------- tests/unit/test_m0_delivery_carriers.py | 54 ++----------------- tests/unit/test_ticket_audience_separation.py | 50 ++++++++++++++++- 6 files changed, 82 insertions(+), 98 deletions(-) diff --git a/eval/catalogs/m0-security-evidence.yaml b/eval/catalogs/m0-security-evidence.yaml index accd237a..44f60943 100644 --- a/eval/catalogs/m0-security-evidence.yaml +++ b/eval/catalogs/m0-security-evidence.yaml @@ -99,7 +99,7 @@ {"id": "FIXTURE-ACCEPT-009", "layer": "runtime", "selector": "tests/unit/test_http_unavailable_capabilities.py::test_accept_009_server_owned_unavailable_source_paths_are_generic_and_zero_io"}, {"id": "FIXTURE-ACCEPT-010", "layer": "runtime", "selector": "tests/unit/test_http_unavailable_capabilities.py::test_accept_010_open_citation_is_generic_and_zero_io"}, {"id": "FIXTURE-ACCEPT-011", "layer": "runtime", "selector": "tests/integration/test_runtime_non_enumeration_integration.py::test_real_postgres_http_denied_and_missing_are_externally_equivalent"}, - {"id": "FIXTURE-ACCEPT-012", "layer": "runtime", "selector": "tests/unit/test_m0_delivery_carriers.py::test_accept_012_action_plane_is_unavailable_before_ticket_or_effect"} + {"id": "FIXTURE-ACCEPT-012", "layer": "runtime", "selector": "tests/unit/test_ticket_audience_separation.py::test_accept_012_context_read_ticket_cannot_create_an_action_effect"} ], "invariantMappings": [ {"invariantRef": "TENANT-OWNERSHIP-001", "evidenceRefs": {"property": ["PROP-TENANT-OWNERSHIP-001"], "postgres": ["PG-RLS-ALL-TENANT-TABLES"], "runtime": ["RUNTIME-TENANT-OWNERSHIP-001"]}}, diff --git a/scripts/security_gate/report.py b/scripts/security_gate/report.py index 12bf9dcd..8dd15ec5 100644 --- a/scripts/security_gate/report.py +++ b/scripts/security_gate/report.py @@ -574,8 +574,12 @@ def build_release_gate_report( and rls_audit.get("passed") is True and not provenance_failures ) + rls_failures = _strings(rls_audit.get("failures")) + if rls_audit.get("passed") is not True and not rls_failures: + rls_failures = ["RLS audit did not pass"] security_failures = [ *_strings(reconciliation.get("failures")), + *rls_failures, *provenance_failures, ] security = { diff --git a/tests/catalog/test_m0_security_gate.py b/tests/catalog/test_m0_security_gate.py index 1d525551..e4fba287 100644 --- a/tests/catalog/test_m0_security_gate.py +++ b/tests/catalog/test_m0_security_gate.py @@ -503,6 +503,31 @@ def test_release_report_keeps_four_gates_independent_and_has_no_score() -> None: assert canonical_digest(report) == canonical_digest(copy.deepcopy(report)) +@pytest.mark.parametrize( + ("rls_audit", "expected_failure"), + [ + ( + {"passed": False, "failures": ["tenant table missing FORCE RLS"]}, + "tenant table missing FORCE RLS", + ), + ({"passed": False}, "RLS audit did not pass"), + ], +) +def test_release_report_surfaces_rls_failure_reason( + rls_audit: dict[str, object], expected_failure: str +) -> None: + report = build_release_gate_report( + reconciliation=reconcile_execution(registry(), raw_execution()), + rls_audit=rls_audit, + provenance=complete_provenance(), + raw_result_digest="b" * 64, + ) + + gates = cast(dict[str, dict[str, Any]], report["gates"]) + assert gates["Security"]["status"] == "fail" + assert expected_failure in gates["Security"]["failures"] + + @pytest.mark.parametrize( "mutation", [ diff --git a/tests/support/security_gate.py b/tests/support/security_gate.py index ef15f35b..b76d7b43 100644 --- a/tests/support/security_gate.py +++ b/tests/support/security_gate.py @@ -2,52 +2,7 @@ from __future__ import annotations -import importlib from collections.abc import Callable -from dataclasses import dataclass - - -@dataclass(frozen=True, slots=True) -class UnavailableCarrierProbe: - """Observed result of attempting one explicitly absent M0 carrier.""" - - request: str - external_status: int - external_code: str - carrier_available: bool - ticket_created: bool - sender_call_count: int - effect_count: int - - -def probe_unavailable_module_carrier( - *, module_name: str, request: str, unavailable_code: str -) -> UnavailableCarrierProbe: - """Attempt an M0 carrier and normalize only its exact module absence.""" - - try: - importlib.import_module(module_name) - except ModuleNotFoundError as error: - if error.name != module_name: - raise - return UnavailableCarrierProbe( - request=request, - external_status=404, - external_code=unavailable_code, - carrier_available=False, - ticket_created=False, - sender_call_count=0, - effect_count=0, - ) - return UnavailableCarrierProbe( - request=request, - external_status=500, - external_code="unexpected_active_carrier", - carrier_available=True, - ticket_created=False, - sender_call_count=0, - effect_count=0, - ) def record_security_oracles( diff --git a/tests/unit/test_m0_delivery_carriers.py b/tests/unit/test_m0_delivery_carriers.py index d0abb08b..87a6a5e4 100644 --- a/tests/unit/test_m0_delivery_carriers.py +++ b/tests/unit/test_m0_delivery_carriers.py @@ -1,71 +1,23 @@ from __future__ import annotations import importlib.util -from collections.abc import Callable import pytest import engine.runtime as runtime -from tests.support.security_gate import ( - probe_unavailable_module_carrier, - record_security_oracles, -) @pytest.mark.security_evidence(id="RUNTIME-EGRESS-011", layer="runtime") def test_m0_egress_carrier_is_unavailable_before_model_or_sender_bytes() -> None: - """M0 cannot construct the future delivery types or their trusted process.""" + """M0 cannot construct the future delivery/action types or trusted process.""" assert importlib.util.find_spec("bot_delivery") is None + assert importlib.util.find_spec("action_plane") is None for public_name in ( + "ActionPlane", "AuthorizedModelInput", "EgressGrant", "ModelGateway", "Sender", ): assert not hasattr(runtime, public_name) - - -@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-012", layer="runtime") -def test_accept_012_action_plane_is_unavailable_before_ticket_or_effect( - record_property: Callable[[str, object], None], -) -> None: - """The documented M0 form has no ActionPlane, ticket, Sender, or effect.""" - - forbidden_runtime_authorities = tuple( - public_name - for public_name in ("ActionPlane", "AuthorizedModelInput", "EgressGrant") - if hasattr(runtime, public_name) - ) - probe = probe_unavailable_module_carrier( - module_name="action_plane", - request="ActionPlane.prepare(send_message, conversation-b, digest-b)", - unavailable_code="action_not_available", - ) - - assert probe.request == ( - "ActionPlane.prepare(send_message, conversation-b, digest-b)" - ) - assert (probe.external_status, probe.external_code) == ( - 404, - "action_not_available", - ) - assert probe.carrier_available is False - assert probe.ticket_created is False - assert probe.sender_call_count == 0 - assert probe.effect_count == 0 - assert importlib.util.find_spec("bot_delivery") is None - assert forbidden_runtime_authorities == () - - record_security_oracles( - record_property, - fixture_ref="ACCEPT-012", - unauthorized_evidence_count=len(forbidden_runtime_authorities), - wrong_organization_effect_count=probe.effect_count, - missing_context_fallback_count=int( - probe.external_status != 404 - or probe.external_code != "action_not_available" - or probe.ticket_created - or probe.sender_call_count != 0 - ), - ) diff --git a/tests/unit/test_ticket_audience_separation.py b/tests/unit/test_ticket_audience_separation.py index 022e4d0e..b20ebc14 100644 --- a/tests/unit/test_ticket_audience_separation.py +++ b/tests/unit/test_ticket_audience_separation.py @@ -2,7 +2,7 @@ import hmac import json import pickle -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from dataclasses import fields from datetime import UTC, datetime, timedelta @@ -61,6 +61,7 @@ TicketRejectionAuditReceipt, TicketRejectionCategory, ) +from tests.support.security_gate import record_security_oracles NOW = datetime(2026, 7, 22, 8, 0, tzinfo=UTC) ORGANIZATION_ID = UUID("81e18bca-86a1-478a-937d-7675c6fe69b0") @@ -604,6 +605,53 @@ def test_each_ticket_is_rejected_by_the_other_plane_and_deserializer( assert provider.effects == channel.effects == 0 +@pytest.mark.security_evidence(id="FIXTURE-ACCEPT-012", layer="runtime") +def test_accept_012_context_read_ticket_cannot_create_an_action_effect( + record_property: Callable[[str, object], None], +) -> None: + """Invoke the highest current action seam with read-only authority.""" + + channel = _Channel() + with _identity() as identity: + read_ticket = ContextAccessTicketIssuer( + keyring=KEYRING, + organization_id=ORGANIZATION_ID, + provider_ref="provider-a", + clock=lambda: NOW, + ).issue(identity) + handler = ActionTicketNoopHandler( + keyring=KEYRING, + organization_id=ORGANIZATION_ID, + channel_ref="conversation-b", + channel=channel, + clock=lambda: NOW, + ) + + with pytest.raises( + TicketNotAvailable, + match="^capability not available$", + ) as error: + handler.perform( + ticket=cast(ActionTicket, read_ticket), + identity=identity, + ) + + rejection = error.value + non_generic_rejection_count = int( + rejection.audit_receipt.category + is not TicketRejectionCategory.CAPABILITY_NOT_AVAILABLE + or rejection.audit_receipt.denied_detail_count != 0 + ) + assert channel.effects == 0 + record_security_oracles( + record_property, + fixture_ref="ACCEPT-012", + unauthorized_evidence_count=0, + wrong_organization_effect_count=channel.effects, + missing_context_fallback_count=non_generic_rejection_count, + ) + + @pytest.mark.parametrize("segment_index", [0, 1, 2]) def test_one_bit_tamper_in_each_signed_segment_is_rejected_before_effect( segment_index: int,